Skip to content

Commit c30955f

Browse files
simensrostadsimensrostad
committed
.github: tests: add nRF Cloud connect hardware test
Add a CI workflow and on-target test suite that provisions nRF Cloud credentials on a fresh device, onboards it via `onboard.csv`, and verifies `"Cloud connected"` appears in the serial log. - `cloud-connect-test.yml`: reusable workflow that builds/flashes firmware, runs the pytest suite, and uploads artifacts on failure - `test.yml`: top-level workflow triggered on push to main; resolves test matrix from `tests.yml` and fans out to the reusable workflow - `tests.yml`: catalog of test cases with board, hardware, and nRF Cloud config - `tests/on_target/`: pytest suite with fixtures for flashing, serial capture (`Uart`), credential installation, and device onboarding/deletion via the nRF Cloud REST API Signed-off-by: Simen S. Røstad <simen.rostad@nordicsemi.no>
1 parent 1ae8c7e commit c30955f

17 files changed

Lines changed: 1044 additions & 0 deletions

‎.github/test/cloud/nrf-cloud-device.sh‎

Lines changed: 108 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,108 @@
1+
#!/usr/bin/env bash
2+
# SPDX-License-Identifier: LicenseRef-Nordic-5-Clause
3+
# nRF Cloud device management for on-target provisioning tests.
4+
#
5+
# delete-if-exists only removes the single device ID passed on the command line
6+
# after validating it against CI_NRF54L15_DEVICE_ID. It never bulk-deletes.
7+
set -eu
8+
9+
: "${NRF_CLOUD_API_KEY:?NRF_CLOUD_API_KEY is required}"
10+
: "${CI_NRF54L15_DEVICE_ID:?CI_NRF54L15_DEVICE_ID is required for delete-if-exists}"
11+
12+
API_HOST="${NRF_CLOUD_API_HOST:-https://api.nrfcloud.com/v1}"
13+
14+
normalize_device_id() {
15+
printf '%s' "$1" | tr '[:lower:]' '[:upper:]'
16+
}
17+
18+
validate_dut_device_id() {
19+
local device_id="$1"
20+
local normalized expected
21+
22+
normalized="$(normalize_device_id "${device_id}")"
23+
expected="$(normalize_device_id "${CI_NRF54L15_DEVICE_ID}")"
24+
25+
if ! printf '%s' "${normalized}" | grep -Eq '^[0-9A-F]{16}$'; then
26+
echo "::error::Refusing nRF Cloud operation: invalid device ID '${device_id}'" >&2
27+
return 1
28+
fi
29+
30+
if [ "${normalized}" != "${expected}" ]; then
31+
echo "::error::Refusing delete: device ID ${normalized}" \
32+
" does not match configured DUT allowlist ${expected}" >&2
33+
return 1
34+
fi
35+
36+
printf '%s' "${normalized}"
37+
}
38+
39+
device_url() {
40+
printf '%s/devices/%s' "${API_HOST}" "$1"
41+
}
42+
43+
device_exists() {
44+
local device_id="$1"
45+
local status
46+
47+
status="$(curl -sS -o /dev/null -w '%{http_code}' \
48+
-H "Authorization: Bearer ${NRF_CLOUD_API_KEY}" \
49+
"$(device_url "${device_id}")")"
50+
51+
case "${status}" in
52+
200) return 0 ;;
53+
404) return 1 ;;
54+
*)
55+
echo "::error::Unexpected status ${status} checking device ${device_id}" >&2
56+
return 2
57+
;;
58+
esac
59+
}
60+
61+
cmd_delete_if_exists() {
62+
local device_id="$1"
63+
local validated
64+
65+
validated="$(validate_dut_device_id "${device_id}")"
66+
67+
if ! device_exists "${validated}"; then
68+
echo "DUT ${validated} is not registered in nRF Cloud; nothing to delete"
69+
return 0
70+
fi
71+
72+
echo "Deleting only the configured DUT ${validated} from nRF Cloud"
73+
curl -sS -f -X DELETE \
74+
-H "Authorization: Bearer ${NRF_CLOUD_API_KEY}" \
75+
"$(device_url "${validated}")"
76+
echo "DUT ${validated} deleted"
77+
}
78+
79+
cmd_onboard() {
80+
local csv_file="$1"
81+
82+
if [ ! -f "${csv_file}" ]; then
83+
echo "::error::Onboarding CSV not found: ${csv_file}" >&2
84+
return 1
85+
fi
86+
87+
echo "Onboarding DUT from ${csv_file}"
88+
nrf_cloud_onboard --api-key "${NRF_CLOUD_API_KEY}" --csv "${csv_file}"
89+
}
90+
91+
usage() {
92+
echo "Usage: $0 {delete-if-exists|onboard} ..." >&2
93+
exit 1
94+
}
95+
96+
case "${1:-}" in
97+
delete-if-exists)
98+
[ "$#" -eq 2 ] || usage
99+
cmd_delete_if_exists "$2"
100+
;;
101+
onboard)
102+
[ "$#" -eq 2 ] || usage
103+
cmd_onboard "$2"
104+
;;
105+
*)
106+
usage
107+
;;
108+
esac

‎.github/test/cloud/run.sh‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
#!/usr/bin/env bash
2+
# SPDX-License-Identifier: LicenseRef-Nordic-5-Clause
3+
set -eu
4+
5+
: "${REPO_ROOT:?REPO_ROOT is required}"
6+
: "${TEST_JSON:?TEST_JSON is required}"
7+
8+
mkdir -p "${REPO_ROOT}/build"
9+
10+
pip install --quiet -r "${REPO_ROOT}/tests/on_target/requirements.txt"
11+
12+
export PYTHONPATH="${REPO_ROOT}/tests/on_target${PYTHONPATH:+:${PYTHONPATH}}"
13+
14+
python3 -m pytest \
15+
"${REPO_ROOT}/tests/on_target/tests/test_cloud/test_cloud_connect.py" \
16+
-c "${REPO_ROOT}/tests/on_target/tests/pytest.ini" \
17+
--html="${REPO_ROOT}/build/hardware-pytest-report.html" \
18+
--self-contained-html \
19+
-v

‎.github/test/scripts/load-test.sh‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
#!/usr/bin/env bash
2+
# SPDX-License-Identifier: LicenseRef-Nordic-5-Clause
3+
set -eu
4+
5+
ROOT="${1:-.}"
6+
TEST_ID="${2:?test id required}"
7+
8+
python3 - "$ROOT" "$TEST_ID" <<'PY'
9+
import json
10+
import sys
11+
from pathlib import Path
12+
13+
import yaml
14+
15+
root = Path(sys.argv[1])
16+
test_id = sys.argv[2]
17+
catalog = yaml.safe_load((root / ".github/test/tests.yml").read_text(encoding="utf-8"))
18+
19+
for test in catalog.get("tests", []):
20+
if test.get("id") == test_id:
21+
print(json.dumps(test))
22+
break
23+
else:
24+
raise SystemExit(f"test id not found: {test_id}")
25+
PY

‎.github/test/scripts/resolve-matrix.sh‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
#!/usr/bin/env bash
2+
# SPDX-License-Identifier: LicenseRef-Nordic-5-Clause
3+
set -eu
4+
5+
ROOT="${ROOT:-.}"
6+
TEST_FILTER="${TEST_FILTER:-all}"
7+
8+
python3 - "$ROOT" "$TEST_FILTER" <<'PY'
9+
import json
10+
import os
11+
import sys
12+
from pathlib import Path
13+
14+
import yaml
15+
16+
root = Path(sys.argv[1])
17+
test_filter = sys.argv[2]
18+
catalog = yaml.safe_load((root / ".github/test/tests.yml").read_text(encoding="utf-8"))
19+
20+
tests = [t for t in catalog.get("tests", []) if t.get("enabled", True)]
21+
if test_filter != "all":
22+
tests = [t for t in tests if t.get("id") == test_filter]
23+
24+
matrix = json.dumps(tests)
25+
26+
if github_output := os.environ.get("GITHUB_OUTPUT"):
27+
with open(github_output, "a", encoding="utf-8") as handle:
28+
handle.write(f"matrix={matrix}\n")
29+
else:
30+
print(matrix)
31+
PY

‎.github/test/tests.yml‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
tests:
2+
- id: 91m1-cloud-connect-nrf54l15
3+
enabled: true
4+
type: cloud_connect
5+
app: 91m1
6+
board: nrf54l15dk/nrf54l15/cpuapp/ns
7+
hardware:
8+
segger_sn_var: CI_NRF54L15_SEGGER_SN
9+
serial_port_var: CI_NRF54L15_SERIAL_PORT
10+
# Required allowlist: only this 16-hex nRF54L15 host ID may be deleted from nRF Cloud.
11+
device_id_var: CI_NRF54L15_DEVICE_ID
12+
# boot + slot0 from nrf54l15_cpuapp_mcuboot_partitions.dtsi (PSA at 0x175000+)
13+
internal_erase_end: "0x16b000"
14+
erase_external_slot: true
15+
erase_credentials: true
16+
nrf_cloud:
17+
api_key_var: NRF_CLOUD_API_KEY
18+
ca_cert_var: NRF_CLOUD_CA_CERT
19+
ca_key_var: NRF_CLOUD_CA_KEY

‎.github/workflows/cloud-connect-test.yml‎

Lines changed: 107 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,107 @@
1+
# Reusable nRF Cloud connect hardware test workflow.
2+
#
3+
# Provisions credentials on a device with no stored credentials, onboards via
4+
# onboard.csv, and verifies "Cloud connected" in the serial log.
5+
#
6+
# When adding tests, update workflow_dispatch options below and .github/test/tests.yml.
7+
8+
name: Cloud Connect Test
9+
10+
on:
11+
workflow_call:
12+
inputs:
13+
test_id:
14+
required: true
15+
type: string
16+
workflow_dispatch:
17+
inputs:
18+
test:
19+
description: Cloud connect test to run
20+
required: true
21+
type: choice
22+
default: 91m1-cloud-connect-nrf54l15
23+
options:
24+
- 91m1-cloud-connect-nrf54l15
25+
26+
permissions:
27+
contents: read
28+
29+
jobs:
30+
cloud_connect:
31+
runs-on: self-hosted
32+
container:
33+
image: ghcr.io/zephyrproject-rtos/ci:v0.29.2
34+
options: --privileged -v /dev:/dev
35+
env:
36+
CMAKE_PREFIX_PATH: /opt/toolchains
37+
NRF_CLOUD_API_KEY: ${{ secrets.NRF_CLOUD_API_KEY }}
38+
NRF_CLOUD_CA_CERT: ${{ secrets.NRF_CLOUD_CA_CERT }}
39+
NRF_CLOUD_CA_KEY: ${{ secrets.NRF_CLOUD_CA_KEY }}
40+
CI_NRF54L15_SEGGER_SN: ${{ vars.CI_NRF54L15_SEGGER_SN }}
41+
CI_NRF54L15_SERIAL_PORT: ${{ vars.CI_NRF54L15_SERIAL_PORT }}
42+
CI_NRF54L15_DEVICE_ID: ${{ vars.CI_NRF54L15_DEVICE_ID }}
43+
steps:
44+
- name: Checkout
45+
uses: actions/checkout@v6
46+
with:
47+
path: serial-modem-host-applications
48+
49+
- name: Load test configuration
50+
id: config
51+
working-directory: serial-modem-host-applications
52+
run: |
53+
set -eu
54+
pip install --quiet pyyaml
55+
56+
if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then
57+
TEST_ID="${{ github.event.inputs.test }}"
58+
else
59+
TEST_ID="${{ inputs.test_id }}"
60+
fi
61+
62+
TEST_JSON="$(ROOT=. .github/test/scripts/load-test.sh "$TEST_ID")"
63+
{
64+
echo "test_json<<EOF"
65+
echo "$TEST_JSON"
66+
echo "EOF"
67+
} >> "$GITHUB_OUTPUT"
68+
69+
- name: Initialize West workspace
70+
uses: ./serial-modem-host-applications/.github/actions/west-init
71+
72+
- name: Install test dependencies
73+
run: |
74+
nrfutil install device
75+
76+
- name: Run cloud connect test
77+
working-directory: serial-modem-host-applications
78+
env:
79+
REPO_ROOT: ${{ github.workspace }}/serial-modem-host-applications
80+
TEST_JSON: ${{ steps.config.outputs.test_json }}
81+
run: |
82+
chmod +x .github/test/cloud/*.sh .github/test/scripts/*.sh
83+
.github/test/cloud/run.sh
84+
85+
- name: Upload test summary
86+
if: always()
87+
uses: actions/upload-artifact@v4
88+
with:
89+
name: hardware-test-summary-${{ github.run_id }}
90+
path: serial-modem-host-applications/build/hardware-test-summary.txt
91+
if-no-files-found: ignore
92+
93+
- name: Upload serial log
94+
if: failure()
95+
uses: actions/upload-artifact@v4
96+
with:
97+
name: hardware-serial-log-${{ github.run_id }}
98+
path: serial-modem-host-applications/build/hardware-serial.log
99+
if-no-files-found: ignore
100+
101+
- name: Upload pytest report
102+
if: failure()
103+
uses: actions/upload-artifact@v4
104+
with:
105+
name: hardware-pytest-report-${{ github.run_id }}
106+
path: serial-modem-host-applications/build/hardware-pytest-report.html
107+
if-no-files-found: ignore

‎.github/workflows/test.yml‎

Lines changed: 78 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,78 @@
1+
# On-target hardware test: nRF Cloud provisioning and connection verification.
2+
#
3+
# Prerequisites:
4+
# - nRF54L15 DK + nRF9151 Serial Modem wired and configured per applications/91m1/doc
5+
# - Self-signed CA certificate and private key stored in GitHub secrets
6+
# - nRF Cloud API key with permission to onboard/delete devices
7+
#
8+
# GitHub variables:
9+
# CI_NRF54L15_SEGGER_SN=<SEGGER serial of the nRF54L15 DK J-Link>
10+
# CI_NRF54L15_DEVICE_ID=<16-hex host device ID allowlist; only this ID may be deleted>
11+
# CI_NRF54L15_SERIAL_PORT=<optional explicit /dev/tty... path; auto-resolved if unset>
12+
#
13+
# GitHub secrets:
14+
# NRF_CLOUD_API_KEY, NRF_CLOUD_CA_CERT, NRF_CLOUD_CA_KEY
15+
#
16+
# Verification flow:
17+
# 1. Clear application firmware and TF-M credential storage.
18+
# 2. Start pyserial capture, build and flash firmware.
19+
# 3. Read device ID from boot log; confirm missing-credentials warning.
20+
# 4. Delete only CI_NRF54L15_DEVICE_ID from nRF Cloud if present; provision and onboard.
21+
# 5. Wait for "Cloud connected" in the serial log.
22+
23+
name: Test
24+
25+
on:
26+
push:
27+
branches:
28+
- main
29+
workflow_dispatch:
30+
inputs:
31+
test:
32+
description: Which test to run
33+
required: true
34+
type: choice
35+
default: all
36+
options:
37+
- all
38+
- 91m1-cloud-connect-nrf54l15
39+
40+
permissions:
41+
contents: read
42+
43+
concurrency:
44+
group: hardware-test-${{ github.event.pull_request.number || github.run_id }}
45+
cancel-in-progress: false
46+
47+
jobs:
48+
plan:
49+
runs-on: self-hosted
50+
outputs:
51+
matrix: ${{ steps.resolve.outputs.matrix }}
52+
steps:
53+
- name: Checkout
54+
uses: actions/checkout@v6
55+
with:
56+
path: serial-modem-host-applications
57+
58+
- name: Resolve test matrix
59+
id: resolve
60+
working-directory: serial-modem-host-applications
61+
env:
62+
TEST_FILTER: ${{ github.event_name == 'workflow_dispatch' && inputs.test || 'all' }}
63+
run: |
64+
pip install --quiet pyyaml
65+
ROOT=. .github/test/scripts/resolve-matrix.sh
66+
67+
cloud_connect:
68+
needs: plan
69+
if: ${{ needs.plan.outputs.matrix != '[]' && needs.plan.outputs.matrix != '' }}
70+
strategy:
71+
fail-fast: false
72+
max-parallel: 1
73+
matrix:
74+
include: ${{ fromJson(needs.plan.outputs.matrix) }}
75+
uses: ./.github/workflows/cloud-connect-test.yml
76+
with:
77+
test_id: ${{ matrix.id }}
78+
secrets: inherit

‎tests/on_target/requirements.txt‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
pytest
2+
pytest-html
3+
pyserial
4+
nrfcloud-utils

0 commit comments

Comments
 (0)