Skip to content

Commit 12e2280

Browse files
simensrostadsimensrostad
committed
tests: on_target: add application FOTA-via-cloud-sync test
Add end-to-end hardware test that builds baseline and update firmware, deploys through Memfault, and verifies automatic OTA via nRF Cloud. Includes Memfault OTA helpers, cloud device cleanup, CI workflow updates, and tightened wait timeouts for reliable CI runs. Signed-off-by: Simen S. Røstad <simen.rostad@nordicsemi.no>
1 parent 351df4c commit 12e2280

12 files changed

Lines changed: 1269 additions & 42 deletions

File tree

‎.github/test/tests.yml‎

Lines changed: 19 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,9 +16,14 @@ tests:
1616
api_key_var: NRF_CLOUD_API_KEY
1717
ca_cert_var: NRF_CLOUD_CA_CERT
1818
ca_key_var: NRF_CLOUD_CA_KEY
19+
memfault:
20+
org_token_var: MEMFAULT_ORG_TOKEN
21+
org_var: MEMFAULT_ORG
22+
project_var: MEMFAULT_PROJECT
23+
fleet_sampling_configuration_var: MEMFAULT_FLEET_SAMPLING_CONFIGURATION
1924

2025
- id: 91m1_ppp-application-fota-nrf54l15
21-
enabled: false
26+
enabled: true
2227
type: fota
2328
pytest_dir: tests/on_target/tests/test_fota/
2429
app: 91m1_ppp
@@ -28,3 +33,16 @@ tests:
2833
serial_port_var: CI_NRF54L15_SERIAL_PORT
2934
console_vcom: 1
3035
device_id_var: CI_NRF54L15_DEVICE_ID
36+
nrf_cloud:
37+
api_key_var: NRF_CLOUD_API_KEY
38+
ca_cert_var: NRF_CLOUD_CA_CERT
39+
ca_key_var: NRF_CLOUD_CA_KEY
40+
memfault:
41+
org_token_var: MEMFAULT_ORG_TOKEN
42+
org_var: MEMFAULT_ORG
43+
project_var: MEMFAULT_PROJECT
44+
fleet_sampling_configuration_var: MEMFAULT_FLEET_SAMPLING_CONFIGURATION
45+
cohort: ci-91m1-fota-nrf54l15
46+
cohort_name: CI 91m1 FOTA nRF54L15
47+
baseline_version: "1.0"
48+
update_version: "1.1"

‎.github/workflows/test.yml‎

Lines changed: 12 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,12 @@
2020
# CI_NRF54L15_SERIAL_PORT=<optional explicit /dev/tty... path; auto-resolved if unset>
2121
#
2222
# GitHub secrets:
23-
# NRF_CLOUD_API_KEY, NRF_CLOUD_CA_CERT, NRF_CLOUD_CA_KEY
23+
# NRF_CLOUD_API_KEY, NRF_CLOUD_CA_CERT, NRF_CLOUD_CA_KEY, MEMFAULT_ORG_TOKEN
24+
#
25+
# GitHub variables (Memfault, linked nRF Cloud project):
26+
# MEMFAULT_ORG, MEMFAULT_PROJECT
27+
# MEMFAULT_COHORT (optional fallback; FOTA tests define a dedicated cohort in tests.yml)
28+
# MEMFAULT_FLEET_SAMPLING_CONFIGURATION (optional fleet sampling config ID for dev mode)
2429

2530
name: Test
2631

@@ -93,6 +98,11 @@ jobs:
9398
CI_NRF54L15_SEGGER_SN: ${{ vars.CI_NRF54L15_SEGGER_SN }}
9499
CI_NRF54L15_SERIAL_PORT: ${{ vars.CI_NRF54L15_SERIAL_PORT }}
95100
CI_NRF54L15_DEVICE_ID: ${{ vars.CI_NRF54L15_DEVICE_ID }}
101+
MEMFAULT_ORG_TOKEN: ${{ secrets.MEMFAULT_ORG_TOKEN }}
102+
MEMFAULT_ORG: ${{ vars.MEMFAULT_ORG }}
103+
MEMFAULT_PROJECT: ${{ vars.MEMFAULT_PROJECT }}
104+
MEMFAULT_COHORT: ${{ vars.MEMFAULT_COHORT }}
105+
MEMFAULT_FLEET_SAMPLING_CONFIGURATION: ${{ vars.MEMFAULT_FLEET_SAMPLING_CONFIGURATION }}
96106
steps:
97107
- name: Checkout
98108
uses: actions/checkout@v6
@@ -162,7 +172,7 @@ jobs:
162172
-v
163173
164174
- name: Upload serial log
165-
if: failure()
175+
if: always()
166176
uses: actions/upload-artifact@v4
167177
with:
168178
name: hardware-serial-log-${{ matrix.id }}-${{ github.run_id }}

‎tests/on_target/requirements.txt‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,3 +3,4 @@ pytest-html
33
pyserial
44
pyyaml
55
nrfcloud-utils
6+
memfault-cli

‎tests/on_target/tests/conftest.py‎

Lines changed: 41 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@
1010

1111
import pytest
1212

13-
from utils.flash_tools import nrfutil_reset, west_build, west_flash
13+
from utils.flash_tools import memfault_fw_version_cmake_arg, nrfutil_reset, west_build, west_flash
1414
from utils.helpers import REPO_ROOT, SERIAL_LOG, load_test_config
1515
from utils.logger import get_logger
1616
from utils.serial_port import resolve_serial_port
@@ -73,6 +73,46 @@ def cloud_connect_dut(request: pytest.FixtureRequest, test_config: dict) -> type
7373
request.node.user_properties.append(("serial_log", str(SERIAL_LOG)))
7474

7575

76+
@pytest.fixture(scope="function")
77+
def fota_dut(request: pytest.FixtureRequest, test_config: dict) -> types.SimpleNamespace:
78+
"""Prepare device with baseline FOTA firmware for application FOTA test."""
79+
segger_sn, board, app_dir = _hardware_context(test_config)
80+
_prepare_serial_log()
81+
82+
memfault = test_config.get("memfault", {})
83+
baseline_version = memfault.get("baseline_version", "0.1.0")
84+
85+
logger.info("Step 1/3 - Build baseline FOTA firmware %s", baseline_version)
86+
west_build(
87+
app_dir,
88+
board,
89+
cmake_args=[memfault_fw_version_cmake_arg(baseline_version)],
90+
)
91+
92+
logger.info("Step 2/3 - Recover and flash firmware (clears all flash including TF-M storage)")
93+
west_flash(app_dir, segger_sn, recover=True)
94+
95+
logger.info("Step 3/3 - Start serial capture and reset device")
96+
time.sleep(2)
97+
serial_port = resolve_serial_port(test_config)
98+
uart = Uart(serial_port, log_path=SERIAL_LOG)
99+
nrfutil_reset(segger_sn)
100+
101+
dut = types.SimpleNamespace(
102+
uart=uart,
103+
segger_sn=segger_sn,
104+
serial_port=serial_port,
105+
app_dir=app_dir,
106+
board=board,
107+
serial_log=SERIAL_LOG,
108+
)
109+
110+
yield dut
111+
112+
dut.uart.stop()
113+
request.node.user_properties.append(("serial_log", str(SERIAL_LOG)))
114+
115+
76116
@pytest.fixture(scope="function")
77117
def nrf_cloud_env(test_config: dict) -> dict:
78118
nrf_cloud = test_config.get("nrf_cloud", {})

‎tests/on_target/tests/pytest.ini‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,7 @@
11
[pytest]
2-
testpaths = test_cloud
2+
testpaths = test_cloud test_fota
33
pythonpath = ..
44
log_cli = true
55
log_cli_level = INFO
6+
markers =
7+
slow: long-running on-target hardware tests (FOTA, etc.)

‎tests/on_target/tests/test_cloud/test_cloud_connect.py‎

Lines changed: 53 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,11 @@
99
wait_for_device_id,
1010
)
1111
from utils.logger import get_logger
12+
from utils.memfault_ota import (
13+
delete_device_if_exists,
14+
enable_server_side_developer_mode,
15+
load_memfault_credentials,
16+
)
1217
from utils.nrf_cloud_device import delete_if_exists, onboard
1318
from utils.nrf_cloud_provision import install_device_credentials
1419
from utils.uart import Uart
@@ -27,35 +32,56 @@ def test_cloud_connect_after_provisioning(
2732
"""Provision credentials on a fresh device, onboard to nRF Cloud, and connect."""
2833
dut = cloud_connect_dut
2934
expected_device_id = load_expected_device_id(test_config)
35+
memfault_env = load_memfault_credentials(test_config)
36+
device_id: str | None = None
3037

31-
logger.info("Phase 1/5 - Wait for device ID in boot log")
32-
device_id = assert_dut_device_id(
33-
wait_for_device_id(dut.uart, timeout=30),
34-
expected_device_id,
35-
)
36-
37-
logger.info("Phase 2/5 - Confirm device boots without nRF Cloud credentials")
38-
dut.uart.wait_for_substring(MISSING_CREDENTIALS_LOG, timeout=300)
38+
try:
39+
logger.info("Phase 1/5 - Wait for device ID in boot log")
40+
device_id = assert_dut_device_id(
41+
wait_for_device_id(dut.uart, timeout=30),
42+
expected_device_id,
43+
)
3944

40-
logger.info(
41-
"Phase 3/5 - Remove only the configured DUT from nRF Cloud if already registered"
42-
)
43-
delete_if_exists(device_id, expected_device_id)
45+
logger.info("Phase 2/5 - Confirm device boots without nRF Cloud credentials")
46+
dut.uart.wait_for_substring(MISSING_CREDENTIALS_LOG, timeout=60)
4447

45-
logger.info("Phase 4/5 - Install credentials and onboard using onboard.csv")
46-
dut.uart.stop()
47-
try:
48-
onboard_csv = install_device_credentials(
49-
work_dir=nrf_cloud_env["work_dir"],
50-
device_id=device_id,
51-
serial_port=dut.serial_port,
52-
ca_cert=nrf_cloud_env["ca_cert"],
53-
ca_key=nrf_cloud_env["ca_key"],
48+
logger.info(
49+
"Phase 3/5 - Remove only the configured DUT from nRF Cloud if already registered"
5450
)
55-
onboard(str(onboard_csv))
56-
finally:
57-
dut.uart = Uart(dut.serial_port, log_path=dut.serial_log)
58-
nrfutil_reset(dut.segger_sn)
51+
delete_if_exists(device_id, expected_device_id)
5952

60-
logger.info("Phase 5/5 - Wait for nRF Cloud connection in serial log")
61-
dut.uart.wait_for_substring(CLOUD_CONNECTED_LOG, timeout=900)
53+
logger.info("Phase 4/5 - Install credentials and onboard using onboard.csv")
54+
dut.uart.stop()
55+
try:
56+
onboard_csv = install_device_credentials(
57+
work_dir=nrf_cloud_env["work_dir"],
58+
device_id=device_id,
59+
serial_port=dut.serial_port,
60+
ca_cert=nrf_cloud_env["ca_cert"],
61+
ca_key=nrf_cloud_env["ca_key"],
62+
segger_sn=dut.segger_sn,
63+
)
64+
onboard(str(onboard_csv))
65+
finally:
66+
dut.uart = Uart(dut.serial_port, log_path=dut.serial_log)
67+
nrfutil_reset(dut.segger_sn)
68+
69+
logger.info("Phase 5/5 - Wait for nRF Cloud connection in serial log")
70+
dut.uart.wait_for_substring(CLOUD_CONNECTED_LOG, timeout=120)
71+
enable_server_side_developer_mode(memfault_env, device_id)
72+
finally:
73+
if device_id is not None:
74+
try:
75+
delete_if_exists(device_id, expected_device_id)
76+
except RuntimeError as exc:
77+
logger.warning(
78+
"Failed to delete DUT from nRF Cloud during cleanup: %s",
79+
exc,
80+
)
81+
try:
82+
delete_device_if_exists(memfault_env, device_id, expected_device_id)
83+
except RuntimeError as exc:
84+
logger.warning(
85+
"Failed to delete DUT from Memfault during cleanup: %s",
86+
exc,
87+
)

0 commit comments

Comments
 (0)