.github: ci: replace shell scripts with Python catalog module

Replace ad-hoc shell scripts (`load-test.sh`, `resolve-matrix.sh`,
`nrf-cloud-device.sh`, `run.sh`) with a proper Python module
`ci/catalog.py` that handles test catalog loading and matrix
resolution.

Move nRF Cloud device management from a shell script into
`utils/nrf_cloud_device.py`, using Python's `urllib` instead of
`curl`. Make hardware env var names configurable via `test_config`
rather than hardcoded to nRF54L15-specific names.

Add a new `fota-test.yml` reusable workflow and a disabled FOTA
test entry in `tests.yml`. Switch `test.yml` to dispatch the
correct workflow based on `matrix.type`, and tighten the concurrency
group to `github.ref`.

Drop the `summary_file` artifact and related `summary_line` helpers;
serial log upload is sufficient for failure triage.

Signed-off-by: Simen S. Røstad <simen.rostad@nordicsemi.no>

Author: simensrostad

.github/test/cloud/nrf-cloud-device.sh

@@ -17,3 +17,16 @@ tests:
       api_key_var: NRF_CLOUD_API_KEY
       ca_cert_var: NRF_CLOUD_CA_CERT
       ca_key_var: NRF_CLOUD_CA_KEY
+
+  - id: 91m1-application-fota-nrf54l15
+    enabled: false
+    type: fota
+    app: 91m1
+    board: nrf54l15dk/nrf54l15/cpuapp/ns
+    hardware:
+      segger_sn_var: CI_NRF54L15_SEGGER_SN
+      serial_port_var: CI_NRF54L15_SERIAL_PORT
+      device_id_var: CI_NRF54L15_DEVICE_ID
+      internal_erase_end: "0x16b000"
+      erase_external_slot: true
+      erase_credentials: false

.github/test/cloud/run.sh

@@ -2,8 +2,6 @@
 #
 # Provisions credentials on a device with no stored credentials, onboards via
 # onboard.csv, and verifies "Cloud connected" in the serial log.
-#
-# When adding tests, update workflow_dispatch options below and .github/test/tests.yml.
 
 name: Cloud Connect Test
 
@@ -13,15 +11,6 @@ on:
       test_id:
         required: true
         type: string
-  workflow_dispatch:
-    inputs:
-      test:
-        description: Cloud connect test to run
-        required: true
-        type: choice
-        default: 91m1-cloud-connect-nrf54l15
-        options:
-          - 91m1-cloud-connect-nrf54l15
 
 permissions:
   contents: read
@@ -51,15 +40,8 @@ jobs:
         working-directory: serial-modem-host-applications
         run: |
           set -eu
-          pip install --quiet pyyaml
-
-          if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then
-            TEST_ID="${{ github.event.inputs.test }}"
-          else
-            TEST_ID="${{ inputs.test_id }}"
-          fi
-
-          TEST_JSON="$(ROOT=. .github/test/scripts/load-test.sh "$TEST_ID")"
+          pip install --quiet -r tests/on_target/requirements.txt
+          TEST_JSON="$(PYTHONPATH=tests/on_target python3 -m ci.catalog load "${{ inputs.test_id }}")"
           {
             echo "test_json<<EOF"
             echo "$TEST_JSON"
@@ -79,16 +61,12 @@ jobs:
           REPO_ROOT: ${{ github.workspace }}/serial-modem-host-applications
           TEST_JSON: ${{ steps.config.outputs.test_json }}
         run: |
-          chmod +x .github/test/cloud/*.sh .github/test/scripts/*.sh
-          .github/test/cloud/run.sh
-
-      - name: Upload test summary
-        if: always()
-        uses: actions/upload-artifact@v4
-        with:
-          name: hardware-test-summary-${{ github.run_id }}
-          path: serial-modem-host-applications/build/hardware-test-summary.txt
-          if-no-files-found: ignore
+          PYTHONPATH=tests/on_target python3 -m pytest \
+            tests/on_target/tests/test_cloud/ \
+            -c tests/on_target/tests/pytest.ini \
+            --html=build/hardware-pytest-report.html \
+            --self-contained-html \
+            -v
 
       - name: Upload serial log
         if: failure()

.github/test/scripts/load-test.sh

@@ -0,0 +1,81 @@
+# Reusable application FOTA hardware test workflow.
+#
+# Enable the matching entry in .github/test/tests.yml when test_fota is restored.
+
+name: FOTA Test
+
+on:
+  workflow_call:
+    inputs:
+      test_id:
+        required: true
+        type: string
+
+permissions:
+  contents: read
+
+jobs:
+  fota:
+    runs-on: self-hosted
+    container:
+      image: ghcr.io/zephyrproject-rtos/ci:v0.29.2
+      options: --privileged -v /dev:/dev
+    env:
+      CMAKE_PREFIX_PATH: /opt/toolchains
+      CI_NRF54L15_SEGGER_SN: ${{ vars.CI_NRF54L15_SEGGER_SN }}
+      CI_NRF54L15_SERIAL_PORT: ${{ vars.CI_NRF54L15_SERIAL_PORT }}
+      CI_NRF54L15_DEVICE_ID: ${{ vars.CI_NRF54L15_DEVICE_ID }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v6
+        with:
+          path: serial-modem-host-applications
+
+      - name: Load test configuration
+        id: config
+        working-directory: serial-modem-host-applications
+        run: |
+          set -eu
+          pip install --quiet -r tests/on_target/requirements.txt
+          TEST_JSON="$(PYTHONPATH=tests/on_target python3 -m ci.catalog load "${{ inputs.test_id }}")"
+          {
+            echo "test_json<<EOF"
+            echo "$TEST_JSON"
+            echo "EOF"
+          } >> "$GITHUB_OUTPUT"
+
+      - name: Initialize West workspace
+        uses: ./serial-modem-host-applications/.github/actions/west-init
+
+      - name: Install test dependencies
+        run: |
+          nrfutil install device
+
+      - name: Run FOTA test
+        working-directory: serial-modem-host-applications
+        env:
+          REPO_ROOT: ${{ github.workspace }}/serial-modem-host-applications
+          TEST_JSON: ${{ steps.config.outputs.test_json }}
+        run: |
+          PYTHONPATH=tests/on_target python3 -m pytest \
+            tests/on_target/tests/test_fota/ \
+            -c tests/on_target/tests/pytest.ini \
+            --html=build/hardware-pytest-report.html \
+            --self-contained-html \
+            -v
+
+      - name: Upload serial log
+        if: failure()
+        uses: actions/upload-artifact@v4
+        with:
+          name: hardware-serial-log-${{ github.run_id }}
+          path: serial-modem-host-applications/build/hardware-serial.log
+          if-no-files-found: ignore
+
+      - name: Upload pytest report
+        if: failure()
+        uses: actions/upload-artifact@v4
+        with:
+          name: hardware-pytest-report-${{ github.run_id }}
+          path: serial-modem-host-applications/build/hardware-pytest-report.html
+          if-no-files-found: ignore

.github/test/scripts/resolve-matrix.sh

@@ -36,17 +36,20 @@ on:
         options:
           - all
           - 91m1-cloud-connect-nrf54l15
+          - 91m1-application-fota-nrf54l15
 
 permissions:
   contents: read
 
 concurrency:
-  group: hardware-test-${{ github.event.pull_request.number || github.run_id }}
+  group: hardware-test-${{ github.ref }}
   cancel-in-progress: false
 
 jobs:
   plan:
     runs-on: self-hosted
+    container:
+      image: ghcr.io/zephyrproject-rtos/ci:v0.29.2
     outputs:
       matrix: ${{ steps.resolve.outputs.matrix }}
     steps:
@@ -61,18 +64,18 @@ jobs:
         env:
           TEST_FILTER: ${{ github.event_name == 'workflow_dispatch' && inputs.test || 'all' }}
         run: |
-          pip install --quiet pyyaml
-          ROOT=. .github/test/scripts/resolve-matrix.sh
+          pip install --quiet -r tests/on_target/requirements.txt
+          PYTHONPATH=tests/on_target python3 -m ci.catalog matrix --filter "$TEST_FILTER"
 
-  cloud_connect:
+  hardware_test:
     needs: plan
     if: ${{ needs.plan.outputs.matrix != '[]' && needs.plan.outputs.matrix != '' }}
     strategy:
       fail-fast: false
       max-parallel: 1
       matrix:
         include: ${{ fromJson(needs.plan.outputs.matrix) }}
-    uses: ./.github/workflows/cloud-connect-test.yml
+    uses: ./.github/workflows/${{ matrix.type }}-test.yml
     with:
       test_id: ${{ matrix.id }}
     secrets: inherit

.github/test/tests.yml

@@ -0,0 +1 @@
+# SPDX-License-Identifier: LicenseRef-Nordic-5-Clause