nightly-build-test-sophon.yml

name: Nightly Sophon Build and Test

on:
  # Trigger daily at 2:00 AM Beijing Time (18:00 UTC previous day)
  schedule:
    - cron: '12 18 * * *'
  # Allow manual trigger for testing purposes
  workflow_dispatch:

permissions:
  contents: read

jobs:
  build-sophon:
    runs-on: ubuntu-latest
    timeout-minutes: 60
    defaults:
      run:
        shell: bash
    outputs:
      candidate_guard_sha256: ${{ steps.candidate_guard_runtime.outputs.sha256 }}
      candidate_tests_sha256: ${{ steps.candidate_test_binary.outputs.sha256 }}
    env:
      COSMO_MODEL_GUARD_BUILD_PROFILE: public-runtime
      # Cargo defaults each HTTP request to 30 seconds and only three retries.
      # The public rsproxy sparse index can exceed that from GitHub runners.
      CARGO_HTTP_TIMEOUT: "120"
      CARGO_NET_RETRY: "5"

    # Specify the public container image for the build environment
    container:
      image: ghcr.io/cosmo-wander-ai/cosmo_edge-build-env_sophon:v1

    steps:
      - name: Checkout code
        uses: actions/checkout@v7
        with:
          submodules: recursive # Recommended if the repository uses git submodules

      # Enforce project standards: use the unified build script instead of cmake directly.
      # A single configure builds the clean public-runtime package and cosmo-tests
      # together, sharing one set of compiled OBJECT libraries; coverage is off
      # by default (COSMO_ENABLE_COVERAGE), so the package binary stays clean.
      - name: Build package and tests
        run: |
          chmod +x scripts/build.sh
          ./scripts/build.sh -T -c bm1688

      # Upload the build artifacts for deployment or download
      - name: Upload Artifacts
        uses: actions/upload-artifact@v7
        with:
          name: sophon-build-package
          # Path where the built binaries/packages are generated by scripts/build.sh
          path: build/install/
          overwrite: true
          retention-days: 3

      - name: Record Candidate Test Binary
        id: candidate_test_binary
        run: |
          set -euo pipefail
          binary="build/cosmo-tests"
          if [ ! -f "$binary" ] || [ -L "$binary" ]; then
            echo "Candidate test binary is not a regular file: $binary" >&2
            exit 1
          fi
          digest="$(sha256sum "$binary" | awk '{print $1}')"
          if [ "${#digest}" -ne 64 ] || [[ "$digest" == *[!0-9a-f]* ]]; then
            echo "Invalid Candidate test binary SHA-256: $digest" >&2
            exit 1
          fi
          echo "sha256=$digest" >> "$GITHUB_OUTPUT"
          echo "Candidate test binary SHA-256: $digest"

      - name: Upload Test Binary
        uses: actions/upload-artifact@v7
        with:
          name: sophon-tests-binary
          path: build/cosmo-tests
          if-no-files-found: error
          overwrite: true
          retention-days: 1

      - name: Record Candidate Guard Runtime
        id: candidate_guard_runtime
        run: |
          set -euo pipefail
          runtime="build/install/lib/libcosmo_model_guard.so.2.0.0"
          if [ ! -f "$runtime" ] || [ -L "$runtime" ]; then
            echo "Candidate Guard runtime is not a regular file: $runtime" >&2
            exit 1
          fi
          digest="$(sha256sum "$runtime" | awk '{print $1}')"
          if [ "${#digest}" -ne 64 ] || [[ "$digest" == *[!0-9a-f]* ]]; then
            echo "Invalid Candidate Guard SHA-256: $digest" >&2
            exit 1
          fi
          echo "sha256=$digest" >> "$GITHUB_OUTPUT"
          echo "Candidate Guard SHA-256: $digest"

      # Keep the candidate Guard runtime independent from the large package and
      # test-binary cache. overwrite=true makes a full job re-run replace the
      # artifact, while a failed-job re-run still downloads the build's artifact.
      - name: Upload Candidate Guard Runtime
        uses: actions/upload-artifact@v7
        with:
          name: sophon-model-guard-runtime
          path: build/install/lib/libcosmo_model_guard.so.2.0.0
          if-no-files-found: error
          overwrite: true
          retention-days: 1

  test-sophon:
    needs: build-sophon
    runs-on: [self-hosted, hosted-sophon-device]
    timeout-minutes: 45   # self-hosted 无默认超时，防挂死占用设备（分片+二分；绿跑仅数分钟，45min 是坏夜二分余量）
    env:
      COSMO_TEST_CASE_TIMEOUT_SECONDS: 120
      COSMO_CATCH2_SHARDS: 12
      COSMO_CATCH2_SHARD_TIMEOUT_SECONDS: 120
      COSMO_CATCH2_RESULTS_DIR: test-results/catch2
      COSMO_CATCH2_REPORTER: compact
      COSMO_SOPHON_LD_LIBRARY_PATH: /appfs/cosmo_wander/cwai_data/lib:/data:/usr/lib
      COSMO_CANDIDATE_RUNTIME_DIR: ${{ github.workspace }}/candidate-runtime
      COSMO_CANDIDATE_GUARD_SHA256: ${{ needs.build-sophon.outputs.candidate_guard_sha256 }}
      COSMO_CANDIDATE_TESTS_SHA256: ${{ needs.build-sophon.outputs.candidate_tests_sha256 }}

    steps:
      # The test binary may be restored from the persistent device cache, but
      # the Guard library must always come from this build's current artifact.
      - name: Prepare Candidate Guard Runtime Directory
        run: |
          set -euo pipefail
          expected_dir="$GITHUB_WORKSPACE/candidate-runtime"
          if [ "$COSMO_CANDIDATE_RUNTIME_DIR" != "$expected_dir" ]; then
            echo "Refusing unexpected candidate runtime directory: $COSMO_CANDIDATE_RUNTIME_DIR" >&2
            exit 1
          fi
          rm -rf -- "$COSMO_CANDIDATE_RUNTIME_DIR"
          mkdir -p -- "$COSMO_CANDIDATE_RUNTIME_DIR"

      - name: Download Candidate Guard Runtime
        uses: actions/download-artifact@v8
        with:
          name: sophon-model-guard-runtime
          path: candidate-runtime

      - name: Verify Candidate Guard Runtime
        run: |
          set -euo pipefail
          runtime="$COSMO_CANDIDATE_RUNTIME_DIR/libcosmo_model_guard.so.2.0.0"
          if [ ! -f "$runtime" ] || [ -L "$runtime" ]; then
            echo "Downloaded Candidate Guard runtime is not a regular file: $runtime" >&2
            exit 1
          fi
          if [ "${#COSMO_CANDIDATE_GUARD_SHA256}" -ne 64 ] \
              || [[ "$COSMO_CANDIDATE_GUARD_SHA256" == *[!0-9a-f]* ]]; then
            echo "Missing or invalid expected Candidate Guard SHA-256." >&2
            exit 1
          fi
          actual_sha256="$(sha256sum "$runtime" | awk '{print $1}')"
          if [ "$actual_sha256" != "$COSMO_CANDIDATE_GUARD_SHA256" ]; then
            echo "Candidate Guard SHA-256 mismatch." >&2
            echo "Expected: $COSMO_CANDIDATE_GUARD_SHA256" >&2
            echo "Actual:   $actual_sha256" >&2
            exit 1
          fi
          ln -s -- "libcosmo_model_guard.so.2.0.0" \
            "$COSMO_CANDIDATE_RUNTIME_DIR/libcosmo_model_guard.so.2"
          ln -s -- "libcosmo_model_guard.so.2" \
            "$COSMO_CANDIDATE_RUNTIME_DIR/libcosmo_model_guard.so"
          test "$(readlink "$COSMO_CANDIDATE_RUNTIME_DIR/libcosmo_model_guard.so.2")" \
            = "libcosmo_model_guard.so.2.0.0"
          test "$(readlink "$COSMO_CANDIDATE_RUNTIME_DIR/libcosmo_model_guard.so")" \
            = "libcosmo_model_guard.so.2"
          echo "Candidate Guard runtime verified: $actual_sha256"

      # The Download step below is the dominant wall-clock cost on this
      # self-hosted device (fetching the artifact from GitHub over a slow link
      # routinely takes 7-13 min, vs seconds-to-minutes for the tests). On
      # "Re-run failed jobs" the whole job re-runs, so we cache the binary on
      # the device's persistent filesystem keyed by both GITHUB_RUN_ID and the
      # build-produced SHA-256. A full job re-run therefore cannot reuse a stale
      # binary, while a failed-job re-run can still skip the large download.
      - name: Restore Test Binary From Device Cache
        id: binary-cache
        run: |
          set -euo pipefail
          if [ "${#COSMO_CANDIDATE_TESTS_SHA256}" -ne 64 ] \
              || [[ "$COSMO_CANDIDATE_TESTS_SHA256" == *[!0-9a-f]* ]]; then
            echo "Missing or invalid expected Candidate test binary SHA-256." >&2
            exit 1
          fi
          # $HOME persists across runs and is outside the per-job workspace
          # GitHub cleans, so a file saved here survives across attempts.
          # Override the dir by setting COSMO_TEST_BINARY_CACHE_DIR in env.
          cache_dir="${COSMO_TEST_BINARY_CACHE_DIR:-$HOME/cosmo-ci-cache}"
          mkdir -p "$cache_dir"
          cached="$cache_dir/cosmo-tests-${GITHUB_RUN_ID}-${COSMO_CANDIDATE_TESTS_SHA256}"
          if [ -f "$cached" ] && [ ! -L "$cached" ]; then
            cached_sha256="$(sha256sum "$cached" | awk '{print $1}')"
            if [ "$cached_sha256" = "$COSMO_CANDIDATE_TESTS_SHA256" ]; then
              cp -- "$cached" ./cosmo-tests
              restored_sha256="$(sha256sum ./cosmo-tests | awk '{print $1}')"
              if [ "$restored_sha256" != "$COSMO_CANDIDATE_TESTS_SHA256" ]; then
                echo "Restored Candidate test binary SHA-256 mismatch." >&2
                exit 1
              fi
              echo "cache-hit=true" >> "$GITHUB_OUTPUT"
              echo "Restored verified cosmo-tests from $cached (skipping artifact download)."
            else
              echo "cache-hit=false" >> "$GITHUB_OUTPUT"
              echo "Ignoring corrupt test-binary cache entry: $cached"
            fi
          else
            echo "cache-hit=false" >> "$GITHUB_OUTPUT"
            echo "No verified cache entry for run ${GITHUB_RUN_ID} and SHA ${COSMO_CANDIDATE_TESTS_SHA256}; will download."
          fi

      - name: Download Test Binary
        if: steps.binary-cache.outputs.cache-hit != 'true' # skipped on device-cache hit
        uses: actions/download-artifact@v8
        with:
          name: sophon-tests-binary
          path: .

      - name: Verify Candidate Test Binary
        run: |
          set -euo pipefail
          if [ ! -f ./cosmo-tests ] || [ -L ./cosmo-tests ]; then
            echo "Candidate test binary is not a regular file." >&2
            exit 1
          fi
          actual_sha256="$(sha256sum ./cosmo-tests | awk '{print $1}')"
          if [ "$actual_sha256" != "$COSMO_CANDIDATE_TESTS_SHA256" ]; then
            echo "Candidate test binary SHA-256 mismatch." >&2
            echo "Expected: $COSMO_CANDIDATE_TESTS_SHA256" >&2
            echo "Actual:   $actual_sha256" >&2
            exit 1
          fi
          echo "Candidate test binary verified: $actual_sha256"

      - name: Save Test Binary To Device Cache
        if: steps.binary-cache.outputs.cache-hit != 'true' && success()
        run: |
          set -euo pipefail
          cache_dir="${COSMO_TEST_BINARY_CACHE_DIR:-$HOME/cosmo-ci-cache}"
          cached="$cache_dir/cosmo-tests-${GITHUB_RUN_ID}-${COSMO_CANDIDATE_TESTS_SHA256}"
          pending="${cached}.pending-${GITHUB_RUN_ATTEMPT}"
          cp --remove-destination -- ./cosmo-tests "$pending"
          pending_sha256="$(sha256sum "$pending" | awk '{print $1}')"
          if [ "$pending_sha256" != "$COSMO_CANDIDATE_TESTS_SHA256" ]; then
            echo "Refusing to cache test binary with mismatched SHA-256." >&2
            exit 1
          fi
          mv -f -- "$pending" "$cached"
          # Prune to the 5 most-recent entries; never touch the current run.
          # `|| true`: grep exits 1 when only the current run's file exists
          # (nothing left after filtering), which under `set -o pipefail` would
          # otherwise fail this step. Pruning is best-effort cleanup.
          ls -t "$cache_dir"/cosmo-tests-* 2>/dev/null \
            | grep -v -F -- "$cached" \
            | tail -n +6 \
            | xargs -r rm -f || true
          echo "Saved cosmo-tests to $cached; cache pruned."

      - name: Add Execution Permission
        run: chmod +x ./cosmo-tests

      - name: Verify Candidate Guard Resolution
        run: |
          set -euo pipefail
          export LD_LIBRARY_PATH="$COSMO_CANDIDATE_RUNTIME_DIR:$COSMO_SOPHON_LD_LIBRARY_PATH:${LD_LIBRARY_PATH:-}"
          if ! command -v ldd >/dev/null 2>&1; then
            echo "Required command is missing: ldd" >&2
            exit 1
          fi
          resolved="$(ldd ./cosmo-tests \
            | awk '$1 == "libcosmo_model_guard.so.2" && $2 == "=>" {print $3; exit}')"
          if [ -z "$resolved" ]; then
            echo "cosmo-tests did not resolve libcosmo_model_guard.so.2." >&2
            exit 1
          fi
          expected_real="$(readlink -f \
            "$COSMO_CANDIDATE_RUNTIME_DIR/libcosmo_model_guard.so.2.0.0")"
          resolved_real="$(readlink -f "$resolved")"
          if [ "$resolved_real" != "$expected_real" ]; then
            echo "cosmo-tests resolved a non-candidate Guard runtime: $resolved" >&2
            exit 1
          fi
          echo "cosmo-tests resolves Candidate Guard runtime: $resolved_real"

      - name: Run Catch2 Tests
        run: |
          set +e
          set -uo pipefail

          export LD_LIBRARY_PATH="$COSMO_CANDIDATE_RUNTIME_DIR:$COSMO_SOPHON_LD_LIBRARY_PATH:${LD_LIBRARY_PATH:-}"

          shard_count="$COSMO_CATCH2_SHARDS"
          shard_timeout_seconds="$COSMO_CATCH2_SHARD_TIMEOUT_SECONDS"
          case_timeout_seconds="$COSMO_TEST_CASE_TIMEOUT_SECONDS"
          reporter="$COSMO_CATCH2_REPORTER"
          results_dir="$COSMO_CATCH2_RESULTS_DIR"
          shards_dir="$results_dir/shards"
          bisect_dir="$results_dir/bisect"
          summary_file="$results_dir/summary.txt"
          gcov_dir="${RUNNER_TEMP:-/tmp}/cosmo-gcov-${GITHUB_RUN_ID:-manual}-${GITHUB_RUN_ATTEMPT:-0}"

          # Validate numeric knobs: a non-positive shard_count would skip every loop and
          # the verdict below would pass 0/0 → green with zero tests run.
          require_pos_int() {
            case "$2" in
              ''|*[!0-9]*) echo "error: $1 must be a positive integer (got '$2')" >&2; exit 2 ;;
            esac
            [ "$2" -ge 1 ] || { echo "error: $1 must be >= 1 (got '$2')" >&2; exit 2; }
          }
          require_pos_int COSMO_CATCH2_SHARDS                "$shard_count"
          require_pos_int COSMO_CATCH2_SHARD_TIMEOUT_SECONDS "$shard_timeout_seconds"
          require_pos_int COSMO_TEST_CASE_TIMEOUT_SECONDS    "$case_timeout_seconds"

          # --input-file parses each line as a Catch2 test specification. Escape
          # grammar characters so punctuation in a literal test name (notably
          # commas) cannot turn the generated filter into an invalid spec.
          write_catch2_test_spec() {
            local escaped="$1"
            escaped="${escaped//\\/\\\\}"
            escaped="${escaped//,/\\,}"
            escaped="${escaped//\"/\\\"}"
            printf '"%s"\n' "$escaped"
          }

          # self-hosted runner 工作目录跨 run 复用,历史 run 的 bisect/shards 文件会累积,
          # 被 Upload step 整个目录打包上传,新旧混杂干扰诊断。每次 run 开始清空这两个目录。
          rm -rf "$shards_dir" "$bisect_dir"
          mkdir -p "$shards_dir" "$bisect_dir" "$gcov_dir"
          : > "$summary_file"

          # cosmo-tests is built WITHOUT --coverage by default; coverage is now
          # opt-in via COSMO_ENABLE_COVERAGE. The GCOV_PREFIX exports below are a
          # no-op in the default nightly build, but are kept so that if coverage
          # is re-enabled, gcov writes redirect to a local writable dir instead
          # of the GitHub /__w workspace path that does not exist on this device
          # (libgcov would otherwise pollute stderr / fail process exit).
          export GCOV_PREFIX="$gcov_dir"
          export GCOV_PREFIX_STRIP=4

          if ! command -v timeout >/dev/null 2>&1; then
            echo "Required command is missing: timeout" >&2
            exit 2
          fi

          # 1) Enumerate all test cases. `--verbosity quiet` makes --list-tests print one
          #    plain name per line (Catch2's listTestNamesOnly), which round-trips through
          #    --input-file. Wrap in timeout in case the binary deadlocks while loading.
          list_file="$results_dir/test-list.txt"
          raw_list_file="$results_dir/test-list.raw.txt"
          list_stderr_file="$results_dir/test-list.stderr.log"
          timeout 120s ./cosmo-tests --list-tests --verbosity quiet > "$raw_list_file" 2> "$list_stderr_file"
          list_status=$?
          if [ "$list_status" -ne 0 ]; then
            echo "Failed to list Catch2 test cases, exit code: $list_status" >&2
            cat "$raw_list_file"
            cat "$list_stderr_file"
            exit "$list_status"
          fi

          awk 'NF && $0 !~ /^profiling:/' "$raw_list_file" > "$list_file"

          mapfile -t all_tests < "$list_file"
          total=${#all_tests[@]}
          if [ "$total" -eq 0 ]; then
            echo "No Catch2 test cases found." >&2
            exit 1
          fi

          # Don't create more shards than tests.
          if [ "$total" -lt "$shard_count" ]; then
            shard_count="$total"
          fi

          echo "Listed $total Catch2 test cases; splitting into $shard_count shards (${shard_timeout_seconds}s budget/shard, ${case_timeout_seconds}s/case on bisection)."

          # 2) Round-robin each test name into its shard's --input-file.
          shard_input=()
          for ((s = 0; s < shard_count; s++)); do
            shard_input[$s]="$(printf '%s/shards/%02d.input' "$results_dir" "$((s + 1))")"
            : > "${shard_input[$s]}"
          done
          for i in "${!all_tests[@]}"; do
            name="${all_tests[$i]}"
            [ -n "$name" ] || continue
            s=$((i % shard_count))
            printf '%s\n' "$name" >> "${shard_input[$s]}"
          done

          # 3) Run each shard in its own process. Per-shard timeout bounds a hung test to
          #    the shard budget instead of the whole job; --warn UnmatchedTestSpec turns a
          #    partial spec match (shouldn't happen, but defensive) into a non-zero exit.
          shards_run=0
          shards_passed=0
          problem_shards=0
          bisect_targets=()

          for ((s = 0; s < shard_count; s++)); do
            input="${shard_input[$s]}"
            # An empty --input-file means "no filter" and would run the WHOLE suite.
            [ -s "$input" ] || continue

            idx="$(printf '%02d' "$((s + 1))")"
            log="$shards_dir/$idx.log"
            mapfile -t shard_tests < "$input"
            n=${#shard_tests[@]}
            shards_run=$((shards_run + 1))

            filter="$shards_dir/$idx.filter"
            : > "$filter"
            for name in "${shard_tests[@]}"; do
              [ -n "$name" ] || continue
              write_catch2_test_spec "$name" >> "$filter"
            done

            echo
            echo "[$idx] shard with $n tests"

            start_seconds="$(date +%s)"
            timeout --kill-after=10s "${shard_timeout_seconds}s" \
              ./cosmo-tests --input-file "$filter" --reporter "$reporter" \
                            --warn UnmatchedTestSpec \
              > "$log" 2>&1
            status=$?
            elapsed_seconds=$(( $(date +%s) - start_seconds ))

            if [ "$status" -eq 0 ]; then
              echo "SHARD PASS in ${elapsed_seconds}s: #$idx ($n tests)"
              printf 'SHARD\tPASS\telapsed_seconds=%s\ttests=%s\tshard=%s\n' "$elapsed_seconds" "$n" "$idx" >> "$summary_file"
              shards_passed=$((shards_passed + 1))
            elif [ "$status" -eq 42 ]; then
              # TestFailureExitCode: an assertion already failed and Catch2 named it.
              echo "SHARD FAIL in ${elapsed_seconds}s: #$idx (assertion failure; last 40 log lines:)"
              tail -n 40 "$log"
              printf 'SHARD\tFAIL\telapsed_seconds=%s\ttests=%s\tshard=%s\n' "$elapsed_seconds" "$n" "$idx" >> "$summary_file"
              problem_shards=$((problem_shards + 1))
            elif [ "$status" -eq 124 ] || [ "$status" -eq 137 ]; then
              echo "SHARD TIMEOUT after ${elapsed_seconds}s: #$idx (will bisect per-test)"
              printf 'SHARD\tTIMEOUT\telapsed_seconds=%s\ttests=%s\tshard=%s\n' "$elapsed_seconds" "$n" "$idx" >> "$summary_file"
              bisect_targets+=("$s")
              problem_shards=$((problem_shards + 1))
            else
              echo "SHARD CRASH($status) in ${elapsed_seconds}s: #$idx (will bisect per-test)"
              printf 'SHARD\tCRASH(%s)\telapsed_seconds=%s\ttests=%s\tshard=%s\n' "$status" "$elapsed_seconds" "$n" "$idx" >> "$summary_file"
              bisect_targets+=("$s")
              problem_shards=$((problem_shards + 1))
            fi
          done

          # 4) For any shard that timed out or crashed, re-run its tests one-per-process to
          #    pinpoint the culprit. Use the same literal-name escaping as the shard run;
          #    --warn UnmatchedTestSpec still catches a no-match.
          if [ "${#bisect_targets[@]}" -gt 0 ]; then
            echo
            echo "Bisecting ${#bisect_targets[@]} shard(s) per-test (${case_timeout_seconds}s/case)."
            for s in "${bisect_targets[@]}"; do
              idx="$(printf '%02d' "$((s + 1))")"
              mapfile -t shard_tests < "${shard_input[$s]}"
              echo
              echo "[bisect #$idx] ${#shard_tests[@]} tests"
              for j in "${!shard_tests[@]}"; do
                name="${shard_tests[$j]}"
                [ -n "$name" ] || continue
                number=$((j + 1))
                safe_name="$(printf '%s' "$name" | tr -cs '[:alnum:]_.-' '_' | cut -c1-120)"
                safe_name="${safe_name:-test}"
                log_file="$bisect_dir/${idx}_$(printf '%03d' "$number")_${safe_name}.log"
                filter_file="$bisect_dir/${idx}_$(printf '%03d' "$number")_${safe_name}.filter"
                write_catch2_test_spec "$name" > "$filter_file"

                echo "[bisect #$idx $number/${#shard_tests[@]}] $name"
                start_seconds="$(date +%s)"
                timeout --kill-after=10s "${case_timeout_seconds}s" \
                  ./cosmo-tests --input-file "$filter_file" --reporter "$reporter" \
                                --warn UnmatchedTestSpec \
                  > "$log_file" 2>&1
                status=$?
                elapsed_seconds=$(( $(date +%s) - start_seconds ))

                if [ "$status" -eq 0 ]; then
                  printf 'BISECT\tPASS\tshard=%s\telapsed_seconds=%s\t%s\n' "$idx" "$elapsed_seconds" "$name" >> "$summary_file"
                elif [ "$status" -eq 124 ] || [ "$status" -eq 137 ]; then
                  echo "  -> TIMEOUT after ${elapsed_seconds}s: $name"
                  printf 'BISECT\tTIMEOUT\tshard=%s\telapsed_seconds=%s\t%s\n' "$idx" "$elapsed_seconds" "$name" >> "$summary_file"
                else
                  echo "  -> FAIL($status) in ${elapsed_seconds}s: $name"
                  printf 'BISECT\tFAIL(%s)\tshard=%s\telapsed_seconds=%s\t%s\n' "$status" "$idx" "$elapsed_seconds" "$name" >> "$summary_file"
                fi
              done
            done
          fi

          # 5) Verdict.
          echo
          echo "Catch2 summary: shards_passed=$shards_passed/$shards_run problem_shards=$problem_shards total_tests=$total"
          echo "Per-shard logs: $shards_dir ; bisect logs: $bisect_dir ; summary: $summary_file"

          # Fail on any problem shard, OR if we somehow ran zero shards (e.g. bad
          # config): a vacuous 0/0 must never pass as green.
          if [ "$problem_shards" -ne 0 ] || [ "$shards_passed" -ne "$shards_run" ] || [ "$shards_run" -eq 0 ]; then
            exit 1
          fi

      - name: Upload Test Results
        if: always()
        uses: actions/upload-artifact@v7
        with:
          name: sophon-catch2-test-results
          path: test-results/catch2/
          retention-days: 3