nightly-build-test-sophon.yml

name: Nightly Sophon Build and Test

on:
  # Trigger daily at 2:00 AM Beijing Time (18:00 UTC previous day)
  schedule:
    - cron: '12 18 * * *'
  # Allow manual trigger for testing purposes
  workflow_dispatch:

permissions:
  contents: read

jobs:
  build-sophon:
    runs-on: ubuntu-latest
    timeout-minutes: 60

    # Specify the public container image for the build environment
    container:
      image: ghcr.io/cosmo-wander-ai/cosmo_edge-build-env_sophon:v1

    steps:
      - name: Checkout code
        uses: actions/checkout@v7
        with:
          submodules: recursive # Recommended if the repository uses git submodules

      # Enforce project standards: use the unified build script instead of cmake directly.
      # A single configure builds the clean production package and cosmo-tests
      # together, sharing one set of compiled OBJECT libraries; coverage is off
      # by default (COSMO_ENABLE_COVERAGE), so the package binary stays clean.
      - name: Build package and tests
        run: |
          chmod +x scripts/build.sh
          ./scripts/build.sh -T

      # Upload the build artifacts for deployment or download
      - name: Upload Artifacts
        uses: actions/upload-artifact@v7
        with:
          name: sophon-build-package
          # Path where the built binaries/packages are generated by scripts/build.sh
          path: build/install/
          retention-days: 3

      - name: Upload Test Binary
        uses: actions/upload-artifact@v7
        with:
          name: sophon-tests-binary
          path: build/cosmo-tests
          retention-days: 1

  test-sophon:
    needs: build-sophon
    runs-on: [self-hosted, hosted-sophon-device]
    timeout-minutes: 45   # self-hosted 无默认超时，防挂死占用设备（分片+二分；绿跑仅数分钟，45min 是坏夜二分余量）
    env:
      COSMO_TEST_CASE_TIMEOUT_SECONDS: 120
      COSMO_CATCH2_SHARDS: 12
      COSMO_CATCH2_SHARD_TIMEOUT_SECONDS: 120
      COSMO_CATCH2_RESULTS_DIR: test-results/catch2
      COSMO_CATCH2_REPORTER: compact
      COSMO_SOPHON_LD_LIBRARY_PATH: /appfs/cosmo_wander/cwai_data/lib:/data:/usr/lib

    steps:
      # The Download step below is the dominant wall-clock cost on this
      # self-hosted device (fetching the artifact from GitHub over a slow link
      # routinely takes 7-13 min, vs seconds-to-minutes for the tests). On
      # "Re-run failed jobs" the whole job re-runs, so we cache the binary on
      # the device's persistent filesystem keyed by GITHUB_RUN_ID (stable across
      # re-run attempts) and skip re-downloading on a cache hit.
      # CAVEAT: "Re-run all jobs" rebuilds the binary under the same artifact
      # name; this cache would then serve a stale binary. Before a full re-run,
      # clear the cache dir on the device (rm -rf $HOME/cosmo-ci-cache).
      - name: Restore Test Binary From Device Cache
        id: binary-cache
        run: |
          set -euo pipefail
          # $HOME persists across runs and is outside the per-job workspace
          # GitHub cleans, so a file saved here survives across attempts.
          # Override the dir by setting COSMO_TEST_BINARY_CACHE_DIR in env.
          cache_dir="${COSMO_TEST_BINARY_CACHE_DIR:-$HOME/cosmo-ci-cache}"
          mkdir -p "$cache_dir"
          cached="$cache_dir/cosmo-tests-${GITHUB_RUN_ID}"
          if [ -f "$cached" ]; then
            cp "$cached" ./cosmo-tests
            echo "cache-hit=true" >> "$GITHUB_OUTPUT"
            echo "Restored cosmo-tests from $cached (skipping artifact download)."
          else
            echo "cache-hit=false" >> "$GITHUB_OUTPUT"
            echo "No cached binary for run ${GITHUB_RUN_ID}; will download."
          fi

      - name: Download Test Binary
        if: steps.binary-cache.outputs.cache-hit != 'true' # skipped on device-cache hit
        uses: actions/download-artifact@v8
        with:
          name: sophon-tests-binary
          path: .

      - name: Save Test Binary To Device Cache
        if: steps.binary-cache.outputs.cache-hit != 'true' && success()
        run: |
          set -euo pipefail
          cache_dir="${COSMO_TEST_BINARY_CACHE_DIR:-$HOME/cosmo-ci-cache}"
          cached="$cache_dir/cosmo-tests-${GITHUB_RUN_ID}"
          cp ./cosmo-tests "$cached"
          # Prune to the 5 most-recent entries; never touch the current run.
          # `|| true`: grep exits 1 when only the current run's file exists
          # (nothing left after filtering), which under `set -o pipefail` would
          # otherwise fail this step. Pruning is best-effort cleanup.
          ls -t "$cache_dir"/cosmo-tests-* 2>/dev/null \
            | grep -v -F -- "cosmo-tests-${GITHUB_RUN_ID}" \
            | tail -n +6 \
            | xargs -r rm -f || true
          echo "Saved cosmo-tests to $cached; cache pruned."

      - name: Add Execution Permission
        run: chmod +x ./cosmo-tests

      - name: Run Catch2 Tests
        run: |
          set +e
          set -uo pipefail

          export LD_LIBRARY_PATH="$COSMO_SOPHON_LD_LIBRARY_PATH:${LD_LIBRARY_PATH:-}"

          shard_count="$COSMO_CATCH2_SHARDS"
          shard_timeout_seconds="$COSMO_CATCH2_SHARD_TIMEOUT_SECONDS"
          case_timeout_seconds="$COSMO_TEST_CASE_TIMEOUT_SECONDS"
          reporter="$COSMO_CATCH2_REPORTER"
          results_dir="$COSMO_CATCH2_RESULTS_DIR"
          shards_dir="$results_dir/shards"
          bisect_dir="$results_dir/bisect"
          summary_file="$results_dir/summary.txt"
          gcov_dir="${RUNNER_TEMP:-/tmp}/cosmo-gcov-${GITHUB_RUN_ID:-manual}-${GITHUB_RUN_ATTEMPT:-0}"

          # Validate numeric knobs: a non-positive shard_count would skip every loop and
          # the verdict below would pass 0/0 → green with zero tests run.
          require_pos_int() {
            case "$2" in
              ''|*[!0-9]*) echo "error: $1 must be a positive integer (got '$2')" >&2; exit 2 ;;
            esac
            [ "$2" -ge 1 ] || { echo "error: $1 must be >= 1 (got '$2')" >&2; exit 2; }
          }
          require_pos_int COSMO_CATCH2_SHARDS                "$shard_count"
          require_pos_int COSMO_CATCH2_SHARD_TIMEOUT_SECONDS "$shard_timeout_seconds"
          require_pos_int COSMO_TEST_CASE_TIMEOUT_SECONDS    "$case_timeout_seconds"

          # self-hosted runner 工作目录跨 run 复用,历史 run 的 bisect/shards 文件会累积,
          # 被 Upload step 整个目录打包上传,新旧混杂干扰诊断。每次 run 开始清空这两个目录。
          rm -rf "$shards_dir" "$bisect_dir"
          mkdir -p "$shards_dir" "$bisect_dir" "$gcov_dir"
          : > "$summary_file"

          # cosmo-tests is built WITHOUT --coverage by default; coverage is now
          # opt-in via COSMO_ENABLE_COVERAGE. The GCOV_PREFIX exports below are a
          # no-op in the default nightly build, but are kept so that if coverage
          # is re-enabled, gcov writes redirect to a local writable dir instead
          # of the GitHub /__w workspace path that does not exist on this device
          # (libgcov would otherwise pollute stderr / fail process exit).
          export GCOV_PREFIX="$gcov_dir"
          export GCOV_PREFIX_STRIP=4

          if ! command -v timeout >/dev/null 2>&1; then
            echo "Required command is missing: timeout" >&2
            exit 2
          fi

          # 1) Enumerate all test cases. `--verbosity quiet` makes --list-tests print one
          #    plain name per line (Catch2's listTestNamesOnly), which round-trips through
          #    --input-file. Wrap in timeout in case the binary deadlocks while loading.
          list_file="$results_dir/test-list.txt"
          raw_list_file="$results_dir/test-list.raw.txt"
          list_stderr_file="$results_dir/test-list.stderr.log"
          timeout 120s ./cosmo-tests --list-tests --verbosity quiet > "$raw_list_file" 2> "$list_stderr_file"
          list_status=$?
          if [ "$list_status" -ne 0 ]; then
            echo "Failed to list Catch2 test cases, exit code: $list_status" >&2
            cat "$raw_list_file"
            cat "$list_stderr_file"
            exit "$list_status"
          fi

          awk 'NF && $0 !~ /^profiling:/' "$raw_list_file" > "$list_file"

          mapfile -t all_tests < "$list_file"
          total=${#all_tests[@]}
          if [ "$total" -eq 0 ]; then
            echo "No Catch2 test cases found." >&2
            exit 1
          fi

          # Don't create more shards than tests.
          if [ "$total" -lt "$shard_count" ]; then
            shard_count="$total"
          fi

          echo "Listed $total Catch2 test cases; splitting into $shard_count shards (${shard_timeout_seconds}s budget/shard, ${case_timeout_seconds}s/case on bisection)."

          # 2) Round-robin each test name into its shard's --input-file.
          shard_input=()
          for ((s = 0; s < shard_count; s++)); do
            shard_input[$s]="$(printf '%s/shards/%02d.input' "$results_dir" "$((s + 1))")"
            : > "${shard_input[$s]}"
          done
          for i in "${!all_tests[@]}"; do
            name="${all_tests[$i]}"
            [ -n "$name" ] || continue
            s=$((i % shard_count))
            printf '%s\n' "$name" >> "${shard_input[$s]}"
          done

          # 3) Run each shard in its own process. Per-shard timeout bounds a hung test to
          #    the shard budget instead of the whole job; --warn UnmatchedTestSpec turns a
          #    partial spec match (shouldn't happen, but defensive) into a non-zero exit.
          shards_run=0
          shards_passed=0
          problem_shards=0
          bisect_targets=()

          for ((s = 0; s < shard_count; s++)); do
            input="${shard_input[$s]}"
            # An empty --input-file means "no filter" and would run the WHOLE suite.
            [ -s "$input" ] || continue

            idx="$(printf '%02d' "$((s + 1))")"
            log="$shards_dir/$idx.log"
            mapfile -t shard_tests < "$input"
            n=${#shard_tests[@]}
            shards_run=$((shards_run + 1))

            echo
            echo "[$idx] shard with $n tests"

            start_seconds="$(date +%s)"
            timeout --kill-after=10s "${shard_timeout_seconds}s" \
              ./cosmo-tests --input-file "$input" --reporter "$reporter" \
                            --warn UnmatchedTestSpec \
              > "$log" 2>&1
            status=$?
            elapsed_seconds=$(( $(date +%s) - start_seconds ))

            if [ "$status" -eq 0 ]; then
              echo "SHARD PASS in ${elapsed_seconds}s: #$idx ($n tests)"
              printf 'SHARD\tPASS\telapsed_seconds=%s\ttests=%s\tshard=%s\n' "$elapsed_seconds" "$n" "$idx" >> "$summary_file"
              shards_passed=$((shards_passed + 1))
            elif [ "$status" -eq 42 ]; then
              # TestFailureExitCode: an assertion already failed and Catch2 named it.
              echo "SHARD FAIL in ${elapsed_seconds}s: #$idx (assertion failure; last 40 log lines:)"
              tail -n 40 "$log"
              printf 'SHARD\tFAIL\telapsed_seconds=%s\ttests=%s\tshard=%s\n' "$elapsed_seconds" "$n" "$idx" >> "$summary_file"
              problem_shards=$((problem_shards + 1))
            elif [ "$status" -eq 124 ] || [ "$status" -eq 137 ]; then
              echo "SHARD TIMEOUT after ${elapsed_seconds}s: #$idx (will bisect per-test)"
              printf 'SHARD\tTIMEOUT\telapsed_seconds=%s\ttests=%s\tshard=%s\n' "$elapsed_seconds" "$n" "$idx" >> "$summary_file"
              bisect_targets+=("$s")
              problem_shards=$((problem_shards + 1))
            else
              echo "SHARD CRASH($status) in ${elapsed_seconds}s: #$idx (will bisect per-test)"
              printf 'SHARD\tCRASH(%s)\telapsed_seconds=%s\ttests=%s\tshard=%s\n' "$status" "$elapsed_seconds" "$n" "$idx" >> "$summary_file"
              bisect_targets+=("$s")
              problem_shards=$((problem_shards + 1))
            fi
          done

          # 4) For any shard that timed out or crashed, re-run its tests one-per-process to
          #    pinpoint the culprit. Single-spec here, so NoTestsRunExitCode=2 already
          #    catches a no-match; --warn UnmatchedTestSpec kept for parity.
          if [ "${#bisect_targets[@]}" -gt 0 ]; then
            echo
            echo "Bisecting ${#bisect_targets[@]} shard(s) per-test (${case_timeout_seconds}s/case)."
            for s in "${bisect_targets[@]}"; do
              idx="$(printf '%02d' "$((s + 1))")"
              mapfile -t shard_tests < "${shard_input[$s]}"
              echo
              echo "[bisect #$idx] ${#shard_tests[@]} tests"
              for j in "${!shard_tests[@]}"; do
                name="${shard_tests[$j]}"
                [ -n "$name" ] || continue
                number=$((j + 1))
                safe_name="$(printf '%s' "$name" | tr -cs '[:alnum:]_.-' '_' | cut -c1-120)"
                safe_name="${safe_name:-test}"
                log_file="$bisect_dir/${idx}_$(printf '%03d' "$number")_${safe_name}.log"
                filter_file="$bisect_dir/${idx}_$(printf '%03d' "$number")_${safe_name}.filter"
                printf '%s\n' "$name" > "$filter_file"

                echo "[bisect #$idx $number/${#shard_tests[@]}] $name"
                start_seconds="$(date +%s)"
                timeout --kill-after=10s "${case_timeout_seconds}s" \
                  ./cosmo-tests --input-file "$filter_file" --reporter "$reporter" \
                                --warn UnmatchedTestSpec \
                  > "$log_file" 2>&1
                status=$?
                elapsed_seconds=$(( $(date +%s) - start_seconds ))

                if [ "$status" -eq 0 ]; then
                  printf 'BISECT\tPASS\tshard=%s\telapsed_seconds=%s\t%s\n' "$idx" "$elapsed_seconds" "$name" >> "$summary_file"
                elif [ "$status" -eq 124 ] || [ "$status" -eq 137 ]; then
                  echo "  -> TIMEOUT after ${elapsed_seconds}s: $name"
                  printf 'BISECT\tTIMEOUT\tshard=%s\telapsed_seconds=%s\t%s\n' "$idx" "$elapsed_seconds" "$name" >> "$summary_file"
                else
                  echo "  -> FAIL($status) in ${elapsed_seconds}s: $name"
                  printf 'BISECT\tFAIL(%s)\tshard=%s\telapsed_seconds=%s\t%s\n' "$status" "$idx" "$elapsed_seconds" "$name" >> "$summary_file"
                fi
              done
            done
          fi

          # 5) Verdict.
          echo
          echo "Catch2 summary: shards_passed=$shards_passed/$shards_run problem_shards=$problem_shards total_tests=$total"
          echo "Per-shard logs: $shards_dir ; bisect logs: $bisect_dir ; summary: $summary_file"

          # Fail on any problem shard, OR if we somehow ran zero shards (e.g. bad
          # config): a vacuous 0/0 must never pass as green.
          if [ "$problem_shards" -ne 0 ] || [ "$shards_passed" -ne "$shards_run" ] || [ "$shards_run" -eq 0 ]; then
            exit 1
          fi

      - name: Upload Test Results
        if: always()
        uses: actions/upload-artifact@v7
        with:
          name: sophon-catch2-test-results
          path: test-results/catch2/
          retention-days: 3