Nightly Sophon Build and Test #58
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Nightly Sophon Build and Test | |
| on: | |
| # Trigger daily at 2:00 AM Beijing Time (18:00 UTC previous day) | |
| schedule: | |
| - cron: '12 18 * * *' | |
| # Allow manual trigger for testing purposes | |
| workflow_dispatch: | |
| permissions: | |
| contents: read | |
| jobs: | |
| build-sophon: | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 60 | |
| defaults: | |
| run: | |
| shell: bash | |
| outputs: | |
| candidate_guard_sha256: ${{ steps.candidate_guard_runtime.outputs.sha256 }} | |
| candidate_tests_sha256: ${{ steps.candidate_test_binary.outputs.sha256 }} | |
| env: | |
| COSMO_MODEL_GUARD_BUILD_PROFILE: public-runtime | |
| # Cargo defaults each HTTP request to 30 seconds and only three retries. | |
| # The public rsproxy sparse index can exceed that from GitHub runners. | |
| CARGO_HTTP_TIMEOUT: "120" | |
| CARGO_NET_RETRY: "5" | |
| # Specify the public container image for the build environment | |
| container: | |
| image: ghcr.io/cosmo-wander-ai/cosmo_edge-build-env_sophon:v1 | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v7 | |
| with: | |
| submodules: recursive # Recommended if the repository uses git submodules | |
| # Enforce project standards: use the unified build script instead of cmake directly. | |
| # A single configure builds the clean public-runtime package and cosmo-tests | |
| # together, sharing one set of compiled OBJECT libraries; coverage is off | |
| # by default (COSMO_ENABLE_COVERAGE), so the package binary stays clean. | |
| - name: Build package and tests | |
| run: | | |
| chmod +x scripts/build.sh | |
| ./scripts/build.sh -T -c bm1688 | |
| # Upload the build artifacts for deployment or download | |
| - name: Upload Artifacts | |
| uses: actions/upload-artifact@v7 | |
| with: | |
| name: sophon-build-package | |
| # Path where the built binaries/packages are generated by scripts/build.sh | |
| path: build/install/ | |
| overwrite: true | |
| retention-days: 3 | |
| - name: Record Candidate Test Binary | |
| id: candidate_test_binary | |
| run: | | |
| set -euo pipefail | |
| binary="build/cosmo-tests" | |
| if [ ! -f "$binary" ] || [ -L "$binary" ]; then | |
| echo "Candidate test binary is not a regular file: $binary" >&2 | |
| exit 1 | |
| fi | |
| digest="$(sha256sum "$binary" | awk '{print $1}')" | |
| if [ "${#digest}" -ne 64 ] || [[ "$digest" == *[!0-9a-f]* ]]; then | |
| echo "Invalid Candidate test binary SHA-256: $digest" >&2 | |
| exit 1 | |
| fi | |
| echo "sha256=$digest" >> "$GITHUB_OUTPUT" | |
| echo "Candidate test binary SHA-256: $digest" | |
| - name: Upload Test Binary | |
| uses: actions/upload-artifact@v7 | |
| with: | |
| name: sophon-tests-binary | |
| path: build/cosmo-tests | |
| if-no-files-found: error | |
| overwrite: true | |
| retention-days: 1 | |
| - name: Record Candidate Guard Runtime | |
| id: candidate_guard_runtime | |
| run: | | |
| set -euo pipefail | |
| runtime="build/install/lib/libcosmo_model_guard.so.2.0.0" | |
| if [ ! -f "$runtime" ] || [ -L "$runtime" ]; then | |
| echo "Candidate Guard runtime is not a regular file: $runtime" >&2 | |
| exit 1 | |
| fi | |
| digest="$(sha256sum "$runtime" | awk '{print $1}')" | |
| if [ "${#digest}" -ne 64 ] || [[ "$digest" == *[!0-9a-f]* ]]; then | |
| echo "Invalid Candidate Guard SHA-256: $digest" >&2 | |
| exit 1 | |
| fi | |
| echo "sha256=$digest" >> "$GITHUB_OUTPUT" | |
| echo "Candidate Guard SHA-256: $digest" | |
| # Keep the candidate Guard runtime independent from the large package and | |
| # test-binary cache. overwrite=true makes a full job re-run replace the | |
| # artifact, while a failed-job re-run still downloads the build's artifact. | |
| - name: Upload Candidate Guard Runtime | |
| uses: actions/upload-artifact@v7 | |
| with: | |
| name: sophon-model-guard-runtime | |
| path: build/install/lib/libcosmo_model_guard.so.2.0.0 | |
| if-no-files-found: error | |
| overwrite: true | |
| retention-days: 1 | |
| test-sophon: | |
| needs: build-sophon | |
| runs-on: [self-hosted, hosted-sophon-device] | |
| timeout-minutes: 45 # self-hosted 无默认超时，防挂死占用设备（分片+二分；绿跑仅数分钟，45min 是坏夜二分余量） | |
| env: | |
| COSMO_TEST_CASE_TIMEOUT_SECONDS: 120 | |
| COSMO_CATCH2_SHARDS: 12 | |
| COSMO_CATCH2_SHARD_TIMEOUT_SECONDS: 120 | |
| COSMO_CATCH2_RESULTS_DIR: test-results/catch2 | |
| COSMO_CATCH2_REPORTER: compact | |
| COSMO_SOPHON_LD_LIBRARY_PATH: /appfs/cosmo_wander/cwai_data/lib:/data:/usr/lib | |
| COSMO_CANDIDATE_RUNTIME_DIR: ${{ github.workspace }}/candidate-runtime | |
| COSMO_CANDIDATE_GUARD_SHA256: ${{ needs.build-sophon.outputs.candidate_guard_sha256 }} | |
| COSMO_CANDIDATE_TESTS_SHA256: ${{ needs.build-sophon.outputs.candidate_tests_sha256 }} | |
| steps: | |
| # The test binary may be restored from the persistent device cache, but | |
| # the Guard library must always come from this build's current artifact. | |
| - name: Prepare Candidate Guard Runtime Directory | |
| run: | | |
| set -euo pipefail | |
| expected_dir="$GITHUB_WORKSPACE/candidate-runtime" | |
| if [ "$COSMO_CANDIDATE_RUNTIME_DIR" != "$expected_dir" ]; then | |
| echo "Refusing unexpected candidate runtime directory: $COSMO_CANDIDATE_RUNTIME_DIR" >&2 | |
| exit 1 | |
| fi | |
| rm -rf -- "$COSMO_CANDIDATE_RUNTIME_DIR" | |
| mkdir -p -- "$COSMO_CANDIDATE_RUNTIME_DIR" | |
| - name: Download Candidate Guard Runtime | |
| uses: actions/download-artifact@v8 | |
| with: | |
| name: sophon-model-guard-runtime | |
| path: candidate-runtime | |
| - name: Verify Candidate Guard Runtime | |
| run: | | |
| set -euo pipefail | |
| runtime="$COSMO_CANDIDATE_RUNTIME_DIR/libcosmo_model_guard.so.2.0.0" | |
| if [ ! -f "$runtime" ] || [ -L "$runtime" ]; then | |
| echo "Downloaded Candidate Guard runtime is not a regular file: $runtime" >&2 | |
| exit 1 | |
| fi | |
| if [ "${#COSMO_CANDIDATE_GUARD_SHA256}" -ne 64 ] \ | |
| || [[ "$COSMO_CANDIDATE_GUARD_SHA256" == *[!0-9a-f]* ]]; then | |
| echo "Missing or invalid expected Candidate Guard SHA-256." >&2 | |
| exit 1 | |
| fi | |
| actual_sha256="$(sha256sum "$runtime" | awk '{print $1}')" | |
| if [ "$actual_sha256" != "$COSMO_CANDIDATE_GUARD_SHA256" ]; then | |
| echo "Candidate Guard SHA-256 mismatch." >&2 | |
| echo "Expected: $COSMO_CANDIDATE_GUARD_SHA256" >&2 | |
| echo "Actual: $actual_sha256" >&2 | |
| exit 1 | |
| fi | |
| ln -s -- "libcosmo_model_guard.so.2.0.0" \ | |
| "$COSMO_CANDIDATE_RUNTIME_DIR/libcosmo_model_guard.so.2" | |
| ln -s -- "libcosmo_model_guard.so.2" \ | |
| "$COSMO_CANDIDATE_RUNTIME_DIR/libcosmo_model_guard.so" | |
| test "$(readlink "$COSMO_CANDIDATE_RUNTIME_DIR/libcosmo_model_guard.so.2")" \ | |
| = "libcosmo_model_guard.so.2.0.0" | |
| test "$(readlink "$COSMO_CANDIDATE_RUNTIME_DIR/libcosmo_model_guard.so")" \ | |
| = "libcosmo_model_guard.so.2" | |
| echo "Candidate Guard runtime verified: $actual_sha256" | |
| # The Download step below is the dominant wall-clock cost on this | |
| # self-hosted device (fetching the artifact from GitHub over a slow link | |
| # routinely takes 7-13 min, vs seconds-to-minutes for the tests). On | |
| # "Re-run failed jobs" the whole job re-runs, so we cache the binary on | |
| # the device's persistent filesystem keyed by both GITHUB_RUN_ID and the | |
| # build-produced SHA-256. A full job re-run therefore cannot reuse a stale | |
| # binary, while a failed-job re-run can still skip the large download. | |
| - name: Restore Test Binary From Device Cache | |
| id: binary-cache | |
| run: | | |
| set -euo pipefail | |
| if [ "${#COSMO_CANDIDATE_TESTS_SHA256}" -ne 64 ] \ | |
| || [[ "$COSMO_CANDIDATE_TESTS_SHA256" == *[!0-9a-f]* ]]; then | |
| echo "Missing or invalid expected Candidate test binary SHA-256." >&2 | |
| exit 1 | |
| fi | |
| # $HOME persists across runs and is outside the per-job workspace | |
| # GitHub cleans, so a file saved here survives across attempts. | |
| # Override the dir by setting COSMO_TEST_BINARY_CACHE_DIR in env. | |
| cache_dir="${COSMO_TEST_BINARY_CACHE_DIR:-$HOME/cosmo-ci-cache}" | |
| mkdir -p "$cache_dir" | |
| cached="$cache_dir/cosmo-tests-${GITHUB_RUN_ID}-${COSMO_CANDIDATE_TESTS_SHA256}" | |
| if [ -f "$cached" ] && [ ! -L "$cached" ]; then | |
| cached_sha256="$(sha256sum "$cached" | awk '{print $1}')" | |
| if [ "$cached_sha256" = "$COSMO_CANDIDATE_TESTS_SHA256" ]; then | |
| cp -- "$cached" ./cosmo-tests | |
| restored_sha256="$(sha256sum ./cosmo-tests | awk '{print $1}')" | |
| if [ "$restored_sha256" != "$COSMO_CANDIDATE_TESTS_SHA256" ]; then | |
| echo "Restored Candidate test binary SHA-256 mismatch." >&2 | |
| exit 1 | |
| fi | |
| echo "cache-hit=true" >> "$GITHUB_OUTPUT" | |
| echo "Restored verified cosmo-tests from $cached (skipping artifact download)." | |
| else | |
| echo "cache-hit=false" >> "$GITHUB_OUTPUT" | |
| echo "Ignoring corrupt test-binary cache entry: $cached" | |
| fi | |
| else | |
| echo "cache-hit=false" >> "$GITHUB_OUTPUT" | |
| echo "No verified cache entry for run ${GITHUB_RUN_ID} and SHA ${COSMO_CANDIDATE_TESTS_SHA256}; will download." | |
| fi | |
| - name: Download Test Binary | |
| if: steps.binary-cache.outputs.cache-hit != 'true' # skipped on device-cache hit | |
| uses: actions/download-artifact@v8 | |
| with: | |
| name: sophon-tests-binary | |
| path: . | |
| - name: Verify Candidate Test Binary | |
| run: | | |
| set -euo pipefail | |
| if [ ! -f ./cosmo-tests ] || [ -L ./cosmo-tests ]; then | |
| echo "Candidate test binary is not a regular file." >&2 | |
| exit 1 | |
| fi | |
| actual_sha256="$(sha256sum ./cosmo-tests | awk '{print $1}')" | |
| if [ "$actual_sha256" != "$COSMO_CANDIDATE_TESTS_SHA256" ]; then | |
| echo "Candidate test binary SHA-256 mismatch." >&2 | |
| echo "Expected: $COSMO_CANDIDATE_TESTS_SHA256" >&2 | |
| echo "Actual: $actual_sha256" >&2 | |
| exit 1 | |
| fi | |
| echo "Candidate test binary verified: $actual_sha256" | |
| - name: Save Test Binary To Device Cache | |
| if: steps.binary-cache.outputs.cache-hit != 'true' && success() | |
| run: | | |
| set -euo pipefail | |
| cache_dir="${COSMO_TEST_BINARY_CACHE_DIR:-$HOME/cosmo-ci-cache}" | |
| cached="$cache_dir/cosmo-tests-${GITHUB_RUN_ID}-${COSMO_CANDIDATE_TESTS_SHA256}" | |
| pending="${cached}.pending-${GITHUB_RUN_ATTEMPT}" | |
| cp --remove-destination -- ./cosmo-tests "$pending" | |
| pending_sha256="$(sha256sum "$pending" | awk '{print $1}')" | |
| if [ "$pending_sha256" != "$COSMO_CANDIDATE_TESTS_SHA256" ]; then | |
| echo "Refusing to cache test binary with mismatched SHA-256." >&2 | |
| exit 1 | |
| fi | |
| mv -f -- "$pending" "$cached" | |
| # Prune to the 5 most-recent entries; never touch the current run. | |
| # `|| true`: grep exits 1 when only the current run's file exists | |
| # (nothing left after filtering), which under `set -o pipefail` would | |
| # otherwise fail this step. Pruning is best-effort cleanup. | |
| ls -t "$cache_dir"/cosmo-tests-* 2>/dev/null \ | |
| | grep -v -F -- "$cached" \ | |
| | tail -n +6 \ | |
| | xargs -r rm -f || true | |
| echo "Saved cosmo-tests to $cached; cache pruned." | |
| - name: Add Execution Permission | |
| run: chmod +x ./cosmo-tests | |
| - name: Verify Candidate Guard Resolution | |
| run: | | |
| set -euo pipefail | |
| export LD_LIBRARY_PATH="$COSMO_CANDIDATE_RUNTIME_DIR:$COSMO_SOPHON_LD_LIBRARY_PATH:${LD_LIBRARY_PATH:-}" | |
| if ! command -v ldd >/dev/null 2>&1; then | |
| echo "Required command is missing: ldd" >&2 | |
| exit 1 | |
| fi | |
| resolved="$(ldd ./cosmo-tests \ | |
| | awk '$1 == "libcosmo_model_guard.so.2" && $2 == "=>" {print $3; exit}')" | |
| if [ -z "$resolved" ]; then | |
| echo "cosmo-tests did not resolve libcosmo_model_guard.so.2." >&2 | |
| exit 1 | |
| fi | |
| expected_real="$(readlink -f \ | |
| "$COSMO_CANDIDATE_RUNTIME_DIR/libcosmo_model_guard.so.2.0.0")" | |
| resolved_real="$(readlink -f "$resolved")" | |
| if [ "$resolved_real" != "$expected_real" ]; then | |
| echo "cosmo-tests resolved a non-candidate Guard runtime: $resolved" >&2 | |
| exit 1 | |
| fi | |
| echo "cosmo-tests resolves Candidate Guard runtime: $resolved_real" | |
| - name: Run Catch2 Tests | |
| run: | | |
| set +e | |
| set -uo pipefail | |
| export LD_LIBRARY_PATH="$COSMO_CANDIDATE_RUNTIME_DIR:$COSMO_SOPHON_LD_LIBRARY_PATH:${LD_LIBRARY_PATH:-}" | |
| shard_count="$COSMO_CATCH2_SHARDS" | |
| shard_timeout_seconds="$COSMO_CATCH2_SHARD_TIMEOUT_SECONDS" | |
| case_timeout_seconds="$COSMO_TEST_CASE_TIMEOUT_SECONDS" | |
| reporter="$COSMO_CATCH2_REPORTER" | |
| results_dir="$COSMO_CATCH2_RESULTS_DIR" | |
| shards_dir="$results_dir/shards" | |
| bisect_dir="$results_dir/bisect" | |
| summary_file="$results_dir/summary.txt" | |
| gcov_dir="${RUNNER_TEMP:-/tmp}/cosmo-gcov-${GITHUB_RUN_ID:-manual}-${GITHUB_RUN_ATTEMPT:-0}" | |
| # Validate numeric knobs: a non-positive shard_count would skip every loop and | |
| # the verdict below would pass 0/0 → green with zero tests run. | |
| require_pos_int() { | |
| case "$2" in | |
| ''|*[!0-9]*) echo "error: $1 must be a positive integer (got '$2')" >&2; exit 2 ;; | |
| esac | |
| [ "$2" -ge 1 ] || { echo "error: $1 must be >= 1 (got '$2')" >&2; exit 2; } | |
| } | |
| require_pos_int COSMO_CATCH2_SHARDS "$shard_count" | |
| require_pos_int COSMO_CATCH2_SHARD_TIMEOUT_SECONDS "$shard_timeout_seconds" | |
| require_pos_int COSMO_TEST_CASE_TIMEOUT_SECONDS "$case_timeout_seconds" | |
| # --input-file parses each line as a Catch2 test specification. Escape | |
| # grammar characters so punctuation in a literal test name (notably | |
| # commas) cannot turn the generated filter into an invalid spec. | |
| write_catch2_test_spec() { | |
| local escaped="$1" | |
| escaped="${escaped//\\/\\\\}" | |
| escaped="${escaped//,/\\,}" | |
| escaped="${escaped//\"/\\\"}" | |
| printf '"%s"\n' "$escaped" | |
| } | |
| # self-hosted runner 工作目录跨 run 复用,历史 run 的 bisect/shards 文件会累积, | |
| # 被 Upload step 整个目录打包上传,新旧混杂干扰诊断。每次 run 开始清空这两个目录。 | |
| rm -rf "$shards_dir" "$bisect_dir" | |
| mkdir -p "$shards_dir" "$bisect_dir" "$gcov_dir" | |
| : > "$summary_file" | |
| # cosmo-tests is built WITHOUT --coverage by default; coverage is now | |
| # opt-in via COSMO_ENABLE_COVERAGE. The GCOV_PREFIX exports below are a | |
| # no-op in the default nightly build, but are kept so that if coverage | |
| # is re-enabled, gcov writes redirect to a local writable dir instead | |
| # of the GitHub /__w workspace path that does not exist on this device | |
| # (libgcov would otherwise pollute stderr / fail process exit). | |
| export GCOV_PREFIX="$gcov_dir" | |
| export GCOV_PREFIX_STRIP=4 | |
| if ! command -v timeout >/dev/null 2>&1; then | |
| echo "Required command is missing: timeout" >&2 | |
| exit 2 | |
| fi | |
| # 1) Enumerate all test cases. `--verbosity quiet` makes --list-tests print one | |
| # plain name per line (Catch2's listTestNamesOnly), which round-trips through | |
| # --input-file. Wrap in timeout in case the binary deadlocks while loading. | |
| list_file="$results_dir/test-list.txt" | |
| raw_list_file="$results_dir/test-list.raw.txt" | |
| list_stderr_file="$results_dir/test-list.stderr.log" | |
| timeout 120s ./cosmo-tests --list-tests --verbosity quiet > "$raw_list_file" 2> "$list_stderr_file" | |
| list_status=$? | |
| if [ "$list_status" -ne 0 ]; then | |
| echo "Failed to list Catch2 test cases, exit code: $list_status" >&2 | |
| cat "$raw_list_file" | |
| cat "$list_stderr_file" | |
| exit "$list_status" | |
| fi | |
| awk 'NF && $0 !~ /^profiling:/' "$raw_list_file" > "$list_file" | |
| mapfile -t all_tests < "$list_file" | |
| total=${#all_tests[@]} | |
| if [ "$total" -eq 0 ]; then | |
| echo "No Catch2 test cases found." >&2 | |
| exit 1 | |
| fi | |
| # Don't create more shards than tests. | |
| if [ "$total" -lt "$shard_count" ]; then | |
| shard_count="$total" | |
| fi | |
| echo "Listed $total Catch2 test cases; splitting into $shard_count shards (${shard_timeout_seconds}s budget/shard, ${case_timeout_seconds}s/case on bisection)." | |
| # 2) Round-robin each test name into its shard's --input-file. | |
| shard_input=() | |
| for ((s = 0; s < shard_count; s++)); do | |
| shard_input[$s]="$(printf '%s/shards/%02d.input' "$results_dir" "$((s + 1))")" | |
| : > "${shard_input[$s]}" | |
| done | |
| for i in "${!all_tests[@]}"; do | |
| name="${all_tests[$i]}" | |
| [ -n "$name" ] || continue | |
| s=$((i % shard_count)) | |
| printf '%s\n' "$name" >> "${shard_input[$s]}" | |
| done | |
| # 3) Run each shard in its own process. Per-shard timeout bounds a hung test to | |
| # the shard budget instead of the whole job; --warn UnmatchedTestSpec turns a | |
| # partial spec match (shouldn't happen, but defensive) into a non-zero exit. | |
| shards_run=0 | |
| shards_passed=0 | |
| problem_shards=0 | |
| bisect_targets=() | |
| for ((s = 0; s < shard_count; s++)); do | |
| input="${shard_input[$s]}" | |
| # An empty --input-file means "no filter" and would run the WHOLE suite. | |
| [ -s "$input" ] || continue | |
| idx="$(printf '%02d' "$((s + 1))")" | |
| log="$shards_dir/$idx.log" | |
| mapfile -t shard_tests < "$input" | |
| n=${#shard_tests[@]} | |
| shards_run=$((shards_run + 1)) | |
| filter="$shards_dir/$idx.filter" | |
| : > "$filter" | |
| for name in "${shard_tests[@]}"; do | |
| [ -n "$name" ] || continue | |
| write_catch2_test_spec "$name" >> "$filter" | |
| done | |
| echo | |
| echo "[$idx] shard with $n tests" | |
| start_seconds="$(date +%s)" | |
| timeout --kill-after=10s "${shard_timeout_seconds}s" \ | |
| ./cosmo-tests --input-file "$filter" --reporter "$reporter" \ | |
| --warn UnmatchedTestSpec \ | |
| > "$log" 2>&1 | |
| status=$? | |
| elapsed_seconds=$(( $(date +%s) - start_seconds )) | |
| if [ "$status" -eq 0 ]; then | |
| echo "SHARD PASS in ${elapsed_seconds}s: #$idx ($n tests)" | |
| printf 'SHARD\tPASS\telapsed_seconds=%s\ttests=%s\tshard=%s\n' "$elapsed_seconds" "$n" "$idx" >> "$summary_file" | |
| shards_passed=$((shards_passed + 1)) | |
| elif [ "$status" -eq 42 ]; then | |
| # TestFailureExitCode: an assertion already failed and Catch2 named it. | |
| echo "SHARD FAIL in ${elapsed_seconds}s: #$idx (assertion failure; last 40 log lines:)" | |
| tail -n 40 "$log" | |
| printf 'SHARD\tFAIL\telapsed_seconds=%s\ttests=%s\tshard=%s\n' "$elapsed_seconds" "$n" "$idx" >> "$summary_file" | |
| problem_shards=$((problem_shards + 1)) | |
| elif [ "$status" -eq 124 ] || [ "$status" -eq 137 ]; then | |
| echo "SHARD TIMEOUT after ${elapsed_seconds}s: #$idx (will bisect per-test)" | |
| printf 'SHARD\tTIMEOUT\telapsed_seconds=%s\ttests=%s\tshard=%s\n' "$elapsed_seconds" "$n" "$idx" >> "$summary_file" | |
| bisect_targets+=("$s") | |
| problem_shards=$((problem_shards + 1)) | |
| else | |
| echo "SHARD CRASH($status) in ${elapsed_seconds}s: #$idx (will bisect per-test)" | |
| printf 'SHARD\tCRASH(%s)\telapsed_seconds=%s\ttests=%s\tshard=%s\n' "$status" "$elapsed_seconds" "$n" "$idx" >> "$summary_file" | |
| bisect_targets+=("$s") | |
| problem_shards=$((problem_shards + 1)) | |
| fi | |
| done | |
| # 4) For any shard that timed out or crashed, re-run its tests one-per-process to | |
| # pinpoint the culprit. Use the same literal-name escaping as the shard run; | |
| # --warn UnmatchedTestSpec still catches a no-match. | |
| if [ "${#bisect_targets[@]}" -gt 0 ]; then | |
| echo | |
| echo "Bisecting ${#bisect_targets[@]} shard(s) per-test (${case_timeout_seconds}s/case)." | |
| for s in "${bisect_targets[@]}"; do | |
| idx="$(printf '%02d' "$((s + 1))")" | |
| mapfile -t shard_tests < "${shard_input[$s]}" | |
| echo | |
| echo "[bisect #$idx] ${#shard_tests[@]} tests" | |
| for j in "${!shard_tests[@]}"; do | |
| name="${shard_tests[$j]}" | |
| [ -n "$name" ] || continue | |
| number=$((j + 1)) | |
| safe_name="$(printf '%s' "$name" | tr -cs '[:alnum:]_.-' '_' | cut -c1-120)" | |
| safe_name="${safe_name:-test}" | |
| log_file="$bisect_dir/${idx}_$(printf '%03d' "$number")_${safe_name}.log" | |
| filter_file="$bisect_dir/${idx}_$(printf '%03d' "$number")_${safe_name}.filter" | |
| write_catch2_test_spec "$name" > "$filter_file" | |
| echo "[bisect #$idx $number/${#shard_tests[@]}] $name" | |
| start_seconds="$(date +%s)" | |
| timeout --kill-after=10s "${case_timeout_seconds}s" \ | |
| ./cosmo-tests --input-file "$filter_file" --reporter "$reporter" \ | |
| --warn UnmatchedTestSpec \ | |
| > "$log_file" 2>&1 | |
| status=$? | |
| elapsed_seconds=$(( $(date +%s) - start_seconds )) | |
| if [ "$status" -eq 0 ]; then | |
| printf 'BISECT\tPASS\tshard=%s\telapsed_seconds=%s\t%s\n' "$idx" "$elapsed_seconds" "$name" >> "$summary_file" | |
| elif [ "$status" -eq 124 ] || [ "$status" -eq 137 ]; then | |
| echo " -> TIMEOUT after ${elapsed_seconds}s: $name" | |
| printf 'BISECT\tTIMEOUT\tshard=%s\telapsed_seconds=%s\t%s\n' "$idx" "$elapsed_seconds" "$name" >> "$summary_file" | |
| else | |
| echo " -> FAIL($status) in ${elapsed_seconds}s: $name" | |
| printf 'BISECT\tFAIL(%s)\tshard=%s\telapsed_seconds=%s\t%s\n' "$status" "$idx" "$elapsed_seconds" "$name" >> "$summary_file" | |
| fi | |
| done | |
| done | |
| fi | |
| # 5) Verdict. | |
| echo | |
| echo "Catch2 summary: shards_passed=$shards_passed/$shards_run problem_shards=$problem_shards total_tests=$total" | |
| echo "Per-shard logs: $shards_dir ; bisect logs: $bisect_dir ; summary: $summary_file" | |
| # Fail on any problem shard, OR if we somehow ran zero shards (e.g. bad | |
| # config): a vacuous 0/0 must never pass as green. | |
| if [ "$problem_shards" -ne 0 ] || [ "$shards_passed" -ne "$shards_run" ] || [ "$shards_run" -eq 0 ]; then | |
| exit 1 | |
| fi | |
| - name: Upload Test Results | |
| if: always() | |
| uses: actions/upload-artifact@v7 | |
| with: | |
| name: sophon-catch2-test-results | |
| path: test-results/catch2/ | |
| retention-days: 3 |