Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .gitattributes
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@ Makefile text eol=lf
*.bmodel binary
*.onnx binary
*.so binary
*.so.* binary
*.a binary
*.png binary
*.jpg binary
Expand Down
198 changes: 182 additions & 16 deletions .github/workflows/nightly-build-test-sophon.yml
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,11 @@ jobs:
build-sophon:
runs-on: ubuntu-latest
timeout-minutes: 60
outputs:
candidate_guard_sha256: ${{ steps.candidate_guard_runtime.outputs.sha256 }}
candidate_tests_sha256: ${{ steps.candidate_test_binary.outputs.sha256 }}
env:
COSMO_MODEL_GUARD_BUILD_PROFILE: public-runtime

# Specify the public container image for the build environment
container:
Expand All @@ -26,7 +31,7 @@ jobs:
submodules: recursive # Recommended if the repository uses git submodules

# Enforce project standards: use the unified build script instead of cmake directly.
# A single configure builds the clean production package and cosmo-tests
# A single configure builds the clean public-runtime package and cosmo-tests
# together, sharing one set of compiled OBJECT libraries; coverage is off
# by default (COSMO_ENABLE_COVERAGE), so the package binary stays clean.
- name: Build package and tests
Expand All @@ -41,13 +46,62 @@ jobs:
name: sophon-build-package
# Path where the built binaries/packages are generated by scripts/build.sh
path: build/install/
overwrite: true
retention-days: 3

- name: Record Candidate Test Binary
id: candidate_test_binary
run: |
set -euo pipefail
binary="build/cosmo-tests"
if [ ! -f "$binary" ] || [ -L "$binary" ]; then
echo "Candidate test binary is not a regular file: $binary" >&2
exit 1
fi
digest="$(sha256sum "$binary" | awk '{print $1}')"
if [ "${#digest}" -ne 64 ] || [[ "$digest" == *[!0-9a-f]* ]]; then
echo "Invalid Candidate test binary SHA-256: $digest" >&2
exit 1
fi
echo "sha256=$digest" >> "$GITHUB_OUTPUT"
echo "Candidate test binary SHA-256: $digest"

- name: Upload Test Binary
uses: actions/upload-artifact@v7
with:
name: sophon-tests-binary
path: build/cosmo-tests
if-no-files-found: error
overwrite: true
retention-days: 1

- name: Record Candidate Guard Runtime
id: candidate_guard_runtime
run: |
set -euo pipefail
runtime="build/install/lib/libcosmo_model_guard.so.2.0.0"
if [ ! -f "$runtime" ] || [ -L "$runtime" ]; then
echo "Candidate Guard runtime is not a regular file: $runtime" >&2
exit 1
fi
digest="$(sha256sum "$runtime" | awk '{print $1}')"
if [ "${#digest}" -ne 64 ] || [[ "$digest" == *[!0-9a-f]* ]]; then
echo "Invalid Candidate Guard SHA-256: $digest" >&2
exit 1
fi
echo "sha256=$digest" >> "$GITHUB_OUTPUT"
echo "Candidate Guard SHA-256: $digest"

# Keep the candidate Guard runtime independent from the large package and
# test-binary cache. overwrite=true makes a full job re-run replace the
# artifact, while a failed-job re-run still downloads the build's artifact.
- name: Upload Candidate Guard Runtime
uses: actions/upload-artifact@v7
with:
name: sophon-model-guard-runtime
path: build/install/lib/libcosmo_model_guard.so.2.0.0
if-no-files-found: error
overwrite: true
retention-days: 1

test-sophon:
Expand All @@ -61,34 +115,100 @@ jobs:
COSMO_CATCH2_RESULTS_DIR: test-results/catch2
COSMO_CATCH2_REPORTER: compact
COSMO_SOPHON_LD_LIBRARY_PATH: /appfs/cosmo_wander/cwai_data/lib:/data:/usr/lib
COSMO_CANDIDATE_RUNTIME_DIR: ${{ github.workspace }}/candidate-runtime
COSMO_CANDIDATE_GUARD_SHA256: ${{ needs.build-sophon.outputs.candidate_guard_sha256 }}
COSMO_CANDIDATE_TESTS_SHA256: ${{ needs.build-sophon.outputs.candidate_tests_sha256 }}

steps:
# The test binary may be restored from the persistent device cache, but
# the Guard library must always come from this build's current artifact.
- name: Prepare Candidate Guard Runtime Directory
run: |
set -euo pipefail
expected_dir="$GITHUB_WORKSPACE/candidate-runtime"
if [ "$COSMO_CANDIDATE_RUNTIME_DIR" != "$expected_dir" ]; then
echo "Refusing unexpected candidate runtime directory: $COSMO_CANDIDATE_RUNTIME_DIR" >&2
exit 1
fi
rm -rf -- "$COSMO_CANDIDATE_RUNTIME_DIR"
mkdir -p -- "$COSMO_CANDIDATE_RUNTIME_DIR"

- name: Download Candidate Guard Runtime
uses: actions/download-artifact@v8
with:
name: sophon-model-guard-runtime
path: candidate-runtime

- name: Verify Candidate Guard Runtime
run: |
set -euo pipefail
runtime="$COSMO_CANDIDATE_RUNTIME_DIR/libcosmo_model_guard.so.2.0.0"
if [ ! -f "$runtime" ] || [ -L "$runtime" ]; then
echo "Downloaded Candidate Guard runtime is not a regular file: $runtime" >&2
exit 1
fi
if [ "${#COSMO_CANDIDATE_GUARD_SHA256}" -ne 64 ] \
|| [[ "$COSMO_CANDIDATE_GUARD_SHA256" == *[!0-9a-f]* ]]; then
echo "Missing or invalid expected Candidate Guard SHA-256." >&2
exit 1
fi
actual_sha256="$(sha256sum "$runtime" | awk '{print $1}')"
if [ "$actual_sha256" != "$COSMO_CANDIDATE_GUARD_SHA256" ]; then
echo "Candidate Guard SHA-256 mismatch." >&2
echo "Expected: $COSMO_CANDIDATE_GUARD_SHA256" >&2
echo "Actual: $actual_sha256" >&2
exit 1
fi
ln -s -- "libcosmo_model_guard.so.2.0.0" \
"$COSMO_CANDIDATE_RUNTIME_DIR/libcosmo_model_guard.so.2"
ln -s -- "libcosmo_model_guard.so.2" \
"$COSMO_CANDIDATE_RUNTIME_DIR/libcosmo_model_guard.so"
test "$(readlink "$COSMO_CANDIDATE_RUNTIME_DIR/libcosmo_model_guard.so.2")" \
= "libcosmo_model_guard.so.2.0.0"
test "$(readlink "$COSMO_CANDIDATE_RUNTIME_DIR/libcosmo_model_guard.so")" \
= "libcosmo_model_guard.so.2"
echo "Candidate Guard runtime verified: $actual_sha256"

# The Download step below is the dominant wall-clock cost on this
# self-hosted device (fetching the artifact from GitHub over a slow link
# routinely takes 7-13 min, vs seconds-to-minutes for the tests). On
# "Re-run failed jobs" the whole job re-runs, so we cache the binary on
# the device's persistent filesystem keyed by GITHUB_RUN_ID (stable across
# re-run attempts) and skip re-downloading on a cache hit.
# CAVEAT: "Re-run all jobs" rebuilds the binary under the same artifact
# name; this cache would then serve a stale binary. Before a full re-run,
# clear the cache dir on the device (rm -rf $HOME/cosmo-ci-cache).
# the device's persistent filesystem keyed by both GITHUB_RUN_ID and the
# build-produced SHA-256. A full job re-run therefore cannot reuse a stale
# binary, while a failed-job re-run can still skip the large download.
- name: Restore Test Binary From Device Cache
id: binary-cache
run: |
set -euo pipefail
if [ "${#COSMO_CANDIDATE_TESTS_SHA256}" -ne 64 ] \
|| [[ "$COSMO_CANDIDATE_TESTS_SHA256" == *[!0-9a-f]* ]]; then
echo "Missing or invalid expected Candidate test binary SHA-256." >&2
exit 1
fi
# $HOME persists across runs and is outside the per-job workspace
# GitHub cleans, so a file saved here survives across attempts.
# Override the dir by setting COSMO_TEST_BINARY_CACHE_DIR in env.
cache_dir="${COSMO_TEST_BINARY_CACHE_DIR:-$HOME/cosmo-ci-cache}"
mkdir -p "$cache_dir"
cached="$cache_dir/cosmo-tests-${GITHUB_RUN_ID}"
if [ -f "$cached" ]; then
cp "$cached" ./cosmo-tests
echo "cache-hit=true" >> "$GITHUB_OUTPUT"
echo "Restored cosmo-tests from $cached (skipping artifact download)."
cached="$cache_dir/cosmo-tests-${GITHUB_RUN_ID}-${COSMO_CANDIDATE_TESTS_SHA256}"
if [ -f "$cached" ] && [ ! -L "$cached" ]; then
cached_sha256="$(sha256sum "$cached" | awk '{print $1}')"
if [ "$cached_sha256" = "$COSMO_CANDIDATE_TESTS_SHA256" ]; then
cp -- "$cached" ./cosmo-tests
restored_sha256="$(sha256sum ./cosmo-tests | awk '{print $1}')"
if [ "$restored_sha256" != "$COSMO_CANDIDATE_TESTS_SHA256" ]; then
echo "Restored Candidate test binary SHA-256 mismatch." >&2
exit 1
fi
echo "cache-hit=true" >> "$GITHUB_OUTPUT"
echo "Restored verified cosmo-tests from $cached (skipping artifact download)."
else
echo "cache-hit=false" >> "$GITHUB_OUTPUT"
echo "Ignoring corrupt test-binary cache entry: $cached"
fi
else
echo "cache-hit=false" >> "$GITHUB_OUTPUT"
echo "No cached binary for run ${GITHUB_RUN_ID}; will download."
echo "No verified cache entry for run ${GITHUB_RUN_ID} and SHA ${COSMO_CANDIDATE_TESTS_SHA256}; will download."
fi

- name: Download Test Binary
Expand All @@ -98,32 +218,78 @@ jobs:
name: sophon-tests-binary
path: .

- name: Verify Candidate Test Binary
run: |
set -euo pipefail
if [ ! -f ./cosmo-tests ] || [ -L ./cosmo-tests ]; then
echo "Candidate test binary is not a regular file." >&2
exit 1
fi
actual_sha256="$(sha256sum ./cosmo-tests | awk '{print $1}')"
if [ "$actual_sha256" != "$COSMO_CANDIDATE_TESTS_SHA256" ]; then
echo "Candidate test binary SHA-256 mismatch." >&2
echo "Expected: $COSMO_CANDIDATE_TESTS_SHA256" >&2
echo "Actual: $actual_sha256" >&2
exit 1
fi
echo "Candidate test binary verified: $actual_sha256"

- name: Save Test Binary To Device Cache
if: steps.binary-cache.outputs.cache-hit != 'true' && success()
run: |
set -euo pipefail
cache_dir="${COSMO_TEST_BINARY_CACHE_DIR:-$HOME/cosmo-ci-cache}"
cached="$cache_dir/cosmo-tests-${GITHUB_RUN_ID}"
cp ./cosmo-tests "$cached"
cached="$cache_dir/cosmo-tests-${GITHUB_RUN_ID}-${COSMO_CANDIDATE_TESTS_SHA256}"
pending="${cached}.pending-${GITHUB_RUN_ATTEMPT}"
cp --remove-destination -- ./cosmo-tests "$pending"
pending_sha256="$(sha256sum "$pending" | awk '{print $1}')"
if [ "$pending_sha256" != "$COSMO_CANDIDATE_TESTS_SHA256" ]; then
echo "Refusing to cache test binary with mismatched SHA-256." >&2
exit 1
fi
mv -f -- "$pending" "$cached"
# Prune to the 5 most-recent entries; never touch the current run.
# `|| true`: grep exits 1 when only the current run's file exists
# (nothing left after filtering), which under `set -o pipefail` would
# otherwise fail this step. Pruning is best-effort cleanup.
ls -t "$cache_dir"/cosmo-tests-* 2>/dev/null \
| grep -v -F -- "cosmo-tests-${GITHUB_RUN_ID}" \
| grep -v -F -- "$cached" \
| tail -n +6 \
| xargs -r rm -f || true
echo "Saved cosmo-tests to $cached; cache pruned."

- name: Add Execution Permission
run: chmod +x ./cosmo-tests

- name: Verify Candidate Guard Resolution
run: |
set -euo pipefail
export LD_LIBRARY_PATH="$COSMO_CANDIDATE_RUNTIME_DIR:$COSMO_SOPHON_LD_LIBRARY_PATH:${LD_LIBRARY_PATH:-}"
if ! command -v ldd >/dev/null 2>&1; then
echo "Required command is missing: ldd" >&2
exit 1
fi
resolved="$(ldd ./cosmo-tests \
| awk '$1 == "libcosmo_model_guard.so.2" && $2 == "=>" {print $3; exit}')"
if [ -z "$resolved" ]; then
echo "cosmo-tests did not resolve libcosmo_model_guard.so.2." >&2
exit 1
fi
expected_real="$(readlink -f \
"$COSMO_CANDIDATE_RUNTIME_DIR/libcosmo_model_guard.so.2.0.0")"
resolved_real="$(readlink -f "$resolved")"
if [ "$resolved_real" != "$expected_real" ]; then
echo "cosmo-tests resolved a non-candidate Guard runtime: $resolved" >&2
exit 1
fi
echo "cosmo-tests resolves Candidate Guard runtime: $resolved_real"

- name: Run Catch2 Tests
run: |
set +e
set -uo pipefail

export LD_LIBRARY_PATH="$COSMO_SOPHON_LD_LIBRARY_PATH:${LD_LIBRARY_PATH:-}"
export LD_LIBRARY_PATH="$COSMO_CANDIDATE_RUNTIME_DIR:$COSMO_SOPHON_LD_LIBRARY_PATH:${LD_LIBRARY_PATH:-}"

shard_count="$COSMO_CATCH2_SHARDS"
shard_timeout_seconds="$COSMO_CATCH2_SHARD_TIMEOUT_SECONDS"
Expand Down
Binary file not shown.
Loading
Loading