trusted-blue-eval #461
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: trusted-blue-eval | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| profile_override: | |
| description: Optional runtime profile override for this evaluation run | |
| required: false | |
| type: string | |
| publish_pr_comment: | |
| description: Publish or update a PR comment with the evaluation result | |
| required: false | |
| default: true | |
| type: boolean | |
| # Auto-fire after blue-pr-sync registers a new patch in the API queue. | |
| # The claim step returns HTTP 204 when the queue is empty, so spurious | |
| # triggers are no-ops. | |
| workflow_run: | |
| workflows: ["blue-pr-sync"] | |
| types: [completed] | |
| permissions: | |
| contents: read | |
| env: | |
| KERNELGUARD_API_BASE_URL: https://kguard.sinatras.dev | |
| # Per-trigger concurrency group so multiple PRs syncing within seconds of | |
| # each other do not cancel each other out (GitHub allows only 1 pending + | |
| # 1 running per group, beyond which new runs auto-cancel). Each | |
| # blue-pr-sync run carries the candidate head SHA on | |
| # event.workflow_run.head_sha; manual workflow_dispatch falls back to the | |
| # unique run id so manual drains never collide with each other either. | |
| concurrency: | |
| group: trusted-blue-eval-${{ github.event.workflow_run.head_sha || github.run_id }} | |
| cancel-in-progress: false | |
| jobs: | |
| claim: | |
| # Skip when triggered by a failed blue-pr-sync (rejected gate, static | |
| # security failure, etc.). Manual workflow_dispatch always passes this | |
| # check because workflow_run context is empty in that case. | |
| if: ${{ github.event_name != 'workflow_run' || github.event.workflow_run.conclusion == 'success' }} | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 10 | |
| environment: trusted-blue-eval | |
| outputs: | |
| claimed: ${{ steps.claim.outputs.claimed }} | |
| evaluation_job_id: ${{ steps.claim.outputs.evaluation_job_id }} | |
| blue_submission_id: ${{ steps.claim.outputs.blue_submission_id }} | |
| repo_full_name: ${{ steps.claim.outputs.repo_full_name }} | |
| pr_number: ${{ steps.claim.outputs.pr_number }} | |
| head_sha: ${{ steps.claim.outputs.head_sha }} | |
| server_profile: ${{ steps.claim.outputs.server_profile }} | |
| claim_lease_token: ${{ steps.claim.outputs.claim_lease_token }} | |
| claim_lease_expires_at: ${{ steps.claim.outputs.claim_lease_expires_at }} | |
| env: | |
| PIP_DISABLE_PIP_VERSION_CHECK: "1" | |
| PYTHONDONTWRITEBYTECODE: "1" | |
| steps: | |
| - name: Checkout trusted default-branch automation code | |
| uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd | |
| with: | |
| ref: refs/heads/${{ github.event.repository.default_branch }} | |
| path: trusted | |
| persist-credentials: false | |
| fetch-depth: 1 | |
| - name: Set up Python | |
| uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 | |
| with: | |
| python-version: "3.12" | |
| - name: Claim next evaluation job | |
| id: claim | |
| env: | |
| KERNELGUARD_API_INTERNAL_TOKEN: ${{ secrets.KERNELGUARD_API_INTERNAL_TOKEN }} | |
| run: | | |
| python trusted/scripts/github/claim_blue_evaluation.py \ | |
| --api-base-url "$KERNELGUARD_API_BASE_URL" \ | |
| --internal-token "$KERNELGUARD_API_INTERNAL_TOKEN" \ | |
| --output-json "$RUNNER_TEMP/kernelguard-claim-ref.json" | |
| evaluate: | |
| needs: claim | |
| if: needs.claim.outputs.claimed == 'true' | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 30 | |
| env: | |
| PIP_DISABLE_PIP_VERSION_CHECK: "1" | |
| PYTHONDONTWRITEBYTECODE: "1" | |
| steps: | |
| - name: Checkout trusted default-branch automation code | |
| uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd | |
| with: | |
| ref: refs/heads/${{ github.event.repository.default_branch }} | |
| path: trusted | |
| persist-credentials: false | |
| fetch-depth: 1 | |
| - name: Set up Python | |
| uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 | |
| with: | |
| python-version: "3.12" | |
| - name: Fetch claimed evaluation payload | |
| run: | | |
| python trusted/scripts/github/fetch_blue_evaluation_claim.py \ | |
| --api-base-url "$KERNELGUARD_API_BASE_URL" \ | |
| --evaluation-job-id "${{ needs.claim.outputs.evaluation_job_id }}" \ | |
| --lease-token "${{ needs.claim.outputs.claim_lease_token }}" \ | |
| --output-json "$RUNNER_TEMP/kernelguard-claim.json" | |
| - name: Checkout candidate PR head | |
| uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd | |
| with: | |
| repository: ${{ needs.claim.outputs.repo_full_name }} | |
| ref: refs/pull/${{ needs.claim.outputs.pr_number }}/head | |
| path: candidate | |
| persist-credentials: false | |
| fetch-depth: 1 | |
| - name: Verify candidate checkout matches claimed head SHA | |
| run: | | |
| actual_sha="$(git -C candidate rev-parse HEAD)" | |
| test "$actual_sha" = "${{ needs.claim.outputs.head_sha }}" | |
| - name: Static RCE security check | |
| run: | | |
| python trusted/scripts/github/blue_pr_static_security.py \ | |
| --base-file trusted/kernelguard.py \ | |
| --candidate-file candidate/kernelguard.py \ | |
| --output-json "$RUNNER_TEMP/blue-pr-static-security.json" | |
| - name: Evaluate candidate PR in isolated runner context | |
| run: | | |
| if [ -n "${{ inputs.profile_override }}" ]; then | |
| python trusted/scripts/github/evaluate_blue_pr.py \ | |
| --claim-json "$RUNNER_TEMP/kernelguard-claim.json" \ | |
| --pr-checkout candidate \ | |
| --output-json "$RUNNER_TEMP/kernelguard-result.json" \ | |
| --profile "${{ inputs.profile_override }}" | |
| else | |
| python trusted/scripts/github/evaluate_blue_pr.py \ | |
| --claim-json "$RUNNER_TEMP/kernelguard-claim.json" \ | |
| --pr-checkout candidate \ | |
| --output-json "$RUNNER_TEMP/kernelguard-result.json" | |
| fi | |
| - name: Post evaluation result back to KernelGuard API | |
| run: | | |
| python trusted/scripts/github/complete_blue_evaluation.py \ | |
| --api-base-url "$KERNELGUARD_API_BASE_URL" \ | |
| --evaluation-job-id "${{ needs.claim.outputs.evaluation_job_id }}" \ | |
| --lease-token "${{ needs.claim.outputs.claim_lease_token }}" \ | |
| --result-json "$RUNNER_TEMP/kernelguard-result.json" \ | |
| --output-json "$RUNNER_TEMP/kernelguard-complete.json" | |
| - name: Write public evaluation summary | |
| run: | | |
| python trusted/scripts/github/write_blue_public_summary.py \ | |
| --completion-json "$RUNNER_TEMP/kernelguard-complete.json" \ | |
| --evaluation-job-id "${{ needs.claim.outputs.evaluation_job_id }}" \ | |
| --blue-submission-id "${{ needs.claim.outputs.blue_submission_id }}" \ | |
| --repo-full-name "${{ needs.claim.outputs.repo_full_name }}" \ | |
| --pr-number "${{ needs.claim.outputs.pr_number }}" \ | |
| --head-sha "${{ needs.claim.outputs.head_sha }}" \ | |
| --output-json "$RUNNER_TEMP/kernelguard-public-summary.json" | |
| - name: Upload public evaluation summary | |
| uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a | |
| with: | |
| name: kernelguard-public-summary-${{ needs.claim.outputs.evaluation_job_id }} | |
| path: ${{ runner.temp }}/kernelguard-public-summary.json | |
| if-no-files-found: error | |
| retention-days: 7 | |
| report: | |
| needs: | |
| - claim | |
| - evaluate | |
| if: always() && needs.claim.outputs.claimed == 'true' | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 15 | |
| permissions: | |
| contents: read | |
| issues: write | |
| pull-requests: write | |
| env: | |
| PIP_DISABLE_PIP_VERSION_CHECK: "1" | |
| PYTHONDONTWRITEBYTECODE: "1" | |
| steps: | |
| - name: Checkout trusted default-branch automation code | |
| uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd | |
| with: | |
| ref: refs/heads/${{ github.event.repository.default_branch }} | |
| path: trusted | |
| persist-credentials: false | |
| fetch-depth: 1 | |
| - name: Set up Python | |
| uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 | |
| with: | |
| python-version: "3.12" | |
| - name: Download public evaluation summary | |
| if: needs.evaluate.result == 'success' | |
| continue-on-error: true | |
| uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c | |
| with: | |
| name: kernelguard-public-summary-${{ needs.claim.outputs.evaluation_job_id }} | |
| path: ${{ runner.temp }}/public-summary | |
| - name: Complete failed evaluation if needed | |
| env: | |
| RESULT_JSON: ${{ runner.temp }}/kernelguard-failure-result.json | |
| COMPLETE_JSON: ${{ runner.temp }}/kernelguard-complete.json | |
| PUBLIC_SUMMARY_JSON: ${{ runner.temp }}/public-summary/kernelguard-public-summary.json | |
| EVALUATE_JOB_RESULT: ${{ needs.evaluate.result }} | |
| run: | | |
| if [ ! -f "$PUBLIC_SUMMARY_JSON" ]; then | |
| mkdir -p "$(dirname "$PUBLIC_SUMMARY_JSON")" | |
| python -c 'import json, os; result_path = os.environ["RESULT_JSON"]; evaluate_job_result = os.environ.get("EVALUATE_JOB_RESULT") or "unknown"; message = f"evaluation job did not produce a public summary artifact: {evaluate_job_result}"; payload = {"status": "failed", "worker_version": "trusted-blue-evaluator/1.0", "error": message, "summary": {"passed": False, "failure_reason": message, "target_red_submission": {"passed": False}, "validation_suite": {"true_positive_passed": 0, "true_positive_total": 0, "false_positive_passed": 0, "false_positive_total": 0, "passed": False}, "surgicalness_score": 0.0}, "case_results": [], "artifacts": [{"kind": "evaluation_error", "value": message}]}; open(result_path, "w", encoding="utf-8").write(json.dumps(payload, indent=2, sort_keys=True) + "\\n")' | |
| python trusted/scripts/github/complete_blue_evaluation.py \ | |
| --api-base-url "$KERNELGUARD_API_BASE_URL" \ | |
| --evaluation-job-id "${{ needs.claim.outputs.evaluation_job_id }}" \ | |
| --lease-token "${{ needs.claim.outputs.claim_lease_token }}" \ | |
| --result-json "$RESULT_JSON" \ | |
| --output-json "$COMPLETE_JSON" | |
| python trusted/scripts/github/write_blue_public_summary.py \ | |
| --completion-json "$COMPLETE_JSON" \ | |
| --evaluation-job-id "${{ needs.claim.outputs.evaluation_job_id }}" \ | |
| --blue-submission-id "${{ needs.claim.outputs.blue_submission_id }}" \ | |
| --repo-full-name "${{ needs.claim.outputs.repo_full_name }}" \ | |
| --pr-number "${{ needs.claim.outputs.pr_number }}" \ | |
| --head-sha "${{ needs.claim.outputs.head_sha }}" \ | |
| --output-json "$PUBLIC_SUMMARY_JSON" | |
| fi | |
| - name: Update PR comment with evaluation summary | |
| # Post the comment by default. inputs.* only resolves for | |
| # workflow_dispatch, so for workflow_run we want the comment; | |
| # for explicit dispatch we honor the publish_pr_comment toggle. | |
| if: github.event_name != 'workflow_dispatch' || inputs.publish_pr_comment | |
| env: | |
| GITHUB_TOKEN: ${{ github.token }} | |
| WORKFLOW_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} | |
| run: | | |
| python trusted/scripts/github/post_blue_evaluation_comment.py \ | |
| --public-summary-json "$RUNNER_TEMP/public-summary/kernelguard-public-summary.json" \ | |
| --github-token "$GITHUB_TOKEN" \ | |
| --workflow-run-url "$WORKFLOW_RUN_URL" |