Add scoreboard freshness check that opens an issue on stale data (#123)

strimo378 · claude · andife · web-flow · commit 28aa7777e1fb · 2026-07-20T16:18:10.000+02:00
* Add scoreboard freshness check that opens an issue on stale data Adds a daily workflow that fetches the published scoreboard page (index.html), reads each backend's last update date from the embedded data and opens a tracking issue when any backend is older than 3 days. To avoid issue spam a single labelled issue is reused: it is created on the first stale detection, refreshed silently while it stays stale, and closed automatically once all backends are up to date again. - scripts/check_scoreboard_freshness.py: parser + freshness evaluation, writes results to GITHUB_OUTPUT (configurable --max-age-days, default 3) - .github/workflows/scoreboard-freshness.yml: daily schedule + manual dispatch, manages the tracking issue via actions/github-script - .gitignore: whitelist the new script (scripts/* is ignored by default) Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01EeeD61kc68kKLbHuSmahUc Signed-off-by: Timo Stripf <timo.stripf@emmtrix.com> * Pin GitHub Actions to commit SHAs Addresses zizmor code-scanning findings: actions must be pinned to a full commit hash per the repo's blanket policy. - actions/checkout -> de0fac2 (v6.0.2, matches existing workflows) - actions/setup-python -> a309ff8 (v6.2.0) - actions/github-script -> f28e40c (v7.1.0) Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com> Signed-off-by: Timo Stripf <timo.stripf@emmtrix.com> * Fix scoreboard freshness review comments Signed-off-by: Andreas Fehlner <fehlner@arcor.de> * Fix CodeQL issue in freshness test Signed-off-by: Andreas Fehlner <fehlner@arcor.de> * Fix Sonar findings in freshness script Signed-off-by: Andreas Fehlner <fehlner@arcor.de> --------- Signed-off-by: Timo Stripf <timo.stripf@emmtrix.com> Signed-off-by: Andreas Fehlner <fehlner@arcor.de> Co-authored-by: Claude Opus 4.8 <noreply@anthropic.com> Co-authored-by: Andreas Fehlner <fehlner@arcor.de>
diff --git a/.github/workflows/scoreboard-freshness.yml b/.github/workflows/scoreboard-freshness.yml
@@ -0,0 +1,118 @@
+# Copyright (c) ONNX Project Contributors
+#
+# SPDX-License-Identifier: Apache-2.0
+
+name: Scoreboard freshness check
+
+on:
+  schedule:
+    # Run a few hours after the website is rebuilt/deployed.
+    - cron: '0 9 * * *'
+  workflow_dispatch:
+
+permissions: {}
+
+jobs:
+  check_freshness:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      issues: write
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+        with:
+          persist-credentials: false
+
+      - name: Set up Python 3.12
+        uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
+        with:
+          python-version: '3.12'
+
+      - name: Check published scoreboard freshness
+        id: check
+        run: |
+          python3 scripts/check_scoreboard_freshness.py \
+            --url "https://onnx.ai/backend-scoreboard/index.html" \
+            --max-age-days 3
+
+      - name: Open, update or close the tracking issue
+        uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7.1.0
+        env:
+          STALE: ${{ steps.check.outputs.stale }}
+          STALE_BACKENDS: ${{ steps.check.outputs.stale_backends }}
+          ISSUE_BODY: ${{ steps.check.outputs.body }}
+          RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
+        with:
+          script: |
+            const label = 'scoreboard-stale';
+            const title = '⚠️ Scoreboard data is out of date';
+            const stale = process.env.STALE === 'true';
+            const footer = `\n\n---\n_Automated by [Scoreboard freshness check](${process.env.RUN_URL})._`;
+            const body = (process.env.ISSUE_BODY || '').trim() + footer;
+
+            try {
+              await github.rest.issues.getLabel({
+                owner: context.repo.owner,
+                repo: context.repo.repo,
+                name: label,
+              });
+            } catch (error) {
+              if (error.status !== 404) {
+                throw error;
+              }
+              await github.rest.issues.createLabel({
+                owner: context.repo.owner,
+                repo: context.repo.repo,
+                name: label,
+                color: 'd73a4a',
+                description: 'Tracking issue for stale backend scoreboard data',
+              });
+            }
+
+            const existing = await github.paginate(github.rest.issues.listForRepo, {
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              state: 'open',
+              labels: label,
+              per_page: 100,
+            });
+            const issue = existing.find(i => i.title === title);
+
+            if (stale) {
+              if (issue) {
+                // Refresh the body silently so we don't spam subscribers daily.
+                await github.rest.issues.update({
+                  owner: context.repo.owner,
+                  repo: context.repo.repo,
+                  issue_number: issue.number,
+                  body,
+                });
+                core.info(`Updated existing issue #${issue.number}`);
+              } else {
+                const created = await github.rest.issues.create({
+                  owner: context.repo.owner,
+                  repo: context.repo.repo,
+                  title,
+                  body,
+                  labels: [label],
+                });
+                core.info(`Created issue #${created.data.number}`);
+              }
+            } else if (issue) {
+              await github.rest.issues.createComment({
+                owner: context.repo.owner,
+                repo: context.repo.repo,
+                issue_number: issue.number,
+                body: `✅ All scoreboard backends are up to date again as of this run.${footer}`,
+              });
+              await github.rest.issues.update({
+                owner: context.repo.owner,
+                repo: context.repo.repo,
+                issue_number: issue.number,
+                state: 'closed',
+              });
+              core.info(`Closed resolved issue #${issue.number}`);
+            } else {
+              core.info('Scoreboard is fresh; nothing to do.');
+            }
diff --git a/.gitignore b/.gitignore
@@ -1,6 +1,7 @@
 # Developer scripts for test only
 scripts/*
 !scripts/merge_trends.py
+!scripts/check_scoreboard_freshness.py
 
 # Byte-compiled / optimized / DLL files
 __pycache__/
diff --git a/scripts/check_scoreboard_freshness.py b/scripts/check_scoreboard_freshness.py
@@ -0,0 +1,224 @@
+# Copyright (c) ONNX Project Contributors
+#
+# SPDX-License-Identifier: Apache-2.0
+
+"""Check whether the published scoreboard page shows up-to-date data.
+
+The script downloads the rendered scoreboard page (by default the public
+``index.html``), reads the per-backend "Date" values that the website renders
+from ``trend[-1].date`` and reports any backend whose latest update is older
+than a configurable threshold (default: 3 days).
+
+It is meant to run in CI: results are written to ``GITHUB_OUTPUT`` so a
+follow-up step can open / update / close a single tracking issue.
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import os
+import re
+import sys
+import urllib.request
+
+from datetime import datetime, timezone
+from pathlib import Path
+from urllib.parse import urlparse
+
+
+DEFAULT_URL = "https://onnx.ai/backend-scoreboard/index.html"
+
+# The website embeds the full scoreboard as JSON in
+# <div id="content" database='{...}'>. The visible "Date" column is rendered
+# from each backend's last trend entry (trend[-1].date).
+DATABASE_RE = re.compile(r"database='(.*?)'>", re.DOTALL)
+
+# Date format produced by website-generator/generator.py:
+# datetime.now().strftime("%m/%d/%Y %H:%M:%S")
+DATE_FORMAT = "%m/%d/%Y %H:%M:%S"
+
+
+def fetch_html(url: str, timeout: int = 60) -> str:
+    """Download the scoreboard page."""
+    parsed = urlparse(url)
+    if parsed.scheme not in {"http", "https"} or not parsed.netloc:
+        raise ValueError(f"Unsupported scoreboard URL: {url!r}")
+    if parsed.username or parsed.password:
+        raise ValueError("Scoreboard URL must not include credentials")
+    headers = {"User-Agent": "scoreboard-freshness-check"}
+    request = urllib.request.Request(url, headers=headers)
+    with urllib.request.urlopen(request, timeout=timeout) as response:  # noqa: S310
+        charset = response.headers.get_content_charset() or "utf-8"
+        return response.read().decode(charset, errors="replace")
+
+
+def load_html_file(path: str) -> str:
+    """Read and return a validated local HTML file."""
+    html_path = Path(path).expanduser().resolve(strict=True)
+    if not html_path.is_file():
+        raise ValueError(f"Scoreboard HTML path must point to a file: {path!r}")
+    return html_path.read_text(encoding="utf-8")
+
+
+def extract_database(html: str) -> dict:
+    """Extract and parse the embedded scoreboard database JSON."""
+    match = DATABASE_RE.search(html)
+    if not match:
+        raise ValueError("Could not find the 'database' payload in the page")
+    return json.loads(match.group(1))
+
+
+def latest_date(backend_data: dict) -> str | None:
+    """Return the most recent trend date string for a backend, if any."""
+    trend = backend_data.get("trend") or []
+    if not trend:
+        return None
+    return trend[-1].get("date")
+
+
+def parse_date(value: str) -> datetime:
+    """Parse a scoreboard date string as a UTC datetime."""
+    return datetime.strptime(value, DATE_FORMAT).replace(tzinfo=timezone.utc)
+
+
+def evaluate(database: dict, max_age_days: float, now: datetime) -> list[dict]:
+    """Return a per-backend freshness report sorted by descending age."""
+    report = []
+    for backend, backend_data in database.items():
+        name = backend_data.get("name", backend)
+        date_str = latest_date(backend_data)
+
+        if not date_str:
+            report.append(
+                {
+                    "backend": backend,
+                    "name": name,
+                    "date": None,
+                    "age_days": None,
+                    "stale": True,
+                    "reason": "no trend data",
+                }
+            )
+            continue
+
+        try:
+            updated = parse_date(date_str)
+        except ValueError:
+            report.append(
+                {
+                    "backend": backend,
+                    "name": name,
+                    "date": date_str,
+                    "age_days": None,
+                    "stale": True,
+                    "reason": "unparseable date",
+                }
+            )
+            continue
+
+        age_days = (now - updated).total_seconds() / 86400.0
+        report.append(
+            {
+                "backend": backend,
+                "name": name,
+                "date": date_str,
+                "age_days": age_days,
+                "stale": age_days > max_age_days,
+                "reason": "",
+            }
+        )
+
+    def sort_key(row: dict) -> float:
+        return -1.0 if row["age_days"] is None else row["age_days"]
+
+    report.sort(key=sort_key, reverse=True)
+    return report
+
+
+def render_markdown(
+    report: list[dict], url: str, max_age_days: float, now: datetime
+) -> str:
+    """Build a Markdown summary suitable for an issue body."""
+    stale = [row for row in report if row["stale"]]
+    lines = [
+        f"The scoreboard page reports data older than **{max_age_days:g} days** "
+        f"for {len(stale)} of {len(report)} backend(s).",
+        "",
+        f"- Source: {url}",
+        f"- Checked at: {now.strftime('%Y-%m-%d %H:%M:%S')} UTC",
+        "",
+        "| Backend | Last update (UTC) | Age | Status |",
+        "| --- | --- | --- | --- |",
+    ]
+    for row in report:
+        if row["age_days"] is None:
+            age = row["reason"] or "unknown"
+        else:
+            age = f"{row['age_days']:.1f} days"
+        status = "⚠️ stale" if row["stale"] else "✅ ok"
+        lines.append(f"| {row['name']} | {row['date'] or '—'} | {age} | {status} |")
+    return "\n".join(lines)
+
+
+def write_outputs(stale: bool, report: list[dict], body: str) -> None:
+    """Expose results to later workflow steps via GITHUB_OUTPUT."""
+    output_path = os.environ.get("GITHUB_OUTPUT")
+    if not output_path:
+        return
+    stale_names = ", ".join(row["name"] for row in report if row["stale"])
+    with open(output_path, "a", encoding="utf-8") as handle:
+        handle.write(f"stale={'true' if stale else 'false'}\n")
+        handle.write(f"stale_count={sum(1 for row in report if row['stale'])}\n")
+        handle.write(f"stale_backends={stale_names}\n")
+        handle.write("body<<FRESHNESS_EOF\n")
+        handle.write(body + "\n")
+        handle.write("FRESHNESS_EOF\n")
+
+
+def main(argv: list[str] | None = None) -> int:
+    """Run the freshness check and emit a report plus CI outputs."""
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--url", default=DEFAULT_URL, help="Scoreboard page URL")
+    parser.add_argument(
+        "--html-file",
+        help="Read the page from a local file instead of fetching it (for testing)",
+    )
+    parser.add_argument(
+        "--max-age-days",
+        type=float,
+        default=3.0,
+        help="A backend is stale if its latest date is older than this many days "
+        "(default: 3)",
+    )
+    args = parser.parse_args(argv)
+
+    source = args.html_file or args.url
+    try:
+        if args.html_file:
+            html = load_html_file(args.html_file)
+        else:
+            html = fetch_html(args.url)
+        database = extract_database(html)
+    except Exception as error:  # noqa: BLE001 - surface any failure as a stale signal
+        message = f"Failed to read scoreboard data from {source}: {error}"
+        print(message, file=sys.stderr)
+        body = (
+            f"The freshness check could not read the scoreboard page.\n\n"
+            f"- Source: {source}\n- Error: `{error}`"
+        )
+        write_outputs(True, [], body)
+        return 1
+
+    now = datetime.now(timezone.utc)
+    report = evaluate(database, args.max_age_days, now)
+    stale = any(row["stale"] for row in report)
+    body = render_markdown(report, source, args.max_age_days, now)
+
+    print(body)
+    write_outputs(stale, report, body)
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())
diff --git a/test/test_scoreboard_freshness.py b/test/test_scoreboard_freshness.py
