ci: run SQL E2E via standalone haybarn + signed vgi community extension

rustyconover · claude · rustyconover · commit 2f0ca15b220f · 2026-06-23T18:26:10.000-04:00
deps: resolve vgi-python from PyPI; track uv.lock

Co-Authored-By: Claude Opus 4.8 (1M context) &lt;noreply@anthropic.com&gt;
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -1,17 +1,28 @@
+# Copyright 2026 Query Farm LLC - https://query.farm
+#
+# Unit tests (pytest), lint, and the sqllogictest extension suite
+# (test/sql/*.test) run against the vgi-news worker through the real signed
+# `vgi` DuckDB community extension via a prebuilt standalone `haybarn-unittest`.
+# The SQL E2E redirects every provider at a local mock news API — no keys, no
+# cost, no live egress. See ci/README.md for the design.
 name: CI
 
 on:
   push:
     branches: [main]
-    paths-ignore: ['README.md', 'CLAUDE.md']
+    paths-ignore: ['README.md']
   pull_request:
     branches: [main]
-    paths-ignore: ['README.md', 'CLAUDE.md']
+    paths-ignore: ['README.md']
   workflow_dispatch:
 
 permissions:
   contents: read
 
+concurrency:
+  group: ci-${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
 jobs:
   test:
     name: Python ${{ matrix.python-version }} on ${{ matrix.os }}
@@ -21,60 +32,105 @@ jobs:
       matrix:
         os: [ubuntu-latest, macos-latest]
         python-version: ["3.13"]
+
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v6
 
       - name: Install uv
-        uses: astral-sh/setup-uv@v3
+        uses: astral-sh/setup-uv@v8.2.0
 
       - name: Set up Python ${{ matrix.python-version }}
         run: uv python install ${{ matrix.python-version }}
 
-      # Install from PyPI (not the local ../vgi-python path used for dev), so CI
-      # is self-contained.
       - name: Install dependencies
-        run: |
-          uv venv
-          uv pip install vgi-python "httpx>=0.27" "pyarrow>=16" pytest ruff mypy
-          uv pip install -e . --no-deps
+        run: uv sync --frozen --all-extras
 
-      - name: Lint (ruff + mypy)
-        run: |
-          uv run --no-sync ruff check .
-          uv run --no-sync mypy vgi_news/ news_worker.py
-
-      # Unit + mock-server + worker-integration tests. Live (network) tests are
-      # excluded by default (pyproject addopts: -m 'not live'), so CI never
-      # touches a real news API.
+      # The `live` marker (real GDELT over the network) is excluded by default
+      # via the pyproject addopts.
       - name: Run tests
         run: uv run --no-sync pytest -q
 
-  e2e-sql:
-    name: SQL end-to-end (haybarn, mocked)
+  lint:
+    name: Lint
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v6
 
       - name: Install uv
-        uses: astral-sh/setup-uv@v3
+        uses: astral-sh/setup-uv@v8.2.0
+
+      - name: Install dependencies
+        run: uv sync --frozen --all-extras
+
+      - name: Check linting
+        run: uv run --no-sync ruff check .
+
+      - name: Type check (mypy)
+        run: uv run --no-sync mypy vgi_news/ news_worker.py
+
+  # Resolve the latest published haybarn release once, so the whole matrix tests
+  # the same version (and we never hardcode/pin it).
+  resolve-haybarn:
+    name: Resolve haybarn release
+    runs-on: ubuntu-latest
+    outputs:
+      release: ${{ steps.r.outputs.release }}
+    steps:
+      - id: r
+        run: |
+          REL=$(gh release view --repo Query-farm-haybarn/haybarn --json tagName --jq .tagName)
+          echo "release=$REL" >> "$GITHUB_OUTPUT"
+          echo "Latest haybarn release: $REL"
+        env:
+          GH_TOKEN: ${{ github.token }}
+
+  integration:
+    name: End-to-end SQL on ${{ matrix.os }}
+    needs: resolve-haybarn
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - { os: ubuntu-latest, asset: haybarn_unittest-linux-amd64.zip }
+          - { os: macos-latest,  asset: haybarn_unittest-osx-arm64.zip }
+    runs-on: ${{ matrix.os }}
+    defaults:
+      run:
+        shell: bash
+    steps:
+      - uses: actions/checkout@v6
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v8.2.0
 
       - name: Set up Python 3.13
         run: uv python install 3.13
 
-      - name: Install worker dependencies
+      - name: Install the worker (from the lockfile)
+        run: uv sync --frozen --python 3.13
+
+      - name: Download haybarn-unittest
         run: |
-          uv venv
-          uv pip install vgi-python "httpx>=0.27" "pyarrow>=16"
-          uv pip install -e . --no-deps
+          gh release download "$HAYBARN_RELEASE" \
+            --repo Query-farm-haybarn/haybarn \
+            --pattern '${{ matrix.asset }}' \
+            --output haybarn-unittest.zip --clobber
+          mkdir -p hb && unzip -o -q haybarn-unittest.zip -d hb
+        env:
+          GH_TOKEN: ${{ github.token }}
+          HAYBARN_RELEASE: ${{ needs.resolve-haybarn.outputs.release }}
 
-      - name: Install haybarn-unittest
+      - name: Resolve runner + worker paths
         run: |
-          uv tool install haybarn-unittest
-          echo "$HOME/.local/bin" >> "$GITHUB_PATH"
+          # Absolute paths: run-integration.sh cd's into a staging dir before
+          # invoking the runner, so relative paths would not resolve. The worker
+          # runs from the synced .venv (deps resolved from PyPI via the lockfile);
+          # plain `.venv/bin/python` ignores the PEP 723 header. run-integration.sh
+          # starts the mock news API and redirects both providers at it.
+          UNITTEST="$PWD/$(find hb -name 'haybarn-unittest' -type f | head -1)"
+          chmod +x "$UNITTEST"
+          echo "HAYBARN_UNITTEST=$UNITTEST" >> "$GITHUB_ENV"
+          echo "VGI_NEWS_WORKER=$PWD/.venv/bin/python $PWD/news_worker.py" >> "$GITHUB_ENV"
 
-      # Drives the worker (as a DuckDB subprocess) against a local mock news API.
-      # No network egress to GDELT/NewsAPI, no API keys.
-      - name: Run SQL end-to-end tests
-        env:
-          VGI_NEWS_WORKER: .venv/bin/python news_worker.py
-        run: uv run --no-sync python scripts/run_sql_e2e.py
+      - name: Run extension integration suite
+        run: ci/run-integration.sh
diff --git a/.gitignore b/.gitignore
@@ -9,4 +9,3 @@ vendor/
 duckdb_unittest_tempdir/
 *.coverage
 .coverage
-uv.lock
diff --git a/ci/README.md b/ci/README.md
@@ -0,0 +1,43 @@
+# CI: the vgi-calendar worker integration suite
+
+[`.github/workflows/ci.yml`](../.github/workflows/ci.yml) runs the unit tests
+and this repo's sqllogictest suite (`test/sql/*.test`) against the vgi-calendar
+VGI worker through the **real DuckDB `vgi` extension** on every push / PR.
+
+## How it works (no C++ build)
+
+Rather than building the vgi DuckDB extension from source, CI drives a
+**prebuilt** standalone `haybarn-unittest` (the DuckDB/Haybarn sqllogictest
+runner, published in Haybarn's releases) and installs the **signed** `vgi`
+extension from the Haybarn community channel:
+
+1. **Install the worker** — `uv sync --frozen` into a venv. `calendar_worker.py`
+   is a self-contained PEP 723 stdio worker the extension can spawn via
+   `uv run calendar_worker.py`.
+2. **Download the runner** — the matching `haybarn_unittest-*` asset per
+   platform from the latest Haybarn release.
+3. **Preprocess** — the standalone runner links none of the extensions the
+   tests gate on, so [`preprocess-require.awk`](preprocess-require.awk) rewrites
+   each `require <ext>` into an explicit signed `INSTALL <ext> FROM
+   {community,core}; LOAD <ext>;`. These tests skip `require vgi` (haybarn
+   silently SKIPs it) and `LOAD vgi;` directly, so the awk also injects an
+   `INSTALL vgi FROM community;` right before each bare `LOAD vgi;`. `require-env`
+   and everything else pass through untouched.
+4. **Run** — [`run-integration.sh`](run-integration.sh) stages the preprocessed
+   tree, points `VGI_CALENDAR_WORKER` at `uv run calendar_worker.py`, warms the
+   extension cache once, then runs the suite in a single `haybarn-unittest`
+   invocation. Any failed assertion exits non-zero and fails the job.
+
+## Run it locally
+
+```bash
+uv sync --python 3.13                       # install the worker + deps
+# point HAYBARN_UNITTEST at a haybarn-unittest binary (or a local DuckDB
+# `unittest` built with the vgi extension), and the worker at the stdio command:
+HAYBARN_UNITTEST=/path/to/haybarn-unittest \
+VGI_CALENDAR_WORKER="uv run --python 3.13 calendar_worker.py" \
+  ci/run-integration.sh
+```
+
+Or use the Makefile target `make test-sql`, which installs `haybarn-unittest`
+as a uv tool and points the worker at `uv run --python 3.13 calendar_worker.py`.
diff --git a/ci/preprocess-require.awk b/ci/preprocess-require.awk
@@ -0,0 +1,25 @@
+# Copyright 2026 Query Farm LLC - https://query.farm
+#
+# Rewrite each `require <ext>` gate (and the bare `LOAD vgi;` these tests use
+# instead — haybarn skips `require vgi`, so the .test files LOAD it directly)
+# in this repo's sqllogictest files into an explicit signed INSTALL+LOAD, so the
+# prebuilt standalone `haybarn-unittest` (which links none of these extensions)
+# can run the suite. The vgi extension comes from the signed community channel;
+# httpfs/json/parquet/spatial from the signed core channel. `require-env` and
+# every other directive pass through untouched. See ci/README.md.
+/^require[ \t]+vgi[ \t]*$/ {
+    print "statement ok"; print "INSTALL vgi FROM community;"; print "";
+    print "statement ok"; print "LOAD vgi;"; next
+}
+/^require[ \t]+(httpfs|json|parquet|spatial)[ \t]*$/ {
+    ext = $2
+    print "statement ok"; print "INSTALL " ext " FROM core;"; print "";
+    print "statement ok"; print "LOAD " ext ";"; next
+}
+# These tests gate the worker with `require-env` and then `LOAD vgi;` directly
+# (haybarn silently SKIPs `require vgi`). Inject a signed community INSTALL right
+# before that bare LOAD so a clean runner can resolve the extension.
+/^[ \t]*LOAD[ \t]+vgi[ \t]*;[ \t]*$/ {
+    print "INSTALL vgi FROM community;"; print $0; next
+}
+{ print }
diff --git a/ci/run-integration.sh b/ci/run-integration.sh
@@ -0,0 +1,92 @@
+#!/usr/bin/env bash
+# Copyright 2026 Query Farm LLC - https://query.farm
+#
+# Run this repo's sqllogictest suite (test/sql/*.test) against the vgi-news
+# VGI worker, using a prebuilt standalone `haybarn-unittest` and the signed
+# community `vgi` extension — no C++ build from source. See ci/README.md.
+#
+# The GDELT / NewsAPI providers are redirected at a local canned-response mock
+# HTTP server (the same routes scripts/run_sql_e2e.py serves) via the
+# VGI_NEWS_GDELT_BASE_URL / VGI_NEWS_NEWSAPI_BASE_URL env vars, so the suite is
+# deterministic and never egresses to a live API.
+#
+# Required environment:
+#   HAYBARN_UNITTEST  path to the haybarn-unittest binary
+#   VGI_NEWS_WORKER   worker LOCATION the .test files ATTACH (a stdio command)
+# Optional:
+#   STAGE             scratch dir for the preprocessed test tree (default: mktemp)
+set -euo pipefail
+
+: "${HAYBARN_UNITTEST:?path to the haybarn-unittest binary}"
+: "${VGI_NEWS_WORKER:?worker LOCATION (stdio command or http:// URL)}"
+
+HERE="$(cd "$(dirname "$0")" && pwd)"
+REPO="$(cd "$HERE/.." && pwd)"
+STAGE="${STAGE:-$(mktemp -d)}"
+
+echo "Staging preprocessed tests into $STAGE ..."
+mkdir -p "$STAGE/test/sql"
+for f in "$REPO"/test/sql/*.test; do
+  awk -f "$HERE/preprocess-require.awk" "$f" > "$STAGE/test/sql/$(basename "$f")"
+done
+
+# Start the canned-response mock news server — reuse the exact routes/fixtures
+# from scripts/run_sql_e2e.py. It prints `URL:<base>` once bound and then blocks;
+# we read the URL, redirect both providers at it, and kill it on exit.
+MOCK_OUT="$(mktemp)"
+( cd "$REPO" && REPO="$REPO" uv run --no-sync python - >"$MOCK_OUT" 2>/dev/null <<'PY' ) &
+import importlib.util, os, threading, time
+from http.server import HTTPServer
+
+spec = importlib.util.spec_from_file_location(
+    "news_sql_e2e", os.path.join(os.environ["REPO"], "scripts", "run_sql_e2e.py")
+)
+mod = importlib.util.module_from_spec(spec)
+spec.loader.exec_module(mod)
+
+srv = HTTPServer(("127.0.0.1", 0), mod._Handler)
+threading.Thread(target=srv.serve_forever, daemon=True).start()
+host, port = srv.server_address
+print(f"URL:http://{host}:{port}", flush=True)
+while True:
+    time.sleep(3600)
+PY
+MOCK_PID=$!
+cleanup() { kill "$MOCK_PID" 2>/dev/null || true; rm -f "$MOCK_OUT"; }
+trap cleanup EXIT
+
+BASE=""
+for _ in $(seq 1 100); do
+  BASE="$(sed -n 's/^URL:\(.*\)$/\1/p' "$MOCK_OUT" | head -n1)"
+  [ -n "$BASE" ] && break
+  sleep 0.1
+done
+if [ -z "$BASE" ]; then
+  echo "ERROR: mock news server did not report a URL" >&2
+  exit 1
+fi
+echo "Mock news server at $BASE"
+export VGI_NEWS_GDELT_BASE_URL="$BASE/doc"
+export VGI_NEWS_NEWSAPI_BASE_URL="$BASE/everything"
+export VGI_NEWS_TIMEOUT="10"
+
+cd "$STAGE"
+
+# Warm the extension cache once: vgi from the signed community channel. A miss
+# here is only a warning — the per-test INSTALL/LOAD (injected by
+# preprocess-require.awk) is what actually gates each file.
+echo "Warming the extension cache (vgi from community) ..."
+mkdir -p "$STAGE/test"
+cat > "$STAGE/test/_warm.test" <<'EOF'
+# name: test/_warm.test
+# group: [warm]
+statement ok
+INSTALL vgi FROM community;
+EOF
+"$HAYBARN_UNITTEST" "test/_warm.test" >/dev/null 2>&1 || echo "::warning::extension warm step did not fully succeed"
+rm -f "$STAGE/test/_warm.test"
+
+# Run the whole suite in one invocation, streaming the runner's native
+# sqllogictest report. Any failed assertion exits non-zero and fails the job.
+echo "Running suite (worker: $VGI_NEWS_WORKER) ..."
+"$HAYBARN_UNITTEST" "test/sql/*"
diff --git a/news_worker.py b/news_worker.py
@@ -1,12 +1,9 @@
 # /// script
 # requires-python = ">=3.13"
 # dependencies = [
-#     "vgi-python",
+#     "vgi-python>=0.8.3",
 #     "httpx>=0.27",
 # ]
-#
-# [tool.uv.sources]
-# vgi-python = { path = "../vgi-python" }
 # ///
 """VGI worker exposing global news search to DuckDB/SQL.
 
diff --git a/pyproject.toml b/pyproject.toml
@@ -8,7 +8,7 @@ license = { file = "LICENSE" }
 authors = [{ name = "Query Farm" }]
 keywords = ["vgi", "duckdb", "news", "gdelt", "newsapi", "search"]
 dependencies = [
-    "vgi-python",
+    "vgi-python>=0.8.3",
     "httpx>=0.27",
     "pyarrow>=16",
 ]
@@ -20,11 +20,6 @@ dev = [
     "mypy>=1.10",
 ]
 
-# vgi-python is distributed on PyPI as `vgi-python`. For local development
-# against an in-tree checkout, uncomment the source override below.
-[tool.uv.sources]
-vgi-python = { path = "../vgi-python", editable = true }
-
 [build-system]
 requires = ["hatchling"]
 build-backend = "hatchling.build"
diff --git a/uv.lock b/uv.lock
