🗄️ Backfill Wheels to GH Releases & PyPI #18
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: "🗄️ Backfill Wheels to GH Releases & PyPI" | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| dry_run: | |
| description: 'Dry run — show what would be uploaded without uploading' | |
| type: boolean | |
| default: true | |
| max_pages: | |
| description: 'Max pages of artifact runs to scan (20 runs/page) — only runs newer than latest release asset' | |
| default: '5' | |
| permissions: | |
| contents: write | |
| actions: read | |
| jobs: | |
| backfill: | |
| name: "⬆️ Backfill artifacts → GH Releases → PyPI" | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Run backfill | |
| env: | |
| GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| PYPI_API_TOKEN: ${{ secrets.OMNIPKG_DEPLOY_KEY }} | |
| REPO: ${{ github.repository }} | |
| DRY_RUN: ${{ inputs.dry_run }} | |
| MAX_PAGES: ${{ inputs.max_pages }} | |
| run: | | |
| python3 - <<'PYEOF' | |
| import os, json, re, time, tempfile, zipfile, subprocess | |
| import urllib.request, urllib.error, http.client, ssl | |
| from pathlib import Path | |
| repo = os.environ["REPO"] | |
| token = os.environ["GH_TOKEN"] | |
| pypi_tok = os.environ.get("PYPI_API_TOKEN", "") | |
| dry_run = os.environ["DRY_RUN"] == "true" | |
| max_pages = int(os.environ["MAX_PAGES"]) | |
| WORKFLOWS = [ | |
| "build-wheels.yml", | |
| "build-wheels-extended.yml", | |
| "build-wheels-exotic.yml", | |
| ] | |
| PYPI_PLAT_KEEP = [ | |
| r"manylinux_2_28_x86_64", r"manylinux_2_28_aarch64", | |
| r"manylinux_2_17_x86_64", r"manylinux_2_17_aarch64", | |
| r"musllinux_1_2_x86_64", r"musllinux_1_2_aarch64", | |
| r"macosx_11_0_arm64", r"macosx_10_12_x86_64", | |
| r"win_amd64", r"win_arm64", | |
| ] | |
| def pypi_eligible(name): | |
| if re.search(r"-pp\d|-graalpy|-cp313t-", name): | |
| return False | |
| if re.search(r"macosx_10_12_x86_64\.whl$", name): | |
| return False | |
| if re.search(r"macosx_11_0_arm64\.whl$", name): | |
| return False | |
| return any(re.search(p, name) for p in PYPI_PLAT_KEEP) | |
| def strip_license_file(whl_path): | |
| import zipfile, shutil | |
| tmp = str(whl_path) + ".tmp" | |
| with zipfile.ZipFile(whl_path, 'r') as zin, \ | |
| zipfile.ZipFile(tmp, 'w', zipfile.ZIP_DEFLATED) as zout: | |
| for item in zin.infolist(): | |
| data = zin.read(item.filename) | |
| if item.filename.endswith('METADATA'): | |
| lines = data.decode('utf-8').splitlines(keepends=True) | |
| data = ''.join(l for l in lines | |
| if not l.lower().startswith('license-file:')).encode('utf-8') | |
| zout.writestr(item, data) | |
| shutil.move(tmp, str(whl_path)) | |
| # ── GH API helpers ─────────────────────────────────────────────── | |
| def gh(path): | |
| url = path if path.startswith("http") else f"https://api.github.com/repos/{repo}/{path}" | |
| req = urllib.request.Request(url, headers={ | |
| "Authorization": f"token {token}", | |
| "Accept": "application/vnd.github+json", | |
| "X-GitHub-Api-Version": "2022-11-28", | |
| "User-Agent": "backfill-bot", | |
| }) | |
| try: | |
| return json.loads(urllib.request.urlopen(req).read()) | |
| except urllib.error.HTTPError as e: | |
| print(f" HTTP {e.code} for {url}") | |
| return None | |
| def gh_upload(release_id, filepath): | |
| name = Path(filepath).name | |
| url = f"https://uploads.github.com/repos/{repo}/releases/{release_id}/assets?name={name}" | |
| with open(filepath, "rb") as f: | |
| data = f.read() | |
| req = urllib.request.Request(url, data=data, method="POST", headers={ | |
| "Authorization": f"token {token}", | |
| "Content-Type": "application/octet-stream", | |
| "Content-Length": str(len(data)), | |
| }) | |
| try: | |
| urllib.request.urlopen(req) | |
| return True | |
| except urllib.error.HTTPError as e: | |
| body = e.read().decode() | |
| if "already_exists" in body: | |
| return True | |
| print(f" upload failed {e.code}: {body[:200]}") | |
| return False | |
| def download_artifact(artifact_id, dest_dir): | |
| ctx = ssl.create_default_context() | |
| conn = http.client.HTTPSConnection("api.github.com", context=ctx) | |
| conn.request("GET", f"/repos/{repo}/actions/artifacts/{artifact_id}/zip", headers={ | |
| "Authorization": f"token {token}", | |
| "Accept": "application/vnd.github+json", | |
| "User-Agent": "backfill-bot", | |
| }) | |
| resp = conn.getresponse() | |
| if resp.status not in (301, 302, 307, 308): | |
| print(f" expected redirect, got {resp.status}") | |
| resp.read(); conn.close() | |
| return False | |
| s3_url = resp.getheader("Location") | |
| resp.read(); conn.close() | |
| try: | |
| s3_req = urllib.request.Request(s3_url, headers={"User-Agent": "backfill-bot"}) | |
| with urllib.request.urlopen(s3_req) as r: | |
| zip_path = os.path.join(dest_dir, f"{artifact_id}.zip") | |
| with open(zip_path, "wb") as f: | |
| f.write(r.read()) | |
| with zipfile.ZipFile(zip_path) as z: | |
| z.extractall(dest_dir) | |
| os.remove(zip_path) | |
| return True | |
| except Exception as e: | |
| print(f" download failed: {e}") | |
| return False | |
| # ── Step 1: load all GH releases + their assets ────────────────── | |
| print("=" * 60) | |
| print("Step 1 — Loading GH releases and assets") | |
| print("=" * 60) | |
| releases_map = {} # tag -> release_id | |
| release_assets = {} # release_id -> set of filenames | |
| latest_asset_ts = "2000-01-01T00:00:00Z" | |
| page = 1 | |
| while True: | |
| rels = gh(f"releases?per_page=100&page={page}") | |
| if not rels: | |
| break | |
| for rel in rels: | |
| rid = rel["id"] | |
| tag = rel["tag_name"] | |
| releases_map[tag] = rid | |
| assets = [] | |
| assets_pg = 1 | |
| while True: | |
| batch = gh(f"releases/{rid}/assets?per_page=100&page={assets_pg}") or [] | |
| assets.extend(batch) | |
| if len(batch) < 100: | |
| break | |
| assets_pg += 1 | |
| names = set() | |
| for a in assets: | |
| if a["name"].endswith(".whl"): | |
| names.add(a["name"]) | |
| if a["updated_at"] > latest_asset_ts: | |
| latest_asset_ts = a["updated_at"] | |
| release_assets[rid] = names | |
| if len(rels) < 100: | |
| break | |
| page += 1 | |
| print(f"Found {len(releases_map)} releases") | |
| print(f"Latest release asset timestamp: {latest_asset_ts}") | |
| total_release_wheels = sum(len(v) for v in release_assets.values()) | |
| print(f"Total wheels already in releases: {total_release_wheels}") | |
| # ── Step 2: artifact → GH Release backfill ─────────────────────── | |
| # Only scan runs created AFTER the latest release asset timestamp | |
| print() | |
| print("=" * 60) | |
| print(f"Step 2 — Artifact → GH Release backfill (runs after {latest_asset_ts})") | |
| print("=" * 60) | |
| art_uploaded = art_skipped = art_failed = 0 | |
| uploaded_names = set() # dedupe across artifacts | |
| with tempfile.TemporaryDirectory() as tmpdir: | |
| for wf in WORKFLOWS: | |
| print(f"\n📋 {wf}") | |
| for page in range(1, max_pages + 1): | |
| runs = gh(f"actions/workflows/{wf}/runs?per_page=20&page={page}&status=success") | |
| if not runs or not runs.get("workflow_runs"): | |
| break | |
| stop = False | |
| for run in runs["workflow_runs"]: | |
| run_ts = run["created_at"] | |
| if run_ts <= latest_asset_ts: | |
| print(f" run {run['id']} at {run_ts} ≤ latest asset — stopping scan") | |
| stop = True | |
| break | |
| run_id = run["id"] | |
| arts = gh(f"actions/runs/{run_id}/artifacts?per_page=100") | |
| if not arts: | |
| continue | |
| for art in arts.get("artifacts", []): | |
| if art["expired"]: | |
| continue | |
| aname = art["name"] | |
| if not (aname.startswith("wheels-") or | |
| aname.startswith("ext-wheels-") or | |
| aname.startswith("exotic-wheels-")): | |
| continue | |
| art_id = art["id"] | |
| art_dir = os.path.join(tmpdir, str(art_id)) | |
| os.makedirs(art_dir, exist_ok=True) | |
| if dry_run: | |
| print(f" [DRY RUN] would process {aname} (id={art_id})") | |
| continue | |
| print(f"\n 📦 {aname} (id={art_id})") | |
| if not download_artifact(art_id, art_dir): | |
| art_failed += 1 | |
| continue | |
| wheels = list(Path(art_dir).rglob("*.whl")) | |
| for whl in wheels: | |
| wname = whl.name | |
| m = re.match(r"uv_ffi-([^-]+)-", wname) | |
| if not m: | |
| continue | |
| tag = f"v{m.group(1)}" | |
| if wname in uploaded_names: | |
| art_skipped += 1 | |
| continue | |
| rid = releases_map.get(tag) | |
| if not rid: | |
| print(f" ⚠️ no release for {tag} — skip {wname}") | |
| continue | |
| existing = release_assets[rid] | |
| if wname in existing: | |
| art_skipped += 1 | |
| uploaded_names.add(wname) | |
| continue | |
| print(f" ⬆️ {tag} ← {wname}") | |
| if gh_upload(rid, str(whl)): | |
| print(f" ✅ {wname}") | |
| existing.add(wname) | |
| uploaded_names.add(wname) | |
| art_uploaded += 1 | |
| else: | |
| art_failed += 1 | |
| time.sleep(0.3) | |
| # clear disk | |
| import shutil | |
| shutil.rmtree(art_dir, ignore_errors=True) | |
| os.makedirs(art_dir, exist_ok=True) | |
| if stop: | |
| break | |
| if len(runs["workflow_runs"]) < 20: | |
| break | |
| if not dry_run: | |
| print(f"\nArtifact→Release: uploaded={art_uploaded} skipped={art_skipped} failed={art_failed}") | |
| # ── Step 3: GH Release → PyPI gap fill ────────────────────────── | |
| # Purely filename comparison — no artifact downloads needed | |
| print() | |
| print("=" * 60) | |
| print("Step 3 — GH Release → PyPI gap fill") | |
| print("=" * 60) | |
| def get_pypi_filenames(version): | |
| try: | |
| req = urllib.request.Request(f"https://pypi.org/pypi/uv-ffi/{version}/json") | |
| return {f["filename"] for f in json.loads(urllib.request.urlopen(req).read()).get("urls", [])} | |
| except: | |
| return set() | |
| def get_asset_download_url(release_id, filename): | |
| assets = gh(f"releases/{release_id}/assets?per_page=100") or [] | |
| for a in assets: | |
| if a["name"] == filename: | |
| return a["browser_download_url"] | |
| return None | |
| if not dry_run and not pypi_tok: | |
| print("No PYPI_API_TOKEN — skipping PyPI upload") | |
| else: | |
| if not dry_run: | |
| subprocess.run(["pip", "install", "twine", "--quiet"], check=True) | |
| # group release wheels by version, filter PyPI-eligible | |
| pypi_todo = {} # version -> [(rid, name)] | |
| for tag, rid in releases_map.items(): | |
| version = tag.lstrip("v") | |
| for name in release_assets[rid]: | |
| if pypi_eligible(name): | |
| pypi_todo.setdefault(version, []).append((rid, name)) | |
| pypi_uploaded = pypi_skipped = pypi_failed = 0 | |
| for version in sorted(pypi_todo): | |
| wheels = pypi_todo[version] | |
| existing = get_pypi_filenames(version) | |
| to_upload = [(rid, n) for rid, n in wheels if n not in existing] | |
| if not to_upload: | |
| print(f" {version}: all {len(wheels)} PyPI-eligible wheels already there") | |
| pypi_skipped += len(wheels) | |
| continue | |
| print(f" {version}: {len(to_upload)} to upload ({len(wheels)-len(to_upload)} already on PyPI)") | |
| if dry_run: | |
| for _, n in to_upload: | |
| print(f" [DRY RUN] would upload {n}") | |
| continue | |
| with tempfile.TemporaryDirectory() as td: | |
| for rid, name in to_upload: | |
| url = get_asset_download_url(rid, name) | |
| if not url: | |
| print(f" no download url for {name}") | |
| pypi_failed += 1 | |
| continue | |
| dest = os.path.join(td, name) | |
| try: | |
| urllib.request.urlretrieve(url, dest) | |
| except Exception as e: | |
| print(f" download failed {name}: {e}") | |
| pypi_failed += 1 | |
| continue | |
| whl_files = list(Path(td).glob("*.whl")) | |
| if not whl_files: | |
| continue | |
| for wf_ in whl_files: | |
| strip_license_file(wf_) | |
| result = subprocess.run( | |
| ["twine", "upload", "--skip-existing", "--non-interactive", | |
| *[str(w) for w in whl_files]], | |
| env={**os.environ, | |
| "TWINE_USERNAME": "__token__", | |
| "TWINE_PASSWORD": pypi_tok}, | |
| capture_output=True, text=True | |
| ) | |
| if result.returncode == 0: | |
| print(f" ✅ uploaded {len(whl_files)} wheels for {version}") | |
| pypi_uploaded += len(whl_files) | |
| else: | |
| print(f" ❌ twine failed (rc={result.returncode})") | |
| print(f" STDOUT: {result.stdout[-2000:]}") | |
| print(f" STDERR: {result.stderr[-2000:]}") | |
| pypi_failed += len(whl_files) | |
| if not dry_run: | |
| print(f"\nPyPI: uploaded={pypi_uploaded} skipped={pypi_skipped} failed={pypi_failed}") | |
| else: | |
| print("\n[DRY RUN] rerun with dry_run=false to actually upload") | |
| PYEOF | |
| update-wheel-index: | |
| name: "📄 Update GH Pages wheel index" | |
| needs: backfill | |
| runs-on: ubuntu-latest | |
| if: inputs.dry_run == false | |
| permissions: | |
| pages: write | |
| id-token: write | |
| environment: | |
| name: github-pages | |
| url: ${{ steps.deploy.outputs.page_url }} | |
| steps: | |
| - name: Regenerate PEP 503 index | |
| env: | |
| GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| REPO: ${{ github.repository }} | |
| run: | | |
| mkdir -p _site/uv-ffi | |
| python3 - <<'EOF' | |
| import os, json, urllib.request | |
| repo = os.environ["REPO"] | |
| token = os.environ["GH_TOKEN"] | |
| def gh(url): | |
| req = urllib.request.Request(url, headers={ | |
| "Authorization": f"token {token}", | |
| "Accept": "application/vnd.github+json"}) | |
| return json.loads(urllib.request.urlopen(req).read()) | |
| def pypi_wheels(pkg): | |
| try: | |
| req = urllib.request.Request(f"https://pypi.org/pypi/{pkg}/json") | |
| data = json.loads(urllib.request.urlopen(req).read()) | |
| return [(v, f["filename"], f["url"]) | |
| for v, files in data["releases"].items() | |
| for f in files if f["filename"].endswith(".whl")] | |
| except Exception as e: | |
| print(f"PyPI fetch failed: {e}") | |
| return [] | |
| gh_wheels = [] | |
| page = 1 | |
| while True: | |
| rels = gh(f"https://api.github.com/repos/{repo}/releases?per_page=50&page={page}") | |
| if not rels: | |
| break | |
| for rel in rels: | |
| assets_pg2 = 1 | |
| while True: | |
| assets_url = f"https://api.github.com/repos/{repo}/releases/{rel['id']}/assets?per_page=100&page={assets_pg2}" | |
| batch = gh(assets_url) or [] | |
| for a in batch: | |
| if a["name"].endswith(".whl"): | |
| gh_wheels.append((rel["tag_name"], a["name"], a["browser_download_url"])) | |
| if len(batch) < 100: | |
| break | |
| assets_pg2 += 1 | |
| page += 1 | |
| pip_wheels = pypi_wheels("uv-ffi") | |
| seen = {} | |
| for tag, name, url in pip_wheels: | |
| seen[name] = (tag, name, url) | |
| for tag, name, url in gh_wheels: | |
| seen[name] = (tag, name, url) | |
| all_wheels = sorted(seen.values(), key=lambda x: (x[0], x[1])) | |
| links = "\n".join( | |
| f' <a href="{url}" data-requires-python=">=3.8">{name}</a><br>' | |
| for _, name, url in all_wheels | |
| ) | |
| with open("_site/uv-ffi/index.html", "w") as fh: | |
| fh.write(f"<!DOCTYPE html>\n<html>\n <head><title>Links for uv-ffi</title></head>\n" | |
| f" <body>\n <h1>Links for uv-ffi</h1>\n{links}\n </body>\n</html>\n") | |
| with open("_site/index.html", "w") as fh: | |
| fh.write(f"<!DOCTYPE html>\n<html>\n <head><title>uv-ffi wheel index</title></head>\n" | |
| f" <body>\n <h1>uv-ffi wheel index</h1>\n" | |
| f' <p>Mainstream wheels: <a href="https://pypi.org/project/uv-ffi/">PyPI</a></p>\n' | |
| f" <p>Exotic/extended platform wheels hosted here.</p>\n" | |
| f" <pre>pip install uv-ffi --extra-index-url https://1minds3t.github.io/uv-ffi/</pre>\n" | |
| f' <p><a href="uv-ffi/">Browse all {len(all_wheels)} wheels</a></p>\n' | |
| f" </body>\n</html>\n") | |
| print(f"Indexed {len(all_wheels)} total wheels ({len(pip_wheels)} PyPI + {len(gh_wheels)} GH Releases)") | |
| EOF | |
| - name: Upload Pages artifact | |
| uses: actions/upload-pages-artifact@v3 | |
| with: | |
| path: _site | |
| - name: Deploy to GitHub Pages | |
| id: deploy | |
| uses: actions/deploy-pages@v4 |