🗄️ Backfill Wheels to GH Releases & PyPI

🗄️ Backfill Wheels to GH Releases & PyPI #32

Workflow file for this run

.github/workflows/backfill_wheels.yml at 1648c4c

	name: "🗄️ Backfill Wheels to GH Releases & PyPI"

	on:
	workflow_dispatch:
	inputs:
	from_tag:
	description: 'Start tag (inclusive) — only scan runs created after this release (e.g. v0.10.8.post11)'
	required: true
	default: ''
	to_tag:
	description: 'End tag (inclusive) — only scan runs created before this release (e.g. v0.10.8.post13). Leave blank to use latest release.'
	required: false
	default: ''
	dry_run:
	description: 'Dry run — show what would be uploaded without uploading'
	type: boolean
	default: true
	max_pages:
	description: 'Max pages of artifact runs to scan per workflow (20 runs/page)'
	default: '5'

	permissions:
	contents: write
	actions: read
	id-token: write

	jobs:
	backfill:
	name: "⬆️ Backfill artifacts → GH Releases → PyPI"
	runs-on: ubuntu-latest
	steps:
	- name: Run backfill
	env:
	GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
	REPO: ${{ github.repository }}
	DRY_RUN: ${{ inputs.dry_run }}
	MAX_PAGES: ${{ inputs.max_pages }}
	FROM_TAG: ${{ inputs.from_tag }}
	TO_TAG: ${{ inputs.to_tag }}
	run: \|
	python3 - <<'PYEOF'
	import os, json, re, time, tempfile, zipfile
	import urllib.request, urllib.error, http.client, ssl
	from pathlib import Path

	repo = os.environ["REPO"]
	token = os.environ["GH_TOKEN"]
	dry_run = os.environ["DRY_RUN"] == "true"
	max_pages = int(os.environ["MAX_PAGES"])
	from_tag = os.environ.get("FROM_TAG", "").strip()
	to_tag = os.environ.get("TO_TAG", "").strip()

	WORKFLOWS = [
	"build-wheels.yml",
	"build-wheels-extended.yml",
	"build-wheels-exotic.yml",
	]

	PYPI_PLAT_KEEP = [
	r"manylinux_2_28_x86_64", r"manylinux_2_28_aarch64",
	r"manylinux_2_17_x86_64", r"manylinux_2_17_aarch64",
	r"musllinux_1_2_x86_64", r"musllinux_1_2_aarch64",
	r"macosx_11_0_arm64", r"macosx_10_12_x86_64",
	r"win_amd64", r"win_arm64",
	]

	def pypi_eligible(name):
	if re.search(r"-pp\d\|-graalpy\|-cp313t-", name):
	return False
	if re.search(r"macosx_10_12_x86_64\.whl$", name):
	return False
	if re.search(r"macosx_11_0_arm64\.whl$", name):
	return False
	return any(re.search(p, name) for p in PYPI_PLAT_KEEP)

	def strip_license_file(whl_path):
	import zipfile, shutil
	tmp = str(whl_path) + ".tmp"
	with zipfile.ZipFile(whl_path, 'r') as zin, \
	zipfile.ZipFile(tmp, 'w', zipfile.ZIP_DEFLATED) as zout:
	for item in zin.infolist():
	data = zin.read(item.filename)
	if item.filename.endswith('METADATA'):
	lines = data.decode('utf-8').splitlines(keepends=True)
	data = ''.join(l for l in lines
	if not l.lower().startswith('license-file:')).encode('utf-8')
	zout.writestr(item, data)
	shutil.move(tmp, str(whl_path))

	# ── GH API helpers ───────────────────────────────────────────────
	def gh(path):
	url = path if path.startswith("http") else f"https://api.github.com/repos/{repo}/{path}"
	req = urllib.request.Request(url, headers={
	"Authorization": f"token {token}",
	"Accept": "application/vnd.github+json",
	"X-GitHub-Api-Version": "2022-11-28",
	"User-Agent": "backfill-bot",
	})
	try:
	return json.loads(urllib.request.urlopen(req).read())
	except urllib.error.HTTPError as e:
	print(f" HTTP {e.code} for {url}")
	return None

	def gh_upload(release_id, filepath):
	name = Path(filepath).name
	url = f"https://uploads.github.com/repos/{repo}/releases/{release_id}/assets?name={name}"
	with open(filepath, "rb") as f:
	data = f.read()
	req = urllib.request.Request(url, data=data, method="POST", headers={
	"Authorization": f"token {token}",
	"Content-Type": "application/octet-stream",
	"Content-Length": str(len(data)),
	})
	try:
	urllib.request.urlopen(req)
	return True
	except urllib.error.HTTPError as e:
	body = e.read().decode()
	if "already_exists" in body:
	return True
	print(f" upload failed {e.code}: {body[:200]}")
	return False

	def download_artifact(artifact_id, dest_dir):
	ctx = ssl.create_default_context()
	conn = http.client.HTTPSConnection("api.github.com", context=ctx)
	conn.request("GET", f"/repos/{repo}/actions/artifacts/{artifact_id}/zip", headers={
	"Authorization": f"token {token}",
	"Accept": "application/vnd.github+json",
	"User-Agent": "backfill-bot",
	})
	resp = conn.getresponse()
	if resp.status not in (301, 302, 307, 308):
	print(f" expected redirect, got {resp.status}")
	resp.read(); conn.close()
	return False
	s3_url = resp.getheader("Location")
	resp.read(); conn.close()
	try:
	s3_req = urllib.request.Request(s3_url, headers={"User-Agent": "backfill-bot"})
	with urllib.request.urlopen(s3_req) as r:
	zip_path = os.path.join(dest_dir, f"{artifact_id}.zip")
	with open(zip_path, "wb") as f:
	f.write(r.read())
	with zipfile.ZipFile(zip_path) as z:
	z.extractall(dest_dir)
	os.remove(zip_path)
	return True
	except Exception as e:
	print(f" download failed: {e}")
	return False

	# ── Resolve from/to timestamps from release dates ────────────────
	def get_release_ts(tag):
	data = gh(f"releases/tags/{tag}")
	if not data:
	raise SystemExit(f"❌ Could not find release for tag {tag}")
	return data["created_at"]

	from_ts = get_release_ts(from_tag)
	# to_tag blank → use now (include everything up to present)
	if to_tag:
	to_ts = get_release_ts(to_tag)
	else:
	import datetime
	to_ts = datetime.datetime.utcnow().strftime("%Y-%m-%dT%H:%M:%SZ")

	print(f"Scanning runs between: {from_ts} ({from_tag}) → {to_ts} ({to_tag or 'now'})")

	# ── Step 1: load all GH releases + their assets ──────────────────
	print("=" * 60)
	print("Step 1 — Loading GH releases and assets")
	print("=" * 60)

	releases_map = {} # tag -> release_id
	release_assets = {} # release_id -> set of filenames

	page = 1
	while True:
	rels = gh(f"releases?per_page=100&page={page}")
	if not rels:
	break
	for rel in rels:
	rid = rel["id"]
	tag = rel["tag_name"]
	releases_map[tag] = rid
	assets = []
	assets_pg = 1
	while True:
	batch = gh(f"releases/{rid}/assets?per_page=100&page={assets_pg}") or []
	assets.extend(batch)
	if len(batch) < 100:
	break
	assets_pg += 1
	names = {}
	for a in assets:
	if a["name"].endswith(".whl"):
	names[a["name"]] = a["browser_download_url"]
	release_assets[rid] = names
	if len(rels) < 100:
	break
	page += 1

	print(f"Found {len(releases_map)} releases")

	total_release_wheels = sum(len(v) for v in release_assets.values())
	print(f"Total wheels already in releases: {total_release_wheels}")

	# ── Step 2: artifact → GH Release backfill ───────────────────────
	# Only scan runs created AFTER the latest release asset timestamp
	print()
	print("=" * 60)
	print(f"Step 2 — Artifact → GH Release backfill")
	print("=" * 60)

	art_uploaded = art_skipped = art_failed = 0
	uploaded_names = set() # dedupe across artifacts

	with tempfile.TemporaryDirectory() as tmpdir:
	for wf in WORKFLOWS:
	print(f"\n📋 {wf}")
	for page in range(1, max_pages + 1):
	runs = gh(f"actions/workflows/{wf}/runs?per_page=20&page={page}&status=completed")
	if not runs or not runs.get("workflow_runs"):
	break
	stop = False
	for run in runs["workflow_runs"]:
	run_ts = run["created_at"]
	if run_ts > to_ts:
	continue # too new, skip
	if run_ts < from_ts:
	print(f" run {run['id']} at {run_ts} < from_ts — stopping scan")
	stop = True
	break
	run_id = run["id"]
	arts = gh(f"actions/runs/{run_id}/artifacts?per_page=100")
	if not arts:
	continue
	for art in arts.get("artifacts", []):
	if art["expired"]:
	continue
	aname = art["name"]
	if not (aname.startswith("wheels-") or
	aname.startswith("ext-wheels-") or
	aname.startswith("exotic-wheels-")):
	continue
	art_id = art["id"]
	art_dir = os.path.join(tmpdir, str(art_id))
	os.makedirs(art_dir, exist_ok=True)

	if dry_run:
	print(f" [DRY RUN] would process {aname} (id={art_id})")
	continue

	print(f"\n 📦 {aname} (id={art_id})")
	if not download_artifact(art_id, art_dir):
	art_failed += 1
	continue

	wheels = list(Path(art_dir).rglob("*.whl"))
	for whl in wheels:
	wname = whl.name
	m = re.match(r"uv_ffi-([^-]+)-", wname)
	if not m:
	continue
	tag = f"v{m.group(1)}"
	if wname in uploaded_names:
	art_skipped += 1
	continue
	rid = releases_map.get(tag)
	if not rid:
	print(f" ⚠️ no release for {tag} — skip {wname}")
	continue
	existing = release_assets[rid]
	if wname not in existing:
	print(f" ⬆️ {tag} ← {wname}")
	if gh_upload(rid, str(whl)):
	print(f" ✅ {wname}")
	existing[wname] = ""
	uploaded_names.add(wname)
	art_uploaded += 1
	else:
	art_failed += 1
	time.sleep(0.3)
	else:
	art_skipped += 1
	uploaded_names.add(wname)

	# clear disk
	import shutil
	shutil.rmtree(art_dir, ignore_errors=True)
	os.makedirs(art_dir, exist_ok=True)

	if stop:
	break
	if len(runs["workflow_runs"]) < 20:
	break

	if not dry_run:
	print(f"\nArtifact→Release: uploaded={art_uploaded} skipped={art_skipped} failed={art_failed}")

	# ── Reload release assets after Step 2 uploads ──────────────────
	print("\nReloading release assets after Step 2 uploads...")
	for tag, rid in releases_map.items():
	assets = []
	assets_pg = 1
	while True:
	batch = gh(f"releases/{rid}/assets?per_page=100&page={assets_pg}") or []
	assets.extend(batch)
	if len(batch) < 100:
	break
	assets_pg += 1
	release_assets[rid] = {a["name"]: a["browser_download_url"] for a in assets if a["name"].endswith(".whl")}

	# ── Step 3: GH Release → PyPI gap fill ──────────────────────────
	# Purely filename comparison — no artifact downloads needed
	print()
	print("=" * 60)
	print("Step 3 — GH Release → PyPI gap fill")
	print("=" * 60)

	def get_pypi_filenames(version):
	try:
	req = urllib.request.Request(f"https://pypi.org/pypi/uv-ffi/{version}/json")
	return {f["filename"] for f in json.loads(urllib.request.urlopen(req).read()).get("urls", [])}
	except:
	return set()

	def get_asset_download_url(release_id, filename):
	return release_assets.get(release_id, {}).get(filename)

	if not dry_run:
	print("No PYPI_API_TOKEN — wheels will be uploaded via OIDC in next job")

	pypi_todo = {}
	for tag, rid in releases_map.items():
	version = tag.lstrip("v")
	for name in release_assets[rid]:
	if pypi_eligible(name):
	pypi_todo.setdefault(version, []).append((rid, name))

	pypi_skipped = 0
	os.makedirs("/tmp/pypi-dist", exist_ok=True)

	for version in sorted(pypi_todo):
	wheels = pypi_todo[version]
	existing = get_pypi_filenames(version)
	to_upload = [(rid, n) for rid, n in wheels if n not in existing]

	if not to_upload:
	print(f" {version}: all {len(wheels)} PyPI-eligible wheels already there")
	pypi_skipped += len(wheels)
	continue

	print(f" {version}: {len(to_upload)} to stage ({len(wheels)-len(to_upload)} already on PyPI)")

	if dry_run:
	for _, n in to_upload:
	print(f" [DRY RUN] would upload {n}")
	continue

	for rid, name in to_upload:
	url = get_asset_download_url(rid, name)
	if not url:
	print(f" no download url for {name}")
	continue
	dest = os.path.join("/tmp/pypi-dist", name)
	if os.path.exists(dest):
	continue
	try:
	urllib.request.urlretrieve(url, dest)
	strip_license_file(Path(dest))
	print(f" staged: {name}")
	except Exception as e:
	print(f" download failed {name}: {e}")

	staged = list(Path("/tmp/pypi-dist").glob("*.whl"))
	print(f"\nStaged {len(staged)} wheels for PyPI OIDC upload")
	for w in staged:
	print(f" {w.name}")

	PYEOF

	- name: Upload staged wheels for OIDC publish
	if: inputs.dry_run == false
	uses: actions/upload-artifact@v4
	with:
	name: backfill-pypi-dist
	path: /tmp/pypi-dist/
	if-no-files-found: ignore

	pypi-publish:
	name: "🚀 Publish backfilled wheels via OIDC"
	needs: backfill
	if: inputs.dry_run == false
	runs-on: ubuntu-latest
	environment:
	name: pypi
	url: https://pypi.org/p/uv-ffi
	permissions:
	id-token: write
	steps:
	- name: Download staged wheels
	uses: actions/download-artifact@v4
	with:
	name: backfill-pypi-dist
	path: dist/

	- name: Check for wheels to upload
	id: check
	run: \|
	COUNT=$(find dist/ -name "*.whl" \| wc -l)
	echo "count=$COUNT" >> $GITHUB_OUTPUT
	echo "Found $COUNT wheels to upload"
	ls dist/ \|\| true

	- name: Publish via OIDC (trusted publisher)
	if: steps.check.outputs.count > 0
	uses: pypa/gh-action-pypi-publish@release/v1
	with:
	packages-dir: dist/
	skip-existing: true

	update-wheel-index:
	name: "📄 Update GH Pages wheel index"
	needs: [backfill, pypi-publish]
	runs-on: ubuntu-latest
	if: inputs.dry_run == false
	permissions:
	pages: write
	id-token: write
	environment:
	name: github-pages
	url: ${{ steps.deploy.outputs.page_url }}
	steps:
	- name: Regenerate PEP 503 index
	env:
	GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
	REPO: ${{ github.repository }}
	run: \|
	mkdir -p _site/uv-ffi
	python3 - <<'EOF'
	import os, json, urllib.request

	repo = os.environ["REPO"]
	token = os.environ["GH_TOKEN"]

	def gh(url):
	req = urllib.request.Request(url, headers={
	"Authorization": f"token {token}",
	"Accept": "application/vnd.github+json"})
	return json.loads(urllib.request.urlopen(req).read())

	def pypi_wheels(pkg):
	try:
	req = urllib.request.Request(f"https://pypi.org/pypi/{pkg}/json")
	data = json.loads(urllib.request.urlopen(req).read())
	return [(v, f["filename"], f["url"])
	for v, files in data["releases"].items()
	for f in files if f["filename"].endswith(".whl")]
	except Exception as e:
	print(f"PyPI fetch failed: {e}")
	return []

	gh_wheels = []
	page = 1
	while True:
	rels = gh(f"https://api.github.com/repos/{repo}/releases?per_page=50&page={page}")
	if not rels:
	break
	for rel in rels:
	assets_pg2 = 1
	while True:
	assets_url = f"https://api.github.com/repos/{repo}/releases/{rel['id']}/assets?per_page=100&page={assets_pg2}"
	batch = gh(assets_url) or []
	for a in batch:
	if a["name"].endswith(".whl"):
	gh_wheels.append((rel["tag_name"], a["name"], a["browser_download_url"]))
	if len(batch) < 100:
	break
	assets_pg2 += 1
	page += 1

	pip_wheels = pypi_wheels("uv-ffi")

	seen = {}
	for tag, name, url in pip_wheels:
	seen[name] = (tag, name, url)
	for tag, name, url in gh_wheels:
	seen[name] = (tag, name, url)

	all_wheels = sorted(seen.values(), key=lambda x: (x[0], x[1]))

	links = "\n".join(
	f' <a href="{url}" data-requires-python=">=3.8">{name}</a><br>'
	for _, name, url in all_wheels
	)

	with open("_site/uv-ffi/index.html", "w") as fh:
	fh.write(f"<!DOCTYPE html>\n<html>\n <head><title>Links for uv-ffi</title></head>\n"
	f" <body>\n <h1>Links for uv-ffi</h1>\n{links}\n </body>\n</html>\n")

	with open("_site/index.html", "w") as fh:
	fh.write(f"<!DOCTYPE html>\n<html>\n <head><title>uv-ffi wheel index</title></head>\n"
	f" <body>\n <h1>uv-ffi wheel index</h1>\n"
	f' <p>Mainstream wheels: <a href="https://pypi.org/project/uv-ffi/">PyPI</a></p>\n'
	f" <p>Exotic/extended platform wheels hosted here.</p>\n"
	f" <pre>pip install uv-ffi --extra-index-url https://exotic-wheels.github.io/</pre>\n"
	f' <p><a href="uv-ffi/">Browse all {len(all_wheels)} wheels</a></p>\n'
	f" </body>\n</html>\n")

	print(f"Indexed {len(all_wheels)} total wheels ({len(pip_wheels)} PyPI + {len(gh_wheels)} GH Releases)")
	EOF

	- name: Upload Pages artifact
	uses: actions/upload-pages-artifact@v3
	with:
	path: _site

	- name: Deploy to GitHub Pages
	id: deploy
	uses: actions/deploy-pages@v4

Navigation Menu

🗄️ Backfill Wheels to GH Releases & PyPI #32

Workflow file

🗄️ Backfill Wheels to GH Releases & PyPI #32

Uh oh!

Workflow file for this run