From d550002d20852c2592e01ed3cc3a9d5818222d37 Mon Sep 17 00:00:00 2001 From: Arash Date: Wed, 22 Oct 2025 20:22:35 +0200 Subject: [PATCH 01/15] Add workflow to fix outdated tools and implement script for uninstallable revisions --- .github/workflows/fix-outdated-tools.yml | 105 ++++++++++++++++++ scripts/fix_outdated.py | 131 +++++++++++++++++++++++ 2 files changed, 236 insertions(+) create mode 100644 .github/workflows/fix-outdated-tools.yml create mode 100644 scripts/fix_outdated.py diff --git a/.github/workflows/fix-outdated-tools.yml b/.github/workflows/fix-outdated-tools.yml new file mode 100644 index 00000000..ec39ceb3 --- /dev/null +++ b/.github/workflows/fix-outdated-tools.yml @@ -0,0 +1,105 @@ +name: Fix Outdated Tools + +on: + workflow_dispatch: + +jobs: + get-lockfiles: + runs-on: ubuntu-latest + outputs: + lockfiles: ${{ steps.set-matrix.outputs.lockfiles }} + steps: + - name: Checkout repository + uses: actions/checkout@v5 + + - name: Get all lock files + id: set-matrix + run: | + lockfiles=$(ls *.yaml.lock | jq -R -s -c 'split("\n")[:-1]') + echo "lockfiles=$lockfiles" >> $GITHUB_OUTPUT + + fix-outdated: + needs: get-lockfiles + runs-on: ubuntu-latest + strategy: + matrix: + lockfile: ${{ fromJson(needs.get-lockfiles.outputs.lockfiles) }} + fail-fast: false + permissions: + contents: write + pull-requests: write + steps: + - name: Checkout repository + uses: actions/checkout@v5 + + - name: Set up Python + uses: actions/setup-python@v6 + with: + python-version: '3.13' + + - name: Install uv + uses: astral-sh/setup-uv@v7 + + - name: Install dependencies + run: uv pip install --system -r requirements.txt + + - name: Fix ${{ matrix.lockfile }} + run: python scripts/fix_outdated.py "${{ matrix.lockfile }}" + + - name: Upload changes + uses: actions/upload-artifact@v4 + if: always() + with: + name: ${{ matrix.lockfile }} + path: ${{ matrix.lockfile }} + + create-pr: + needs: fix-outdated + if: always() + runs-on: ubuntu-latest + permissions: + contents: write + pull-requests: write + steps: + - name: Checkout repository + uses: actions/checkout@v5 + with: + fetch-depth: 0 + + - name: Download all artifacts + uses: actions/download-artifact@v5 + with: + merge-multiple: true + + - name: Check for changes + id: check_changes + run: | + if [[ -n $(git status --porcelain) ]]; then + echo "changes=true" >> $GITHUB_OUTPUT + echo "Changes detected in lock files" + else + echo "changes=false" >> $GITHUB_OUTPUT + echo "No changes detected" + fi + + - name: Create or update Pull Request + id: cpr + if: steps.check_changes.outputs.changes == 'true' + uses: peter-evans/create-pull-request@v7 + with: + branch: fix-outdated-tools + commit-message: Fix uninstallable tool revisions + title: 'Fix uninstallable tool revisions' + body: | + This PR was automatically generated by the `fix-outdated-tools` workflow. + Workflow run: [${{ github.run_id }}](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}) + delete-branch: true + + - name: Comment on updated PR + if: steps.cpr.outputs.pull-request-operation == 'update' + uses: peter-evans/create-or-update-comment@v5 + with: + issue-number: ${{ steps.cpr.outputs.pull-request-number }} + body: | + 🔄 This PR was automatically updated with new lockfile changes. + Workflow run: [${{ github.run_id }}](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}) diff --git a/scripts/fix_outdated.py b/scripts/fix_outdated.py new file mode 100644 index 00000000..04725fc2 --- /dev/null +++ b/scripts/fix_outdated.py @@ -0,0 +1,131 @@ +import argparse +import logging +import yaml +from concurrent.futures import ThreadPoolExecutor, as_completed + +from bioblend import toolshed + +logging.basicConfig( + level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s" +) +logger = logging.getLogger(__name__) + + +def get_tool_versions(ts, name, owner, revision): + versions = set() + + try: + repo_metadata = ts.repositories.get_repository_revision_install_info( + name, owner, revision + ) + if isinstance(repo_metadata, list) and len(repo_metadata) > 1: + for tool in repo_metadata[1].get("valid_tools", []): + if "id" in tool and "version" in tool: + versions.add((tool["id"], tool["version"])) + except Exception as e: + logger.warning(f"{name},{owner}: failed to fetch {revision} ({e})") + return versions + + +def fetch_versions_parallel(ts, name, owner, revisions, max_workers=10): + version_cache = {} + with ThreadPoolExecutor(max_workers=max_workers) as executor: + futures = { + executor.submit(get_tool_versions, ts, name, owner, rev): rev + for rev in revisions + } + for future in as_completed(futures): + rev = futures[future] + try: + version_cache[rev] = future.result() + except Exception as e: + logger.warning(f"{name},{owner}: error fetching {rev} ({e})") + version_cache[rev] = set() + return version_cache + + +def fix_uninstallable(lockfile_name, toolshed_url): + ts = toolshed.ToolShedInstance(url=toolshed_url) + with open(lockfile_name) as f: + lockfile = yaml.safe_load(f) or {} + locked_tools = lockfile.get("tools", []) + total = len(locked_tools) + + logger.info(f"Processing {total} tools from {lockfile_name}...") + changed, skipped = 0, 0 + + for i, tool in enumerate(locked_tools): + if i % 10 == 0: + logger.info( + f"Progress: {i}/{total} tools ({skipped} skipped, {changed} changed)" + ) + + name, owner = tool.get("name"), tool.get("owner") + revisions = tool.get("revisions", []) + try: + installable = ts.repositories.get_ordered_installable_revisions(name, owner) + except Exception as e: + logger.warning(f"{name},{owner}: could not get installable revisions ({e})") + continue + + uninstallable = set(revisions) - set(installable) + if not uninstallable: + skipped += 1 + continue + + all_revs = list(uninstallable) + list(installable) + version_cache = fetch_versions_parallel(ts, name, owner, all_revs) + + to_remove = [] + for cur in uninstallable: + cur_versions = version_cache.get(cur, set()) + if not cur_versions: + if installable: + nxt = installable[0] + logger.info(f"{name},{owner}: unverifiable {cur}, keeping {nxt}") + to_remove.append(cur) + continue + + nxt = next( + ( + cand + for cand in installable + if version_cache.get(cand) == cur_versions + ), + None, + ) + + if not nxt: + logger.warning( + f"{name},{owner}: no matching installable revision for {cur}" + ) + continue + + logger.info( + f"{name},{owner}: removing {cur} {'in favor of ' + nxt if nxt in revisions else 'with no installable alternative found'}" + ) + to_remove.append(cur) + + if to_remove: + changed += 1 + tool["revisions"] = sorted(set(r for r in revisions if r not in to_remove)) + + logger.info( + f"Completed: {total} tools processed, {skipped} skipped, {changed} changed" + ) + + with open(lockfile_name, "w") as f: + yaml.dump(lockfile, f, sort_keys=False, default_flow_style=False) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument( + "lockfile", type=argparse.FileType("r"), help="Tool.yaml.lock file" + ) + parser.add_argument( + "--toolshed", default="https://toolshed.g2.bx.psu.edu", help="Toolshed base URL" + ) + args = parser.parse_args() + + fix_uninstallable(args.lockfile.name, args.toolshed) From 0b0e5a64e6a761b0220448c9df568a443e59f59e Mon Sep 17 00:00:00 2001 From: Arash Date: Thu, 23 Oct 2025 11:15:05 +0200 Subject: [PATCH 02/15] Handle errors by exiting the script in fix_outdated.py --- ephemeris | 1 + scripts/fix_outdated.py | 8 +++++--- 2 files changed, 6 insertions(+), 3 deletions(-) create mode 160000 ephemeris diff --git a/ephemeris b/ephemeris new file mode 160000 index 00000000..ea30c38e --- /dev/null +++ b/ephemeris @@ -0,0 +1 @@ +Subproject commit ea30c38ed570455fe8c6d699305977b49068b5d9 diff --git a/scripts/fix_outdated.py b/scripts/fix_outdated.py index 04725fc2..b698dc20 100644 --- a/scripts/fix_outdated.py +++ b/scripts/fix_outdated.py @@ -1,8 +1,9 @@ import argparse import logging -import yaml +import sys from concurrent.futures import ThreadPoolExecutor, as_completed +import yaml from bioblend import toolshed logging.basicConfig( @@ -24,6 +25,7 @@ def get_tool_versions(ts, name, owner, revision): versions.add((tool["id"], tool["version"])) except Exception as e: logger.warning(f"{name},{owner}: failed to fetch {revision} ({e})") + sys.exit(1) return versions @@ -40,7 +42,7 @@ def fetch_versions_parallel(ts, name, owner, revisions, max_workers=10): version_cache[rev] = future.result() except Exception as e: logger.warning(f"{name},{owner}: error fetching {rev} ({e})") - version_cache[rev] = set() + sys.exit(1) return version_cache @@ -99,7 +101,7 @@ def fix_uninstallable(lockfile_name, toolshed_url): logger.warning( f"{name},{owner}: no matching installable revision for {cur}" ) - continue + sys.exit(1) logger.info( f"{name},{owner}: removing {cur} {'in favor of ' + nxt if nxt in revisions else 'with no installable alternative found'}" From 9d953a35047e771f19e7a1ca7957cabd6d5b38cd Mon Sep 17 00:00:00 2001 From: Arash Date: Fri, 24 Oct 2025 10:25:09 +0200 Subject: [PATCH 03/15] Remove ephemeris subproject reference --- ephemeris | 1 - 1 file changed, 1 deletion(-) delete mode 160000 ephemeris diff --git a/ephemeris b/ephemeris deleted file mode 160000 index ea30c38e..00000000 --- a/ephemeris +++ /dev/null @@ -1 +0,0 @@ -Subproject commit ea30c38ed570455fe8c6d699305977b49068b5d9 From 6263fb601a231f38892f7d4988e2865238966e68 Mon Sep 17 00:00:00 2001 From: Arash Date: Fri, 24 Oct 2025 11:05:58 +0200 Subject: [PATCH 04/15] Implement retry logic with exponential backoff for repository fetch operations --- scripts/fix_outdated.py | 40 +++++++++++++++++++++++++++++++++++++--- 1 file changed, 37 insertions(+), 3 deletions(-) diff --git a/scripts/fix_outdated.py b/scripts/fix_outdated.py index b698dc20..fdaed895 100644 --- a/scripts/fix_outdated.py +++ b/scripts/fix_outdated.py @@ -1,6 +1,7 @@ import argparse import logging import sys +import time from concurrent.futures import ThreadPoolExecutor, as_completed import yaml @@ -12,12 +13,43 @@ logger = logging.getLogger(__name__) +def retry_with_backoff(func, *args, **kwargs): + MAX_RETRIES = 5 + backoff = 2 + last_exception = None + + for attempt in range(MAX_RETRIES): + try: + return func(*args, **kwargs) + except Exception as e: + last_exception = e + error_msg = str(e) + if any( + code in error_msg + for code in ["502", "503", "504", "timed out", "timeout", "Connection"] + ): + if attempt < MAX_RETRIES - 1: + logger.warning( + f"Attempt {attempt + 1}/{MAX_RETRIES} failed: {error_msg}. Retrying in {backoff}s..." + ) + time.sleep(backoff) + backoff = min(backoff * 2, 60) + else: + logger.error(f"All {MAX_RETRIES} attempts failed") + else: + raise + + if last_exception: + raise last_exception + raise Exception("Retry failed with no exception captured") + + def get_tool_versions(ts, name, owner, revision): versions = set() try: - repo_metadata = ts.repositories.get_repository_revision_install_info( - name, owner, revision + repo_metadata = retry_with_backoff( + ts.repositories.get_repository_revision_install_info, name, owner, revision ) if isinstance(repo_metadata, list) and len(repo_metadata) > 1: for tool in repo_metadata[1].get("valid_tools", []): @@ -65,7 +97,9 @@ def fix_uninstallable(lockfile_name, toolshed_url): name, owner = tool.get("name"), tool.get("owner") revisions = tool.get("revisions", []) try: - installable = ts.repositories.get_ordered_installable_revisions(name, owner) + installable = retry_with_backoff( + ts.repositories.get_ordered_installable_revisions, name, owner + ) except Exception as e: logger.warning(f"{name},{owner}: could not get installable revisions ({e})") continue From f926b662dd8f76acb6a79cb5b4addbffaca199eb Mon Sep 17 00:00:00 2001 From: Arash Date: Fri, 24 Oct 2025 12:26:37 +0200 Subject: [PATCH 05/15] Remove comment step from PR update workflow in fix-outdated-tools.yml --- .github/workflows/fix-outdated-tools.yml | 9 --------- 1 file changed, 9 deletions(-) diff --git a/.github/workflows/fix-outdated-tools.yml b/.github/workflows/fix-outdated-tools.yml index ec39ceb3..b32bc8f9 100644 --- a/.github/workflows/fix-outdated-tools.yml +++ b/.github/workflows/fix-outdated-tools.yml @@ -94,12 +94,3 @@ jobs: This PR was automatically generated by the `fix-outdated-tools` workflow. Workflow run: [${{ github.run_id }}](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}) delete-branch: true - - - name: Comment on updated PR - if: steps.cpr.outputs.pull-request-operation == 'update' - uses: peter-evans/create-or-update-comment@v5 - with: - issue-number: ${{ steps.cpr.outputs.pull-request-number }} - body: | - 🔄 This PR was automatically updated with new lockfile changes. - Workflow run: [${{ github.run_id }}](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}) From 732e484b0e7bc6f7d92ae5645fb3badb202cdd20 Mon Sep 17 00:00:00 2001 From: Arash Date: Tue, 28 Oct 2025 12:24:22 +0100 Subject: [PATCH 06/15] Refactor tool removal logic to ensure revisions are updated correctly and improve logging clarity --- scripts/fix_outdated.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/scripts/fix_outdated.py b/scripts/fix_outdated.py index fdaed895..5b716017 100644 --- a/scripts/fix_outdated.py +++ b/scripts/fix_outdated.py @@ -137,14 +137,14 @@ def fix_uninstallable(lockfile_name, toolshed_url): ) sys.exit(1) - logger.info( - f"{name},{owner}: removing {cur} {'in favor of ' + nxt if nxt in revisions else 'with no installable alternative found'}" - ) + logger.info(f"{name},{owner}: removing {cur} in favor of {nxt}") + if nxt not in revisions: + revisions.append(nxt) to_remove.append(cur) if to_remove: changed += 1 - tool["revisions"] = sorted(set(r for r in revisions if r not in to_remove)) + tool["revisions"] = sorted(set(revisions) - set(to_remove)) logger.info( f"Completed: {total} tools processed, {skipped} skipped, {changed} changed" From ac1664cebd17b1297168d6802de42bd07339677d Mon Sep 17 00:00:00 2001 From: Arash Date: Fri, 7 Nov 2025 12:54:47 +0100 Subject: [PATCH 07/15] reverse over installable to get the latest revisions --- scripts/fix_outdated.py | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/scripts/fix_outdated.py b/scripts/fix_outdated.py index 5b716017..1a1cdec4 100644 --- a/scripts/fix_outdated.py +++ b/scripts/fix_outdated.py @@ -122,14 +122,11 @@ def fix_uninstallable(lockfile_name, toolshed_url): to_remove.append(cur) continue - nxt = next( - ( - cand - for cand in installable - if version_cache.get(cand) == cur_versions - ), - None, - ) + nxt = None + for cand in reversed(installable): + if version_cache.get(cand) == cur_versions: + nxt = cand + break if not nxt: logger.warning( From 8c920f8b2bef0e070e9a64cc275219794213ac0c Mon Sep 17 00:00:00 2001 From: Arash Date: Mon, 24 Nov 2025 12:19:29 +0100 Subject: [PATCH 08/15] Fix: Add missing github-token to uv installation step --- .github/workflows/fix-outdated-tools.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/fix-outdated-tools.yml b/.github/workflows/fix-outdated-tools.yml index b32bc8f9..b361fac7 100644 --- a/.github/workflows/fix-outdated-tools.yml +++ b/.github/workflows/fix-outdated-tools.yml @@ -39,6 +39,8 @@ jobs: - name: Install uv uses: astral-sh/setup-uv@v7 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} - name: Install dependencies run: uv pip install --system -r requirements.txt From c7696df83c9a494c6ca3da26a7d5285f171009d2 Mon Sep 17 00:00:00 2001 From: Arash Date: Mon, 24 Nov 2025 17:05:23 +0100 Subject: [PATCH 09/15] Fix: Refactor fix_uninstallable function for improved error handling and path management --- scripts/fix_outdated.py | 95 +++++++++++++++++++++++++---------------- 1 file changed, 58 insertions(+), 37 deletions(-) diff --git a/scripts/fix_outdated.py b/scripts/fix_outdated.py index 1a1cdec4..d05537df 100644 --- a/scripts/fix_outdated.py +++ b/scripts/fix_outdated.py @@ -2,7 +2,9 @@ import logging import sys import time +from pathlib import Path from concurrent.futures import ThreadPoolExecutor, as_completed +from collections import defaultdict import yaml from bioblend import toolshed @@ -16,13 +18,11 @@ def retry_with_backoff(func, *args, **kwargs): MAX_RETRIES = 5 backoff = 2 - last_exception = None for attempt in range(MAX_RETRIES): try: return func(*args, **kwargs) except Exception as e: - last_exception = e error_msg = str(e) if any( code in error_msg @@ -34,14 +34,9 @@ def retry_with_backoff(func, *args, **kwargs): ) time.sleep(backoff) backoff = min(backoff * 2, 60) - else: - logger.error(f"All {MAX_RETRIES} attempts failed") - else: - raise - - if last_exception: - raise last_exception - raise Exception("Retry failed with no exception captured") + continue + raise e + raise Exception("Retry failed after max attempts") def get_tool_versions(ts, name, owner, revision): @@ -80,12 +75,28 @@ def fetch_versions_parallel(ts, name, owner, revisions, max_workers=10): def fix_uninstallable(lockfile_name, toolshed_url): ts = toolshed.ToolShedInstance(url=toolshed_url) - with open(lockfile_name) as f: + lockfile_path = Path(lockfile_name) + with open(lockfile_path) as f: lockfile = yaml.safe_load(f) or {} locked_tools = lockfile.get("tools", []) total = len(locked_tools) - logger.info(f"Processing {total} tools from {lockfile_name}...") + uninstallable_file = lockfile_path.with_name( + lockfile_path.name.replace(".yaml.lock", ".uninstallable_revisions.yaml") + ) + + removed_map = defaultdict(set) + try: + with open(uninstallable_file) as f: + uninstallable_data = yaml.safe_load(f) or {} + for t in uninstallable_data.get("tools", []): + removed_map[(t["name"], t["owner"])] = set( + t.get("removed_revisions", []) + ) + except FileNotFoundError: + pass + + logger.info(f"Processing {total} tools from {lockfile_path.name}...") changed, skipped = 0, 0 for i, tool in enumerate(locked_tools): @@ -95,38 +106,40 @@ def fix_uninstallable(lockfile_name, toolshed_url): ) name, owner = tool.get("name"), tool.get("owner") - revisions = tool.get("revisions", []) + current_revisions = set(tool.get("revisions", [])) try: - installable = retry_with_backoff( + installable_list = retry_with_backoff( ts.repositories.get_ordered_installable_revisions, name, owner ) except Exception as e: logger.warning(f"{name},{owner}: could not get installable revisions ({e})") continue - uninstallable = set(revisions) - set(installable) + uninstallable = current_revisions - set(installable_list) if not uninstallable: skipped += 1 continue - all_revs = list(uninstallable) + list(installable) + all_revs = list(uninstallable) + installable_list version_cache = fetch_versions_parallel(ts, name, owner, all_revs) - to_remove = [] + installable_signatures = {} + for rev in installable_list: + sig = frozenset(version_cache.get(rev, [])) + if sig: + installable_signatures[sig] = rev + to_remove = set() + for cur in uninstallable: - cur_versions = version_cache.get(cur, set()) - if not cur_versions: - if installable: - nxt = installable[0] + cur_sig = frozenset(version_cache.get(cur, [])) + if not cur_sig: + if installable_list: + nxt = installable_list[-1] logger.info(f"{name},{owner}: unverifiable {cur}, keeping {nxt}") - to_remove.append(cur) - continue + to_remove.add(cur) + continue - nxt = None - for cand in reversed(installable): - if version_cache.get(cand) == cur_versions: - nxt = cand - break + nxt = installable_signatures.get(cur_sig) if not nxt: logger.warning( @@ -135,30 +148,38 @@ def fix_uninstallable(lockfile_name, toolshed_url): sys.exit(1) logger.info(f"{name},{owner}: removing {cur} in favor of {nxt}") - if nxt not in revisions: - revisions.append(nxt) - to_remove.append(cur) + if nxt not in current_revisions: + tool["revisions"].append(nxt) + to_remove.add(cur) if to_remove: changed += 1 - tool["revisions"] = sorted(set(revisions) - set(to_remove)) + tool["revisions"] = sorted(set(tool["revisions"]) - to_remove) + removed_map[(name, owner)].update(to_remove) logger.info( f"Completed: {total} tools processed, {skipped} skipped, {changed} changed" ) - with open(lockfile_name, "w") as f: + with open(lockfile_path, "w") as f: yaml.dump(lockfile, f, sort_keys=False, default_flow_style=False) + uninstallable_output = { + "tools": [ + {"name": n, "owner": o, "removed_revisions": sorted(revs)} + for (n, o), revs in removed_map.items() + ] + } + with open(uninstallable_file, "w") as f: + yaml.dump(uninstallable_output, f, sort_keys=False, default_flow_style=False) + if __name__ == "__main__": parser = argparse.ArgumentParser() - parser.add_argument( - "lockfile", type=argparse.FileType("r"), help="Tool.yaml.lock file" - ) + parser.add_argument("lockfile", help="Tool.yaml.lock file path") parser.add_argument( "--toolshed", default="https://toolshed.g2.bx.psu.edu", help="Toolshed base URL" ) args = parser.parse_args() - fix_uninstallable(args.lockfile.name, args.toolshed) + fix_uninstallable(args.lockfile, args.toolshed) From 84418c8e199f4a7b65e56a796ef2c2b9bc7f961a Mon Sep 17 00:00:00 2001 From: Arash Date: Tue, 25 Nov 2025 11:16:05 +0100 Subject: [PATCH 10/15] Fix: Update upload artifact step to include uninstallable revisions and ignore if no files found --- .github/workflows/fix-outdated-tools.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/fix-outdated-tools.yml b/.github/workflows/fix-outdated-tools.yml index b361fac7..6b619912 100644 --- a/.github/workflows/fix-outdated-tools.yml +++ b/.github/workflows/fix-outdated-tools.yml @@ -53,7 +53,10 @@ jobs: if: always() with: name: ${{ matrix.lockfile }} - path: ${{ matrix.lockfile }} + path: | + ${{ matrix.lockfile }} + *.uninstallable_revisions.yaml + if-no-files-found: ignore create-pr: needs: fix-outdated From 7991eb4e1b460ba01f009d6670235c6f4eaee392 Mon Sep 17 00:00:00 2001 From: Arash Date: Tue, 25 Nov 2025 13:10:06 +0100 Subject: [PATCH 11/15] Fix: Only write uninstallable revisions file if there are removed revisions --- scripts/fix_outdated.py | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/scripts/fix_outdated.py b/scripts/fix_outdated.py index d05537df..ac15c462 100644 --- a/scripts/fix_outdated.py +++ b/scripts/fix_outdated.py @@ -164,14 +164,17 @@ def fix_uninstallable(lockfile_name, toolshed_url): with open(lockfile_path, "w") as f: yaml.dump(lockfile, f, sort_keys=False, default_flow_style=False) - uninstallable_output = { - "tools": [ - {"name": n, "owner": o, "removed_revisions": sorted(revs)} - for (n, o), revs in removed_map.items() - ] - } - with open(uninstallable_file, "w") as f: - yaml.dump(uninstallable_output, f, sort_keys=False, default_flow_style=False) + if removed_map: + uninstallable_output = { + "tools": [ + {"name": n, "owner": o, "removed_revisions": sorted(revs)} + for (n, o), revs in removed_map.items() + ] + } + with open(uninstallable_file, "w") as f: + yaml.dump( + uninstallable_output, f, sort_keys=False, default_flow_style=False + ) if __name__ == "__main__": From d30bb8d4c6121393aa626688311845df2b429e5f Mon Sep 17 00:00:00 2001 From: Arash Date: Fri, 28 Nov 2025 12:08:28 +0100 Subject: [PATCH 12/15] Add scheduled trigger for the fix outdated tools workflow --- .github/workflows/fix-outdated-tools.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/fix-outdated-tools.yml b/.github/workflows/fix-outdated-tools.yml index 6b619912..d63408e7 100644 --- a/.github/workflows/fix-outdated-tools.yml +++ b/.github/workflows/fix-outdated-tools.yml @@ -2,6 +2,8 @@ name: Fix Outdated Tools on: workflow_dispatch: + schedule: + - cron: '0 9 1 * *' jobs: get-lockfiles: From 7548fad8ac66b275289f5aff512ec8f445a24063 Mon Sep 17 00:00:00 2001 From: Arash Date: Wed, 3 Dec 2025 12:44:12 +0100 Subject: [PATCH 13/15] Rename uninstallable revisions to not-installable revisions in workflow and script --- .github/workflows/fix-outdated-tools.yml | 6 +++--- scripts/fix_outdated.py | 22 ++++++++++------------ 2 files changed, 13 insertions(+), 15 deletions(-) diff --git a/.github/workflows/fix-outdated-tools.yml b/.github/workflows/fix-outdated-tools.yml index d63408e7..447d1580 100644 --- a/.github/workflows/fix-outdated-tools.yml +++ b/.github/workflows/fix-outdated-tools.yml @@ -57,7 +57,7 @@ jobs: name: ${{ matrix.lockfile }} path: | ${{ matrix.lockfile }} - *.uninstallable_revisions.yaml + *.not-installable-revisions.yaml if-no-files-found: ignore create-pr: @@ -95,8 +95,8 @@ jobs: uses: peter-evans/create-pull-request@v7 with: branch: fix-outdated-tools - commit-message: Fix uninstallable tool revisions - title: 'Fix uninstallable tool revisions' + commit-message: Remove not-installable tool revisions + title: 'Remove not-installable tool revisions' body: | This PR was automatically generated by the `fix-outdated-tools` workflow. Workflow run: [${{ github.run_id }}](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}) diff --git a/scripts/fix_outdated.py b/scripts/fix_outdated.py index ac15c462..ae4e25aa 100644 --- a/scripts/fix_outdated.py +++ b/scripts/fix_outdated.py @@ -81,18 +81,16 @@ def fix_uninstallable(lockfile_name, toolshed_url): locked_tools = lockfile.get("tools", []) total = len(locked_tools) - uninstallable_file = lockfile_path.with_name( - lockfile_path.name.replace(".yaml.lock", ".uninstallable_revisions.yaml") + not_installable_file = lockfile_path.with_name( + lockfile_path.name.replace(".yaml.lock", ".not-installable-revisions.yaml") ) removed_map = defaultdict(set) try: - with open(uninstallable_file) as f: - uninstallable_data = yaml.safe_load(f) or {} - for t in uninstallable_data.get("tools", []): - removed_map[(t["name"], t["owner"])] = set( - t.get("removed_revisions", []) - ) + with open(not_installable_file) as f: + not_installable_data = yaml.safe_load(f) or {} + for t in not_installable_data.get("tools", []): + removed_map[(t["name"], t["owner"])] = set(t.get("revisions", [])) except FileNotFoundError: pass @@ -165,15 +163,15 @@ def fix_uninstallable(lockfile_name, toolshed_url): yaml.dump(lockfile, f, sort_keys=False, default_flow_style=False) if removed_map: - uninstallable_output = { + not_installable_output = { "tools": [ - {"name": n, "owner": o, "removed_revisions": sorted(revs)} + {"name": n, "owner": o, "revisions": sorted(revs)} for (n, o), revs in removed_map.items() ] } - with open(uninstallable_file, "w") as f: + with open(not_installable_file, "w") as f: yaml.dump( - uninstallable_output, f, sort_keys=False, default_flow_style=False + not_installable_output, f, sort_keys=False, default_flow_style=False ) From b976dbccf30330870681110216d833fa29b72332 Mon Sep 17 00:00:00 2001 From: Arash Kadkhodaei Date: Wed, 7 Jan 2026 14:09:43 +0100 Subject: [PATCH 14/15] Apply suggestions from code review Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- scripts/fix_outdated.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/fix_outdated.py b/scripts/fix_outdated.py index ae4e25aa..4c4f18dc 100644 --- a/scripts/fix_outdated.py +++ b/scripts/fix_outdated.py @@ -15,8 +15,9 @@ logger = logging.getLogger(__name__) +MAX_RETRIES = 5 + def retry_with_backoff(func, *args, **kwargs): - MAX_RETRIES = 5 backoff = 2 for attempt in range(MAX_RETRIES): From c499cd02f36d8c6fa2c4fbc287419ff21bbbbfbc Mon Sep 17 00:00:00 2001 From: Arash Date: Wed, 7 Jan 2026 15:22:47 +0100 Subject: [PATCH 15/15] Refactor retry_with_backoff to use a local max_retries variable instead of a global constant --- scripts/fix_outdated.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/scripts/fix_outdated.py b/scripts/fix_outdated.py index 4c4f18dc..0e540633 100644 --- a/scripts/fix_outdated.py +++ b/scripts/fix_outdated.py @@ -15,12 +15,11 @@ logger = logging.getLogger(__name__) -MAX_RETRIES = 5 - def retry_with_backoff(func, *args, **kwargs): backoff = 2 + max_retries = 5 - for attempt in range(MAX_RETRIES): + for attempt in range(max_retries): try: return func(*args, **kwargs) except Exception as e: @@ -29,9 +28,9 @@ def retry_with_backoff(func, *args, **kwargs): code in error_msg for code in ["502", "503", "504", "timed out", "timeout", "Connection"] ): - if attempt < MAX_RETRIES - 1: + if attempt < max_retries - 1: logger.warning( - f"Attempt {attempt + 1}/{MAX_RETRIES} failed: {error_msg}. Retrying in {backoff}s..." + f"Attempt {attempt + 1}/{max_retries} failed: {error_msg}. Retrying in {backoff}s..." ) time.sleep(backoff) backoff = min(backoff * 2, 60)