fix(ci): improve caching

Author: conradoplg

.github/workflows/README.md

@@ -138,7 +138,45 @@ _The diagram above illustrates the parallel execution patterns in our CI/CD syst
 
 **Note**: Self-hosted Runners are just used to keep the logs running in the GitHub Actions UI for over 6 hours, the Integration Tests are not run in the Self-hosted Runner itself, but in the deployed VMs in GCP through GitHub Actions.
 
-### 5. Queue Management
+### 5. Rust build caching
+
+Rust jobs cache `~/.cargo` and dependency artifacts in `target/` through
+[`actions-rust-lang/setup-rust-toolchain`](https://github.com/actions-rust-lang/setup-rust-toolchain),
+which wraps [`Swatinem/rust-cache`](https://github.com/Swatinem/rust-cache). Caching is on by
+default, so a job opts *out* with `cache: false` rather than opting in.
+
+GitHub gives each repository a 10 GB cache quota and evicts least-recently-used entries when it is
+exceeded. Caches are also branch-scoped: a branch can read its own caches and the default branch's,
+but never another branch's. Those two rules together mean an unmanaged setup is self-defeating -
+caches written by PRs can never be read by anyone else, but they still evict main's caches, which
+are the only ones every PR does restore from.
+
+Three rules keep the quota usable:
+
+1. **Only main writes.** Every job that builds the workspace sets
+   `cache-save-if: ${{ github.ref == 'refs/heads/main' }}`. PRs restore from main and write nothing.
+2. **Wide matrices share one key.** The `test-crates.yml` matrices use `cache-shared-key` and seed
+   the shared cache from a single leg (`zebra-rpc`, which has the widest dependency closure), rather
+   than one ~1 GB near-duplicate cache per crate.
+3. **Jobs that don't build don't cache.** `fmt`, `no-test-deps`, `deny` (12 jobs wide), and the
+   crate-matrix generator in `test-crates.yml` set `cache: false`.
+
+Some jobs are deliberate exceptions to rule 1, because they never run on main and so would get no
+cache at all under it: `check-no-git-dependencies` in `tests-unit.yml` and `release-readiness` in
+`pr-gate.yml` (both A-release PRs only), and `benchmarks.yml` (label-gated). They do build, and
+they reuse their own cache across pushes to the same PR, so they keep saving. All three are
+infrequent enough not to dominate the quota.
+
+To inspect the current state:
+
+```bash
+gh api repos/ZcashFoundation/zebra/actions/cache/usage
+gh api 'repos/ZcashFoundation/zebra/actions/caches?per_page=100' \
+  --jq '[.actions_caches[] | {ref, mb: (.size_in_bytes/1048576|round)}]
+        | group_by(.ref)[] | "\(.[0].ref) n=\(length) \(map(.mb)|add)MB"'
+```
+
+### 6. Queue Management
 
 [Mergify](https://mergify.com)
 

.github/workflows/lint.yml

@@ -70,7 +70,16 @@ jobs:
         with:
           components: clippy
           toolchain: 1.97.0
+          # Both matrix legs build the same workspace, so they share one cache.
+          cache-shared-key: clippy
           cache-on-failure: true
+          # Restore on every ref, but only write from main. Actions caches are
+          # branch-scoped, so a cache written by a PR can never be read by any
+          # other branch, yet it still counts against the repo-wide 10 GB quota
+          # and evicts main's caches - the only ones PRs can restore from.
+          # The `tests` leg seeds it, because its feature set is a superset.
+          # See "Rust build caching" in .github/workflows/README.md.
+          cache-save-if: ${{ github.ref == 'refs/heads/main' && matrix.type == 'tests' }}
       - uses: ./.github/actions/setup-zebra-build
       - name: Run clippy
         run: cargo clippy ${{ matrix.args }} --features "${{ matrix.features }}"
@@ -90,6 +99,8 @@ jobs:
       - uses: actions-rust-lang/setup-rust-toolchain@46268bd060767258de96ed93c1251119784f2ab6 #v1.16.1
         with:
           cache-on-failure: true
+          # Only main writes caches, see the clippy job above.
+          cache-save-if: ${{ github.ref == 'refs/heads/main' }}
       - uses: taiki-e/install-action@65851e10cd6c377f11a60e600abc07cb08643468 #v2.79.3
         with:
           tool: cargo-hack
@@ -118,6 +129,8 @@ jobs:
         with:
           toolchain: 1.91.0 # MSRV
           cache-on-failure: true
+          # Only main writes caches, see the clippy job above.
+          cache-save-if: ${{ github.ref == 'refs/heads/main' }}
       - uses: ./.github/actions/setup-zebra-build
       - run: cargo build --bin "${{ matrix.binary }}" --workspace
 
@@ -138,6 +151,8 @@ jobs:
         with:
           toolchain: 1.91.0
           components: rustfmt
+          # `cargo fmt` builds nothing, so a cache here only consumes quota.
+          cache: false
       - name: Run fmt
         run: cargo fmt --all -- --check
 
@@ -154,7 +169,8 @@ jobs:
       - uses: actions-rust-lang/setup-rust-toolchain@46268bd060767258de96ed93c1251119784f2ab6 #v1.16.1
         with:
           toolchain: 1.97.0
-          cache-on-failure: true
+          # `cargo tree` builds nothing, so a cache here only consumes quota.
+          cache: false
       - name: Ensure no arbitrary or proptest dependency on default build
         run: cargo tree --package zebrad -e=features,no-dev | grep -Eq "arbitrary|proptest" && exit 1 || exit 0
 
@@ -175,6 +191,8 @@ jobs:
         with:
           toolchain: 1.97.0
           cache-on-failure: true
+          # Only main writes caches, see the clippy job above.
+          cache-save-if: ${{ github.ref == 'refs/heads/main' }}
       - uses: ./.github/actions/setup-zebra-build
       - run: cargo check --locked --all-features --all-targets
 
@@ -211,7 +229,9 @@ jobs:
           persist-credentials: false
       - uses: actions-rust-lang/setup-rust-toolchain@46268bd060767258de96ed93c1251119784f2ab6 #v1.16.1
         with:
-          cache-on-failure: true
+          # cargo-deny reads the dependency graph without building it, and this
+          # matrix is 12 jobs wide, so caching here is pure quota consumption.
+          cache: false
       - name: Check ${{ matrix.checks }} with features ${{ matrix.features }}
         uses: EmbarkStudios/cargo-deny-action@6c8f9facfa5047ec02d8485b6bf52b587b7777d1 #v2.0.18
         with:

.github/workflows/test-crates.yml

@@ -63,7 +63,9 @@ jobs:
         with:
           toolchain: 1.97.0
           components: clippy
-          cache-on-failure: true
+          # This job only runs `cargo tree` to build the matrix, so a cache here
+          # only consumes quota.
+          cache: false
       - uses: ./.github/actions/setup-zebra-build
 
       # This step dynamically creates a JSON containing the values of each crate
@@ -117,8 +119,16 @@ jobs:
         with:
           toolchain: 1.97.0
           components: clippy
-          cache-key: crate-build-${{ matrix.crate }}
+          # One cache for the whole matrix instead of one per crate. These jobs
+          # compile overlapping subsets of the same dependency graph, so 12
+          # per-crate caches stored ~12 GB of near-duplicate artifacts and blew
+          # through the repo-wide 10 GB quota on their own.
+          cache-shared-key: crate-build
           cache-on-failure: true
+          # Only main writes the shared cache, and only from the crate with the
+          # widest dependency closure, so the other legs restore a superset
+          # rather than whichever leg happened to finish first.
+          cache-save-if: ${{ github.ref == 'refs/heads/main' && matrix.crate == 'zebra-rpc' }}
 
       - uses: ./.github/actions/setup-zebra-build
 
@@ -179,8 +189,12 @@ jobs:
       - uses: actions-rust-lang/setup-rust-toolchain@46268bd060767258de96ed93c1251119784f2ab6 #v1.16.1
         with:
           toolchain: ${{ steps.msrv.outputs.version }}
-          cache-key: crate-build-msrv-${{ matrix.crate }}
+          # One cache for the whole matrix, see the `build` job above.
+          cache-shared-key: crate-build-msrv
           cache-on-failure: true
+          # `zebra-rpc` is also the seed here: this matrix skips `zebrad`, and
+          # `zebra-rpc` pulls in the rest of the workspace.
+          cache-save-if: ${{ github.ref == 'refs/heads/main' && matrix.crate == 'zebra-rpc' }}
 
       - uses: ./.github/actions/setup-zebra-build
 

.github/workflows/tests-unit.yml

@@ -69,6 +69,10 @@ jobs:
           toolchain: 1.97.0
           cache-key: unit-tests-1.97.0-default-release-binaries
           cache-on-failure: true
+          # Restore on every ref, but only write from main. This is the longest
+          # job in CI, so it is the one that suffers most when PR-scoped caches
+          # evict main's copy. See "Rust build caching" in README.md.
+          cache-save-if: ${{ github.ref == 'refs/heads/main' }}
       - uses: taiki-e/install-action@65851e10cd6c377f11a60e600abc07cb08643468 #v2.79.3
         with:
           tool: cargo-nextest