fix(zebrad): key the mempool per-peer download cap on IpAddr #2549
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # Deploy Zebra nodes to Google Cloud Platform. | |
| # | |
| # One zonal MIG per (environment, branch, network, zone). MIG names: | |
| # - release: zebrad-${network}-${zone-letter} | |
| # - push to main: zebrad-main-${network}-${zone-letter} | |
| # - workflow_dispatch: zebrad-${branch}-${network}-${zone-letter} | |
| # | |
| # Push and release fan out to 6 cells (2 networks × 3 zones); | |
| # workflow_dispatch deploys one cell (user picks network + zone). | |
| # | |
| # Design rationale: docs/decisions/devops/0006-gcp-deployment-naming.md | |
| # Operations: book/src/dev/gcp-deployment-operations.md | |
| # | |
| # See ADR docs/decisions/devops/0006-gcp-deployment-naming.md for the design | |
| # rationale and book/src/dev/gcp-deployment-operations.md for operational | |
| # procedures (PR-deploy cleanup, disk-corruption recovery, DB-format-version-break | |
| # release). | |
| name: Deploy Nodes to GCP | |
| # Ensures that only one workflow task will run at a time. Previous deployments, if | |
| # already in process, won't get cancelled. Instead, we let the first to complete | |
| # then queue the latest pending workflow, cancelling any workflows in between. | |
| # | |
| # Since the different event types each use a different Managed Instance Group or instance, | |
| # we can run different event types concurrently. | |
| # | |
| # For pull requests, we only run the tests from this workflow, and don't do any deployments. | |
| # So an in-progress pull request gets cancelled, just like other tests. | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.ref }}-${{ inputs.environment }}-${{ inputs.network }}-${{ inputs.zone }} | |
| cancel-in-progress: ${{ github.event_name == 'pull_request' }} | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| # Deployment configuration | |
| network: | |
| description: "Network to deploy: Mainnet or Testnet" | |
| required: true | |
| type: choice | |
| default: Mainnet | |
| options: | |
| - Mainnet | |
| - Testnet | |
| zone: | |
| description: "GCP zone for the workflow_dispatch deploy (single zone)" | |
| required: true | |
| type: choice | |
| default: us-east1-b | |
| options: | |
| - us-east1-b | |
| - us-east1-c | |
| - us-east1-d | |
| environment: | |
| description: "Environment to deploy to" | |
| required: true | |
| type: choice | |
| default: dev | |
| options: | |
| - dev | |
| - prod | |
| # Disk configuration | |
| need_cached_disk: | |
| description: Use a cached state disk | |
| type: boolean | |
| default: true | |
| cached_disk_type: | |
| description: Type of cached disk to use | |
| required: true | |
| type: choice | |
| default: tip | |
| options: | |
| - tip | |
| - checkpoint | |
| # Build configuration | |
| no_cache: | |
| description: Disable the Docker cache for this build | |
| type: boolean | |
| default: false | |
| # Logging configuration | |
| log_file: | |
| description: Log to a file path rather than standard output | |
| default: "" | |
| push: | |
| # Skip main branch updates where Rust code and dependencies aren't modified. | |
| branches: | |
| - main | |
| paths: | |
| # code and tests | |
| - "**/*.rs" | |
| # hard-coded checkpoints and proptest regressions | |
| - "**/*.txt" | |
| # dependencies | |
| - "**/Cargo.toml" | |
| - "**/Cargo.lock" | |
| # configuration files | |
| - .cargo/config.toml | |
| - "**/clippy.toml" | |
| # workflow definitions | |
| - docker/** | |
| - .dockerignore | |
| - .github/workflows/zfnd-deploy-nodes-gcp.yml | |
| - .github/workflows/zfnd-build-docker-image.yml | |
| # Only runs the Docker image tests, doesn't deploy any instances | |
| pull_request: | |
| # Skip PRs where Rust code and dependencies aren't modified. | |
| paths: | |
| # code and tests | |
| - "**/*.rs" | |
| # hard-coded checkpoints and proptest regressions | |
| - "**/*.txt" | |
| # dependencies | |
| - "**/Cargo.toml" | |
| - "**/Cargo.lock" | |
| # configuration files | |
| - .cargo/config.toml | |
| - "**/clippy.toml" | |
| # workflow definitions | |
| - docker/** | |
| - .dockerignore | |
| - .github/workflows/zfnd-deploy-nodes-gcp.yml | |
| - .github/workflows/zfnd-build-docker-image.yml | |
| release: | |
| types: | |
| - published | |
| permissions: | |
| contents: read | |
| jobs: | |
| # Build the (network, zone) matrix and resolve the target environment from | |
| # the trigger. Downstream jobs read `set-matrix` outputs so the event → | |
| # environment mapping is computed once. | |
| set-matrix: | |
| # Deployment image tests run on source PRs, not temporary Mergify queue candidates. | |
| if: >- | |
| ${{ | |
| github.event_name != 'pull_request' || | |
| github.event.pull_request.user.login != 'mergify[bot]' || | |
| github.event.pull_request.head.repo.full_name != github.repository || | |
| !startsWith(github.event.pull_request.head.ref, 'mergify/merge-queue/') | |
| }} | |
| runs-on: ubuntu-latest | |
| outputs: | |
| networks: ${{ steps.set-matrix.outputs.networks }} | |
| zones: ${{ steps.set-matrix.outputs.zones }} | |
| environment: ${{ steps.set-matrix.outputs.environment }} | |
| steps: | |
| - id: set-matrix | |
| run: | | |
| case "${{ github.event_name }}" in | |
| release) ENV="prod" ;; | |
| workflow_dispatch) ENV="${{ inputs.environment }}" ;; | |
| push) ENV="stage" ;; | |
| *) ENV="dev" ;; | |
| esac | |
| if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then | |
| NETWORKS='["${{ inputs.network }}"]' | |
| ZONES='["${{ inputs.zone }}"]' | |
| else | |
| NETWORKS='["Mainnet","Testnet"]' | |
| ZONES='["us-east1-b","us-east1-c","us-east1-d"]' | |
| fi | |
| { | |
| echo "networks=${NETWORKS}" | |
| echo "zones=${ZONES}" | |
| echo "environment=${ENV}" | |
| } >> "$GITHUB_OUTPUT" | |
| # Per-network cache-disk lookup. Cache images are network-specific | |
| # (`zebrad-cache-…-mainnet-tip` vs `…-testnet-tip`), so the lookup must | |
| # run once per network the matrix deploys to. Running a single workflow- | |
| # level lookup with `inputs.network || vars.ZCASH_NETWORK` would return | |
| # the wrong image for the other matrix row. | |
| # | |
| # Skipped for releases (they do not use cached images) and for | |
| # workflow_dispatch with `need_cached_disk=false`. | |
| # One-shot upsert of the HTTP health checks used by the zonal MIGs. | |
| # These are global, network-scoped resources (one per network), so running | |
| # them once per push instead of once per zonal-MIG cell saves 2/3 of the | |
| # upsert calls and avoids concurrent create/update races. | |
| ensure-health-checks: | |
| name: Ensure health checks exist | |
| needs: [set-matrix] | |
| runs-on: ubuntu-latest | |
| environment: ${{ needs.set-matrix.outputs.environment }} | |
| permissions: | |
| contents: read | |
| id-token: write | |
| if: >- | |
| ${{ | |
| github.event_name != 'pull_request' && | |
| (github.event_name != 'release' || startsWith(github.event.release.tag_name, 'v')) | |
| }} | |
| steps: | |
| - uses: google-github-actions/auth@7c6bc770dae815cd3e89ee6cdf493a5fab2cc093 #v3.0.0 | |
| with: | |
| workload_identity_provider: "${{ vars.GCP_WIF }}" | |
| service_account: "${{ vars.GCP_DEPLOYMENTS_SA }}" | |
| - uses: google-github-actions/setup-gcloud@aa5489c8933f4cc7a4f7d45035b3b1440c9c10db #v3.0.1 | |
| - run: | | |
| for NET in $(echo '${{ needs.set-matrix.outputs.networks }}' | jq -r '.[] | ascii_downcase'); do | |
| gcloud compute health-checks describe "zebra-${NET}-health" --global &>/dev/null \ | |
| || gcloud compute health-checks create http "zebra-${NET}-health" \ | |
| --port=8080 --request-path=/healthy \ | |
| --check-interval=60s --timeout=10s \ | |
| --unhealthy-threshold=3 --healthy-threshold=2 \ | |
| --global | |
| done | |
| get-disk-name-mainnet: | |
| name: Get Mainnet cached disk | |
| needs: [set-matrix] | |
| permissions: | |
| contents: read | |
| id-token: write | |
| uses: ./.github/workflows/zfnd-find-cached-disks.yml | |
| if: ${{ github.event_name != 'release' | |
| && !(github.event.pull_request.head.repo.fork) | |
| && (github.event_name != 'workflow_dispatch' || inputs.need_cached_disk) | |
| && contains(fromJSON(needs.set-matrix.outputs.networks), 'Mainnet') }} | |
| with: | |
| network: Mainnet | |
| environment: ${{ needs.set-matrix.outputs.environment }} | |
| disk_prefix: zebrad-cache | |
| disk_suffix: ${{ inputs.cached_disk_type || 'tip' }} | |
| get-disk-name-testnet: | |
| name: Get Testnet cached disk | |
| needs: [set-matrix] | |
| permissions: | |
| contents: read | |
| id-token: write | |
| uses: ./.github/workflows/zfnd-find-cached-disks.yml | |
| if: ${{ github.event_name != 'release' | |
| && !(github.event.pull_request.head.repo.fork) | |
| && (github.event_name != 'workflow_dispatch' || inputs.need_cached_disk) | |
| && contains(fromJSON(needs.set-matrix.outputs.networks), 'Testnet') }} | |
| with: | |
| network: Testnet | |
| environment: ${{ needs.set-matrix.outputs.environment }} | |
| disk_prefix: zebrad-cache | |
| disk_suffix: ${{ inputs.cached_disk_type || 'tip' }} | |
| # Each time this workflow is executed, a build will be triggered to create a new image | |
| # with the corresponding tags using information from Git | |
| # | |
| # The image will be commonly named `zebrad:<short-hash | github-ref | semver>` | |
| build: | |
| name: Build CD Docker | |
| needs: [set-matrix] | |
| permissions: | |
| contents: read | |
| id-token: write | |
| pull-requests: write | |
| attestations: write # required by the called build workflow's merge job | |
| uses: ./.github/workflows/zfnd-build-docker-image.yml | |
| # Build for: | |
| # - Pull requests | |
| # - Manual workflow_dispatch | |
| # - Push to main branch | |
| # - Releases | |
| if: >- | |
| ${{ | |
| (github.event_name == 'pull_request' && !github.event.pull_request.head.repo.fork) || | |
| github.event_name == 'workflow_dispatch' || | |
| (github.event_name == 'push' && github.ref_name == 'main') || | |
| (github.event_name == 'release' && startsWith(github.event.release.tag_name, 'v')) | |
| }} | |
| with: | |
| source_sha: ${{ github.sha }} | |
| dockerfile_path: ./docker/Dockerfile | |
| dockerfile_target: runtime | |
| image_name: zebrad | |
| no_cache: ${{ inputs.no_cache || false }} | |
| rust_log: info | |
| features: ${{ format('{0} {1}', vars.RUST_PROD_FEATURES, vars.RUST_TEST_FEATURES) }} | |
| environment: ${{ needs.set-matrix.outputs.environment }} | |
| # This step needs access to Docker Hub secrets to run successfully | |
| secrets: | |
| DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} | |
| DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} | |
| # Create or update one zonal MIG per matrix cell. Each cell is a | |
| # (network, zone) tuple. MIG identity is the tuple plus environment + | |
| # branch. No two MIGs ever share a disk; rolling updates are per-zone. | |
| # | |
| # `fail-fast: false` keeps each (network, zone) independent: one cell's | |
| # failure must not cancel the five sister cells. | |
| deploy-nodes: | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| network: ${{ fromJSON(needs.set-matrix.outputs.networks) }} | |
| zone: ${{ fromJSON(needs.set-matrix.outputs.zones) }} | |
| name: Deploy ${{ matrix.network }} ${{ matrix.zone }} | |
| needs: [set-matrix, build, get-disk-name-mainnet, get-disk-name-testnet, ensure-health-checks] | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 60 | |
| env: | |
| # Pick the cache image for this matrix row's network. One image seeds | |
| # all three zones for a given network. Released deploys skip the | |
| # cache lookup entirely and get an empty value here. | |
| CACHED_DISK_NAME: >- | |
| ${{ matrix.network == 'Mainnet' | |
| && needs.get-disk-name-mainnet.outputs.cached_disk_name | |
| || needs.get-disk-name-testnet.outputs.cached_disk_name }} | |
| # Use the workflow environment consistently across GitHub, GCP labels, and runtime metadata. | |
| environment: ${{ needs.set-matrix.outputs.environment }} | |
| permissions: | |
| contents: read | |
| id-token: write | |
| # Deploy when: | |
| # - Build job succeeded (needs.build.result == 'success') | |
| # - Running in ZcashFoundation repo (not a fork) | |
| # - Event is one of: push to main, release, or workflow_dispatch | |
| # - Workflow not cancelled or failed | |
| if: >- | |
| ${{ | |
| !cancelled() && !failure() && | |
| needs.build.result == 'success' && | |
| github.repository_owner == 'ZcashFoundation' && | |
| ( | |
| (github.event_name == 'push' && github.ref_name == 'main') || | |
| (github.event_name == 'release' && startsWith(github.event.release.tag_name, 'v')) || | |
| github.event_name == 'workflow_dispatch' | |
| ) | |
| }} | |
| steps: | |
| - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd #v6.0.2 | |
| with: | |
| persist-credentials: false | |
| - name: Inject slug/short variables | |
| uses: rlespinasse/github-slug-action@e6f261660910b273384c5c42b17a0217881b217a #v5.6.0 | |
| with: | |
| short-length: 7 | |
| - name: Extract matrix values | |
| run: | | |
| ZONE="${{ matrix.zone }}" | |
| NET_CAPS="${{ matrix.network }}" | |
| { | |
| echo "NETWORK=${NET_CAPS,,}" | |
| echo "ZONE=${ZONE}" | |
| echo "ZONE_LETTER=${ZONE##*-}" | |
| } >> "$GITHUB_ENV" | |
| - name: Authenticate to Google Cloud | |
| id: auth | |
| uses: google-github-actions/auth@7c6bc770dae815cd3e89ee6cdf493a5fab2cc093 #v3.0.0 | |
| with: | |
| workload_identity_provider: "${{ vars.GCP_WIF }}" | |
| service_account: "${{ vars.GCP_DEPLOYMENTS_SA }}" | |
| - name: Set up Cloud SDK | |
| uses: google-github-actions/setup-gcloud@aa5489c8933f4cc7a4f7d45035b3b1440c9c10db #v3.0.1 | |
| with: | |
| install_components: 'beta' | |
| - name: Compute MIG and disk naming | |
| env: | |
| ENV: ${{ needs.set-matrix.outputs.environment }} | |
| REF_NAME: ${{ github.ref_name }} | |
| REF_SLUG: ${{ env.GITHUB_REF_SLUG_URL }} | |
| SHA_SHORT: ${{ env.GITHUB_SHA_SHORT }} | |
| run: | | |
| if [ "${ENV}" = "prod" ]; then | |
| PREFIX="" | |
| elif [ "${REF_NAME}" = "main" ]; then | |
| PREFIX="main-" | |
| else | |
| PREFIX="${REF_SLUG}-" | |
| fi | |
| { | |
| echo "MIG_NAME=zebrad-${PREFIX}${NETWORK}-${ZONE_LETTER}" | |
| echo "DISK_NAME=zebrad-cache-${PREFIX}${NETWORK}-${ZONE_LETTER}" | |
| echo "TEMPLATE_NAME=zebrad-${PREFIX}${SHA_SHORT}-${NETWORK}-${ZONE_LETTER}" | |
| } >> "$GITHUB_ENV" | |
| # Reject early if the zonal disk is held by an instance from another MIG. | |
| - name: Pre-flight check for stateful disk squatter | |
| run: | | |
| users=$(gcloud compute disks describe "${DISK_NAME}" --zone="${ZONE}" \ | |
| --format="value(users.basename())" 2>/dev/null || true) | |
| for user in ${users}; do | |
| owner=$(gcloud compute instances describe "${user}" --zone="${ZONE}" \ | |
| --format="value(metadata.items.filter(key:created-by).extract(value))" 2>/dev/null \ | |
| | grep -oE 'instanceGroupManagers/[a-z0-9-]+' | cut -d/ -f2 || true) | |
| if [ -n "${owner}" ] && [ "${owner}" != "${MIG_NAME}" ]; then | |
| echo "::error::${DISK_NAME} in ${ZONE} is held by ${user} (MIG ${owner}). See gcp-deployment-operations.md." | |
| exit 1 | |
| fi | |
| done | |
| # Create zonal disk from cache image on first deploy; attach existing on | |
| # subsequent deploys or after a manual bootstrap. | |
| - name: Ensure zonal disk exists | |
| env: | |
| ENV: ${{ needs.set-matrix.outputs.environment }} | |
| run: | | |
| if gcloud compute disks describe "${DISK_NAME}" --zone="${ZONE}" &>/dev/null; then | |
| exit 0 | |
| fi | |
| if [ -z "${CACHED_DISK_NAME}" ]; then | |
| echo "::error::No ${DISK_NAME} and no cache image. Seed via integration-tests or manual snapshot." | |
| exit 1 | |
| fi | |
| gcloud compute disks create "${DISK_NAME}" --zone="${ZONE}" \ | |
| --image="${CACHED_DISK_NAME}" \ | |
| --size=400 --type=pd-balanced \ | |
| --labels="app=zebrad,environment=${ENV},network=${NETWORK},zone=${ZONE_LETTER},created_by=${{ github.event_name }},github_ref=${{ env.GITHUB_REF_SLUG_URL }},github_sha=${{ env.GITHUB_SHA_SHORT }}" | |
| # Single source of truth for the zone-suffix to reserved-IP mapping. | |
| # Gated to stable deploys so feature-branch dispatches to dev can't | |
| # advertise a prod IP that isn't actually attached to the MIG. | |
| - name: Resolve reserved external IP | |
| if: ${{ needs.set-matrix.outputs.environment == 'prod' || github.ref_name == 'main' }} | |
| env: | |
| GCP_REGION: ${{ vars.GCP_REGION }} | |
| run: | | |
| case "${ZONE_LETTER}" in | |
| b) SUFFIX="" ;; | |
| c) SUFFIX="-secondary" ;; | |
| d) SUFFIX="-tertiary" ;; | |
| esac | |
| IP_NAME="zebra-${NETWORK}${SUFFIX}" | |
| IP_ADDRESS=$(gcloud compute addresses describe "${IP_NAME}" \ | |
| --region="${GCP_REGION}" --format='value(address)' 2>/dev/null || true) | |
| echo "IP_NAME=${IP_NAME}" >> "$GITHUB_ENV" | |
| echo "IP_ADDRESS=${IP_ADDRESS}" >> "$GITHUB_ENV" | |
| - name: Create instance template | |
| run: | | |
| if [ "${{ github.event_name }}" = "workflow_dispatch" ] && [ -n "${{ inputs.log_file }}" ]; then | |
| LOG_FILE="${{ inputs.log_file }}" | |
| else | |
| LOG_FILE="${{ vars.CD_LOG_FILE }}" | |
| fi | |
| if [ "${{ matrix.network }}" = "Mainnet" ]; then | |
| P2P=8233; RPC=8232 | |
| else | |
| P2P=18233; RPC=18232 | |
| fi | |
| if gcloud compute instance-templates describe "${TEMPLATE_NAME}" &>/dev/null; then | |
| exit 0 | |
| fi | |
| EXTERNAL_ADDR_ENV="" | |
| if [ -n "${IP_ADDRESS:-}" ]; then | |
| EXTERNAL_ADDR_ENV=",ZEBRA_NETWORK__EXTERNAL_ADDR=${IP_ADDRESS}:${P2P}" | |
| fi | |
| gcloud compute instance-templates create-with-container "${TEMPLATE_NAME}" \ | |
| --machine-type=${{ vars.GCP_SMALL_MACHINE }} \ | |
| --provisioning-model=SPOT \ | |
| --boot-disk-size=10GB --boot-disk-type=pd-standard \ | |
| --image-project=cos-cloud --image-family=cos-stable \ | |
| --subnet=${{ vars.GCP_SUBNETWORK }} --no-address \ | |
| --disk="name=${DISK_NAME},device-name=${DISK_NAME},mode=rw,auto-delete=no,boot=no" \ | |
| --container-mount-disk="mount-path=/home/zebra/.cache/zebra,name=${DISK_NAME},mode=rw" \ | |
| --container-stdin --container-tty \ | |
| --container-image="${{ vars.GAR_BASE }}/zebrad@${{ needs.build.outputs.image_digest }}" \ | |
| --container-env="ZEBRA_NETWORK__NETWORK=${{ matrix.network }},ZEBRA_NETWORK__LISTEN_ADDR=0.0.0.0:${P2P},LOG_FILE=${LOG_FILE},SENTRY_DSN=${{ vars.SENTRY_DSN }},SENTRY_ENVIRONMENT=${{ needs.set-matrix.outputs.environment }},GITHUB_ACTIONS=${GITHUB_ACTIONS},GITHUB_EVENT_NAME=${GITHUB_EVENT_NAME},GITHUB_REF_POINT_SLUG_URL=${GITHUB_REF_POINT_SLUG_URL},GITHUB_SHA=${GITHUB_SHA},GITHUB_RUN_ID=${GITHUB_RUN_ID},GITHUB_RUN_ATTEMPT=${GITHUB_RUN_ATTEMPT},GITHUB_WORKFLOW=${GITHUB_WORKFLOW},GITHUB_JOB=${GITHUB_JOB},CI_PR_NUMBER=${{ github.event.pull_request.number || '' }},ZEBRA_HEALTH__LISTEN_ADDR=0.0.0.0:8080,ZEBRA_HEALTH__MIN_CONNECTED_PEERS=1,ZEBRA_RPC__LISTEN_ADDR=0.0.0.0:${RPC}${EXTERNAL_ADDR_ENV}" \ | |
| --service-account=${{ vars.GCP_DEPLOYMENTS_SA }} --scopes=cloud-platform \ | |
| --metadata=google-logging-enabled=true,google-logging-use-fluentbit=true,google-monitoring-enabled=true \ | |
| --labels="app=zebrad,environment=${{ needs.set-matrix.outputs.environment }},network=${NETWORK},zone=${ZONE_LETTER},created_by=${{ github.event_name }},github_ref=${{ env.GITHUB_REF_SLUG_URL }},github_sha=${{ env.GITHUB_SHA_SHORT }}" \ | |
| --tags=zebrad | |
| - name: Check if zonal MIG exists | |
| id: does-group-exist | |
| continue-on-error: true | |
| run: | | |
| gcloud compute instance-groups managed describe "${MIG_NAME}" --zone="${ZONE}" >/dev/null 2>&1 | |
| # Fresh MIG: size=1 (one instance per zonal MIG). The template's | |
| # `--disk=name=…` attaches the pre-created zonal disk. | |
| - name: Create zonal MIG | |
| if: steps.does-group-exist.outcome == 'failure' | |
| run: | | |
| gcloud compute instance-groups managed create "${MIG_NAME}" \ | |
| --template="${TEMPLATE_NAME}" \ | |
| --zone="${ZONE}" \ | |
| --size=1 \ | |
| --health-check="zebra-${NETWORK}-health" \ | |
| --initial-delay=3600 | |
| - name: Apply stateful disk policy (fresh MIG) | |
| if: steps.does-group-exist.outcome == 'failure' | |
| run: | | |
| gcloud compute instance-groups managed update "${MIG_NAME}" \ | |
| --stateful-disk="device-name=${DISK_NAME},auto-delete=on-permanent-instance-deletion" \ | |
| --zone="${ZONE}" | |
| # Bind the reserved IP resolved earlier to the fresh MIG instance. | |
| # `instance-configs create --stateful-external-ip` accepts STAGING / | |
| # RUNNING-UNKNOWN instances; the short poll handles the async gap | |
| # between MIG-create returning and list-instances reporting. | |
| - name: Assign static IP (fresh MIG, stable deploy) | |
| if: ${{ steps.does-group-exist.outcome == 'failure' && (needs.set-matrix.outputs.environment == 'prod' || github.ref_name == 'main') }} | |
| run: | | |
| if [ -z "${IP_ADDRESS:-}" ]; then | |
| echo "::warning::${IP_NAME:-zebra-${NETWORK}} not reserved; skipping" | |
| exit 0 | |
| fi | |
| for _ in $(seq 1 30); do | |
| INSTANCE=$(gcloud compute instance-groups managed list-instances "${MIG_NAME}" \ | |
| --zone="${ZONE}" --format='value(instance.basename())' | head -1) | |
| [ -n "${INSTANCE}" ] && break | |
| sleep 2 | |
| done | |
| [ -z "${INSTANCE}" ] && { echo "::error::instance did not appear within 60s"; exit 1; } | |
| gcloud compute instance-groups managed instance-configs create "${MIG_NAME}" \ | |
| --instance="${INSTANCE}" --zone="${ZONE}" \ | |
| --stateful-external-ip="address=${IP_ADDRESS},interface-name=nic0,auto-delete=never" | |
| # Rolling update waits only for the new template to start rolling out; | |
| # full health convergence is the verify-nodes job's concern. | |
| - name: Rolling update on existing MIG | |
| if: steps.does-group-exist.outcome == 'success' | |
| run: | | |
| gcloud compute instance-groups managed rolling-action start-update "${MIG_NAME}" \ | |
| --version=template="${TEMPLATE_NAME}" \ | |
| --replacement-method=recreate \ | |
| --max-surge=0 --max-unavailable=1 \ | |
| --zone="${ZONE}" | |
| # Waits for each zonal MIG to reach HEALTHY (app-level: peer mesh + chain | |
| # tip). Runs async from deploy-nodes. Skipped for workflow_dispatch. | |
| verify-nodes: | |
| name: Verify ${{ matrix.network }} ${{ matrix.zone }} | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| network: ${{ fromJSON(needs.set-matrix.outputs.networks) }} | |
| zone: ${{ fromJSON(needs.set-matrix.outputs.zones) }} | |
| needs: [set-matrix, deploy-nodes] | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 120 | |
| environment: ${{ needs.set-matrix.outputs.environment }} | |
| permissions: | |
| contents: read | |
| id-token: write | |
| if: >- | |
| ${{ | |
| !cancelled() && !failure() && | |
| needs.deploy-nodes.result == 'success' && | |
| github.event_name != 'workflow_dispatch' && | |
| (github.event_name != 'release' || startsWith(github.event.release.tag_name, 'v')) && | |
| github.repository_owner == 'ZcashFoundation' | |
| }} | |
| steps: | |
| - name: Authenticate to Google Cloud | |
| uses: google-github-actions/auth@7c6bc770dae815cd3e89ee6cdf493a5fab2cc093 #v3.0.0 | |
| with: | |
| workload_identity_provider: "${{ vars.GCP_WIF }}" | |
| service_account: "${{ vars.GCP_DEPLOYMENTS_SA }}" | |
| - name: Set up Cloud SDK | |
| uses: google-github-actions/setup-gcloud@aa5489c8933f4cc7a4f7d45035b3b1440c9c10db #v3.0.1 | |
| - name: Wait for MIG stable | |
| run: | | |
| ZONE="${{ matrix.zone }}" | |
| NET_CAPS="${{ matrix.network }}" | |
| NETWORK="${NET_CAPS,,}" | |
| ZONE_LETTER="${ZONE##*-}" | |
| case "${{ github.event_name }}" in | |
| release) PREFIX="" ;; | |
| push) PREFIX="main-" ;; | |
| *) echo "::error::unsupported event"; exit 1 ;; | |
| esac | |
| MIG_NAME="zebrad-${PREFIX}${NETWORK}-${ZONE_LETTER}" | |
| { | |
| echo "MIG_NAME=${MIG_NAME}" | |
| echo "ZONE=${ZONE}" | |
| } >> "$GITHUB_ENV" | |
| gcloud compute instance-groups managed wait-until "${MIG_NAME}" \ | |
| --stable --zone="${ZONE}" --timeout=5400 | |
| # Surface the failing instance's health state and container logs. | |
| - name: Diagnose verify failure | |
| if: failure() | |
| env: | |
| GCP_PROJECT: ${{ vars.GCP_PROJECT }} | |
| run: | | |
| gcloud compute instance-groups managed list-instances "${MIG_NAME}" \ | |
| --zone="${ZONE}" \ | |
| --format='table(instance.basename(),instanceStatus,currentAction,instanceHealth[0].detailedHealthState,lastAttempt.errors.errors[0].message)' || true | |
| INSTANCE=$(gcloud compute instance-groups managed list-instances "${MIG_NAME}" \ | |
| --zone="${ZONE}" --format='value(instance.basename())' | head -1 || true) | |
| if [ -z "${INSTANCE}" ]; then | |
| echo "MIG ${MIG_NAME} has no instance" | |
| exit 0 | |
| fi | |
| INSTANCE_ID=$(gcloud compute instances describe "${INSTANCE}" \ | |
| --zone="${ZONE}" --format='value(id)' || true) | |
| gcloud logging read "logName=\"projects/${GCP_PROJECT}/logs/cos_containers\" AND resource.labels.instance_id=\"${INSTANCE_ID}\"" \ | |
| --freshness=2h --limit=50 --format='value(jsonPayload.message)' | tac || true | |
| deploy-nodes-success: | |
| name: Deploy nodes success | |
| runs-on: ubuntu-latest | |
| # Only run when the deployment job actually executed | |
| if: >- | |
| ${{ | |
| always() && | |
| needs.deploy-nodes.result != 'skipped' | |
| }} | |
| needs: [set-matrix, get-disk-name-mainnet, get-disk-name-testnet, ensure-health-checks, build, deploy-nodes] | |
| timeout-minutes: 1 | |
| steps: | |
| - uses: re-actors/alls-green@05ac9388f0aebcb5727afa17fcccfecd6f8ec5fe #v1.2.2 | |
| with: | |
| jobs: ${{ toJSON(needs) }} | |
| allowed-skips: get-disk-name-mainnet, get-disk-name-testnet, build | |
| failure-issue: | |
| name: Open or update issues for deploy failures | |
| # When a new job is added to this workflow, add it to this list. | |
| needs: [build, deploy-nodes] | |
| # Only open tickets for failed or cancelled jobs that are not coming from PRs. | |
| # (PR statuses are already reported in the PR jobs list, and checked by GitHub's Merge Queue.) | |
| if: (failure() && github.event.pull_request == null) || (cancelled() && github.event.pull_request == null) | |
| runs-on: ubuntu-latest | |
| permissions: | |
| contents: read | |
| issues: write | |
| steps: | |
| - uses: jayqi/failed-build-issue-action@1a893bbf43ef1c2a8705e2b115cd4f0fe3c5649b #v1.2.0 | |
| with: | |
| title-template: "{{refname}} branch CI failed: {{eventName}} in {{workflow}}" | |
| # New failures open an issue with this label. | |
| label-name: S-ci-fail-release-auto-issue | |
| # If there is already an open issue with this label, any failures become comments on that issue. | |
| always-create-new-issue: false | |
| github-token: ${{ secrets.GITHUB_TOKEN }} | |
| verify-failure-issue: | |
| name: Open or update issues for verify failures | |
| needs: [verify-nodes] | |
| # Deploy succeeded but the node did not reach HEALTHY within the verify | |
| # window. Separate from `failure-issue` so on-call can distinguish an | |
| # infrastructure problem (deploy-nodes) from a node-level warmup/sync | |
| # problem (verify-nodes). | |
| if: (failure() && github.event.pull_request == null) || (cancelled() && github.event.pull_request == null) | |
| runs-on: ubuntu-latest | |
| permissions: | |
| contents: read | |
| issues: write | |
| steps: | |
| - uses: jayqi/failed-build-issue-action@1a893bbf43ef1c2a8705e2b115cd4f0fe3c5649b #v1.2.0 | |
| with: | |
| title-template: "{{refname}} verify failed: {{eventName}} in {{workflow}}" | |
| label-name: S-ci-fail-verify-auto-issue | |
| always-create-new-issue: false | |
| github-token: ${{ secrets.GITHUB_TOKEN }} |