zfnd-deploy-nodes-gcp.yml

# Deploy Zebra nodes to Google Cloud Platform.
#
# One zonal MIG per (environment, branch, network, zone). MIG names:
#   - release:          zebrad-${network}-${zone-letter}
#   - push to main:     zebrad-main-${network}-${zone-letter}
#   - workflow_dispatch: zebrad-${branch}-${network}-${zone-letter}
#
# Push and release fan out to 6 cells (2 networks × 3 zones);
# workflow_dispatch deploys one cell (user picks network + zone).
#
# Design rationale: docs/decisions/devops/0006-gcp-deployment-naming.md
# Operations: book/src/dev/gcp-deployment-operations.md
#
# See ADR docs/decisions/devops/0006-gcp-deployment-naming.md for the design
# rationale and book/src/dev/gcp-deployment-operations.md for operational
# procedures (PR-deploy cleanup, disk-corruption recovery, DB-format-version-break
# release).
name: Deploy Nodes to GCP

# Ensures that only one workflow task will run at a time. Previous deployments, if
# already in process, won't get cancelled. Instead, we let the first to complete
# then queue the latest pending workflow, cancelling any workflows in between.
#
# Since the different event types each use a different Managed Instance Group or instance,
# we can run different event types concurrently.
#
# For pull requests, we only run the tests from this workflow, and don't do any deployments.
# So an in-progress pull request gets cancelled, just like other tests.
concurrency:
  group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.ref }}-${{ inputs.environment }}-${{ inputs.network }}-${{ inputs.zone }}
  cancel-in-progress: ${{ github.event_name == 'pull_request' }}

on:

  workflow_dispatch:
    inputs:
      # Deployment configuration
      network:
        description: "Network to deploy: Mainnet or Testnet"
        required: true
        type: choice
        default: Mainnet
        options:
          - Mainnet
          - Testnet
      zone:
        description: "GCP zone for the workflow_dispatch deploy (single zone)"
        required: true
        type: choice
        default: us-east1-b
        options:
          - us-east1-b
          - us-east1-c
          - us-east1-d
      environment:
        description: "Environment to deploy to"
        required: true
        type: choice
        default: dev
        options:
          - dev
          - prod
      # Disk configuration
      need_cached_disk:
        description: Use a cached state disk
        type: boolean
        default: true
      cached_disk_type:
        description: Type of cached disk to use
        required: true
        type: choice
        default: tip
        options:
          - tip
          - checkpoint
      # Build configuration
      no_cache:
        description: Disable the Docker cache for this build
        type: boolean
        default: false
      # Logging configuration
      log_file:
        description: Log to a file path rather than standard output
        default: ""

  push:
    # Skip main branch updates where Rust code and dependencies aren't modified.
    branches:
      - main
    paths:
      # code and tests
      - "**/*.rs"
      # hard-coded checkpoints and proptest regressions
      - "**/*.txt"
      # dependencies
      - "**/Cargo.toml"
      - "**/Cargo.lock"
      # configuration files
      - .cargo/config.toml
      - "**/clippy.toml"
      # workflow definitions
      - docker/**
      - .dockerignore
      - .github/workflows/zfnd-deploy-nodes-gcp.yml
      - .github/workflows/zfnd-build-docker-image.yml

  # Only runs the Docker image tests, doesn't deploy any instances
  pull_request:
    # Skip PRs where Rust code and dependencies aren't modified.
    paths:
      # code and tests
      - "**/*.rs"
      # hard-coded checkpoints and proptest regressions
      - "**/*.txt"
      # dependencies
      - "**/Cargo.toml"
      - "**/Cargo.lock"
      # configuration files
      - .cargo/config.toml
      - "**/clippy.toml"
      # workflow definitions
      - docker/**
      - .dockerignore
      - .github/workflows/zfnd-deploy-nodes-gcp.yml
      - .github/workflows/zfnd-build-docker-image.yml

  release:
    types:
      - published

permissions:
  contents: read

jobs:
  # Build the (network, zone) matrix and resolve the target environment from
  # the trigger. Downstream jobs read `set-matrix` outputs so the event →
  # environment mapping is computed once.
  set-matrix:
    # Deployment image tests run on source PRs, not temporary Mergify queue candidates.
    if: >-
      ${{
        github.event_name != 'pull_request' ||
        github.event.pull_request.user.login != 'mergify[bot]' ||
        github.event.pull_request.head.repo.full_name != github.repository ||
        !startsWith(github.event.pull_request.head.ref, 'mergify/merge-queue/')
      }}
    runs-on: ubuntu-latest
    outputs:
      networks: ${{ steps.set-matrix.outputs.networks }}
      zones: ${{ steps.set-matrix.outputs.zones }}
      environment: ${{ steps.set-matrix.outputs.environment }}
    steps:
      - id: set-matrix
        run: |
          case "${{ github.event_name }}" in
            release)           ENV="prod" ;;
            workflow_dispatch) ENV="${{ inputs.environment }}" ;;
            push)              ENV="stage" ;;
            *)                 ENV="dev" ;;
          esac

          if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then
            NETWORKS='["${{ inputs.network }}"]'
            ZONES='["${{ inputs.zone }}"]'
          else
            NETWORKS='["Mainnet","Testnet"]'
            ZONES='["us-east1-b","us-east1-c","us-east1-d"]'
          fi

          {
            echo "networks=${NETWORKS}"
            echo "zones=${ZONES}"
            echo "environment=${ENV}"
          } >> "$GITHUB_OUTPUT"

  # Per-network cache-disk lookup. Cache images are network-specific
  # (`zebrad-cache-…-mainnet-tip` vs `…-testnet-tip`), so the lookup must
  # run once per network the matrix deploys to. Running a single workflow-
  # level lookup with `inputs.network || vars.ZCASH_NETWORK` would return
  # the wrong image for the other matrix row.
  #
  # Skipped for releases (they do not use cached images) and for
  # workflow_dispatch with `need_cached_disk=false`.
  # One-shot upsert of the HTTP health checks used by the zonal MIGs.
  # These are global, network-scoped resources (one per network), so running
  # them once per push instead of once per zonal-MIG cell saves 2/3 of the
  # upsert calls and avoids concurrent create/update races.
  ensure-health-checks:
    name: Ensure health checks exist
    needs: [set-matrix]
    runs-on: ubuntu-latest
    environment: ${{ needs.set-matrix.outputs.environment }}
    permissions:
      contents: read
      id-token: write
    if: >-
      ${{
        github.event_name != 'pull_request' &&
        (github.event_name != 'release' || startsWith(github.event.release.tag_name, 'v'))
      }}
    steps:
      - uses: google-github-actions/auth@7c6bc770dae815cd3e89ee6cdf493a5fab2cc093 #v3.0.0
        with:
          workload_identity_provider: "${{ vars.GCP_WIF }}"
          service_account: "${{ vars.GCP_DEPLOYMENTS_SA }}"
      - uses: google-github-actions/setup-gcloud@aa5489c8933f4cc7a4f7d45035b3b1440c9c10db #v3.0.1
      - run: |
          for NET in $(echo '${{ needs.set-matrix.outputs.networks }}' | jq -r '.[] | ascii_downcase'); do
            gcloud compute health-checks describe "zebra-${NET}-health" --global &>/dev/null \
            || gcloud compute health-checks create http "zebra-${NET}-health" \
              --port=8080 --request-path=/healthy \
              --check-interval=60s --timeout=10s \
              --unhealthy-threshold=3 --healthy-threshold=2 \
              --global
          done

  get-disk-name-mainnet:
    name: Get Mainnet cached disk
    needs: [set-matrix]
    permissions:
      contents: read
      id-token: write
    uses: ./.github/workflows/zfnd-find-cached-disks.yml
    if: ${{ github.event_name != 'release'
      && !(github.event.pull_request.head.repo.fork)
      && (github.event_name != 'workflow_dispatch' || inputs.need_cached_disk)
      && contains(fromJSON(needs.set-matrix.outputs.networks), 'Mainnet') }}
    with:
      network: Mainnet
      environment: ${{ needs.set-matrix.outputs.environment }}
      disk_prefix: zebrad-cache
      disk_suffix: ${{ inputs.cached_disk_type || 'tip' }}

  get-disk-name-testnet:
    name: Get Testnet cached disk
    needs: [set-matrix]
    permissions:
      contents: read
      id-token: write
    uses: ./.github/workflows/zfnd-find-cached-disks.yml
    if: ${{ github.event_name != 'release'
      && !(github.event.pull_request.head.repo.fork)
      && (github.event_name != 'workflow_dispatch' || inputs.need_cached_disk)
      && contains(fromJSON(needs.set-matrix.outputs.networks), 'Testnet') }}
    with:
      network: Testnet
      environment: ${{ needs.set-matrix.outputs.environment }}
      disk_prefix: zebrad-cache
      disk_suffix: ${{ inputs.cached_disk_type || 'tip' }}

  # Each time this workflow is executed, a build will be triggered to create a new image
  # with the corresponding tags using information from Git
  #
  # The image will be commonly named `zebrad:<short-hash | github-ref | semver>`
  build:
    name: Build CD Docker
    needs: [set-matrix]
    permissions:
      contents: read
      id-token: write
      pull-requests: write
      attestations: write # required by the called build workflow's merge job
    uses: ./.github/workflows/zfnd-build-docker-image.yml
    # Build for:
    # - Pull requests
    # - Manual workflow_dispatch
    # - Push to main branch
    # - Releases
    if: >-
      ${{
        (github.event_name == 'pull_request' && !github.event.pull_request.head.repo.fork) ||
        github.event_name == 'workflow_dispatch' ||
        (github.event_name == 'push' && github.ref_name == 'main') ||
        (github.event_name == 'release' && startsWith(github.event.release.tag_name, 'v'))
      }}
    with:
      source_sha: ${{ github.sha }}
      dockerfile_path: ./docker/Dockerfile
      dockerfile_target: runtime
      image_name: zebrad
      no_cache: ${{ inputs.no_cache || false }}
      rust_log: info
      features: ${{ format('{0} {1}', vars.RUST_PROD_FEATURES, vars.RUST_TEST_FEATURES) }}
      environment: ${{ needs.set-matrix.outputs.environment }}
    # This step needs access to Docker Hub secrets to run successfully
    secrets:
      DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}
      DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }}

  # Create or update one zonal MIG per matrix cell. Each cell is a
  # (network, zone) tuple. MIG identity is the tuple plus environment +
  # branch. No two MIGs ever share a disk; rolling updates are per-zone.
  #
  # `fail-fast: false` keeps each (network, zone) independent: one cell's
  # failure must not cancel the five sister cells.
  deploy-nodes:
    strategy:
      fail-fast: false
      matrix:
        network: ${{ fromJSON(needs.set-matrix.outputs.networks) }}
        zone: ${{ fromJSON(needs.set-matrix.outputs.zones) }}
    name: Deploy ${{ matrix.network }} ${{ matrix.zone }}
    needs: [set-matrix, build, get-disk-name-mainnet, get-disk-name-testnet, ensure-health-checks]
    runs-on: ubuntu-latest
    timeout-minutes: 60
    env:
      # Pick the cache image for this matrix row's network. One image seeds
      # all three zones for a given network. Released deploys skip the
      # cache lookup entirely and get an empty value here.
      CACHED_DISK_NAME: >-
        ${{ matrix.network == 'Mainnet'
            && needs.get-disk-name-mainnet.outputs.cached_disk_name
            || needs.get-disk-name-testnet.outputs.cached_disk_name }}
    # Use the workflow environment consistently across GitHub, GCP labels, and runtime metadata.
    environment: ${{ needs.set-matrix.outputs.environment }}
    permissions:
      contents: read
      id-token: write
    # Deploy when:
    # - Build job succeeded (needs.build.result == 'success')
    # - Running in ZcashFoundation repo (not a fork)
    # - Event is one of: push to main, release, or workflow_dispatch
    # - Workflow not cancelled or failed
    if: >-
      ${{
        !cancelled() && !failure() &&
        needs.build.result == 'success' &&
        github.repository_owner == 'ZcashFoundation' &&
        (
          (github.event_name == 'push' && github.ref_name == 'main') ||
          (github.event_name == 'release' && startsWith(github.event.release.tag_name, 'v')) ||
          github.event_name == 'workflow_dispatch'
        )
      }}

    steps:
      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd #v6.0.2
        with:
          persist-credentials: false

      - name: Inject slug/short variables
        uses: rlespinasse/github-slug-action@e6f261660910b273384c5c42b17a0217881b217a #v5.6.0
        with:
          short-length: 7

      - name: Extract matrix values
        run: |
          ZONE="${{ matrix.zone }}"
          NET_CAPS="${{ matrix.network }}"
          {
            echo "NETWORK=${NET_CAPS,,}"
            echo "ZONE=${ZONE}"
            echo "ZONE_LETTER=${ZONE##*-}"
          } >> "$GITHUB_ENV"

      - name: Authenticate to Google Cloud
        id: auth
        uses: google-github-actions/auth@7c6bc770dae815cd3e89ee6cdf493a5fab2cc093 #v3.0.0
        with:
          workload_identity_provider: "${{ vars.GCP_WIF }}"
          service_account: "${{ vars.GCP_DEPLOYMENTS_SA }}"

      - name: Set up Cloud SDK
        uses: google-github-actions/setup-gcloud@aa5489c8933f4cc7a4f7d45035b3b1440c9c10db #v3.0.1
        with:
          install_components: 'beta'

      - name: Compute MIG and disk naming
        env:
          ENV: ${{ needs.set-matrix.outputs.environment }}
          REF_NAME: ${{ github.ref_name }}
          REF_SLUG: ${{ env.GITHUB_REF_SLUG_URL }}
          SHA_SHORT: ${{ env.GITHUB_SHA_SHORT }}
        run: |
          if [ "${ENV}" = "prod" ]; then
            PREFIX=""
          elif [ "${REF_NAME}" = "main" ]; then
            PREFIX="main-"
          else
            PREFIX="${REF_SLUG}-"
          fi
          {
            echo "MIG_NAME=zebrad-${PREFIX}${NETWORK}-${ZONE_LETTER}"
            echo "DISK_NAME=zebrad-cache-${PREFIX}${NETWORK}-${ZONE_LETTER}"
            echo "TEMPLATE_NAME=zebrad-${PREFIX}${SHA_SHORT}-${NETWORK}-${ZONE_LETTER}"
          } >> "$GITHUB_ENV"

      # Reject early if the zonal disk is held by an instance from another MIG.
      - name: Pre-flight check for stateful disk squatter
        run: |
          users=$(gcloud compute disks describe "${DISK_NAME}" --zone="${ZONE}" \
            --format="value(users.basename())" 2>/dev/null || true)
          for user in ${users}; do
            owner=$(gcloud compute instances describe "${user}" --zone="${ZONE}" \
              --format="value(metadata.items.filter(key:created-by).extract(value))" 2>/dev/null \
              | grep -oE 'instanceGroupManagers/[a-z0-9-]+' | cut -d/ -f2 || true)
            if [ -n "${owner}" ] && [ "${owner}" != "${MIG_NAME}" ]; then
              echo "::error::${DISK_NAME} in ${ZONE} is held by ${user} (MIG ${owner}). See gcp-deployment-operations.md."
              exit 1
            fi
          done

      # Create zonal disk from cache image on first deploy; attach existing on
      # subsequent deploys or after a manual bootstrap.
      - name: Ensure zonal disk exists
        env:
          ENV: ${{ needs.set-matrix.outputs.environment }}
        run: |
          if gcloud compute disks describe "${DISK_NAME}" --zone="${ZONE}" &>/dev/null; then
            exit 0
          fi
          if [ -z "${CACHED_DISK_NAME}" ]; then
            echo "::error::No ${DISK_NAME} and no cache image. Seed via integration-tests or manual snapshot."
            exit 1
          fi
          gcloud compute disks create "${DISK_NAME}" --zone="${ZONE}" \
            --image="${CACHED_DISK_NAME}" \
            --size=400 --type=pd-balanced \
            --labels="app=zebrad,environment=${ENV},network=${NETWORK},zone=${ZONE_LETTER},created_by=${{ github.event_name }},github_ref=${{ env.GITHUB_REF_SLUG_URL }},github_sha=${{ env.GITHUB_SHA_SHORT }}"

      # Single source of truth for the zone-suffix to reserved-IP mapping.
      # Gated to stable deploys so feature-branch dispatches to dev can't
      # advertise a prod IP that isn't actually attached to the MIG.
      - name: Resolve reserved external IP
        if: ${{ needs.set-matrix.outputs.environment == 'prod' || github.ref_name == 'main' }}
        env:
          GCP_REGION: ${{ vars.GCP_REGION }}
        run: |
          case "${ZONE_LETTER}" in
            b) SUFFIX="" ;;
            c) SUFFIX="-secondary" ;;
            d) SUFFIX="-tertiary" ;;
          esac
          IP_NAME="zebra-${NETWORK}${SUFFIX}"
          IP_ADDRESS=$(gcloud compute addresses describe "${IP_NAME}" \
            --region="${GCP_REGION}" --format='value(address)' 2>/dev/null || true)
          echo "IP_NAME=${IP_NAME}" >> "$GITHUB_ENV"
          echo "IP_ADDRESS=${IP_ADDRESS}" >> "$GITHUB_ENV"

      - name: Create instance template
        run: |
          if [ "${{ github.event_name }}" = "workflow_dispatch" ] && [ -n "${{ inputs.log_file }}" ]; then
            LOG_FILE="${{ inputs.log_file }}"
          else
            LOG_FILE="${{ vars.CD_LOG_FILE }}"
          fi
          if [ "${{ matrix.network }}" = "Mainnet" ]; then
            P2P=8233; RPC=8232
          else
            P2P=18233; RPC=18232
          fi
          if gcloud compute instance-templates describe "${TEMPLATE_NAME}" &>/dev/null; then
            exit 0
          fi

          EXTERNAL_ADDR_ENV=""
          if [ -n "${IP_ADDRESS:-}" ]; then
            EXTERNAL_ADDR_ENV=",ZEBRA_NETWORK__EXTERNAL_ADDR=${IP_ADDRESS}:${P2P}"
          fi

          gcloud compute instance-templates create-with-container "${TEMPLATE_NAME}" \
            --machine-type=${{ vars.GCP_SMALL_MACHINE }} \
            --provisioning-model=SPOT \
            --boot-disk-size=10GB --boot-disk-type=pd-standard \
            --image-project=cos-cloud --image-family=cos-stable \
            --subnet=${{ vars.GCP_SUBNETWORK }} --no-address \
            --disk="name=${DISK_NAME},device-name=${DISK_NAME},mode=rw,auto-delete=no,boot=no" \
            --container-mount-disk="mount-path=/home/zebra/.cache/zebra,name=${DISK_NAME},mode=rw" \
            --container-stdin --container-tty \
            --container-image="${{ vars.GAR_BASE }}/zebrad@${{ needs.build.outputs.image_digest }}" \
            --container-env="ZEBRA_NETWORK__NETWORK=${{ matrix.network }},ZEBRA_NETWORK__LISTEN_ADDR=0.0.0.0:${P2P},LOG_FILE=${LOG_FILE},SENTRY_DSN=${{ vars.SENTRY_DSN }},SENTRY_ENVIRONMENT=${{ needs.set-matrix.outputs.environment }},GITHUB_ACTIONS=${GITHUB_ACTIONS},GITHUB_EVENT_NAME=${GITHUB_EVENT_NAME},GITHUB_REF_POINT_SLUG_URL=${GITHUB_REF_POINT_SLUG_URL},GITHUB_SHA=${GITHUB_SHA},GITHUB_RUN_ID=${GITHUB_RUN_ID},GITHUB_RUN_ATTEMPT=${GITHUB_RUN_ATTEMPT},GITHUB_WORKFLOW=${GITHUB_WORKFLOW},GITHUB_JOB=${GITHUB_JOB},CI_PR_NUMBER=${{ github.event.pull_request.number || '' }},ZEBRA_HEALTH__LISTEN_ADDR=0.0.0.0:8080,ZEBRA_HEALTH__MIN_CONNECTED_PEERS=1,ZEBRA_RPC__LISTEN_ADDR=0.0.0.0:${RPC}${EXTERNAL_ADDR_ENV}" \
            --service-account=${{ vars.GCP_DEPLOYMENTS_SA }} --scopes=cloud-platform \
            --metadata=google-logging-enabled=true,google-logging-use-fluentbit=true,google-monitoring-enabled=true \
            --labels="app=zebrad,environment=${{ needs.set-matrix.outputs.environment }},network=${NETWORK},zone=${ZONE_LETTER},created_by=${{ github.event_name }},github_ref=${{ env.GITHUB_REF_SLUG_URL }},github_sha=${{ env.GITHUB_SHA_SHORT }}" \
            --tags=zebrad

      - name: Check if zonal MIG exists
        id: does-group-exist
        continue-on-error: true
        run: |
          gcloud compute instance-groups managed describe "${MIG_NAME}" --zone="${ZONE}" >/dev/null 2>&1

      # Fresh MIG: size=1 (one instance per zonal MIG). The template's
      # `--disk=name=…` attaches the pre-created zonal disk.
      - name: Create zonal MIG
        if: steps.does-group-exist.outcome == 'failure'
        run: |
          gcloud compute instance-groups managed create "${MIG_NAME}" \
            --template="${TEMPLATE_NAME}" \
            --zone="${ZONE}" \
            --size=1 \
            --health-check="zebra-${NETWORK}-health" \
            --initial-delay=3600

      - name: Apply stateful disk policy (fresh MIG)
        if: steps.does-group-exist.outcome == 'failure'
        run: |
          gcloud compute instance-groups managed update "${MIG_NAME}" \
            --stateful-disk="device-name=${DISK_NAME},auto-delete=on-permanent-instance-deletion" \
            --zone="${ZONE}"

      # Bind the reserved IP resolved earlier to the fresh MIG instance.
      # `instance-configs create --stateful-external-ip` accepts STAGING /
      # RUNNING-UNKNOWN instances; the short poll handles the async gap
      # between MIG-create returning and list-instances reporting.
      - name: Assign static IP (fresh MIG, stable deploy)
        if: ${{ steps.does-group-exist.outcome == 'failure' && (needs.set-matrix.outputs.environment == 'prod' || github.ref_name == 'main') }}
        run: |
          if [ -z "${IP_ADDRESS:-}" ]; then
            echo "::warning::${IP_NAME:-zebra-${NETWORK}} not reserved; skipping"
            exit 0
          fi
          for _ in $(seq 1 30); do
            INSTANCE=$(gcloud compute instance-groups managed list-instances "${MIG_NAME}" \
              --zone="${ZONE}" --format='value(instance.basename())' | head -1)
            [ -n "${INSTANCE}" ] && break
            sleep 2
          done
          [ -z "${INSTANCE}" ] && { echo "::error::instance did not appear within 60s"; exit 1; }
          gcloud compute instance-groups managed instance-configs create "${MIG_NAME}" \
            --instance="${INSTANCE}" --zone="${ZONE}" \
            --stateful-external-ip="address=${IP_ADDRESS},interface-name=nic0,auto-delete=never"

      # Rolling update waits only for the new template to start rolling out;
      # full health convergence is the verify-nodes job's concern.
      - name: Rolling update on existing MIG
        if: steps.does-group-exist.outcome == 'success'
        run: |
          gcloud compute instance-groups managed rolling-action start-update "${MIG_NAME}" \
            --version=template="${TEMPLATE_NAME}" \
            --replacement-method=recreate \
            --max-surge=0 --max-unavailable=1 \
            --zone="${ZONE}"

  # Waits for each zonal MIG to reach HEALTHY (app-level: peer mesh + chain
  # tip). Runs async from deploy-nodes. Skipped for workflow_dispatch.
  verify-nodes:
    name: Verify ${{ matrix.network }} ${{ matrix.zone }}
    strategy:
      fail-fast: false
      matrix:
        network: ${{ fromJSON(needs.set-matrix.outputs.networks) }}
        zone: ${{ fromJSON(needs.set-matrix.outputs.zones) }}
    needs: [set-matrix, deploy-nodes]
    runs-on: ubuntu-latest
    timeout-minutes: 120
    environment: ${{ needs.set-matrix.outputs.environment }}
    permissions:
      contents: read
      id-token: write
    if: >-
      ${{
        !cancelled() && !failure() &&
        needs.deploy-nodes.result == 'success' &&
        github.event_name != 'workflow_dispatch' &&
        (github.event_name != 'release' || startsWith(github.event.release.tag_name, 'v')) &&
        github.repository_owner == 'ZcashFoundation'
      }}
    steps:
      - name: Authenticate to Google Cloud
        uses: google-github-actions/auth@7c6bc770dae815cd3e89ee6cdf493a5fab2cc093 #v3.0.0
        with:
          workload_identity_provider: "${{ vars.GCP_WIF }}"
          service_account: "${{ vars.GCP_DEPLOYMENTS_SA }}"

      - name: Set up Cloud SDK
        uses: google-github-actions/setup-gcloud@aa5489c8933f4cc7a4f7d45035b3b1440c9c10db #v3.0.1

      - name: Wait for MIG stable
        run: |
          ZONE="${{ matrix.zone }}"
          NET_CAPS="${{ matrix.network }}"
          NETWORK="${NET_CAPS,,}"
          ZONE_LETTER="${ZONE##*-}"
          case "${{ github.event_name }}" in
            release) PREFIX="" ;;
            push)    PREFIX="main-" ;;
            *) echo "::error::unsupported event"; exit 1 ;;
          esac
          MIG_NAME="zebrad-${PREFIX}${NETWORK}-${ZONE_LETTER}"
          {
            echo "MIG_NAME=${MIG_NAME}"
            echo "ZONE=${ZONE}"
          } >> "$GITHUB_ENV"
          gcloud compute instance-groups managed wait-until "${MIG_NAME}" \
            --stable --zone="${ZONE}" --timeout=5400

      # Surface the failing instance's health state and container logs.
      - name: Diagnose verify failure
        if: failure()
        env:
          GCP_PROJECT: ${{ vars.GCP_PROJECT }}
        run: |
          gcloud compute instance-groups managed list-instances "${MIG_NAME}" \
            --zone="${ZONE}" \
            --format='table(instance.basename(),instanceStatus,currentAction,instanceHealth[0].detailedHealthState,lastAttempt.errors.errors[0].message)' || true
          INSTANCE=$(gcloud compute instance-groups managed list-instances "${MIG_NAME}" \
            --zone="${ZONE}" --format='value(instance.basename())' | head -1 || true)
          if [ -z "${INSTANCE}" ]; then
            echo "MIG ${MIG_NAME} has no instance"
            exit 0
          fi
          INSTANCE_ID=$(gcloud compute instances describe "${INSTANCE}" \
            --zone="${ZONE}" --format='value(id)' || true)
          gcloud logging read "logName=\"projects/${GCP_PROJECT}/logs/cos_containers\" AND resource.labels.instance_id=\"${INSTANCE_ID}\"" \
            --freshness=2h --limit=50 --format='value(jsonPayload.message)' | tac || true

  deploy-nodes-success:
    name: Deploy nodes success
    runs-on: ubuntu-latest
    # Only run when the deployment job actually executed
    if: >-
      ${{
        always() &&
        needs.deploy-nodes.result != 'skipped'
      }}
    needs: [set-matrix, get-disk-name-mainnet, get-disk-name-testnet, ensure-health-checks, build, deploy-nodes]
    timeout-minutes: 1
    steps:
      - uses: re-actors/alls-green@05ac9388f0aebcb5727afa17fcccfecd6f8ec5fe #v1.2.2
        with:
          jobs: ${{ toJSON(needs) }}
          allowed-skips: get-disk-name-mainnet, get-disk-name-testnet, build

  failure-issue:
    name: Open or update issues for deploy failures
    # When a new job is added to this workflow, add it to this list.
    needs: [build, deploy-nodes]

    # Only open tickets for failed or cancelled jobs that are not coming from PRs.
    # (PR statuses are already reported in the PR jobs list, and checked by GitHub's Merge Queue.)
    if: (failure() && github.event.pull_request == null) || (cancelled() && github.event.pull_request == null)
    runs-on: ubuntu-latest
    permissions:
      contents: read
      issues: write
    steps:
      - uses: jayqi/failed-build-issue-action@1a893bbf43ef1c2a8705e2b115cd4f0fe3c5649b #v1.2.0
        with:
          title-template: "{{refname}} branch CI failed: {{eventName}} in {{workflow}}"
          # New failures open an issue with this label.
          label-name: S-ci-fail-release-auto-issue
          # If there is already an open issue with this label, any failures become comments on that issue.
          always-create-new-issue: false
          github-token: ${{ secrets.GITHUB_TOKEN }}

  verify-failure-issue:
    name: Open or update issues for verify failures
    needs: [verify-nodes]
    # Deploy succeeded but the node did not reach HEALTHY within the verify
    # window. Separate from `failure-issue` so on-call can distinguish an
    # infrastructure problem (deploy-nodes) from a node-level warmup/sync
    # problem (verify-nodes).
    if: (failure() && github.event.pull_request == null) || (cancelled() && github.event.pull_request == null)
    runs-on: ubuntu-latest
    permissions:
      contents: read
      issues: write
    steps:
      - uses: jayqi/failed-build-issue-action@1a893bbf43ef1c2a8705e2b115cd4f0fe3c5649b #v1.2.0
        with:
          title-template: "{{refname}} verify failed: {{eventName}} in {{workflow}}"
          label-name: S-ci-fail-verify-auto-issue
          always-create-new-issue: false
          github-token: ${{ secrets.GITHUB_TOKEN }}