schedule-daily.yml

name: Schedule Daily

on:
  schedule:
    - cron: "47 1 * * *" # 1:47 UTC - slightly offset to avoid high-load times of other scheduled workflows
  workflow_dispatch:

env:
  CI_JOB_NAME: ${{ github.job }}

jobs:
  fi-tests-nightly:
    name: Bazel Test FI Nightly
    runs-on: &dind-large-setup
      labels: dind-large
    container: &container-setup
      image: ghcr.io/dfinity/ic-build@sha256:1ce83af4fd022cf0938f929a90c11d59ea1d84221fd73b35e9eefa2c89622cc0
      options: >-
        -e NODE_NAME --privileged --cgroupns host --mount type=tmpfs,target="/tmp/containers"
    timeout-minutes: 720 # 12 hours
    environment: Nightly Tests
    steps:
      - &checkout
        name: Checkout
        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
      - uses: ./.github/actions/netrc
      - &backup-pod-access
        name: Set up backup pod access
        run: |
          # Start a dedicated ssh-agent with an explicit socket path under /tmp (outside ~/.ssh, so it
          # survives the `rm -rf ~/.ssh` below) and load the backup-pod private key so CI can SSH into
          # the backup pod. We don't reuse any pre-existing agent because its socket may live under
          # ~/.ssh (as in the 26.04 image) and would be removed below.
          eval "$(ssh-agent -s -a /tmp/backup-pod-agent.sock)"
          echo "SSH_AUTH_SOCK=$SSH_AUTH_SOCK" >> "$GITHUB_ENV"
          ssh-add - <<'EOF'
          ${{ secrets.SSH_PRIVATE_KEY_BACKUP_POD }}
          EOF

          rm -rf ~/.ssh
          mkdir -p ~/.ssh
          chmod 0700 ~/.ssh
          echo -e "Host *\nUser github-runner\n" > ~/.ssh/config
      - name: Run FI Tests Nightly
        uses: ./.github/actions/bazel
        with:
          run: |
            bazel test \
              --config=stamped \
              --test_tag_filters=fi_tests_nightly \
              //rs/ledger_suite/... \
              --test_env=SSH_AUTH_SOCK \
              --keep_going --test_timeout=43200
          GPG_PASSPHRASE: ${{ secrets.GPG_PASSPHRASE }}

  nns-recovery-mainnet-state:
    name: Bazel Test NNS Recovery with Mainnet State
    runs-on:
      group: zh1
      labels: dind-large
    container: *container-setup
    timeout-minutes: 120
    environment: Nightly Tests
    steps:
      - *checkout
      - uses: ./.github/actions/netrc
      - *backup-pod-access
      - name: Run NNS Recovery Test with Mainnet State
        uses: ./.github/actions/bazel
        with:
          run: |
            bazel test \
              --config=flaky_retry \
              --config=stamped \
              //rs/tests/nested/nns_recovery:nr_large_with_mainnet_state_colocate \
              --keep_going --test_timeout=7200
          GPG_PASSPHRASE: ${{ secrets.GPG_PASSPHRASE }}

  bazel-test-bare-metal:
    name: Bazel Test Bare Metal
    runs-on: *dind-large-setup
    container: *container-setup
    environment: Nightly Tests
    steps:
      - *checkout
      - name: Run Bazel Launch Bare Metal
        uses: ./.github/actions/bazel
        with:
          run: |
            echo "$ZH2_DLL01_INI_SECRET" > file1
            echo "$ZH2_FILE_SHARE_KEY" > file2 && chmod 400 file2

            launch_bare_metal() {

              # shellcheck disable=SC2046,SC2086
              bazel run \
                //ic-os/setupos/envs/dev:launch_bare_metal -- \
                  --config_path "$(realpath  ./ic-os/dev-tools/bare_metal_deployment/zh2-dll01.yaml)" \
                  --ini_filename "$(realpath file1)" \
                  --file_share_ssh_key "$(realpath file2)" \
                  --inject_image_pub_key "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIK3gjE/2K5nxIBbk3ohgs8J5LW+XiObwA+kGtSaF5+4c" \
                  --file_share_username ci_interim \
                  --ci_mode \
                  $@
            }

            # Run bare metal installation test
            launch_bare_metal --hsm

            # Run bare metal node performance benchmarks
            launch_bare_metal --benchmark

            # Run bare metal node hostOS metrics check
            launch_bare_metal --check_hostos_metrics

            # Run SEV tests
            bazel test //rs/tests/nested:guestos_upgrade_from_current_to_current_test_sev --test_env=BARE_METAL_HOST_SECRETS="$(realpath file1)"
            bazel test //rs/tests/nested:guestos_upgrade_from_latest_release_to_current_sev --test_env=BARE_METAL_HOST_SECRETS="$(realpath file1)"
            bazel test //rs/tests/nested:hostos_upgrade_from_latest_release_to_current_sev --test_env=BARE_METAL_HOST_SECRETS="$(realpath file1)"
            bazel test //rs/tests/nested:sev_recovery --test_env=BARE_METAL_HOST_SECRETS="$(realpath file1)"

            bazel clean
          GPG_PASSPHRASE: ${{ secrets.GPG_PASSPHRASE }}
        env:
          ZH2_DLL01_INI_SECRET: "${{ secrets.ZH2_DLL01_INI_SECRET }}"
          ZH2_FILE_SHARE_KEY: "${{ secrets.ZH2_FILE_SHARE_KEY }}"

  nns-tests-nightly:
    name: Bazel Test NNS Nightly
    runs-on: *dind-large-setup
    container: *container-setup
    timeout-minutes: 30
    environment: Nightly Tests
    steps:
      - *checkout
      - uses: ./.github/actions/netrc
      - *backup-pod-access
      - name: Run NNS Tests Nightly
        uses: ./.github/actions/bazel
        with:
          run: |
            bazel test \
              --config=stamped \
              --test_tag_filters=nns_tests_nightly \
              //... \
              --test_env=SSH_AUTH_SOCK --test_env=NNS_CANISTER_UPGRADE_SEQUENCE=all \
              --keep_going
          GPG_PASSPHRASE: ${{ secrets.GPG_PASSPHRASE }}
      - name: Post Slack Notification
        uses: slackapi/slack-github-action@6c661ce58804a1a20f6dc5fbee7f0381b469e001 # v1.25.0
        if: failure()
        with:
          channel-id: eng-nns
          slack-message: "${{ github.job }} failed :disappointed: - <${{github.server_url}}/${{github.repository}}/actions/runs/${{github.run_id}}|Run#${{github.run_id}}>"
        env:
          SLACK_BOT_TOKEN: ${{ secrets.SLACK_API_TOKEN }}

  pocketic-tests-nightly:
    name: Bazel Test PocketIC Nightly
    runs-on: *dind-large-setup
    container: *container-setup
    timeout-minutes: 30
    steps:
      - *checkout
      - uses: ./.github/actions/netrc
      - name: Run PocketIC Tests Nightly
        uses: ./.github/actions/bazel
        with:
          run: |
            bazel test \
              --config=stamped \
              --test_tag_filters=pocketic_tests_nightly \
              //rs/pocket_ic_server/... \
              --test_env=SSH_AUTH_SOCK \
              --keep_going
          GPG_PASSPHRASE: ${{ secrets.GPG_PASSPHRASE }}
      - name: Post Slack Notification
        uses: slackapi/slack-github-action@6c661ce58804a1a20f6dc5fbee7f0381b469e001 # v1.25.0
        if: failure()
        with:
          channel-id: pocket-ic
          slack-message: "${{ github.job }} failed :disappointed: - <${{github.server_url}}/${{github.repository}}/actions/runs/${{github.run_id}}|Run#${{github.run_id}}>"
        env:
          SLACK_BOT_TOKEN: ${{ secrets.SLACK_API_TOKEN }}

  pocket-ic-tests-windows:
    uses: ./.github/workflows/pocket-ic-tests-windows.yml
    with:
      commit-sha: ${{ github.sha }}