build-test.yaml

# Workflow triggered by pushes to the main branch and PRs targeting it
# (as long as the commits contain code changes).
#
# Jobs:
# - rustfmt: Checks formatting of the Rust sources using the 'rustfmt'
#            tool. The job is skipped if the workflow was triggered by a
#            PR marked as a draft.
# - fourmolu: Checks formatting of the Haskell sources using the
#             'fourmolu' tool. The job is skipped if the workflow was
#             triggered by a PR marked as a draft.
# - build-test: Build and test both the Haskell and Rust sources.
#               The job is skipped if the workflow was triggered by a PR
#               marked as a draft.
# 
# The steps in 'build-test' are ordered to fail as fast as possible
# and restore caches as late as possible.
# The dependencies between the steps are described in inline comments below
# along with a few suggestions for improving parallelization.

name: Check formatting, build and run tests

on:
  push:
    branches:
        - main
    paths:
    - '.github/workflows/build-test.yaml'
    - '**/Cargo.toml'
    - '**/Cargo.lock'
    - '**.rs'
    - '**.yaml'
    - '**.hs'
    - 'concordium-base'
    - 'concordium-consensus/smart-contracts'
    - 'concordium-consensus/haskell-lmdb'
    - 'concordium-node/rustfmt.toml'

  pull_request:
    types: [opened, synchronize, reopened, ready_for_review]
    paths:
    - '.github/workflows/build-test.yaml'
    - '**/Cargo.toml'
    - '**/Cargo.lock'
    - '**.rs'
    - '**.yaml'
    - '**.hs'
    - 'concordium-base'
    - 'concordium-consensus/smart-contracts'
    - 'concordium-consensus/haskell-lmdb'
    - 'concordium-node/rustfmt.toml'
  workflow_dispatch: # allow manual trigger

env:
  dummy: 17                # change to force cache invalidation
  CARGO_TERM_COLOR: always # implicitly adds '--color=always' to all cargo commands
  GHC_VERSION: 9.10.2

jobs:

  fourmolu:
    runs-on: ubuntu-latest
    steps:

    - name: Download fourmolu
      uses: supplypike/setup-bin@v1
      with:
        uri: 'https://github.com/fourmolu/fourmolu/releases/download/v0.18.0.0/fourmolu-0.18.0.0-linux-x86_64'
        name: 'fourmolu'
        version: '0.18.0.0'
    
    - name: Download hpack
      run: |
        curl -L https://github.com/sol/hpack/releases/download/0.38.1/hpack_linux.gz -o hpack.gz
        gunzip hpack.gz
        chmod +x hpack
        mv hpack /usr/local/bin/

    - name: Checkout project
      uses: actions/checkout@v2
      with:
        submodules: recursive

    # Since fourmolu uses the project cabal file to find default language extensions, this step is
    # needed to generate a cabal file from the package.yaml.
    - name: Setup the project
      run: |
        cd concordium-consensus
        hpack package.yaml

    - name: Run fourmolu
      run: |
        fourmolu --color always --mode check $(git ls-files '*.hs')

  rustfmt:
    runs-on: ubuntu-latest
    steps:
    - name: Checkout
      uses: actions/checkout@v2
      with:
        submodules: recursive

    - name: Run rustfmt
      run: |
        rustup component add rustfmt        
        cargo fmt --manifest-path concordium-node/Cargo.toml -- --check
        cargo fmt --manifest-path collector-backend/Cargo.toml -- --check
        cargo fmt --manifest-path collector/Cargo.toml -- --check
        cargo fmt --manifest-path service/windows/Cargo.toml -- --check
        cargo fmt --manifest-path service/windows/installer/custom-actions/Cargo.toml -- --check

  build:
    needs: [fourmolu, rustfmt]
    # Use fixed OS version because we install packages on the system.
    runs-on: ubuntu-22.04

    steps:
    - name: Remove unnecessary files
      run: |
        sudo rm -rf /usr/share/dotnet
        sudo rm -rf "$AGENT_TOOLSDIRECTORY"    
    - name: Checkout
      uses: actions/checkout@v2
      with:
        #token: ${{ secrets.CONCORDIUM_CI }}
        submodules: recursive
    - name: Install system packages and protoc
      run: |
        sudo apt-get update && sudo apt-get -y install liblmdb-dev gcc-mingw-w64-x86-64 binutils-mingw-w64-x86-64
        wget https://github.com/google/flatbuffers/releases/download/v22.12.06/Linux.flatc.binary.g++-10.zip
        unzip Linux.flatc.binary.g++-10.zip
        sudo mv ./flatc /usr/bin/flatc
        flatc --version
        wget https://github.com/protocolbuffers/protobuf/releases/download/v3.15.3/protoc-3.15.3-linux-x86_64.zip
        unzip protoc-3.15.3-linux-x86_64.zip
        sudo mv ./bin/protoc /usr/bin/protoc

    # Set up Rust and restore dependencies and targets from cache.
    # This must be done before checking the Rust sources (obviously)
    # but also before building the Haskell sources because the Haskell
    # build kicks of a Rust build.
    - name: Install Rust tools
      run: |
        rustup component add clippy llvm-tools
        rustup target add x86_64-pc-windows-gnu
        cargo install cargo-llvm-cov --locked
    - name: Read Rust version
      id: read-rust-version
      run: |
        echo "RUST_VERSION=$(cargo --version | awk '{print $2}')" >> $GITHUB_OUTPUT
    - name: Print Rust version
      id: print-rust-version
      run: |
        echo "Rust version: ${{ steps.read-rust-version.outputs.RUST_VERSION }}"
    - name: Cache cargo dependencies and targets
      uses: actions/cache@v4
      with:
        path: |
          ~/.cargo/registry
          ~/.cargo/git
          concordium-base/rust-src/target
          concordium-base/lib
          concordium-base/smart-contracts/wasm-chain-integration/target
          concordium-base/smart-contracts/lib
          concordium-node/target
          collector/target
        key: ${{ runner.os }}-${{ env.dummy }}-rust-deps-${{ steps.read-rust-version.outputs.RUST_VERSION }}-${{ hashFiles('**/Cargo.toml', '**/Cargo.lock') }}-${{ hashFiles('concordium-base/rust-src/**/*.rs','concordium-base/smart-contracts/wasm-chain-integration/**/*.rs')}}
        restore-keys: |
          ${{ runner.os }}-${{ env.dummy }}-rust-deps-${{ steps.read-rust-version.outputs.RUST_VERSION }}-${{ hashFiles('**/Cargo.toml', '**/Cargo.lock') }}
          ${{ runner.os }}-${{ env.dummy }}-rust-deps-${{ steps.read-rust-version.outputs.RUST_VERSION }}

    # HASKELL #

    # Set up Haskell by caching '~/.stack', '.stack-work', and '~/.local/bin' separately.
    # This must be done before compiling the Haskell sources
    # (which in turns compiles certain Rust sources).
    # The cache entry keys depend on the GHC version and contents of 'package.yaml' and 'stack.yaml'
    # but will fall back to cache entries from different versions if no match is found.

    - name: Compute cache keys
      id: cache-keys
      run: |
        echo "proto_hash=${{ hashFiles('**/*.proto') }}" >> $GITHUB_OUTPUT
        echo "base_hash=${{ hashFiles('concordium-base/**/*.hs') }}" >> $GITHUB_OUTPUT
      
    - name: Cache stack global package DB
      id: stack-global
      uses: actions/cache@v4
      with:
        path: ~/.stack
        key: ${{ runner.os }}-${{ env.dummy }}-stack-global-${{ env.GHC_VERSION }}-${{ hashFiles('**.yaml') }}
        restore-keys: |
          ${{ runner.os }}-${{ env.dummy }}-stack-global-${{ env.GHC_VERSION }}
    - name: Cache '.stack-work'
      uses: actions/cache@v4
      with:
        path: |
          .stack-work
          concordium-base/.stack-work
          concordium-consensus/.stack-work
          concordium-consensus/haskell-lmdb/.stack-work

        key: ${{ runner.os }}-${{ env.dummy }}-stack-work-${{ env.GHC_VERSION }}-${{ hashFiles('**.yaml') }}-${{ steps.cache-keys.outputs.proto_hash }}-${{ steps.cache-keys.outputs.base_hash }}
        restore-keys: |
          ${{ runner.os }}-${{ env.dummy }}-stack-work-${{ env.GHC_VERSION }}-${{ hashFiles('**.yaml') }}-${{ steps.cache-keys.outputs.proto_hash }}-
          ${{ runner.os }}-${{ env.dummy }}-stack-work-${{ env.GHC_VERSION }}-${{ hashFiles('**.yaml') }}-
          ${{ runner.os }}-${{ env.dummy }}-stack-work-${{ env.GHC_VERSION }}

    # Compile Haskell sources. This must be done before running checks or tests on the Rust sources.
    - name: Build consensus and run tests (with code coverage)
      run: |
        stack build concordium-consensus --test --bench --force-dirty --no-run-benchmarks --ghc-options "-Werror" --ghc-options -j --coverage

    - name: Generate Haskell codecov report
      run: |
        stack install hpc-codecov
        hpc-codecov stack:all -o codecov-haskell.json

    # RUST #

    # Check, compile, and test Rust sources. All the steps below could be run in parallel in separate jobs.
    - name: Check that Rust target compiles
      run: |
        cargo check --locked --manifest-path concordium-node/Cargo.toml --workspace
    - name: Check that the collector-backend compiles
      run: |
        cargo check --locked --manifest-path collector-backend/Cargo.toml --workspace
    - name: Check that the collector compiles
      run: |
        cargo check --locked --manifest-path collector/Cargo.toml
    - name: Check that the Windows node runner service compiles (gnu toolchain, disable embedded resources)
      run: |
        cargo check --locked --target x86_64-pc-windows-gnu --manifest-path service/windows/Cargo.toml --workspace --no-default-features
    - name: Check that the Windows installer custom action library compiles (gnu toolchain)
      run: |
        cargo check --locked --target x86_64-pc-windows-gnu --manifest-path service/windows/installer/custom-actions/Cargo.toml --workspace
    - name: Run clippy (without extra features)
      run: |
        cargo clippy --locked --manifest-path concordium-node/Cargo.toml --all -- -D warnings
    - name: Run clippy (with feature 'network_dump')
      run: |
        cargo clippy --locked --manifest-path concordium-node/Cargo.toml --features=network_dump --all -- -D warnings
    - name: Run clippy on collector backend
      run: |
        cargo clippy --locked --manifest-path collector-backend/Cargo.toml -- -D warnings
    - name: Run clippy on collector
      run: |
        cargo clippy --locked --manifest-path collector/Cargo.toml -- -D warnings
    - name: Run clippy on Windows node runner service
      run: |
        cargo clippy --locked --target x86_64-pc-windows-gnu --manifest-path service/windows/Cargo.toml --all --no-default-features -- -D warnings
    - name: Run clippy on Windows installer custom action library
      run: |
        cargo clippy --locked --target x86_64-pc-windows-gnu --manifest-path service/windows/installer/custom-actions/Cargo.toml --all -- -D warnings
    - name: Test Rust crates (without extra features)
      run: |
        cargo llvm-cov test --manifest-path concordium-node/Cargo.toml --all --no-report
    - name: Test Rust crates (with feature 'network_dump')
      run: |
        cargo llvm-cov test --manifest-path concordium-node/Cargo.toml --all --features=network_dump --no-report
    - name: Generate a composite report of the tests above.
      run: |
        cargo llvm-cov report --manifest-path concordium-node/Cargo.toml  --lcov --output-path lcov.info
      # Options documented here: https://github.com/codecov/codecov-action
    - name: Upload coverage to Codecov
      uses: codecov/codecov-action@v5
      with:
        disable_telem: true
        fail_ci_if_error: false
        token: ${{ secrets.CODECOV_TOKEN }}
        verbose: true
        files: lcov.info,codecov-haskell.json

  all-checks-passed:
    needs: [build]
    runs-on: ubuntu-latest
    steps:
      - run: echo "All matrix builds passed!"