Run benchmarks #554
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Run benchmarks | |
| on: | |
| schedule: | |
| - cron: '0 0 * * *' | |
| workflow_dispatch: | |
| jobs: | |
| onnxruntime_stable: | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 90 | |
| permissions: | |
| contents: read | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 | |
| with: | |
| persist-credentials: false | |
| - name: Build docker image | |
| run: docker build -t scoreboard/ort -f runtimes/ort/stable/Dockerfile . | |
| - name: Run docker container | |
| run: docker run --name ort --env-file setup/env.list -v ${{ github.workspace }}/results/ort/stable:/root/results scoreboard/ort || true | |
| - name: Upload results | |
| if: ${{ github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' }} | |
| uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 | |
| with: | |
| name: results-ort-stable | |
| path: results/ort/stable/ | |
| onnx_tf_stable: | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 90 | |
| permissions: | |
| contents: read | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 | |
| with: | |
| persist-credentials: false | |
| - name: Build docker image | |
| run: docker build -t scoreboard/onnxtf -f runtimes/onnx-tf/stable/Dockerfile . | |
| - name: Run docker container | |
| run: docker run --name onnxtf --env-file setup/env.list -v ${{ github.workspace }}/results/onnx-tf/stable:/root/results scoreboard/onnxtf || true | |
| - name: Upload results | |
| if: ${{ github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' }} | |
| uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 | |
| with: | |
| name: results-onnxtf-stable | |
| path: results/onnx-tf/stable/ | |
| onnx_tf_dev: | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 90 | |
| permissions: | |
| contents: read | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 | |
| with: | |
| persist-credentials: false | |
| - name: Build docker image | |
| run: docker build -t scoreboard/onnxtf-dev -f runtimes/onnx-tf/development/Dockerfile . | |
| - name: Run docker container | |
| run: docker run --name onnxtf --env-file setup/env.list -v ${{ github.workspace }}/results/onnx-tf/development:/root/results scoreboard/onnxtf-dev || true | |
| - name: Upload results | |
| if: ${{ github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' }} | |
| uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 | |
| with: | |
| name: results-onnxtf-dev | |
| path: results/onnx-tf/development/ | |
| jaxonnxruntime_stable: | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 90 | |
| permissions: | |
| contents: read | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 | |
| with: | |
| persist-credentials: false | |
| - name: Build docker image | |
| run: docker build -t scoreboard/jaxonnxruntime -f runtimes/jaxonnxruntime/stable/Dockerfile . | |
| - name: Run docker container | |
| run: docker run --name jaxonnxruntime --env-file setup/env.list -v ${{ github.workspace }}/results/jaxonnxruntime/stable:/root/results scoreboard/jaxonnxruntime || true | |
| - name: Upload results | |
| if: ${{ github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' }} | |
| uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 | |
| with: | |
| name: results-jaxonnxruntime-stable | |
| path: results/jaxonnxruntime/stable/ | |
| jaxonnxruntime_dev: | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 90 | |
| permissions: | |
| contents: read | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 | |
| with: | |
| persist-credentials: false | |
| fetch-depth: 0 | |
| clean: true | |
| - name: Build docker image | |
| run: docker build -t scoreboard/jaxonnxruntime-dev -f runtimes/jaxonnxruntime/development/Dockerfile . | |
| - name: Run docker container | |
| run: docker run --name jaxonnxruntime --env-file setup/env.list -v ${{ github.workspace }}/results/jaxonnxruntime/development:/root/results scoreboard/jaxonnxruntime-dev || true | |
| - name: Upload results | |
| if: ${{ github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' }} | |
| uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 | |
| with: | |
| name: results-jaxonnxruntime-dev | |
| path: results/jaxonnxruntime/development/ | |
| onnxruntime_dev: | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 90 | |
| permissions: | |
| contents: read | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 | |
| with: | |
| persist-credentials: false | |
| - name: Build docker image | |
| run: docker build -t scoreboard/ort-dev -f runtimes/ort/development/Dockerfile . | |
| - name: Prepare results directory | |
| run: mkdir -p ${{ github.workspace }}/results/ort/development && chmod 777 ${{ github.workspace }}/results/ort/development | |
| - name: Run docker container | |
| run: docker run --name ort-dev --env-file setup/env.list -v ${{ github.workspace }}/results/ort/development:/root/results scoreboard/ort-dev || true | |
| - name: Upload results | |
| if: ${{ github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' }} | |
| uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 | |
| with: | |
| name: results-ort-dev | |
| path: results/ort/development/ | |
| emx_onnx_cgen_stable: | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 90 | |
| permissions: | |
| contents: read | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 | |
| with: | |
| persist-credentials: false | |
| - name: Build docker image | |
| run: docker build -t scoreboard/emx-onnx-cgen -f runtimes/emx-onnx-cgen/stable/Dockerfile . | |
| - name: Prepare results directory | |
| run: mkdir -p ${{ github.workspace }}/results/emx-onnx-cgen/stable && chmod 777 ${{ github.workspace }}/results/emx-onnx-cgen/stable | |
| - name: Run docker container | |
| run: docker run --name emx-onnx-cgen --env-file setup/env.list -v ${{ github.workspace }}/results/emx-onnx-cgen/stable:/root/results scoreboard/emx-onnx-cgen || true | |
| - name: Upload results | |
| if: ${{ github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' }} | |
| uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 | |
| with: | |
| name: results-emx-onnx-cgen-stable | |
| path: results/emx-onnx-cgen/stable/ | |
| tract_stable: | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 90 | |
| permissions: | |
| contents: read | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 | |
| with: | |
| persist-credentials: false | |
| - name: Build docker image | |
| run: docker build -t scoreboard/tract -f runtimes/tract/stable/Dockerfile . | |
| - name: Prepare results directory | |
| run: mkdir -p ${{ github.workspace }}/results/tract/stable && chmod 777 ${{ github.workspace }}/results/tract/stable | |
| - name: Run docker container | |
| run: docker run --name tract --env-file setup/env.list -v ${{ github.workspace }}/results/tract/stable:/root/results scoreboard/tract || true | |
| - name: Upload results | |
| if: ${{ github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' }} | |
| uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 | |
| with: | |
| name: results-tract-stable | |
| path: results/tract/stable/ | |
| onnx_reference_stable: | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 90 | |
| permissions: | |
| contents: read | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 | |
| with: | |
| persist-credentials: false | |
| - name: Build docker image | |
| run: docker build -t scoreboard/onnx-reference -f runtimes/onnx-reference/stable/Dockerfile . | |
| - name: Prepare results directory | |
| run: mkdir -p ${{ github.workspace }}/results/onnx-reference/stable && chmod 777 ${{ github.workspace }}/results/onnx-reference/stable | |
| - name: Run docker container | |
| run: docker run --name onnx-reference --env-file setup/env.list -v ${{ github.workspace }}/results/onnx-reference/stable:/root/results scoreboard/onnx-reference || true | |
| - name: Upload results | |
| if: ${{ github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' }} | |
| uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 | |
| with: | |
| name: results-onnx-reference-stable | |
| path: results/onnx-reference/stable/ | |
| opencv_stable: | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 90 | |
| permissions: | |
| contents: read | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 | |
| with: | |
| persist-credentials: false | |
| - name: Build docker image | |
| run: docker build -t scoreboard/opencv -f runtimes/opencv/stable/Dockerfile . | |
| - name: Prepare results directory | |
| run: mkdir -p ${{ github.workspace }}/results/opencv/stable && chmod 777 ${{ github.workspace }}/results/opencv/stable | |
| - name: Run docker container | |
| run: docker run --name opencv --env-file setup/env.list -v ${{ github.workspace }}/results/opencv/stable:/root/results scoreboard/opencv || true | |
| - name: Upload results | |
| if: ${{ github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' }} | |
| uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 | |
| with: | |
| name: results-opencv-stable | |
| path: results/opencv/stable/ | |
| tvm_stable: | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 90 | |
| permissions: | |
| contents: read | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 | |
| with: | |
| persist-credentials: false | |
| - name: Build docker image | |
| run: docker build -t scoreboard/tvm -f runtimes/tvm/stable/Dockerfile . | |
| - name: Prepare results directory | |
| run: mkdir -p ${{ github.workspace }}/results/tvm/stable && chmod 777 ${{ github.workspace }}/results/tvm/stable | |
| - name: Run docker container | |
| run: docker run --name tvm --env-file setup/env.list -v ${{ github.workspace }}/results/tvm/stable:/root/results scoreboard/tvm || true | |
| - name: Upload results | |
| if: ${{ github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' }} | |
| uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 | |
| with: | |
| name: results-tvm-stable | |
| path: results/tvm/stable/ | |
| onnxruntime_openvino_stable: | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 90 | |
| permissions: | |
| contents: read | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 | |
| with: | |
| persist-credentials: false | |
| - name: Build docker image | |
| run: docker build -t scoreboard/onnxruntime-openvino -f runtimes/onnxruntime-openvino/stable/Dockerfile . | |
| - name: Prepare results directory | |
| run: mkdir -p ${{ github.workspace }}/results/onnxruntime-openvino/stable && chmod 777 ${{ github.workspace }}/results/onnxruntime-openvino/stable | |
| - name: Run docker container | |
| run: | | |
| docker run --name onnxruntime-openvino --env-file setup/env.list \ | |
| -e RESULTS_DIR=/home/onnx/results \ | |
| -e CSVDIR=/home/onnx/results \ | |
| -v ${{ github.workspace }}/results/onnxruntime-openvino/stable:/home/onnx/results \ | |
| scoreboard/onnxruntime-openvino || true | |
| - name: Upload results | |
| if: ${{ github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' }} | |
| uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 | |
| with: | |
| name: results-onnxruntime-openvino-stable | |
| path: results/onnxruntime-openvino/stable/ | |
| collect_and_deploy: | |
| runs-on: ubuntu-latest | |
| permissions: | |
| contents: write | |
| needs: [onnxruntime_stable, onnxruntime_dev, onnx_tf_stable, onnx_tf_dev, jaxonnxruntime_stable, jaxonnxruntime_dev, emx_onnx_cgen_stable, tract_stable, onnx_reference_stable, opencv_stable, tvm_stable, onnxruntime_openvino_stable] | |
| # Run even if some backend jobs failed, so the scoreboard is still deployed | |
| # with the results that did succeed (missing artifacts are skipped below). | |
| if: ${{ always() && (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch') }} | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 | |
| with: | |
| persist-credentials: false | |
| - name: Set up Python 3.12 | |
| uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 | |
| with: | |
| python-version: '3.12' | |
| - name: Preserve merge inputs | |
| run: | | |
| mkdir -p /tmp/scoreboard-merge | |
| cp scripts/merge_trends.py /tmp/scoreboard-merge/merge_trends.py | |
| cp setup/config.json /tmp/scoreboard-merge/config.json | |
| - name: Switch to benchmark-results branch | |
| run: | | |
| git config --global user.name "GitHub Action" | |
| git config --global user.email "action@github.com" | |
| git fetch origin | |
| if git ls-remote --exit-code --heads origin benchmark-results > /dev/null 2>&1; then | |
| git checkout -B benchmark-results origin/benchmark-results | |
| else | |
| git checkout --orphan benchmark-results | |
| git rm -rf --cached . 2>/dev/null || true | |
| fi | |
| - name: Download ort stable results | |
| uses: actions/download-artifact@70fc10c6e5e1ce46ad2ea6f2b72d43f7d47b13c3 # v8.0.0 | |
| continue-on-error: true | |
| with: | |
| name: results-ort-stable | |
| path: results/ort/stable/ | |
| - name: Download onnx-tf stable results | |
| uses: actions/download-artifact@70fc10c6e5e1ce46ad2ea6f2b72d43f7d47b13c3 # v8.0.0 | |
| continue-on-error: true | |
| with: | |
| name: results-onnxtf-stable | |
| path: results/onnx-tf/stable/ | |
| - name: Download onnx-tf dev results | |
| uses: actions/download-artifact@70fc10c6e5e1ce46ad2ea6f2b72d43f7d47b13c3 # v8.0.0 | |
| continue-on-error: true | |
| with: | |
| name: results-onnxtf-dev | |
| path: results/onnx-tf/development/ | |
| - name: Download jaxonnxruntime stable results | |
| uses: actions/download-artifact@70fc10c6e5e1ce46ad2ea6f2b72d43f7d47b13c3 # v8.0.0 | |
| continue-on-error: true | |
| with: | |
| name: results-jaxonnxruntime-stable | |
| path: results/jaxonnxruntime/stable/ | |
| - name: Download jaxonnxruntime dev results | |
| uses: actions/download-artifact@70fc10c6e5e1ce46ad2ea6f2b72d43f7d47b13c3 # v8.0.0 | |
| continue-on-error: true | |
| with: | |
| name: results-jaxonnxruntime-dev | |
| path: results/jaxonnxruntime/development/ | |
| - name: Download ort dev results | |
| uses: actions/download-artifact@70fc10c6e5e1ce46ad2ea6f2b72d43f7d47b13c3 # v8.0.0 | |
| continue-on-error: true | |
| with: | |
| name: results-ort-dev | |
| path: results/ort/development/ | |
| - name: Download emx-onnx-cgen stable results | |
| uses: actions/download-artifact@70fc10c6e5e1ce46ad2ea6f2b72d43f7d47b13c3 # v8.0.0 | |
| continue-on-error: true | |
| with: | |
| name: results-emx-onnx-cgen-stable | |
| path: results/emx-onnx-cgen/stable/ | |
| - name: Download tract stable results | |
| uses: actions/download-artifact@70fc10c6e5e1ce46ad2ea6f2b72d43f7d47b13c3 # v8.0.0 | |
| continue-on-error: true | |
| with: | |
| name: results-tract-stable | |
| path: results/tract/stable/ | |
| - name: Download ONNX reference stable results | |
| uses: actions/download-artifact@70fc10c6e5e1ce46ad2ea6f2b72d43f7d47b13c3 # v8.0.0 | |
| continue-on-error: true | |
| with: | |
| name: results-onnx-reference-stable | |
| path: results/onnx-reference/stable/ | |
| - name: Download opencv stable results | |
| uses: actions/download-artifact@70fc10c6e5e1ce46ad2ea6f2b72d43f7d47b13c3 # v8.0.0 | |
| continue-on-error: true | |
| with: | |
| name: results-opencv-stable | |
| path: results/opencv/stable/ | |
| - name: Download tvm stable results | |
| uses: actions/download-artifact@70fc10c6e5e1ce46ad2ea6f2b72d43f7d47b13c3 # v8.0.0 | |
| continue-on-error: true | |
| with: | |
| name: results-tvm-stable | |
| path: results/tvm/stable/ | |
| - name: Download onnxruntime-openvino stable results | |
| uses: actions/download-artifact@70fc10c6e5e1ce46ad2ea6f2b72d43f7d47b13c3 # v8.0.0 | |
| continue-on-error: true | |
| with: | |
| name: results-onnxruntime-openvino-stable | |
| path: results/onnxruntime-openvino/stable/ | |
| - name: Merge trend history | |
| run: python3 /tmp/scoreboard-merge/merge_trends.py --config /tmp/scoreboard-merge/config.json --base-ref HEAD | |
| - name: Deploy results | |
| run: | | |
| git add results | |
| if ! git diff --cached --quiet; then | |
| git commit -s -m "Scoreboard results [skip ci]" | |
| git push origin benchmark-results | |
| else | |
| echo "No changes to commit" | |
| fi |