Update DANDI Data #158
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Update DANDI Data | |
| on: | |
| schedule: | |
| - cron: '0 3 * * *' # 3 AM UTC nightly | |
| workflow_dispatch: | |
| inputs: | |
| mode: | |
| description: 'Update mode' | |
| required: true | |
| default: 'incremental' | |
| type: choice | |
| options: | |
| - incremental | |
| - full | |
| permissions: | |
| contents: write | |
| # A run that is still going when the next night's run starts should yield to | |
| # it rather than have both push to main. | |
| concurrency: | |
| group: update-data | |
| cancel-in-progress: false | |
| jobs: | |
| update-data: | |
| runs-on: ubuntu-latest | |
| # The atlas steps are individually fault-tolerant (see continue-on-error | |
| # below) and the cache is saved even on failure, so a long run costs a | |
| # slow night rather than a lost one. 350 leaves headroom under the | |
| # 360-minute ceiling for hosted runners. | |
| timeout-minutes: 350 | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| lfs: true | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: '3.11' | |
| - name: Install dependencies | |
| run: pip install -r scripts/requirements.txt | |
| # Restore and save are split so that the save can run with | |
| # `if: always()`. With the combined `actions/cache` action the save | |
| # happens in a post-step that is skipped when the job fails or times | |
| # out. That is what broke this workflow: every run since May timed out, | |
| # so no cache was ever written, the old one aged out after 7 days, and | |
| # every subsequent run started cold and timed out again. | |
| - name: Restore cache | |
| id: cache-restore | |
| uses: actions/cache/restore@v4 | |
| with: | |
| path: | | |
| scripts/label_cache.jsonl | |
| scripts/electrode_cache.jsonl | |
| scripts/d99_electrode_cache.jsonl | |
| scripts/nmt_electrode_cache.jsonl | |
| scripts/mebrains_electrode_cache.jsonl | |
| scripts/whs_sd_electrode_cache.jsonl | |
| scripts/whs_sd_sweep_cache.jsonl | |
| scripts/macaque_locations_cache.jsonl | |
| scripts/macaque_sweep_cache.jsonl | |
| key: data-cache-${{ github.run_id }} | |
| restore-keys: data-cache- | |
| - name: Determine mode | |
| id: mode | |
| run: | | |
| if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then | |
| echo "mode=${{ inputs.mode }}" >> $GITHUB_OUTPUT | |
| else | |
| echo "mode=incremental" >> $GITHUB_OUTPUT | |
| fi | |
| # Each atlas is allowed to fail without taking the others down. One | |
| # atlas hitting a transient DANDI or Allen API error should not stop the | |
| # other five from publishing. Failures are surfaced in the summary step. | |
| - name: Update Allen CCF data | |
| id: allen | |
| continue-on-error: true | |
| run: python scripts/update_data.py --mode ${{ steps.mode.outputs.mode }} --workers 4 | |
| env: | |
| PYTHONUNBUFFERED: "1" | |
| - name: Update macaque atlas data (D99) | |
| id: d99 | |
| continue-on-error: true | |
| run: python scripts/update_macaque_data.py --atlas d99 --mode ${{ steps.mode.outputs.mode }} | |
| env: | |
| PYTHONUNBUFFERED: "1" | |
| - name: Update macaque atlas data (NMT) | |
| id: nmt | |
| continue-on-error: true | |
| run: python scripts/update_macaque_data.py --atlas nmt --mode ${{ steps.mode.outputs.mode }} | |
| env: | |
| PYTHONUNBUFFERED: "1" | |
| - name: Update macaque atlas data (MEBRAINS) | |
| id: mebrains | |
| continue-on-error: true | |
| run: python scripts/update_macaque_data.py --atlas mebrains --mode ${{ steps.mode.outputs.mode }} | |
| env: | |
| PYTHONUNBUFFERED: "1" | |
| - name: Update rat atlas data (WHS-SD) | |
| id: whs_sd | |
| continue-on-error: true | |
| run: python scripts/update_rat_data.py --mode ${{ steps.mode.outputs.mode }} | |
| env: | |
| PYTHONUNBUFFERED: "1" | |
| # Save before the index build and commit so that partial progress | |
| # survives even if something later goes wrong. | |
| # NOTE: this path list must stay in sync with the restore step above. | |
| # GitHub Actions does not support YAML anchors, so it has to be repeated. | |
| - name: Save cache | |
| if: always() | |
| uses: actions/cache/save@v4 | |
| with: | |
| path: | | |
| scripts/label_cache.jsonl | |
| scripts/electrode_cache.jsonl | |
| scripts/d99_electrode_cache.jsonl | |
| scripts/nmt_electrode_cache.jsonl | |
| scripts/mebrains_electrode_cache.jsonl | |
| scripts/whs_sd_electrode_cache.jsonl | |
| scripts/whs_sd_sweep_cache.jsonl | |
| scripts/macaque_locations_cache.jsonl | |
| scripts/macaque_sweep_cache.jsonl | |
| key: data-cache-${{ github.run_id }} | |
| - name: Build landing-page atlas index | |
| if: always() | |
| run: python scripts/build_atlases_index.py | |
| env: | |
| PYTHONUNBUFFERED: "1" | |
| - name: Commit and push if changed | |
| if: always() | |
| run: | | |
| git diff --quiet data/atlases/ data/atlases_index.json data/last_updated.json && exit 0 | |
| git config user.name "github-actions[bot]" | |
| git config user.email "github-actions[bot]@users.noreply.github.com" | |
| git add data/atlases/ data/atlases_index.json data/last_updated.json | |
| git commit -m "Update DANDI data (${{ steps.mode.outputs.mode }})" | |
| git pull --rebase | |
| git push | |
| - name: Report per-atlas outcome | |
| if: always() | |
| run: | | |
| { | |
| echo "| Atlas | Outcome |" | |
| echo "|---|---|" | |
| echo "| Allen CCF | ${{ steps.allen.outcome }} |" | |
| echo "| D99 | ${{ steps.d99.outcome }} |" | |
| echo "| NMT | ${{ steps.nmt.outcome }} |" | |
| echo "| MEBRAINS | ${{ steps.mebrains.outcome }} |" | |
| echo "| WHS-SD | ${{ steps.whs_sd.outcome }} |" | |
| } >> "$GITHUB_STEP_SUMMARY" | |
| # Fail the run if every atlas failed, so a total outage is not | |
| # reported as green. | |
| if [ "${{ steps.allen.outcome }}" != "success" ] \ | |
| && [ "${{ steps.d99.outcome }}" != "success" ] \ | |
| && [ "${{ steps.nmt.outcome }}" != "success" ] \ | |
| && [ "${{ steps.mebrains.outcome }}" != "success" ] \ | |
| && [ "${{ steps.whs_sd.outcome }}" != "success" ]; then | |
| echo "All atlas updates failed." >&2 | |
| exit 1 | |
| fi |