Update DANDI Data #87
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Update DANDI Data | |
| on: | |
| schedule: | |
| - cron: '0 3 * * *' # 3 AM UTC nightly | |
| workflow_dispatch: | |
| inputs: | |
| mode: | |
| description: 'Update mode' | |
| required: true | |
| default: 'incremental' | |
| type: choice | |
| options: | |
| - incremental | |
| - full | |
| permissions: | |
| contents: write | |
| jobs: | |
| update-data: | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 120 | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| lfs: true | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: '3.11' | |
| - name: Install dependencies | |
| run: pip install -r scripts/requirements.txt | |
| - name: Restore cache | |
| uses: actions/cache@v4 | |
| with: | |
| path: | | |
| scripts/label_cache.jsonl | |
| scripts/electrode_cache.jsonl | |
| scripts/d99_electrode_cache.jsonl | |
| scripts/nmt_electrode_cache.jsonl | |
| scripts/mebrains_electrode_cache.jsonl | |
| key: data-cache-${{ github.run_id }} | |
| restore-keys: data-cache- | |
| - name: Determine mode | |
| id: mode | |
| run: | | |
| if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then | |
| echo "mode=${{ inputs.mode }}" >> $GITHUB_OUTPUT | |
| else | |
| echo "mode=incremental" >> $GITHUB_OUTPUT | |
| fi | |
| - name: Update Allen CCF data | |
| run: python scripts/update_data.py --mode ${{ steps.mode.outputs.mode }} --workers 4 | |
| env: | |
| PYTHONUNBUFFERED: "1" | |
| - name: Update macaque atlas data (D99) | |
| run: python scripts/update_macaque_data.py --atlas d99 --mode ${{ steps.mode.outputs.mode }} | |
| env: | |
| PYTHONUNBUFFERED: "1" | |
| - name: Update macaque atlas data (NMT) | |
| run: python scripts/update_macaque_data.py --atlas nmt --mode ${{ steps.mode.outputs.mode }} | |
| env: | |
| PYTHONUNBUFFERED: "1" | |
| - name: Update macaque atlas data (MEBRAINS) | |
| run: python scripts/update_macaque_data.py --atlas mebrains --mode ${{ steps.mode.outputs.mode }} | |
| env: | |
| PYTHONUNBUFFERED: "1" | |
| - name: Build landing-page atlas index | |
| run: python scripts/build_atlases_index.py | |
| env: | |
| PYTHONUNBUFFERED: "1" | |
| - name: Commit and push if changed | |
| run: | | |
| git diff --quiet data/atlases/ data/atlases_index.json data/last_updated.json && exit 0 | |
| git config user.name "github-actions[bot]" | |
| git config user.email "github-actions[bot]@users.noreply.github.com" | |
| git add data/atlases/ data/atlases_index.json data/last_updated.json | |
| git commit -m "Update DANDI data (${{ steps.mode.outputs.mode }})" | |
| git pull --rebase | |
| git push |