Discover Awesome OSINT Tools candidates #8
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Discover Awesome OSINT Tools candidates | |
| on: | |
| schedule: | |
| - cron: "43 4 * * *" | |
| workflow_dispatch: | |
| concurrency: | |
| group: catalogue-candidate-discovery | |
| cancel-in-progress: false | |
| permissions: | |
| contents: write | |
| pull-requests: write | |
| jobs: | |
| discover: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Check out repository | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| - name: Set up Python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: "3.12" | |
| - name: Preserve candidates from an open review pull request | |
| env: | |
| GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| run: | | |
| branch="automation/catalog-candidates" | |
| pull_request=$(gh pr list --head "$branch" --state open --json number --jq '.[0].number // empty') | |
| if [ -n "$pull_request" ]; then | |
| git fetch origin "+refs/heads/$branch:refs/remotes/origin/$branch" | |
| git show "refs/remotes/origin/$branch:.catalog/data/candidates.csv" > "$RUNNER_TEMP/previous-candidates.csv" | |
| fi | |
| - name: Scan configured public sources | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| run: | | |
| seed=() | |
| if [ -s "$RUNNER_TEMP/previous-candidates.csv" ]; then | |
| seed=(--seed-candidates "$RUNNER_TEMP/previous-candidates.csv") | |
| fi | |
| python .catalog/scripts/discover_candidates.py \ | |
| --write \ | |
| --lookback-days 14 \ | |
| --max-per-source 100 \ | |
| --delay 2.1 \ | |
| "${seed[@]}" \ | |
| --report "$RUNNER_TEMP/discovery-report.md" | |
| python .catalog/scripts/validate_catalog.py | |
| - name: Open or update candidate review pull request | |
| env: | |
| GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| BASE_BRANCH: ${{ github.event.repository.default_branch }} | |
| run: | | |
| branch="automation/catalog-candidates" | |
| if [ -s "$RUNNER_TEMP/previous-candidates.csv" ] && cmp -s .catalog/data/candidates.csv "$RUNNER_TEMP/previous-candidates.csv"; then | |
| exit 0 | |
| fi | |
| git config user.name "github-actions[bot]" | |
| git config user.email "41898282+github-actions[bot]@users.noreply.github.com" | |
| git fetch origin "+refs/heads/$branch:refs/remotes/origin/$branch" || true | |
| git checkout -B "$branch" | |
| git add .catalog/data/candidates.csv | |
| if git diff --cached --quiet; then | |
| exit 0 | |
| fi | |
| git commit -m "chore: add discovered catalogue candidates" | |
| git push --force-with-lease origin "HEAD:refs/heads/$branch" | |
| pull_request=$(gh pr list --head "$branch" --state open --json number --jq '.[0].number // empty') | |
| if [ -z "$pull_request" ]; then | |
| gh pr create \ | |
| --base "$BASE_BRANCH" \ | |
| --head "$branch" \ | |
| --title "chore: review discovered OSINT candidates" \ | |
| --body-file "$RUNNER_TEMP/discovery-report.md" | |
| else | |
| gh pr edit "$pull_request" \ | |
| --title "chore: review discovered OSINT candidates" \ | |
| --body-file "$RUNNER_TEMP/discovery-report.md" | |
| fi |