Google Search Scraper #82
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Google Search Scraper | |
| on: | |
| schedule: | |
| # Run daily at 2 AM UTC | |
| - cron: '0 2 * * *' | |
| workflow_dispatch: # Allow manual trigger | |
| concurrency: | |
| group: google-search-scraper-${{ github.ref }} | |
| cancel-in-progress: false | |
| jobs: | |
| scrape: | |
| runs-on: ubuntu-latest | |
| continue-on-error: true | |
| permissions: | |
| contents: write # Allow pushing changes back to repo | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v6 | |
| - name: Set up Python | |
| uses: actions/setup-python@v6 | |
| with: | |
| python-version: '3.11' | |
| cache: 'pip' | |
| - name: Install system dependencies | |
| run: | | |
| sudo apt-get update | |
| sudo apt-get install -y ca-certificates curl gnupg xvfb ffmpeg | |
| google-chrome --version | |
| - name: Install Python dependencies | |
| run: | | |
| pip install --upgrade pip | |
| pip install -r requirements-googlesearch.txt | |
| - name: Run Google search scraper (first attempt without Tor) | |
| id: scrape_normal | |
| continue-on-error: true | |
| run: | | |
| set -o pipefail | |
| xvfb-run --auto-servernum --server-args='-screen 0 1920x1080x24' python3 src/run_googlesearch.py | tee scraper_output.log | |
| - name: Install and configure Tor (if first attempt failed) | |
| id: install_tor | |
| if: steps.scrape_normal.outcome == 'failure' | |
| run: | | |
| echo "First attempt failed, installing and configuring Tor..." | |
| sudo apt-get update | |
| sudo apt-get install -y tor | |
| # Start Tor service | |
| sudo systemctl start tor | |
| sudo systemctl status tor --no-pager | |
| # Wait for Tor to be ready | |
| sleep 10 | |
| # Verify Tor is working | |
| curl --socks5-hostname localhost:9050 https://check.torproject.org/ | grep -q "Congratulations" || echo "Tor may not be working properly" | |
| - name: Run Google search scraper with Tor (if first attempt failed) | |
| id: scrape_tor | |
| if: steps.scrape_normal.outcome == 'failure' | |
| run: | | |
| set -o pipefail | |
| echo "Retrying with Tor proxy..." | |
| xvfb-run --auto-servernum --server-args='-screen 0 1920x1080x24' python3 src/run_googlesearch.py --use-tor | tee scraper_output.log | |
| - name: Check for changes | |
| id: check_changes | |
| if: always() | |
| run: | | |
| if git diff --quiet data/supplements.csv data/supplements_links.txt data/timeout.csv data/timeout_links.txt 2>/dev/null; then | |
| echo "changed=false" >> $GITHUB_OUTPUT | |
| echo "No changes detected" | |
| else | |
| echo "changed=true" >> $GITHUB_OUTPUT | |
| echo "Changes detected" | |
| fi | |
| - name: Extract summary for commit message | |
| id: extract_summary | |
| if: steps.check_changes.outputs.changed == 'true' | |
| run: | | |
| # Extract the summary lines from the scraper output | |
| if [ -f scraper_output.log ]; then | |
| SUPPLEMENTS_LINE=$(grep "Supplements:" scraper_output.log | tail -1 || echo "Supplements: stats unavailable") | |
| TIMEOUT_LINE=$(grep "Timeout:" scraper_output.log | tail -1 || echo "Timeout: stats unavailable") | |
| echo "supplements_summary=$SUPPLEMENTS_LINE" >> $GITHUB_OUTPUT | |
| echo "timeout_summary=$TIMEOUT_LINE" >> $GITHUB_OUTPUT | |
| else | |
| echo "supplements_summary=Supplements: stats unavailable" >> $GITHUB_OUTPUT | |
| echo "timeout_summary=Timeout: stats unavailable" >> $GITHUB_OUTPUT | |
| fi | |
| - name: Commit and push results | |
| if: steps.check_changes.outputs.changed == 'true' | |
| run: | | |
| git config --local user.email "github-actions[bot]@users.noreply.github.com" | |
| git config --local user.name "github-actions[bot]" | |
| git pull | |
| git commit -am "Update Google search results [skip ci] | |
| ${{ steps.extract_summary.outputs.supplements_summary }} | |
| ${{ steps.extract_summary.outputs.timeout_summary }}" | |
| git push | |
| - name: Upload results as artifact | |
| uses: actions/upload-artifact@v7 | |
| if: always() | |
| with: | |
| name: search-results | |
| path: | | |
| data/supplements.csv | |
| data/supplements_links.txt | |
| data/timeout.csv | |
| data/timeout_links.txt | |
| error-screenshot-*.png | |
| retention-days: 30 |