Crawler scheduled maintenance #495
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Crawler scheduled maintenance | |
| # Periodic ephemeral crawler CLI runs. Kept in-repo so the cadence is | |
| # reviewable and version-controlled (see #2630). Dispatches to the Hetzner | |
| # crawler host via SSH and runs the task as `docker run --rm` using the | |
| # image tag currently recorded in `/home/deploy/.env`. | |
| # | |
| # What runs here (vs. elsewhere): | |
| # refresh-typesense — this workflow, every 4h + at every deploy (inline | |
| # via `crawler sync` in apps/crawler/deploy.sh). | |
| # notify-indexnow — Retired in #2821. Companies are no longer indexed, | |
| # so there is no surface left for the notifier to | |
| # push. Module + CLI subcommand kept for revival; | |
| # no scheduled invocation anywhere in the repo. | |
| # reconcile — NOT this workflow. A deploy-independent systemd | |
| # timer on the crawler host resumes bounded repairs; | |
| # see docs/03-crawler-architecture.md. | |
| # backfill-typesense — NOT scheduled. Full re-index, invoked manually via | |
| # workflow_dispatch below when a schema change or | |
| # count drift requires it. | |
| on: | |
| schedule: | |
| # 00:17, 04:17, 08:17, 12:17, 16:17, 20:17 UTC — off-the-hour to avoid | |
| # clustering with other cron jobs, 4h cadence matches the count-staleness | |
| # tolerance (exporter keeps job_posting fresh; counts on company + taxonomy | |
| # collections only advance on refresh-typesense). | |
| - cron: "17 */4 * * *" | |
| workflow_dispatch: | |
| inputs: | |
| task: | |
| description: Which maintenance command to run | |
| type: choice | |
| default: refresh-typesense | |
| options: | |
| - refresh-typesense | |
| - backfill-typesense | |
| concurrency: | |
| # Serialize scheduled runs. Deploys have a separate concurrency group, but | |
| # deploy.sh refuses to overlap these named maintenance containers because | |
| # its inline sync also refreshes Typesense counts. | |
| group: crawler-scheduled-maintenance | |
| cancel-in-progress: false | |
| permissions: | |
| contents: read | |
| jobs: | |
| run: | |
| runs-on: ubuntu-latest | |
| environment: production | |
| steps: | |
| - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 | |
| - name: Validate host hygiene check | |
| run: python3 -m py_compile scripts/crawler-host-hygiene.py | |
| - name: Copy host hygiene check | |
| uses: appleboy/scp-action@ff85246acaad7bdce478db94a363cd2bf7c90345 # v1.0.0 | |
| with: | |
| host: ${{ secrets.HETZNER_HOST }} | |
| username: deploy | |
| key: ${{ secrets.HETZNER_SSH_KEY }} | |
| source: scripts/crawler-host-hygiene.py | |
| target: /tmp/jobseek-crawler-maintenance/${{ github.sha }} | |
| strip_components: 1 | |
| - name: Resolve task | |
| id: task | |
| run: | | |
| if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then | |
| echo "task=${{ inputs.task }}" >> "$GITHUB_OUTPUT" | |
| else | |
| echo "task=refresh-typesense" >> "$GITHUB_OUTPUT" | |
| fi | |
| - name: Run via SSH | |
| uses: appleboy/ssh-action@0ff4204d59e8e51228ff73bce53f80d53301dee2 # v1 | |
| with: | |
| host: ${{ secrets.HETZNER_HOST }} | |
| username: deploy | |
| key: ${{ secrets.HETZNER_SSH_KEY }} | |
| envs: GITHUB_SHA | |
| # A full production re-index currently takes about 45 minutes. | |
| # Allow the two-hour host-lock wait plus the bounded two-hour | |
| # operation and leave cleanup headroom beyond ssh-action's default. | |
| command_timeout: 5h | |
| script: | | |
| set -euo pipefail | |
| exec 9>/run/lock/jobseek-crawler-mutation.lock | |
| flock -w 7200 9 || { | |
| echo "Timed out waiting for the crawler mutation lock" >&2 | |
| exit 1 | |
| } | |
| TASK="${{ steps.task.outputs.task }}" | |
| NAME="crawler-${TASK}-$(date -u +%Y%m%dT%H%M%SZ)" | |
| tag="$(grep -E '^CRAWLER_IMAGE_TAG=' /home/deploy/.env 2>/dev/null | tail -n1 | cut -d= -f2- || true)" | |
| tag="${tag:-latest}" | |
| maintenance_status=0 | |
| cleanup() { | |
| status=$? | |
| trap - EXIT HUP INT TERM | |
| if docker ps -a --format '{{.Names}}' | grep -Fxq "$NAME"; then | |
| docker rm -f "$NAME" >/dev/null 2>&1 || true | |
| fi | |
| exit "$status" | |
| } | |
| trap cleanup EXIT HUP INT TERM | |
| # The GitHub concurrency lock cannot see a container orphaned by | |
| # a cancelled or timed-out older SSH session. Refuse to overlap it | |
| # with another Typesense maintenance operation. | |
| if docker ps --format '{{.Names}}' | grep -Eq '^crawler-(backfill|refresh)-typesense-'; then | |
| echo "Another Typesense maintenance container is already running" >&2 | |
| maintenance_status=1 | |
| else | |
| timeout --foreground --signal=TERM --kill-after=90s 2h docker run --rm \ | |
| --name "$NAME" \ | |
| --env-file /home/deploy/.env \ | |
| --network host \ | |
| --label com.docker.compose.project=deploy \ | |
| --label "com.docker.compose.service=${TASK}" \ | |
| --label com.docker.compose.container-number=1 \ | |
| --label com.docker.compose.oneoff=True \ | |
| --label "jobseek.maintenance.operation=${TASK}" \ | |
| --label jobseek.maintenance.issue=2630 \ | |
| --label "jobseek.maintenance.revision=${GITHUB_SHA}" \ | |
| --label jobseek.maintenance.budget-seconds=7200 \ | |
| "ghcr.io/colophon-group/jobseek-crawler:${tag}" \ | |
| uv run --no-sync crawler "$TASK" || maintenance_status=$? | |
| fi | |
| hygiene_status=0 | |
| python3 "/tmp/jobseek-crawler-maintenance/${GITHUB_SHA}/crawler-host-hygiene.py" || hygiene_status=$? | |
| if (( maintenance_status != 0 )); then | |
| echo "Maintenance task failed with status ${maintenance_status}" >&2 | |
| fi | |
| if (( hygiene_status != 0 )); then | |
| echo "Host hygiene check failed with status ${hygiene_status}" >&2 | |
| fi | |
| if (( maintenance_status != 0 || hygiene_status != 0 )); then | |
| exit 1 | |
| fi |