Runner Health #6368
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # Self-hosted runner health monitor | |
| # Runs every 15 minutes to check runner status and alert if offline | |
| name: Runner Health | |
| on: | |
| schedule: | |
| # Every 15 minutes | |
| - cron: '*/15 * * * *' | |
| workflow_dispatch: | |
| permissions: | |
| contents: read | |
| actions: read | |
| issues: write | |
| jobs: | |
| health-check: | |
| name: Check Self-Hosted Runner | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5.0.1 | |
| with: | |
| persist-credentials: false | |
| - name: Check runner status | |
| id: check | |
| env: | |
| # Repository runner listing may require a broader token, so keep that | |
| # optional token scoped to runner-status lookup instead of reusing it | |
| # for queue visibility. | |
| RUNNER_STATUS_TOKEN: ${{ secrets.RUNNER_HEALTH_TOKEN || github.token }} | |
| QUEUE_TOKEN: ${{ github.token }} | |
| RUNNER_NAME: assay-bpf-runner | |
| REQUIRED_RUNNER_LABEL: assay-bpf-runner | |
| run: | | |
| bash scripts/ci/check-runner-health.sh | |
| - name: Create issue if unhealthy | |
| if: steps.check.outcome == 'failure' | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| REPO: ${{ github.repository }} | |
| RUNNER_STATUS: ${{ steps.check.outputs.runner_status }} | |
| RUNNER_STATUS_ERROR: ${{ steps.check.outputs.runner_status_error }} | |
| RUNNER_BUSY: ${{ steps.check.outputs.runner_busy }} | |
| RUNNER_LABELS: ${{ steps.check.outputs.runner_labels }} | |
| REQUIRED_RUNNER_LABEL: ${{ steps.check.outputs.required_runner_label }} | |
| GENERAL_QUEUED_RUNS: ${{ steps.check.outputs.general_queued_runs }} | |
| INSPECTED_WORKFLOW_RUNS: ${{ steps.check.outputs.inspected_workflow_runs }} | |
| MATCHING_QUEUED_JOBS: ${{ steps.check.outputs.matching_queued_jobs }} | |
| QUEUE_STATUS_ERROR: ${{ steps.check.outputs.queue_status_error }} | |
| HEALTH_REASON: ${{ steps.check.outputs.health_reason }} | |
| run: | | |
| set -euo pipefail | |
| TITLE="🚨 Self-hosted runner health needs attention" | |
| # Only the exact alert issue participates in auto-create/auto-close. | |
| # The runner-health label is also used by follow-up design issues, and | |
| # those must never block or be closed by this scheduled monitor. | |
| if ISSUES_JSON="$(gh issue list --repo "$REPO" --state open --label "runner-health" --limit 100 --json number,title 2>/dev/null)"; then | |
| EXISTING="$(printf '%s' "$ISSUES_JSON" | jq --arg title "$TITLE" '[.[] | select(.title == $title)] | length')" | |
| else | |
| echo "::warning::Issue lookup by label failed; falling back to title-based lookup." | |
| if EXISTING="$(gh issue list --repo "$REPO" --state open --search "$TITLE in:title" --limit 100 --json number,title 2>/dev/null \ | |
| | jq --arg title "$TITLE" '[.[] | select(.title == $title)] | length')"; then | |
| : | |
| else | |
| echo "::warning::Issue lookup by title failed; defaulting to no existing alert issue." | |
| EXISTING=0 | |
| fi | |
| fi | |
| if [[ "$EXISTING" -eq 0 ]]; then | |
| detected_at="$(date -u '+%Y-%m-%d %H:%M:%S UTC')" | |
| ISSUE_BODY="$(cat <<EOF | |
| ## Runner Health Alert | |
| The self-hosted runner \`assay-bpf-runner\` needs attention for label-specific queue health. | |
| ### Details | |
| - **Runner:** assay-bpf-runner | |
| - **Status:** ${RUNNER_STATUS} | |
| - **Status Error:** ${RUNNER_STATUS_ERROR} | |
| - **Busy:** ${RUNNER_BUSY} | |
| - **Runner Labels:** ${RUNNER_LABELS} | |
| - **Required Label:** ${REQUIRED_RUNNER_LABEL} | |
| - **General Queued Workflow Runs:** ${GENERAL_QUEUED_RUNS} | |
| - **Inspected Workflow Runs:** ${INSPECTED_WORKFLOW_RUNS} | |
| - **Matching Queued Jobs:** ${MATCHING_QUEUED_JOBS} | |
| - **Queue Classification Error:** ${QUEUE_STATUS_ERROR} | |
| - **Health Reason:** ${HEALTH_REASON} | |
| - **Detected:** ${detected_at} | |
| ### Recovery Steps | |
| From a checkout of this repository on the host that owns the Multipass runner: | |
| \`\`\`bash | |
| git clone https://github.com/Rul1an/assay.git | |
| cd assay | |
| ./infra/bpf-runner/health_check.sh --recover | |
| \`\`\` | |
| If the repository is already checked out locally, run the same script from the repository root. | |
| Or manually: | |
| 1. Check VM: \`multipass list\` | |
| 2. Start VM if needed: \`multipass start assay-bpf-runner\` | |
| 3. Sync time: \`multipass exec assay-bpf-runner -- sudo timedatectl set-ntp true\` | |
| 4. Re-register runner with fresh token | |
| ### Common Causes | |
| - VM clock drift (NTP not synced) | |
| - VM suspended or stopped | |
| - Runner token expired | |
| - Network issues | |
| This issue will auto-close when the label-specific health alert condition clears. | |
| EOF | |
| )" | |
| if gh issue create --repo "$REPO" \ | |
| --title "$TITLE" \ | |
| --label "runner-health,urgent" \ | |
| --body "$ISSUE_BODY"; then | |
| echo "Created health alert issue with labels" | |
| else | |
| echo "::warning::Failed to apply labels (missing label or insufficient permission); creating issue without labels." | |
| if gh issue create --repo "$REPO" --title "$TITLE" --body "$ISSUE_BODY"; then | |
| echo "Created health alert issue without labels" | |
| else | |
| echo "::warning::Failed to create health alert issue without labels; continuing without failing the workflow." | |
| fi | |
| fi | |
| else | |
| echo "Health alert issue already exists" | |
| fi | |
| - name: Close issue if healthy | |
| if: success() | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| REPO: ${{ github.repository }} | |
| run: | | |
| set -euo pipefail | |
| TITLE="🚨 Self-hosted runner health needs attention" | |
| # Close only the exact auto-created alert issue. Other runner-health | |
| # labeled issues can be design or follow-up work and must remain open. | |
| ISSUES="$(gh issue list --repo "$REPO" --state open --label "runner-health" --limit 100 --json number,title 2>/dev/null \ | |
| | jq -r --arg title "$TITLE" '.[] | select(.title == $title) | .number' || true)" | |
| if [[ -z "$ISSUES" ]]; then | |
| ISSUES="$(gh issue list --repo "$REPO" --state open --search "$TITLE in:title" --limit 100 --json number,title \ | |
| | jq -r --arg title "$TITLE" '.[] | select(.title == $title) | .number' || true)" | |
| fi | |
| for ISSUE in $ISSUES; do | |
| gh issue close "$ISSUE" --repo "$REPO" \ | |
| --comment "✅ Runner health alert condition cleared. Auto-closing this alert." | |
| echo "Closed issue #$ISSUE" | |
| done |