sentry-smoke-test.yaml

name: Sentry Smoke Test

on:
  pull_request:
    paths:
      - '**.go'
      - 'go.mod'
      - 'go.sum'
      - 'Dockerfile'
      - 'sentry-logs/**'
      - 'docker-compose.yml'
      - 'deploy/**'
      - '.github/sentry/**'
      - '.github/workflows/sentry-smoke-test.yaml'
  workflow_dispatch:

concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: true
jobs:
  sentry-smoke-test:
    timeout-minutes: 15
    runs-on: ubuntu-latest
    steps:
    - name: Checkout repository
      uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
    - name: Install dependencies
      run: |
        # Install clickhouse client
        sudo apt-get install -y apt-transport-https ca-certificates curl gnupg
        curl -fsSL 'https://packages.clickhouse.com/rpm/lts/repodata/repomd.xml.key' | sudo gpg --dearmor -o /usr/share/keyrings/clickhouse-keyring.gpg
        echo "deb [signed-by=/usr/share/keyrings/clickhouse-keyring.gpg] https://packages.clickhouse.com/deb stable main" | sudo tee \
        /etc/apt/sources.list.d/clickhouse.list
        sudo apt-get update
        sudo apt-get install -y clickhouse-client
        clickhouse-client --version

        # Install yq for YAML parsing
        sudo wget -qO /usr/local/bin/yq https://github.com/mikefarah/yq/releases/latest/download/yq_linux_amd64
        sudo chmod +x /usr/local/bin/yq
        yq --version
    - name: Canary health check on beacon node
      id: canary
      run: |
        BEACON_NODE="https://mainnet-lb-bn.utility.production.platform.ethpandaops.io"

        echo "Checking beacon node health at $BEACON_NODE..."
        HTTP_CODE=$(curl -s -o /dev/null -w "%{http_code}" --max-time 10 \
          -H "Authorization: ${{ secrets.PLATFORM_UTILITY_NODE_AUTHORIZATION_2 }}" \
          "$BEACON_NODE/eth/v1/node/health" || echo "000")

        if [ "$HTTP_CODE" -lt 200 ] || [ "$HTTP_CODE" -ge 400 ]; then
          echo "::warning::Beacon node unreachable (HTTP $HTTP_CODE). Skipping sentry smoke test."
          echo "skip=true" >> $GITHUB_OUTPUT
        else
          echo "Beacon node is healthy (HTTP $HTTP_CODE)."
          echo "skip=false" >> $GITHUB_OUTPUT
        fi
    - name: Pull docker compose images in the background
      if: steps.canary.outputs.skip != 'true'
      run: |
        docker compose pull &
    - name: Set up Docker Buildx
      if: steps.canary.outputs.skip != 'true'
      uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v3.11.1
    - name: Build xatu image
      if: steps.canary.outputs.skip != 'true'
      uses: docker/build-push-action@14487ce63c7a62a4a324b0bfb37086795e31c6c1 # v6.16.0
      with:
        context: .
        load: true
        tags: ethpandaops/xatu:local
        cache-from: type=gha
        cache-to: type=gha,mode=max
    - name: Build sentry-logs image
      if: steps.canary.outputs.skip != 'true'
      uses: docker/build-push-action@14487ce63c7a62a4a324b0bfb37086795e31c6c1 # v6.16.0
      with:
        context: ./sentry-logs
        load: true
        tags: ethpandaops/xatu-sentry-logs:local
        cache-from: type=gha,scope=sentry-logs
        cache-to: type=gha,mode=max,scope=sentry-logs
    - name: Create sentry-logs directory and log file
      if: steps.canary.outputs.skip != 'true'
      run: |
        mkdir -p deploy/local/docker-compose/sentry-logs/logs
        chmod 777 deploy/local/docker-compose/sentry-logs/logs
        touch deploy/local/docker-compose/sentry-logs/logs/geth.log
        chmod 666 deploy/local/docker-compose/sentry-logs/logs/geth.log
    - name: Run Xatu stack
      if: steps.canary.outputs.skip != 'true'
      timeout-minutes: 5
      run: |
        docker compose up --detach --quiet-pull
    - name: Wait for Xatu stack to be ready
      if: steps.canary.outputs.skip != 'true'
      run: |
        echo "Waiting for xatu-server gRPC to be ready on port 8080..."
        for i in $(seq 1 30); do
          if bash -c 'echo > /dev/tcp/localhost/8080' 2>/dev/null; then
            echo "xatu-server is ready."
            break
          fi
          if [ "$i" -eq 30 ]; then
            echo "::error::xatu-server failed to become ready"
            echo "=== Container status ==="
            docker ps -a --format 'table {{.Names}}\t{{.Status}}\t{{.Ports}}'
            echo "=== xatu-server logs ==="
            docker logs xatu-server 2>&1 | tail -50
            exit 1
          fi
          echo "  attempt $i/30..."
          sleep 2
        done

        echo ""
        echo "=== Container status ==="
        docker ps -a --format 'table {{.Names}}\t{{.Status}}\t{{.Ports}}'
    - name: Inject sentry-logs test data
      if: steps.canary.outputs.skip != 'true'
      run: |
        echo "Injecting fake log entries early so they process in background..."

        # Format 1: Raw JSON (geth format from PR #33655)
        for i in {1..5}; do
          echo '{"level":"warn","msg":"Slow block","block":{"number":'${i}',"hash":"0x'$(printf '%064d' ${i})'","gas_used":15000000,"tx_count":150},"timing":{"execution_ms":45.2,"state_read_ms":12.1,"state_hash_ms":8.3,"commit_ms":5.4,"total_ms":71.0},"throughput":{"mgas_per_sec":211.27},"state_reads":{"accounts":1234,"storage_slots":5678,"code":89,"code_bytes":45000},"state_writes":{"accounts":234,"accounts_deleted":5,"storage_slots":890,"storage_slots_deleted":12,"code":3,"code_bytes":15000},"cache":{"account":{"hits":1000,"misses":234,"hit_rate":81.0},"storage":{"hits":5000,"misses":678,"hit_rate":88.0},"code":{"hits":80,"misses":9,"hit_rate":89.9,"hit_bytes":40000,"miss_bytes":5000}}}' >> deploy/local/docker-compose/sentry-logs/logs/geth.log
        done

        # Format 2: Terminal format (--log.format terminal, geth default)
        for i in {6..10}; do
          echo 'WARN [01-28|12:58:41.123] {"level":"warn","msg":"Slow block","block":{"number":'${i}',"hash":"0x'$(printf '%064d' ${i})'","gas_used":15000000,"tx_count":150},"timing":{"execution_ms":45.2,"state_read_ms":12.1,"state_hash_ms":8.3,"commit_ms":5.4,"total_ms":71.0},"throughput":{"mgas_per_sec":211.27},"state_reads":{"accounts":1234,"storage_slots":5678,"code":89,"code_bytes":45000},"state_writes":{"accounts":234,"accounts_deleted":5,"storage_slots":890,"storage_slots_deleted":12,"code":3,"code_bytes":15000},"cache":{"account":{"hits":1000,"misses":234,"hit_rate":81.0},"storage":{"hits":5000,"misses":678,"hit_rate":88.0},"code":{"hits":80,"misses":9,"hit_rate":89.9,"hit_bytes":40000,"miss_bytes":5000}}}' >> deploy/local/docker-compose/sentry-logs/logs/geth.log
        done

        # Format 3: Logfmt format (--log.format logfmt)
        for i in {11..15}; do
          echo 't=2026-01-28T12:58:41+0000 lvl=warn msg="{\"level\":\"warn\",\"msg\":\"Slow block\",\"block\":{\"number\":'${i}',\"hash\":\"0x'$(printf '%064d' ${i})'\",\"gas_used\":15000000,\"tx_count\":150},\"timing\":{\"execution_ms\":45.2,\"state_read_ms\":12.1,\"state_hash_ms\":8.3,\"commit_ms\":5.4,\"total_ms\":71.0},\"throughput\":{\"mgas_per_sec\":211.27},\"state_reads\":{\"accounts\":1234,\"storage_slots\":5678,\"code\":89,\"code_bytes\":45000},\"state_writes\":{\"accounts\":234,\"accounts_deleted\":5,\"storage_slots\":890,\"storage_slots_deleted\":12,\"code\":3,\"code_bytes\":15000},\"cache\":{\"account\":{\"hits\":1000,\"misses\":234,\"hit_rate\":81.0},\"storage\":{\"hits\":5000,\"misses\":678,\"hit_rate\":88.0},\"code\":{\"hits\":80,\"misses\":9,\"hit_rate\":89.9,\"hit_bytes\":40000,\"miss_bytes\":5000}}}"' >> deploy/local/docker-compose/sentry-logs/logs/geth.log
        done

        echo "Injected 15 test log entries (5 raw JSON, 5 terminal, 5 logfmt)"
    - name: Create sentry config
      if: steps.canary.outputs.skip != 'true'
      run: |
        NETWORK_NAME=$(yq '.network.name' .github/sentry/seeding.yaml)

        cat <<EOF > /tmp/sentry_config.yaml
        logging: debug
        name: sentry-smoke-test
        labels:
          ethpandaops: rocks

        ntpServer: time.google.com

        ethereum:
          beaconNodeAddress: "https://mainnet-lb-bn.utility.production.platform.ethpandaops.io"
          beaconNodeHeaders:
            Authorization: "AUTH_HEADER"
          overrideNetworkName: "$NETWORK_NAME"
          beaconSubscriptions:
          - head
          - block
          - attestation
          - contribution_and_proof
          - finalized_checkpoint

        beaconCommittees:
          enabled: false

        proposerDuty:
          enabled: false

        forkChoice:
          enabled: false

        attestationData:
          enabled: false

        outputs:
          - name: xatu
            type: xatu
            config:
              address: xatu-server:8080
              tls: false
              maxQueueSize: 51200
              batchTimeout: 1s
              exportTimeout: 30s
              maxExportBatchSize: 512
        EOF
    - name: Set authorization header
      if: steps.canary.outputs.skip != 'true'
      run: |
        sed -i 's/AUTH_HEADER/${{ secrets.PLATFORM_UTILITY_NODE_AUTHORIZATION_2 }}/' /tmp/sentry_config.yaml
    - name: Start sentry
      if: steps.canary.outputs.skip != 'true'
      run: |
        docker run -d --network xatu_xatu-net --name xatu-sentry -v /tmp/sentry_config.yaml:/etc/xatu/config.yaml ethpandaops/xatu:local sentry --config /etc/xatu/config.yaml
    - name: Wait for sentry to start
      if: steps.canary.outputs.skip != 'true'
      run: |
        sleep 5
        docker logs xatu-sentry
    - name: Verify ClickHouse has sentry data
      if: steps.canary.outputs.skip != 'true'
      timeout-minutes: 7
      env:
        SENTRY_NAME: sentry-smoke-test
      run: |
        .github/sentry/assert_clickhouse.sh .github/sentry/seeding.yaml
    - name: Verify sentry-logs (execution_block_metrics) pipeline
      if: steps.canary.outputs.skip != 'true'
      timeout-minutes: 3
      run: |
        echo "Testing sentry-logs execution_block_metrics pipeline"
        echo "Log data was injected earlier and should have been processing in background."

        # Debug: show sentry-logs container logs
        echo "=== sentry-logs container logs ==="
        docker logs xatu-sentry-logs 2>&1 | tail -30 || echo "Container not found or no logs"

        # Wait for data to reach ClickHouse
        max_retries=30
        retry_count=0
        while true; do
          data_count=$(docker exec xatu-clickhouse-01 clickhouse-client --query "SELECT COUNT(*) FROM default.execution_block_metrics" 2>/dev/null || echo "0")
          if [[ "$data_count" -ge 15 ]]; then
            echo "SUCCESS: execution_block_metrics has $data_count entries"
            break
          fi
          echo "Waiting for execution_block_metrics data... (current: $data_count, attempt: $((retry_count + 1))/$max_retries)"
          retry_count=$((retry_count + 1))
          if [[ $retry_count -ge $max_retries ]]; then
            echo "FAILED: Timeout waiting for execution_block_metrics data"
            echo "=== Final sentry-logs logs ==="
            docker logs xatu-sentry-logs 2>&1 | tail -100 || echo "Container not found"
            echo "=== Final xatu-server logs ==="
            docker logs xatu-server 2>&1 | tail -50 || echo "Container not found"
            echo "=== Kafka topics ==="
            docker exec xatu-kafka kafka-topics --list --bootstrap-server localhost:9092 2>/dev/null | grep execution || echo "No execution topics"
            exit 1
          fi
          sleep 5
        done

        # Verify data content
        echo "Sample execution_block_metrics data:"
        docker exec xatu-clickhouse-01 clickhouse-client --query "SELECT block_number, total_ms, mgas_per_sec, meta_client_name FROM default.execution_block_metrics LIMIT 5"

    - name: Dump diagnostic info on failure
      if: failure() || cancelled()
      run: |
        echo "=== Container status ==="
        docker ps -a --format 'table {{.Names}}\t{{.Status}}\t{{.Ports}}'
        echo ""
        echo "=== Docker network ==="
        docker network inspect xatu_xatu-net --format '{{range .Containers}}{{.Name}} {{.IPv4Address}}{{"\n"}}{{end}}' 2>/dev/null || echo "Network not found"
        echo ""
        echo "=== xatu-server logs (last 50) ==="
        docker logs xatu-server 2>&1 | tail -50 || echo "Container not found"
        echo ""
        echo "=== xatu-sentry logs (last 50) ==="
        docker logs xatu-sentry 2>&1 | tail -50 || echo "Container not found"
        echo ""
        echo "=== xatu-sentry-logs logs (last 30) ==="
        docker logs xatu-sentry-logs 2>&1 | tail -30 || echo "Container not found"
    - name: Collect docker logs on failure
      if: failure() || cancelled()
      uses: jwalton/gh-docker-logs@2741064ab9d7af54b0b1ffb6076cf64c16f0220e # v2.2.2
      with:
        dest: './logs'
    - name: Tar logs
      if: failure() || cancelled()
      run: tar cvzf ./logs.tgz ./logs
    - name: Upload logs to GitHub
      if: failure() || cancelled()
      uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
      with:
        name: logs.tgz
        path: ./logs.tgz