Skip to content
Sentry Smoke Test

ci: strengthen CI with race detection, govulncheck, and more #1521

Sign in to view logs

ci: strengthen CI with race detection, govulncheck, and more

ci: strengthen CI with race detection, govulncheck, and more #1521

Workflow file for this run

.github/workflows/sentry-smoke-test.yaml at 5a47407
name: Sentry Smoke Test
on:
pull_request:
paths:
- '**.go'
- 'go.mod'
- 'go.sum'
- 'Dockerfile'
- 'sentry-logs/**'
- 'docker-compose.yml'
- 'deploy/**'
- '.github/sentry/**'
- '.github/workflows/sentry-smoke-test.yaml'
workflow_dispatch:
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: true
jobs:
sentry-smoke-test:
timeout-minutes: 15
runs-on: ubuntu-latest
steps:
- name: Checkout repository
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
- name: Install dependencies
run: |
# Install clickhouse client
sudo apt-get install -y apt-transport-https ca-certificates curl gnupg
curl -fsSL 'https://packages.clickhouse.com/rpm/lts/repodata/repomd.xml.key' | sudo gpg --dearmor -o /usr/share/keyrings/clickhouse-keyring.gpg
echo "deb [signed-by=/usr/share/keyrings/clickhouse-keyring.gpg] https://packages.clickhouse.com/deb stable main" | sudo tee \
/etc/apt/sources.list.d/clickhouse.list
sudo apt-get update
sudo apt-get install -y clickhouse-client
clickhouse-client --version
# Install yq for YAML parsing
sudo wget -qO /usr/local/bin/yq https://github.com/mikefarah/yq/releases/latest/download/yq_linux_amd64
sudo chmod +x /usr/local/bin/yq
yq --version
- name: Canary health check on beacon node
id: canary
run: |
BEACON_NODE="https://mainnet-lb-bn.utility.production.platform.ethpandaops.io"
echo "Checking beacon node health at $BEACON_NODE..."
HTTP_CODE=$(curl -s -o /dev/null -w "%{http_code}" --max-time 10 \
-H "Authorization: ${{ secrets.PLATFORM_UTILITY_NODE_AUTHORIZATION_2 }}" \
"$BEACON_NODE/eth/v1/node/health" || echo "000")
if [ "$HTTP_CODE" -lt 200 ] || [ "$HTTP_CODE" -ge 400 ]; then
echo "::warning::Beacon node unreachable (HTTP $HTTP_CODE). Skipping sentry smoke test."
echo "skip=true" >> $GITHUB_OUTPUT
else
echo "Beacon node is healthy (HTTP $HTTP_CODE)."
echo "skip=false" >> $GITHUB_OUTPUT
fi
- name: Pull docker compose images in the background
if: steps.canary.outputs.skip != 'true'
run: |
docker compose pull &
- name: Set up Docker Buildx
if: steps.canary.outputs.skip != 'true'
uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v3.11.1
- name: Build xatu image
if: steps.canary.outputs.skip != 'true'
uses: docker/build-push-action@14487ce63c7a62a4a324b0bfb37086795e31c6c1 # v6.16.0
with:
context: .
load: true
tags: ethpandaops/xatu:local
cache-from: type=gha
cache-to: type=gha,mode=max
- name: Build sentry-logs image
if: steps.canary.outputs.skip != 'true'
uses: docker/build-push-action@14487ce63c7a62a4a324b0bfb37086795e31c6c1 # v6.16.0
with:
context: ./sentry-logs
load: true
tags: ethpandaops/xatu-sentry-logs:local
cache-from: type=gha,scope=sentry-logs
cache-to: type=gha,mode=max,scope=sentry-logs
- name: Create sentry-logs directory and log file
if: steps.canary.outputs.skip != 'true'
run: |
mkdir -p deploy/local/docker-compose/sentry-logs/logs
chmod 777 deploy/local/docker-compose/sentry-logs/logs
touch deploy/local/docker-compose/sentry-logs/logs/geth.log
chmod 666 deploy/local/docker-compose/sentry-logs/logs/geth.log
- name: Run Xatu stack
if: steps.canary.outputs.skip != 'true'
timeout-minutes: 5
run: |
docker compose up --detach --quiet-pull
- name: Wait for Xatu stack to be ready
if: steps.canary.outputs.skip != 'true'
run: |
echo "Waiting for xatu-server gRPC to be ready on port 8080..."
for i in $(seq 1 30); do
if bash -c 'echo > /dev/tcp/localhost/8080' 2>/dev/null; then
echo "xatu-server is ready."
break
fi
if [ "$i" -eq 30 ]; then
echo "::error::xatu-server failed to become ready"
echo "=== Container status ==="
docker ps -a --format 'table {{.Names}}\t{{.Status}}\t{{.Ports}}'
echo "=== xatu-server logs ==="
docker logs xatu-server 2>&1 | tail -50
exit 1
fi
echo " attempt $i/30..."
sleep 2
done
echo ""
echo "=== Container status ==="
docker ps -a --format 'table {{.Names}}\t{{.Status}}\t{{.Ports}}'
- name: Inject sentry-logs test data
if: steps.canary.outputs.skip != 'true'
run: |
echo "Injecting fake log entries early so they process in background..."
# Format 1: Raw JSON (geth format from PR #33655)
for i in {1..5}; do
echo '{"level":"warn","msg":"Slow block","block":{"number":'${i}',"hash":"0x'$(printf '%064d' ${i})'","gas_used":15000000,"tx_count":150},"timing":{"execution_ms":45.2,"state_read_ms":12.1,"state_hash_ms":8.3,"commit_ms":5.4,"total_ms":71.0},"throughput":{"mgas_per_sec":211.27},"state_reads":{"accounts":1234,"storage_slots":5678,"code":89,"code_bytes":45000},"state_writes":{"accounts":234,"accounts_deleted":5,"storage_slots":890,"storage_slots_deleted":12,"code":3,"code_bytes":15000},"cache":{"account":{"hits":1000,"misses":234,"hit_rate":81.0},"storage":{"hits":5000,"misses":678,"hit_rate":88.0},"code":{"hits":80,"misses":9,"hit_rate":89.9,"hit_bytes":40000,"miss_bytes":5000}}}' >> deploy/local/docker-compose/sentry-logs/logs/geth.log
done
# Format 2: Terminal format (--log.format terminal, geth default)
for i in {6..10}; do
echo 'WARN [01-28|12:58:41.123] {"level":"warn","msg":"Slow block","block":{"number":'${i}',"hash":"0x'$(printf '%064d' ${i})'","gas_used":15000000,"tx_count":150},"timing":{"execution_ms":45.2,"state_read_ms":12.1,"state_hash_ms":8.3,"commit_ms":5.4,"total_ms":71.0},"throughput":{"mgas_per_sec":211.27},"state_reads":{"accounts":1234,"storage_slots":5678,"code":89,"code_bytes":45000},"state_writes":{"accounts":234,"accounts_deleted":5,"storage_slots":890,"storage_slots_deleted":12,"code":3,"code_bytes":15000},"cache":{"account":{"hits":1000,"misses":234,"hit_rate":81.0},"storage":{"hits":5000,"misses":678,"hit_rate":88.0},"code":{"hits":80,"misses":9,"hit_rate":89.9,"hit_bytes":40000,"miss_bytes":5000}}}' >> deploy/local/docker-compose/sentry-logs/logs/geth.log
done
# Format 3: Logfmt format (--log.format logfmt)
for i in {11..15}; do
echo 't=2026-01-28T12:58:41+0000 lvl=warn msg="{\"level\":\"warn\",\"msg\":\"Slow block\",\"block\":{\"number\":'${i}',\"hash\":\"0x'$(printf '%064d' ${i})'\",\"gas_used\":15000000,\"tx_count\":150},\"timing\":{\"execution_ms\":45.2,\"state_read_ms\":12.1,\"state_hash_ms\":8.3,\"commit_ms\":5.4,\"total_ms\":71.0},\"throughput\":{\"mgas_per_sec\":211.27},\"state_reads\":{\"accounts\":1234,\"storage_slots\":5678,\"code\":89,\"code_bytes\":45000},\"state_writes\":{\"accounts\":234,\"accounts_deleted\":5,\"storage_slots\":890,\"storage_slots_deleted\":12,\"code\":3,\"code_bytes\":15000},\"cache\":{\"account\":{\"hits\":1000,\"misses\":234,\"hit_rate\":81.0},\"storage\":{\"hits\":5000,\"misses\":678,\"hit_rate\":88.0},\"code\":{\"hits\":80,\"misses\":9,\"hit_rate\":89.9,\"hit_bytes\":40000,\"miss_bytes\":5000}}}"' >> deploy/local/docker-compose/sentry-logs/logs/geth.log
done
echo "Injected 15 test log entries (5 raw JSON, 5 terminal, 5 logfmt)"
- name: Create sentry config
if: steps.canary.outputs.skip != 'true'
run: |
NETWORK_NAME=$(yq '.network.name' .github/sentry/seeding.yaml)
cat <<EOF > /tmp/sentry_config.yaml
logging: debug
name: sentry-smoke-test
labels:
ethpandaops: rocks
ntpServer: time.google.com
ethereum:
beaconNodeAddress: "https://mainnet-lb-bn.utility.production.platform.ethpandaops.io"
beaconNodeHeaders:
Authorization: "AUTH_HEADER"
overrideNetworkName: "$NETWORK_NAME"
beaconSubscriptions:
- head
- block
- attestation
- contribution_and_proof
- finalized_checkpoint
beaconCommittees:
enabled: false
proposerDuty:
enabled: false
forkChoice:
enabled: false
attestationData:
enabled: false
outputs:
- name: xatu
type: xatu
config:
address: xatu-server:8080
tls: false
maxQueueSize: 51200
batchTimeout: 1s
exportTimeout: 30s
maxExportBatchSize: 512
EOF
- name: Set authorization header
if: steps.canary.outputs.skip != 'true'
run: |
sed -i 's/AUTH_HEADER/${{ secrets.PLATFORM_UTILITY_NODE_AUTHORIZATION_2 }}/' /tmp/sentry_config.yaml
- name: Start sentry
if: steps.canary.outputs.skip != 'true'
run: |
docker run -d --network xatu_xatu-net --name xatu-sentry -v /tmp/sentry_config.yaml:/etc/xatu/config.yaml ethpandaops/xatu:local sentry --config /etc/xatu/config.yaml
- name: Wait for sentry to start
if: steps.canary.outputs.skip != 'true'
run: |
sleep 5
docker logs xatu-sentry
- name: Verify ClickHouse has sentry data
if: steps.canary.outputs.skip != 'true'
timeout-minutes: 7
env:
SENTRY_NAME: sentry-smoke-test
run: |
.github/sentry/assert_clickhouse.sh .github/sentry/seeding.yaml
- name: Verify sentry-logs (execution_block_metrics) pipeline
if: steps.canary.outputs.skip != 'true'
timeout-minutes: 3
run: |
echo "Testing sentry-logs execution_block_metrics pipeline"
echo "Log data was injected earlier and should have been processing in background."
# Debug: show sentry-logs container logs
echo "=== sentry-logs container logs ==="
docker logs xatu-sentry-logs 2>&1 | tail -30 || echo "Container not found or no logs"
# Wait for data to reach ClickHouse
max_retries=30
retry_count=0
while true; do
data_count=$(docker exec xatu-clickhouse-01 clickhouse-client --query "SELECT COUNT(*) FROM default.execution_block_metrics" 2>/dev/null || echo "0")
if [[ "$data_count" -ge 15 ]]; then
echo "SUCCESS: execution_block_metrics has $data_count entries"
break
fi
echo "Waiting for execution_block_metrics data... (current: $data_count, attempt: $((retry_count + 1))/$max_retries)"
retry_count=$((retry_count + 1))
if [[ $retry_count -ge $max_retries ]]; then
echo "FAILED: Timeout waiting for execution_block_metrics data"
echo "=== Final sentry-logs logs ==="
docker logs xatu-sentry-logs 2>&1 | tail -100 || echo "Container not found"
echo "=== Final xatu-server logs ==="
docker logs xatu-server 2>&1 | tail -50 || echo "Container not found"
echo "=== Kafka topics ==="
docker exec xatu-kafka kafka-topics --list --bootstrap-server localhost:9092 2>/dev/null | grep execution || echo "No execution topics"
exit 1
fi
sleep 5
done
# Verify data content
echo "Sample execution_block_metrics data:"
docker exec xatu-clickhouse-01 clickhouse-client --query "SELECT block_number, total_ms, mgas_per_sec, meta_client_name FROM default.execution_block_metrics LIMIT 5"
- name: Dump diagnostic info on failure
if: failure() || cancelled()
run: |
echo "=== Container status ==="
docker ps -a --format 'table {{.Names}}\t{{.Status}}\t{{.Ports}}'
echo ""
echo "=== Docker network ==="
docker network inspect xatu_xatu-net --format '{{range .Containers}}{{.Name}} {{.IPv4Address}}{{"\n"}}{{end}}' 2>/dev/null || echo "Network not found"
echo ""
echo "=== xatu-server logs (last 50) ==="
docker logs xatu-server 2>&1 | tail -50 || echo "Container not found"
echo ""
echo "=== xatu-sentry logs (last 50) ==="
docker logs xatu-sentry 2>&1 | tail -50 || echo "Container not found"
echo ""
echo "=== xatu-sentry-logs logs (last 30) ==="
docker logs xatu-sentry-logs 2>&1 | tail -30 || echo "Container not found"
- name: Collect docker logs on failure
if: failure() || cancelled()
uses: jwalton/gh-docker-logs@2741064ab9d7af54b0b1ffb6076cf64c16f0220e # v2.2.2
with:
dest: './logs'
- name: Tar logs
if: failure() || cancelled()
run: tar cvzf ./logs.tgz ./logs
- name: Upload logs to GitHub
if: failure() || cancelled()
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
with:
name: logs.tgz
path: ./logs.tgz