feat: add typed ClickHouse read package (pkg/proto/clickhouse) #1862
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Sentry Smoke Test | |
| on: | |
| pull_request: | |
| paths: | |
| - '**.go' | |
| - 'go.mod' | |
| - 'go.sum' | |
| - 'Dockerfile' | |
| - 'sentry-logs/**' | |
| - 'docker-compose.yml' | |
| - 'deploy/**' | |
| - '.github/sentry/**' | |
| - '.github/workflows/sentry-smoke-test.yaml' | |
| workflow_dispatch: | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.ref }} | |
| cancel-in-progress: true | |
| jobs: | |
| sentry-smoke-test: | |
| timeout-minutes: 20 | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 | |
| - name: Install dependencies | |
| run: | | |
| # Install clickhouse client | |
| sudo apt-get install -y apt-transport-https ca-certificates curl gnupg | |
| curl -fsSL 'https://packages.clickhouse.com/rpm/lts/repodata/repomd.xml.key' | sudo gpg --dearmor -o /usr/share/keyrings/clickhouse-keyring.gpg | |
| echo "deb [signed-by=/usr/share/keyrings/clickhouse-keyring.gpg] https://packages.clickhouse.com/deb stable main" | sudo tee \ | |
| /etc/apt/sources.list.d/clickhouse.list | |
| sudo apt-get update | |
| sudo apt-get install -y clickhouse-client | |
| clickhouse-client --version | |
| # Install yq for YAML parsing | |
| sudo wget -qO /usr/local/bin/yq https://github.com/mikefarah/yq/releases/latest/download/yq_linux_amd64 | |
| sudo chmod +x /usr/local/bin/yq | |
| yq --version | |
| - name: Canary health check on beacon node | |
| id: canary | |
| run: | | |
| BEACON_NODE="https://mainnet-lb-bn.utility.production.platform.ethpandaops.io" | |
| echo "Checking beacon node health at $BEACON_NODE..." | |
| HTTP_CODE=$(curl -s -o /dev/null -w "%{http_code}" --max-time 10 \ | |
| -H "Authorization: ${{ secrets.PLATFORM_UTILITY_NODE_AUTHORIZATION_2 }}" \ | |
| "$BEACON_NODE/eth/v1/node/health" || echo "000") | |
| if [ "$HTTP_CODE" -lt 200 ] || [ "$HTTP_CODE" -ge 400 ]; then | |
| echo "::warning::Beacon node unreachable (HTTP $HTTP_CODE). Skipping sentry smoke test." | |
| echo "skip=true" >> $GITHUB_OUTPUT | |
| else | |
| echo "Beacon node is healthy (HTTP $HTTP_CODE)." | |
| echo "skip=false" >> $GITHUB_OUTPUT | |
| fi | |
| - name: Pull docker compose images in the background | |
| if: steps.canary.outputs.skip != 'true' | |
| run: | | |
| docker compose pull & | |
| - name: Set up Docker Buildx | |
| if: steps.canary.outputs.skip != 'true' | |
| uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v3.11.1 | |
| - name: Build xatu image | |
| if: steps.canary.outputs.skip != 'true' | |
| uses: docker/build-push-action@14487ce63c7a62a4a324b0bfb37086795e31c6c1 # v6.16.0 | |
| with: | |
| context: . | |
| load: true | |
| tags: ethpandaops/xatu:local | |
| # Pull the cryo-builder layer from the release pipeline's registry | |
| # buildcache so a cold PR branch reuses the prebuilt cryo instead of | |
| # recompiling it (~10 min). Read-only: PRs keep writing only gha cache. | |
| cache-from: | | |
| type=gha | |
| type=registry,ref=ethpandaops/xatu:cryo-buildcache | |
| cache-to: type=gha,mode=max | |
| - name: Build sentry-logs image | |
| if: steps.canary.outputs.skip != 'true' | |
| uses: docker/build-push-action@14487ce63c7a62a4a324b0bfb37086795e31c6c1 # v6.16.0 | |
| with: | |
| context: ./sentry-logs | |
| load: true | |
| tags: ethpandaops/xatu-sentry-logs:local | |
| cache-from: type=gha,scope=sentry-logs | |
| cache-to: type=gha,mode=max,scope=sentry-logs | |
| - name: Create sentry-logs directory and log file | |
| if: steps.canary.outputs.skip != 'true' | |
| run: | | |
| mkdir -p deploy/local/docker-compose/sentry-logs/logs | |
| chmod 777 deploy/local/docker-compose/sentry-logs/logs | |
| touch deploy/local/docker-compose/sentry-logs/logs/geth.log | |
| chmod 666 deploy/local/docker-compose/sentry-logs/logs/geth.log | |
| - name: Run Xatu stack | |
| if: steps.canary.outputs.skip != 'true' | |
| timeout-minutes: 10 | |
| run: | | |
| docker compose up --detach --quiet-pull | |
| - name: Wait for Xatu stack to be ready | |
| if: steps.canary.outputs.skip != 'true' | |
| run: | | |
| echo "Waiting for xatu-server gRPC to be ready on port 8080..." | |
| for i in $(seq 1 30); do | |
| if bash -c 'echo > /dev/tcp/localhost/8080' 2>/dev/null; then | |
| echo "xatu-server is ready." | |
| break | |
| fi | |
| if [ "$i" -eq 30 ]; then | |
| echo "::error::xatu-server failed to become ready" | |
| echo "=== Container status ===" | |
| docker ps -a --format 'table {{.Names}}\t{{.Status}}\t{{.Ports}}' | |
| echo "=== xatu-server logs ===" | |
| docker logs xatu-server 2>&1 | tail -50 | |
| exit 1 | |
| fi | |
| echo " attempt $i/30..." | |
| sleep 2 | |
| done | |
| # xatu-consumoor reads from kafka and writes to clickhouse. The | |
| # downstream "Verify ClickHouse has data" step polls with retries, | |
| # so settling time for kafka topic auto-create + consumoor | |
| # discovery (~60s topic refresh) is absorbed there. | |
| echo "Waiting for xatu-consumoor to be running..." | |
| for i in $(seq 1 30); do | |
| if [ "$(docker inspect -f '{{.State.Status}}' xatu-consumoor 2>/dev/null)" = "running" ]; then | |
| echo "xatu-consumoor is running." | |
| break | |
| fi | |
| if [ "$i" -eq 30 ]; then | |
| echo "::error::xatu-consumoor failed to start" | |
| docker logs xatu-consumoor 2>&1 | tail -50 | |
| exit 1 | |
| fi | |
| echo " attempt $i/30..." | |
| sleep 2 | |
| done | |
| echo "" | |
| echo "=== Container status ===" | |
| docker ps -a --format 'table {{.Names}}\t{{.Status}}\t{{.Ports}}' | |
| - name: Inject sentry-logs test data | |
| if: steps.canary.outputs.skip != 'true' | |
| run: | | |
| echo "Injecting fake log entries early so they process in background..." | |
| # Format 1: Raw JSON (geth format from PR #33655) | |
| for i in {1..5}; do | |
| echo '{"level":"warn","msg":"Slow block","block":{"number":'${i}',"hash":"0x'$(printf '%064d' ${i})'","gas_used":15000000,"tx_count":150},"timing":{"execution_ms":45.2,"state_read_ms":12.1,"state_hash_ms":8.3,"commit_ms":5.4,"total_ms":71.0},"throughput":{"mgas_per_sec":211.27},"state_reads":{"accounts":1234,"storage_slots":5678,"code":89,"code_bytes":45000},"state_writes":{"accounts":234,"accounts_deleted":5,"storage_slots":890,"storage_slots_deleted":12,"code":3,"code_bytes":15000},"cache":{"account":{"hits":1000,"misses":234,"hit_rate":81.0},"storage":{"hits":5000,"misses":678,"hit_rate":88.0},"code":{"hits":80,"misses":9,"hit_rate":89.9,"hit_bytes":40000,"miss_bytes":5000}}}' >> deploy/local/docker-compose/sentry-logs/logs/geth.log | |
| done | |
| # Format 2: Terminal format (--log.format terminal, geth default) | |
| for i in {6..10}; do | |
| echo 'WARN [01-28|12:58:41.123] {"level":"warn","msg":"Slow block","block":{"number":'${i}',"hash":"0x'$(printf '%064d' ${i})'","gas_used":15000000,"tx_count":150},"timing":{"execution_ms":45.2,"state_read_ms":12.1,"state_hash_ms":8.3,"commit_ms":5.4,"total_ms":71.0},"throughput":{"mgas_per_sec":211.27},"state_reads":{"accounts":1234,"storage_slots":5678,"code":89,"code_bytes":45000},"state_writes":{"accounts":234,"accounts_deleted":5,"storage_slots":890,"storage_slots_deleted":12,"code":3,"code_bytes":15000},"cache":{"account":{"hits":1000,"misses":234,"hit_rate":81.0},"storage":{"hits":5000,"misses":678,"hit_rate":88.0},"code":{"hits":80,"misses":9,"hit_rate":89.9,"hit_bytes":40000,"miss_bytes":5000}}}' >> deploy/local/docker-compose/sentry-logs/logs/geth.log | |
| done | |
| # Format 3: Logfmt format (--log.format logfmt) | |
| for i in {11..15}; do | |
| echo 't=2026-01-28T12:58:41+0000 lvl=warn msg="{\"level\":\"warn\",\"msg\":\"Slow block\",\"block\":{\"number\":'${i}',\"hash\":\"0x'$(printf '%064d' ${i})'\",\"gas_used\":15000000,\"tx_count\":150},\"timing\":{\"execution_ms\":45.2,\"state_read_ms\":12.1,\"state_hash_ms\":8.3,\"commit_ms\":5.4,\"total_ms\":71.0},\"throughput\":{\"mgas_per_sec\":211.27},\"state_reads\":{\"accounts\":1234,\"storage_slots\":5678,\"code\":89,\"code_bytes\":45000},\"state_writes\":{\"accounts\":234,\"accounts_deleted\":5,\"storage_slots\":890,\"storage_slots_deleted\":12,\"code\":3,\"code_bytes\":15000},\"cache\":{\"account\":{\"hits\":1000,\"misses\":234,\"hit_rate\":81.0},\"storage\":{\"hits\":5000,\"misses\":678,\"hit_rate\":88.0},\"code\":{\"hits\":80,\"misses\":9,\"hit_rate\":89.9,\"hit_bytes\":40000,\"miss_bytes\":5000}}}"' >> deploy/local/docker-compose/sentry-logs/logs/geth.log | |
| done | |
| echo "Injected 15 test log entries (5 raw JSON, 5 terminal, 5 logfmt)" | |
| # State metrics fixture for execution_state_size_delta + execution_mpt_depth. | |
| # Raw JSON format only; the three-format coverage above already exercises | |
| # the parser path. State root / parent state root are 66-char hex strings. | |
| for i in {1..5}; do | |
| echo '{"level":"info","msg":"State metrics","block_number":'${i}',"state_root":"0x'$(printf '%064d' ${i})'","parent_state_root":"0x'$(printf '%064d' $((i - 1)))'","writes":{"account":5,"account_bytes":305,"account_trienode":23,"account_trienode_bytes":8379,"contract_code":0,"contract_code_bytes":0,"storage":0,"storage_bytes":0,"storage_trienode":0,"storage_trienode_bytes":0},"deletes":{"account":5,"account_bytes":296,"account_trienode":23,"account_trienode_bytes":8370,"contract_code":0,"contract_code_bytes":0,"storage":0,"storage_bytes":0,"storage_trienode":0,"storage_trienode_bytes":0},"depth":{"total_account_written_nodes":23,"total_account_written_bytes":8379,"total_account_deleted_nodes":23,"total_account_deleted_bytes":8370,"total_storage_written_nodes":0,"total_storage_written_bytes":0,"total_storage_deleted_nodes":0,"total_storage_deleted_bytes":0,"account_written_nodes":{"0":1,"1":5,"2":5,"3":5,"4":5,"5":2},"account_written_bytes":{"0":532,"1":2660,"2":2660,"3":1636,"4":577,"5":314},"account_deleted_nodes":{"0":1,"1":5,"2":5,"3":5,"4":5,"5":2},"account_deleted_bytes":{"0":532,"1":2660,"2":2660,"3":1636,"4":577,"5":305},"storage_written_nodes":{},"storage_written_bytes":{},"storage_deleted_nodes":{},"storage_deleted_bytes":{}}}' >> deploy/local/docker-compose/sentry-logs/logs/geth.log | |
| done | |
| echo "Injected 5 state metrics log entries" | |
| - name: Create sentry config | |
| if: steps.canary.outputs.skip != 'true' | |
| run: | | |
| NETWORK_NAME=$(yq '.network.name' .github/sentry/seeding.yaml) | |
| cat <<EOF > /tmp/sentry_config.yaml | |
| logging: debug | |
| name: sentry-smoke-test | |
| labels: | |
| ethpandaops: rocks | |
| ntpServer: time.google.com | |
| ethereum: | |
| beaconNodeAddress: "https://mainnet-lb-bn.utility.production.platform.ethpandaops.io" | |
| beaconNodeHeaders: | |
| Authorization: "AUTH_HEADER" | |
| overrideNetworkName: "$NETWORK_NAME" | |
| beaconSubscriptions: | |
| - head | |
| - block | |
| - single_attestation | |
| - finalized_checkpoint | |
| beaconCommittees: | |
| enabled: false | |
| proposerDuty: | |
| enabled: false | |
| forkChoice: | |
| enabled: false | |
| attestationData: | |
| enabled: false | |
| outputs: | |
| - name: xatu | |
| type: xatu | |
| config: | |
| address: xatu-server:8080 | |
| tls: false | |
| maxQueueSize: 51200 | |
| batchTimeout: 1s | |
| exportTimeout: 30s | |
| maxExportBatchSize: 512 | |
| EOF | |
| - name: Set authorization header | |
| if: steps.canary.outputs.skip != 'true' | |
| run: | | |
| sed -i 's/AUTH_HEADER/${{ secrets.PLATFORM_UTILITY_NODE_AUTHORIZATION_2 }}/' /tmp/sentry_config.yaml | |
| - name: Start sentry | |
| if: steps.canary.outputs.skip != 'true' | |
| run: | | |
| docker run -d --network xatu_xatu-net --name xatu-sentry -v /tmp/sentry_config.yaml:/etc/xatu/config.yaml ethpandaops/xatu:local sentry --config /etc/xatu/config.yaml | |
| - name: Wait for sentry to start | |
| if: steps.canary.outputs.skip != 'true' | |
| run: | | |
| sleep 5 | |
| docker logs xatu-sentry | |
| - name: Verify ClickHouse has sentry data | |
| if: steps.canary.outputs.skip != 'true' | |
| timeout-minutes: 7 | |
| env: | |
| SENTRY_NAME: sentry-smoke-test | |
| run: | | |
| .github/sentry/assert_clickhouse.sh .github/sentry/seeding.yaml | |
| - name: Verify sentry-logs (execution_block_metrics) pipeline | |
| if: steps.canary.outputs.skip != 'true' | |
| timeout-minutes: 3 | |
| run: | | |
| echo "Testing sentry-logs execution_block_metrics pipeline" | |
| echo "Log data was injected earlier and should have been processing in background." | |
| # Debug: show sentry-logs container logs | |
| echo "=== sentry-logs container logs ===" | |
| docker logs xatu-sentry-logs 2>&1 | tail -30 || echo "Container not found or no logs" | |
| # Wait for data to reach ClickHouse | |
| max_retries=30 | |
| retry_count=0 | |
| while true; do | |
| data_count=$(docker exec xatu-clickhouse-01 clickhouse-client --query "SELECT COUNT(*) FROM default.execution_block_metrics" 2>/dev/null || echo "0") | |
| if [[ "$data_count" -ge 15 ]]; then | |
| echo "SUCCESS: execution_block_metrics has $data_count entries" | |
| break | |
| fi | |
| echo "Waiting for execution_block_metrics data... (current: $data_count, attempt: $((retry_count + 1))/$max_retries)" | |
| retry_count=$((retry_count + 1)) | |
| if [[ $retry_count -ge $max_retries ]]; then | |
| echo "FAILED: Timeout waiting for execution_block_metrics data" | |
| echo "=== Final sentry-logs logs ===" | |
| docker logs xatu-sentry-logs 2>&1 | tail -100 || echo "Container not found" | |
| echo "=== Final xatu-server logs ===" | |
| docker logs xatu-server 2>&1 | tail -50 || echo "Container not found" | |
| echo "=== Kafka topics ===" | |
| docker exec xatu-kafka kafka-topics --list --bootstrap-server localhost:9092 2>/dev/null | grep xatu-protobuf || echo "No xatu-protobuf topics" | |
| echo "=== xatu-consumoor logs ===" | |
| docker logs xatu-consumoor 2>&1 | tail -50 || echo "Container not found" | |
| exit 1 | |
| fi | |
| sleep 5 | |
| done | |
| # Verify data content | |
| echo "Sample execution_block_metrics data:" | |
| docker exec xatu-clickhouse-01 clickhouse-client --query "SELECT block_number, total_ms, mgas_per_sec, meta_client_name FROM default.execution_block_metrics LIMIT 5" | |
| - name: Verify sentry-logs (execution_state_size_delta + execution_mpt_depth) pipelines | |
| if: steps.canary.outputs.skip != 'true' | |
| timeout-minutes: 3 | |
| run: | | |
| echo "Testing sentry-logs state metrics pipelines" | |
| # Each "State metrics" log fans out to both tables, so we expect rows in | |
| # each independently. | |
| for table in execution_state_size_delta execution_mpt_depth; do | |
| max_retries=30 | |
| retry_count=0 | |
| while true; do | |
| data_count=$(docker exec xatu-clickhouse-01 clickhouse-client --query "SELECT COUNT(*) FROM default.${table}" 2>/dev/null || echo "0") | |
| if [[ "$data_count" -ge 5 ]]; then | |
| echo "SUCCESS: ${table} has $data_count entries" | |
| break | |
| fi | |
| echo "Waiting for ${table} data... (current: $data_count, attempt: $((retry_count + 1))/$max_retries)" | |
| retry_count=$((retry_count + 1)) | |
| if [[ $retry_count -ge $max_retries ]]; then | |
| echo "FAILED: Timeout waiting for ${table} data" | |
| echo "=== Final sentry-logs logs ===" | |
| docker logs xatu-sentry-logs 2>&1 | tail -100 || echo "Container not found" | |
| echo "=== Final xatu-server logs ===" | |
| docker logs xatu-server 2>&1 | tail -50 || echo "Container not found" | |
| echo "=== xatu-consumoor logs ===" | |
| docker logs xatu-consumoor 2>&1 | tail -50 || echo "Container not found" | |
| exit 1 | |
| fi | |
| sleep 5 | |
| done | |
| done | |
| echo "Sample execution_state_size_delta data:" | |
| docker exec xatu-clickhouse-01 clickhouse-client --query "SELECT block_number, account_writes, account_deletes, account_delta, meta_client_name FROM default.execution_state_size_delta LIMIT 5" | |
| echo "Sample execution_mpt_depth data:" | |
| docker exec xatu-clickhouse-01 clickhouse-client --query "SELECT block_number, total_account_written_nodes, total_account_deleted_nodes, account_written_nodes, meta_client_name FROM default.execution_mpt_depth LIMIT 5" | |
| - name: Dump diagnostic info on failure | |
| if: failure() || cancelled() | |
| run: | | |
| echo "=== Container status ===" | |
| docker ps -a --format 'table {{.Names}}\t{{.Status}}\t{{.Ports}}' | |
| echo "" | |
| echo "=== Docker network ===" | |
| docker network inspect xatu_xatu-net --format '{{range .Containers}}{{.Name}} {{.IPv4Address}}{{"\n"}}{{end}}' 2>/dev/null || echo "Network not found" | |
| echo "" | |
| echo "=== xatu-server logs (last 50) ===" | |
| docker logs xatu-server 2>&1 | tail -50 || echo "Container not found" | |
| echo "" | |
| echo "=== xatu-sentry logs (last 50) ===" | |
| docker logs xatu-sentry 2>&1 | tail -50 || echo "Container not found" | |
| echo "" | |
| echo "=== xatu-sentry-logs logs (last 30) ===" | |
| docker logs xatu-sentry-logs 2>&1 | tail -30 || echo "Container not found" | |
| - name: Collect docker logs on failure | |
| if: failure() || cancelled() | |
| uses: jwalton/gh-docker-logs@2741064ab9d7af54b0b1ffb6076cf64c16f0220e # v2.2.2 | |
| with: | |
| dest: './logs' | |
| - name: Tar logs | |
| if: failure() || cancelled() | |
| run: tar cvzf ./logs.tgz ./logs | |
| - name: Upload logs to GitHub | |
| if: failure() || cancelled() | |
| uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 | |
| with: | |
| name: logs.tgz | |
| path: ./logs.tgz |