Skip to content
Cannon Smoke Test

feat: add typed ClickHouse read package (pkg/proto/clickhouse) #1420

Sign in to view logs

feat: add typed ClickHouse read package (pkg/proto/clickhouse)

feat: add typed ClickHouse read package (pkg/proto/clickhouse) #1420

Workflow file for this run

.github/workflows/cannon-smoke-test.yaml at 751aa23
name: Cannon Smoke Test
on:
pull_request:
paths:
- '**.go'
- 'go.mod'
- 'go.sum'
- 'Dockerfile'
- 'docker-compose.yml'
- 'deploy/**'
- '.github/cannon/**'
- '.github/workflows/cannon-smoke-test.yaml'
workflow_dispatch:
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: true
jobs:
cannon-smoke-test:
timeout-minutes: 25
runs-on: ubuntu-latest
steps:
- name: Checkout repository
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
- name: Install dependencies
run: |
# Install clickhouse client
sudo apt-get install -y apt-transport-https ca-certificates curl gnupg
curl -fsSL 'https://packages.clickhouse.com/rpm/lts/repodata/repomd.xml.key' | sudo gpg --dearmor -o /usr/share/keyrings/clickhouse-keyring.gpg
echo "deb [signed-by=/usr/share/keyrings/clickhouse-keyring.gpg] https://packages.clickhouse.com/deb stable main" | sudo tee \
/etc/apt/sources.list.d/clickhouse.list
sudo apt-get update
sudo apt-get install -y clickhouse-client
clickhouse-client --version
# Install yq for YAML parsing
sudo wget -qO /usr/local/bin/yq https://github.com/mikefarah/yq/releases/latest/download/yq_linux_amd64
sudo chmod +x /usr/local/bin/yq
yq --version
- name: Pull docker compose images in the background
run: |
docker compose pull &
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v3.11.1
- name: Build xatu image
uses: docker/build-push-action@14487ce63c7a62a4a324b0bfb37086795e31c6c1 # v6.16.0
with:
context: .
load: true
tags: ethpandaops/xatu:local
# Pull the cryo-builder layer from the release pipeline's registry
# buildcache so a cold PR branch reuses the prebuilt cryo instead of
# recompiling it (~10 min). Read-only: PRs keep writing only gha cache.
cache-from: |
type=gha
type=registry,ref=ethpandaops/xatu:cryo-buildcache
cache-to: type=gha,mode=max
- name: Fetch finalized epoch from beacon node
id: beacon
run: |
# Read config
BEACON_NODE="https://mainnet-archive-lb-bn.utility.production.platform.ethpandaops.io"
EPOCH_OFFSET=$(yq '.epoch_offset' .github/cannon/seeding.yaml)
NETWORK_ID=$(yq '.network.id' .github/cannon/seeding.yaml)
NETWORK_NAME=$(yq '.network.name' .github/cannon/seeding.yaml)
echo "Fetching finalized checkpoint from $BEACON_NODE..."
FINALIZED_RESPONSE=$(curl -s -H "Authorization: ${{ secrets.PLATFORM_UTILITY_NODE_AUTHORIZATION_2 }}" \
"$BEACON_NODE/eth/v1/beacon/states/finalized/finality_checkpoints")
FINALIZED_EPOCH=$(echo "$FINALIZED_RESPONSE" | jq -r '.data.finalized.epoch')
if [ -z "$FINALIZED_EPOCH" ] || [ "$FINALIZED_EPOCH" == "null" ]; then
echo "Failed to fetch finalized epoch"
echo "Response: $FINALIZED_RESPONSE"
exit 1
fi
# Calculate seed epoch (finalized - offset - 1 because cannon processes FROM this epoch ascending)
SEED_EPOCH=$((FINALIZED_EPOCH - EPOCH_OFFSET - 1))
# Target epoch is what we expect to see data for
TARGET_EPOCH=$((FINALIZED_EPOCH - EPOCH_OFFSET))
echo "Finalized epoch: $FINALIZED_EPOCH"
echo "Epoch offset: $EPOCH_OFFSET"
echo "Seed epoch: $SEED_EPOCH"
echo "Target epoch: $TARGET_EPOCH"
echo "Network: $NETWORK_NAME (ID: $NETWORK_ID)"
# Set outputs
echo "finalized_epoch=$FINALIZED_EPOCH" >> $GITHUB_OUTPUT
echo "seed_epoch=$SEED_EPOCH" >> $GITHUB_OUTPUT
echo "target_epoch=$TARGET_EPOCH" >> $GITHUB_OUTPUT
echo "network_id=$NETWORK_ID" >> $GITHUB_OUTPUT
echo "network_name=$NETWORK_NAME" >> $GITHUB_OUTPUT
- name: Run Xatu stack
timeout-minutes: 10
shell: bash
run: |
# Direct-CH path only — cannon talks to ClickHouse via ch-go and uses
# xatu-server only as the coordinator gRPC endpoint. Kafka, vector,
# consumoor, sentry-logs, tempo, prometheus, grafana, nginx are all
# off the path; bringing them up just adds startup contention and
# CI flakes (the "xatu-clickhouse-02 exit 138" / vector readiness
# races we kept rerunning to dodge). Listing leaf services pulls
# their depends_on chains: postgres + zookeepers + clickhouse-01/02
# come up automatically.
docker compose up --detach --quiet-pull \
xatu-postgres-migrator \
xatu-server \
xatu-clickhouse-migrator
- name: Start MinIO + create blob bucket
run: |
# MinIO is the S3-compatible target for the s3blobstore output sink.
# Attached to xatu's docker network so cannon can reach it as
# "xatu-minio:9000". Not exposed on the host — the docker-compose
# ClickHouse already maps :9000 — and nothing outside the network
# needs to reach MinIO.
docker run -d \
--name xatu-minio \
--network xatu_xatu-net \
--hostname xatu-minio \
-e MINIO_ROOT_USER=minioadmin \
-e MINIO_ROOT_PASSWORD=minioadmin \
minio/minio:RELEASE.2024-09-13T20-26-02Z \
server /data
echo "Waiting for MinIO and creating bucket..."
for i in $(seq 1 30); do
if docker run --rm --network xatu_xatu-net \
--entrypoint sh \
minio/mc:RELEASE.2024-09-16T17-43-14Z \
-c "mc alias set local http://xatu-minio:9000 minioadmin minioadmin && mc mb --ignore-existing local/xatu-blobs" \
>/dev/null 2>&1; then
echo "MinIO is ready and bucket created."
break
fi
if [ "$i" -eq 30 ]; then
echo "::error::MinIO failed to become ready (could not create bucket)"
docker logs xatu-minio 2>&1 | tail -50
exit 1
fi
sleep 2
done
- name: Wait for Xatu stack to be ready
run: |
echo "Waiting for xatu-server gRPC to be ready on port 8080..."
for i in $(seq 1 30); do
if bash -c 'echo > /dev/tcp/localhost/8080' 2>/dev/null; then
echo "xatu-server is ready."
break
fi
if [ "$i" -eq 30 ]; then
echo "::error::xatu-server failed to become ready"
echo "=== Container status ==="
docker ps -a --format 'table {{.Names}}\t{{.Status}}\t{{.Ports}}'
echo "=== xatu-server logs ==="
docker logs xatu-server 2>&1 | tail -50
exit 1
fi
echo " attempt $i/30..."
sleep 2
done
# Cannon's Writer.Start runs ValidateTables/ValidateColumns against
# the target CH and fails fatal if the schema isn't applied. So
# block until xatu-clickhouse-migrator has exited successfully.
echo "Waiting for xatu-clickhouse-migrator to apply the schema..."
for i in $(seq 1 120); do
status=$(docker inspect --format='{{.State.Status}}' xatu-clickhouse-migrator 2>/dev/null || echo "missing")
exit_code=$(docker inspect --format='{{.State.ExitCode}}' xatu-clickhouse-migrator 2>/dev/null || echo "")
if [ "$status" = "exited" ] && [ "$exit_code" = "0" ]; then
echo "xatu-clickhouse-migrator completed successfully."
break
fi
if [ "$status" = "exited" ] && [ "$exit_code" != "0" ]; then
echo "::error::xatu-clickhouse-migrator failed (exit_code=$exit_code)"
docker logs xatu-clickhouse-migrator 2>&1 | tail -100
exit 1
fi
if [ "$i" -eq 120 ]; then
echo "::error::xatu-clickhouse-migrator did not finish within 240s (status=$status)"
docker logs xatu-clickhouse-migrator 2>&1 | tail -100
exit 1
fi
echo " migrator status=$status (attempt $i/120)..."
sleep 2
done
echo ""
echo "=== Container status ==="
docker ps -a --format 'table {{.Names}}\t{{.Status}}\t{{.Ports}}'
- name: Seed Postgres with cannon data
run: |
set -e
NETWORK_ID="${{ steps.beacon.outputs.network_id }}"
SEED_EPOCH="${{ steps.beacon.outputs.seed_epoch }}"
# Wait for Postgres to be ready
echo "Waiting for Postgres to be ready..."
docker exec xatu-postgres sh -c 'while ! pg_isready; do sleep 1; done'
echo "Postgres is ready."
# Wait for the cannon_location table to be created
echo "Waiting for cannon_location table to be created..."
until docker exec xatu-postgres psql -U user -d xatu -c "SELECT 1 FROM cannon_location LIMIT 1" &> /dev/null
do
echo "Table not ready yet. Waiting 5 seconds..."
sleep 5
done
echo "cannon_location table is now available."
# Generate and execute seeding commands for each event type
echo "Seeding cannon_location table..."
EVENT_TYPES=$(yq '.event_types[].name' .github/cannon/seeding.yaml)
for TYPE in $EVENT_TYPES; do
VALUE="{\"backfillingCheckpointMarker\":{\"finalizedEpoch\":\"$SEED_EPOCH\"}}"
SQL="INSERT INTO cannon_location (network_id, type, value) VALUES ('$NETWORK_ID', '$TYPE', '$VALUE') ON CONFLICT (network_id, type) DO UPDATE SET value = EXCLUDED.value, update_time = now();"
echo "Seeding $TYPE at epoch $SEED_EPOCH"
docker exec xatu-postgres psql -U user -d xatu -c "$SQL"
done
echo "Seeding completed successfully."
- name: Show the state of the database
run: |
docker exec xatu-postgres psql -U user -d xatu -c "SELECT * FROM cannon_location;"
- name: Create Xatu Cannon config
run: |
NETWORK_NAME="${{ steps.beacon.outputs.network_name }}"
cat <<EOF > /tmp/cannon_config.yaml
logging: debug
name: cannon-smoke-test
labels:
ethpandaops: rocks
# Derivers are grouped by layer (consensus/execution). Consensus
# derivers default to enabled:true, so every deriver this test does NOT
# assert must be explicitly disabled. Execution (cryo) derivers default
# to enabled:false and are omitted entirely — this test is consensus-only,
# so no ethereum.execution / EL RPC is required.
derivers:
consensus:
withdrawal:
enabled: true
executionTransaction:
enabled: true
beaconBlock:
enabled: true
beaconBlobSidecar:
enabled: true
proposerDuty:
enabled: true
beaconCommittee:
enabled: true
attesterSlashing:
enabled: false
proposerSlashing:
enabled: false
blsToExecutionChange:
enabled: false
deposit:
enabled: false
voluntaryExit:
enabled: false
elaboratedAttestation:
enabled: false
beaconValidators:
enabled: false
beaconSyncCommittee:
enabled: false
beaconBlockSyncAggregate:
enabled: false
ntpServer: time.google.com
ethereum:
overrideNetworkName: "$NETWORK_NAME"
beacon:
address: "https://mainnet-archive-lb-bn.utility.production.platform.ethpandaops.io"
headers:
Authorization: "AUTH_HEADER"
coordinator:
address: xatu-server:8080
outputs:
- name: ch-direct
type: clickhouse
config:
dsn: clickhouse://default:@xatu-clickhouse-01:9000/default
chgo:
adaptiveLimiter:
enabled: false
- name: blob-archive
type: s3blobstore
config:
endpoint: xatu-minio:9000
bucket: xatu-blobs
region: us-east-1
accessKeyId: minioadmin
secretAccessKey: minioadmin
insecure: true
EOF
- name: Show config
run: |
cat /tmp/cannon_config.yaml
- name: Set authorization header
run: |
sed -i 's/AUTH_HEADER/${{ secrets.PLATFORM_UTILITY_NODE_AUTHORIZATION_2 }}/' /tmp/cannon_config.yaml
- name: Start Xatu cannon
run: |
docker run -d --network xatu_xatu-net --name xatu-cannon -v /tmp/cannon_config.yaml:/etc/xatu/config.yaml ethpandaops/xatu:local cannon --config /etc/xatu/config.yaml
- name: Wait for Xatu cannon to start
run: |
sleep 5
docker logs xatu-cannon
- name: Verify Clickhouse has data
timeout-minutes: 15
env:
TARGET_EPOCH: ${{ steps.beacon.outputs.target_epoch }}
NETWORK_NAME: ${{ steps.beacon.outputs.network_name }}
run: |
.github/cannon/assert_clickhouse.sh .github/cannon/seeding.yaml
- name: Verify s3blobstore output landed the CH-sampled blob in MinIO
timeout-minutes: 5
env:
TARGET_EPOCH: ${{ steps.beacon.outputs.target_epoch }}
NETWORK_NAME: ${{ steps.beacon.outputs.network_name }}
run: |
# Tie the MinIO assertion to a real versioned_hash that cannon
# actually derived and wrote to ClickHouse for this epoch. A
# sink that always wrote one hard-coded key would still match
# "*.gz exists" but cannot match a CH-sampled hash. We also run
# the full `gunzip -t` integrity check and sanity-check the
# payload size against the mainnet 4096-byte-field × 32-field
# blob shape (131072 bytes raw → 262146 chars of "0x"-hex).
set -euo pipefail
# 1. Sample a versioned_hash cannon actually produced for the
# target epoch. CH side has already been asserted above so
# this will return a row.
SAMPLE_HASH=$(clickhouse-client \
-h localhost --port 9000 \
-u default --password "" \
-d default \
-q "SELECT versioned_hash FROM canonical_beacon_blob_sidecar FINAL WHERE meta_network_name = '${NETWORK_NAME}' AND epoch >= ${TARGET_EPOCH} LIMIT 1" \
</dev/null | tr -d '\r\n')
if [ -z "$SAMPLE_HASH" ]; then
echo "::error::No versioned_hash sample available in canonical_beacon_blob_sidecar — cannot tie MinIO assertion to a real cannon output"
exit 1
fi
EXPECTED_KEY="${NETWORK_NAME}/${SAMPLE_HASH}.gz"
echo "CH-sampled versioned_hash: $SAMPLE_HASH"
echo "Expecting MinIO object: local/xatu-blobs/$EXPECTED_KEY"
# 2. Wait for that exact key to land in MinIO. The s3blobstore
# sink runs concurrent to the CH sink under cannon's fan-out
# but each has its own batching cadence, so retry briefly.
MAX_ATTEMPTS=30
FOUND_BYTES=""
for i in $(seq 1 "$MAX_ATTEMPTS"); do
STAT=$(docker run --rm --network xatu_xatu-net \
--entrypoint sh \
minio/mc:RELEASE.2024-09-16T17-43-14Z \
-c "mc alias set local http://xatu-minio:9000 minioadmin minioadmin >/dev/null && mc stat --json local/xatu-blobs/${EXPECTED_KEY} 2>/dev/null" \
|| true)
if echo "$STAT" | jq -e '.size' >/dev/null 2>&1; then
FOUND_BYTES=$(echo "$STAT" | jq -r '.size')
echo "Object found ($FOUND_BYTES bytes gzipped) after $i attempt(s)"
break
fi
if [ "$i" -eq "$MAX_ATTEMPTS" ]; then
echo "::error::CH-sampled key '$EXPECTED_KEY' did not appear in MinIO after $MAX_ATTEMPTS attempts"
echo "=== Available keys under ${NETWORK_NAME}/ ==="
docker run --rm --network xatu_xatu-net \
--entrypoint sh \
minio/mc:RELEASE.2024-09-16T17-43-14Z \
-c "mc alias set local http://xatu-minio:9000 minioadmin minioadmin >/dev/null && mc ls --recursive local/xatu-blobs/${NETWORK_NAME}/" || true
echo "=== xatu-cannon logs (last 100) ==="
docker logs xatu-cannon 2>&1 | tail -100
echo "=== xatu-minio logs (last 50) ==="
docker logs xatu-minio 2>&1 | tail -50
exit 1
fi
echo " not yet present (attempt $i/$MAX_ATTEMPTS)..."
sleep 5
done
# 3. Pull the object, run full gunzip -t over the whole stream,
# and inspect the decompressed payload.
TMP_GZ=$(mktemp /tmp/blob.XXXXXX.gz)
TMP_RAW=$(mktemp /tmp/blob.XXXXXX.raw)
trap 'rm -f "$TMP_GZ" "$TMP_RAW"' EXIT
docker run --rm --network xatu_xatu-net \
--entrypoint sh \
minio/mc:RELEASE.2024-09-16T17-43-14Z \
-c "mc alias set local http://xatu-minio:9000 minioadmin minioadmin >/dev/null && mc cat local/xatu-blobs/${EXPECTED_KEY}" \
> "$TMP_GZ"
if ! gunzip -t "$TMP_GZ"; then
echo "::error::gunzip -t failed on $EXPECTED_KEY — object is not a valid gzip stream"
exit 1
fi
gunzip -c "$TMP_GZ" > "$TMP_RAW"
RAW_LEN=$(wc -c < "$TMP_RAW" | tr -d ' ')
# mainnet blob = 4096 field elements × 32 bytes = 131072 bytes raw,
# hex-encoded as "0x" + 2 chars/byte = 262146 chars.
EXPECTED_LEN=262146
if [ "$RAW_LEN" -ne "$EXPECTED_LEN" ]; then
echo "::error::decompressed payload is $RAW_LEN bytes, expected $EXPECTED_LEN for a mainnet 131072-byte blob"
exit 1
fi
PREFIX=$(head -c 4 "$TMP_RAW")
if [[ "$PREFIX" != 0x* ]]; then
echo "::error::decompressed payload starts with $(printf '%q' "$PREFIX") — expected leading 0x"
exit 1
fi
# Body must be 0x + lowercase hex only. Inspect a small window;
# a full regex over 262kB hurts log readability without adding
# information beyond the length + prefix asserts above.
if ! head -c 130 "$TMP_RAW" | grep -qE '^0x[0-9a-f]+$'; then
echo "::error::decompressed payload does not begin with 0x + lowercase hex"
head -c 130 "$TMP_RAW"
echo
exit 1
fi
echo "✓ CH-sampled blob $SAMPLE_HASH archived to MinIO at $EXPECTED_KEY"
echo " gzipped size: $FOUND_BYTES bytes"
echo " decompressed size: $RAW_LEN bytes (matches mainnet blob shape)"
echo " payload prefix: $PREFIX..."
# Direct-CH path only: assert the Distributed forward queue on
# xatu-clickhouse-01 is fully drained.
#
# Cannon writes to a Distributed table. The Distributed engine
# *acks the INSERT* once rows are appended to the local
# `data/<table>/<distributed_dir>/` on disk; the actual forward to
# the target shard's _local table happens async. So a row can be
# "acked back to cannon" — and the postgres checkpoint advanced —
# while it is still waiting in the local forward queue. If the node
# crashes before that flush, the row is lost without any signal
# surfacing to cannon.
#
# This step waits briefly for the queue to drain, then asserts it
# is empty (and error_count == 0). The local docker-compose is
# cluster_2S_1R (2 shards, 1 replica each) — there are no actual
# replicas, which makes a cross-replica + select_sequential_consistency
# assertion meaningless here. The forward-queue assertion is the
# CI-topology-appropriate equivalent and tests the same durability
# invariant directly.
- name: Verify Distributed forward queue is drained
timeout-minutes: 3
run: |
set -e
# Allow async forwards to flush. Default monitor sleep is 100ms.
sleep 10
QUEUE_FILES=$(clickhouse-client \
-h localhost --port 9000 \
-u default --password "" \
-d default \
-q "SELECT coalesce(sum(data_files), 0) FROM system.distribution_queue WHERE database = 'default' AND table LIKE 'canonical_%'" \
</dev/null)
QUEUE_ERRORS=$(clickhouse-client \
-h localhost --port 9000 \
-u default --password "" \
-d default \
-q "SELECT coalesce(sum(error_count), 0) FROM system.distribution_queue WHERE database = 'default' AND table LIKE 'canonical_%'" \
</dev/null)
echo "Distributed forward queue (xatu-clickhouse-01):"
echo " pending data_files: $QUEUE_FILES"
echo " error_count: $QUEUE_ERRORS"
# Per-table breakdown for diagnostics on failure.
clickhouse-client \
-h localhost --port 9000 \
-u default --password "" \
-d default \
-q "SELECT table, data_files, data_compressed_bytes, error_count, last_exception FROM system.distribution_queue WHERE database = 'default' AND table LIKE 'canonical_%' FORMAT PrettyCompact" \
</dev/null
FAILED=0
if [[ ! "$QUEUE_FILES" =~ ^[0-9]+$ ]] || [ "$QUEUE_FILES" -ne 0 ]; then
echo "✗ Forward queue is NOT drained — $QUEUE_FILES file(s) still pending"
echo " Cannon's checkpoint advanced for rows still in the queue."
FAILED=1
fi
if [[ ! "$QUEUE_ERRORS" =~ ^[0-9]+$ ]] || [ "$QUEUE_ERRORS" -ne 0 ]; then
echo "✗ Forward queue has $QUEUE_ERRORS forward error(s)"
FAILED=1
fi
if [ "$FAILED" -ne 0 ]; then
exit 1
fi
echo "✓ All canonical_* Distributed forwards drained — every acked row is durably in a target shard"
- name: Dump diagnostic info on failure
if: failure() || cancelled()
run: |
echo "=== Container status ==="
docker ps -a --format 'table {{.Names}}\t{{.Status}}\t{{.Ports}}'
echo ""
echo "=== Docker network ==="
docker network inspect xatu_xatu-net --format '{{range .Containers}}{{.Name}} {{.IPv4Address}}{{"\n"}}{{end}}' 2>/dev/null || echo "Network not found"
echo ""
echo "=== xatu-server logs (last 50) ==="
docker logs xatu-server 2>&1 | tail -50 || echo "Container not found"
echo ""
echo "=== xatu-cannon logs (last 50) ==="
docker logs xatu-cannon 2>&1 | tail -50 || echo "Container not found"
echo ""
echo "=== xatu-minio logs (last 30) ==="
docker logs xatu-minio 2>&1 | tail -30 || echo "Container not found"
- name: Collect docker logs on failure
if: failure() || cancelled()
uses: jwalton/gh-docker-logs@2741064ab9d7af54b0b1ffb6076cf64c16f0220e # v2.2.2
with:
dest: './logs'
- name: Tar logs
if: failure() || cancelled()
run: tar cvzf ./logs.tgz ./logs
- name: Upload logs to GitHub
if: failure() || cancelled()
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
with:
name: logs.tgz
path: ./logs.tgz