Skip to content

Commit 548d27c

Browse files
samcmsamcm
committed
Merge master into release/glamsterdam-devnet-7: gossipsub message payload event
Brings LIBP2P_TRACE_GOSSIPSUB_MESSAGE_PAYLOAD (#877) onto the devnet branch. Proto numbering reconciled: this branch had already assigned the numbers master used, so the event takes 136 (enum), 124 (ClientMeta oneof) and 257 (DecoratedEvent oneof) here, with notes for eventual convergence. Generated protos rebuilt; categorizer union of the ePBS and payload events. Claude-Session: https://claude.ai/code/session_01KDvdo7uVQpxKWgk2gLnWpE
2 parents bf08ee0 + fc20a34 commit 548d27c

35 files changed

Lines changed: 6476 additions & 4162 deletions

‎deploy/migrations/clickhouse/xatu/006_libp2p_gossipsub_message_payload.down.sql‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
DROP TABLE IF EXISTS libp2p_gossipsub_message_payload ON CLUSTER '{cluster}';
2+
DROP TABLE IF EXISTS libp2p_gossipsub_message_payload_local ON CLUSTER '{cluster}';

‎deploy/migrations/clickhouse/xatu/006_libp2p_gossipsub_message_payload.up.sql‎

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
CREATE TABLE IF NOT EXISTS libp2p_gossipsub_message_payload_local ON CLUSTER '{cluster}'
2+
(
3+
`updated_date_time` DateTime COMMENT 'Timestamp when the record was last updated' CODEC(DoubleDelta, ZSTD(1)),
4+
`event_date_time` DateTime64(3) COMMENT 'Timestamp of the event' CODEC(DoubleDelta, ZSTD(1)),
5+
`wallclock_slot` UInt32 COMMENT 'Slot number of the wall clock when the message was received' CODEC(DoubleDelta, ZSTD(1)),
6+
`wallclock_slot_start_date_time` DateTime COMMENT 'Start date and time of the wall clock slot when the message was received' CODEC(DoubleDelta, ZSTD(1)),
7+
`wallclock_epoch` UInt32 COMMENT 'Epoch number of the wall clock when the message was received' CODEC(DoubleDelta, ZSTD(1)),
8+
`wallclock_epoch_start_date_time` DateTime COMMENT 'Start date and time of the wall clock epoch when the message was received' CODEC(DoubleDelta, ZSTD(1)),
9+
`topic_layer` LowCardinality(String) COMMENT 'Layer of the topic',
10+
`topic_fork_digest_value` LowCardinality(String) COMMENT 'Fork digest value of the topic',
11+
`topic_name` LowCardinality(String) COMMENT 'Name of the topic',
12+
`topic_encoding` LowCardinality(String) COMMENT 'Encoding of the topic',
13+
`message_id` String COMMENT 'Gossipsub message ID, derived from the message contents' CODEC(ZSTD(1)),
14+
`message_size` UInt32 COMMENT 'Size of the message payload in bytes' CODEC(ZSTD(1)),
15+
`message_data` String COMMENT 'Raw gossipsub message payload as received off the wire (snappy-framed SSZ)' CODEC(ZSTD(1)),
16+
`meta_client_name` LowCardinality(String) COMMENT 'Name of the client that generated the event',
17+
`meta_client_version` LowCardinality(String) COMMENT 'Version of the client that generated the event',
18+
`meta_client_implementation` LowCardinality(String) COMMENT 'Implementation of the client that generated the event',
19+
`meta_client_os` LowCardinality(String) COMMENT 'Operating system of the client that generated the event',
20+
`meta_client_ip` Nullable(IPv6) COMMENT 'IP address of the client that generated the event' CODEC(ZSTD(1)),
21+
`meta_client_geo_city` LowCardinality(String) COMMENT 'City of the client that generated the event' CODEC(ZSTD(1)),
22+
`meta_client_geo_country` LowCardinality(String) COMMENT 'Country of the client that generated the event' CODEC(ZSTD(1)),
23+
`meta_client_geo_country_code` LowCardinality(String) COMMENT 'Country code of the client that generated the event' CODEC(ZSTD(1)),
24+
`meta_client_geo_continent_code` LowCardinality(String) COMMENT 'Continent code of the client that generated the event' CODEC(ZSTD(1)),
25+
`meta_client_geo_longitude` Nullable(Float64) COMMENT 'Longitude of the client that generated the event' CODEC(ZSTD(1)),
26+
`meta_client_geo_latitude` Nullable(Float64) COMMENT 'Latitude of the client that generated the event' CODEC(ZSTD(1)),
27+
`meta_client_geo_autonomous_system_number` Nullable(UInt32) COMMENT 'Autonomous system number of the client that generated the event' CODEC(ZSTD(1)),
28+
`meta_client_geo_autonomous_system_organization` Nullable(String) COMMENT 'Autonomous system organization of the client that generated the event' CODEC(ZSTD(1)),
29+
`meta_network_name` LowCardinality(String) COMMENT 'Ethereum network name'
30+
)
31+
ENGINE = ReplicatedReplacingMergeTree('/clickhouse/{installation}/{cluster}/tables/{shard}/{database}/{table}', '{replica}', updated_date_time)
32+
PARTITION BY (meta_network_name, toDate(wallclock_slot_start_date_time))
33+
ORDER BY (meta_network_name, wallclock_slot_start_date_time, topic_fork_digest_value, topic_name, message_id)
34+
COMMENT 'Contains raw gossipsub message payloads keyed by message ID. Message IDs are content-derived, and the sorting key deliberately excludes observation-specific columns (peer, client, receive time, validation outcome) so identical messages captured by multiple clients deduplicate on merge. Deduplication is best-effort: clients that receive the same message on opposite sides of a wallclock slot or partition boundary keep one row per side. Per-observation detail lives in the libp2p_deliver_message and libp2p_reject_message tables.';
35+
36+
CREATE TABLE IF NOT EXISTS libp2p_gossipsub_message_payload ON CLUSTER '{cluster}'
37+
AS libp2p_gossipsub_message_payload_local
38+
ENGINE = Distributed('{cluster}', currentDatabase(), 'libp2p_gossipsub_message_payload_local', cityHash64(meta_network_name, topic_fork_digest_value, topic_name, message_id))
39+
COMMENT 'Contains raw gossipsub message payloads keyed by message ID. The sharding key is content-derived so duplicates from multiple capture clients land on the same shard and deduplicate.';

‎example-cl-mimicry.yaml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -130,6 +130,7 @@ outputs:
130130
# gossipSubAttestationEnabled: false
131131
# gossipSubBlobSidecarEnabled: false
132132
# gossipSubDataColumnSidecarEnabled: false
133+
# gossipSubMessagePayloadEnabled: false
133134

134135
tracing:
135136
enabled: false

‎example_discovery.yaml‎

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -50,8 +50,10 @@ p2p:
5050
restart: 2m
5151
# execution layer discovery settings (optional, defaults shown)
5252
# execution:
53-
# retryAttempts: 5 # max retry attempts for dialing a peer
54-
# retryDelay: 5s # delay between retry attempts
53+
# retryAttempts: 1 # max retry attempts for dialing a peer; EL clients
54+
# # throttle same-IP reconnects (30s-5m), so retries
55+
# # spaced closer than that are rejected pre-handshake
56+
# retryDelay: 60s # delay between retry attempts, when enabled
5557
# dialTimeout: 15s # timeout for dialing a peer
5658
# bootstrapRpcUrl: "" # optional EL JSON-RPC. When set, discovery serves
5759
# # Status/headers/bodies/receipts from this endpoint
@@ -124,8 +126,8 @@ p2p:
124126
# forkIdHashes: [0xf0afd0e3] # execution layer
125127
# forkDigests: [0x09fb0a12] # consensus layer
126128
# execution: # optional
127-
# retryAttempts: 5
128-
# retryDelay: 5s
129+
# retryAttempts: 1
130+
# retryDelay: 60s
129131
# dialTimeout: 15s
130132
# bootstrapRpcUrl: "" # optional EL JSON-RPC for Status/headers/bodies/receipts
131133
# consensus: # optional

‎go.mod‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,8 @@ module github.com/ethpandaops/xatu
22

33
go 1.26.2
44

5+
toolchain go1.26.5
6+
57
// release-xatu branch.
68

79
// Match tysm's tablewriter version requirement

‎pkg/clickhouse/route/libp2p/libp2p_gossipsub_message_payload.gen.go‎

Lines changed: 216 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

0 commit comments

Comments
 (0)