From 95fab48de5887aa1cbd9984aed2476f2330cda62 Mon Sep 17 00:00:00 2001 From: Gezi-lzq Date: Fri, 3 Apr 2026 15:51:20 +0800 Subject: [PATCH 1/2] feat(devkit): add HOST listener for host-side client access (#3280) Add a HOST listener on port 29092 (container) mapped to NODE_ID*10000+9192 (host) so that host-side Kafka clients (Python, Java) can connect without /etc/hosts hacks. Previously advertised.listeners used container hostnames (node-0:9092) which are unresolvable from the host. --- devkit/config/defaults.properties | 2 +- devkit/config/role/broker.properties | 4 ++-- devkit/config/role/server.properties | 4 ++-- devkit/docker-compose.yml | 5 +++++ devkit/justfile | 1 + 5 files changed, 11 insertions(+), 5 deletions(-) diff --git a/devkit/config/defaults.properties b/devkit/config/defaults.properties index 9b3cd7f622..a024b3fd4b 100644 --- a/devkit/config/defaults.properties +++ b/devkit/config/defaults.properties @@ -4,7 +4,7 @@ # KRaft mode controller.listener.names=CONTROLLER inter.broker.listener.name=PLAINTEXT -listener.security.protocol.map=CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT +listener.security.protocol.map=CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT,HOST:PLAINTEXT # S3 storage (MinIO) elasticstream.enable=true diff --git a/devkit/config/role/broker.properties b/devkit/config/role/broker.properties index 6f99b34884..a2ee93943b 100644 --- a/devkit/config/role/broker.properties +++ b/devkit/config/role/broker.properties @@ -3,5 +3,5 @@ # Variable: ${NODE_ID} is replaced by generate-config process.roles=broker -listeners=PLAINTEXT://:9092 -advertised.listeners=PLAINTEXT://node-${NODE_ID}:9092 +listeners=PLAINTEXT://:9092,HOST://:29092 +advertised.listeners=PLAINTEXT://node-${NODE_ID}:9092,HOST://localhost:${HOST_PORT} diff --git a/devkit/config/role/server.properties b/devkit/config/role/server.properties index a02aac934d..464e50498b 100644 --- a/devkit/config/role/server.properties +++ b/devkit/config/role/server.properties @@ -3,5 +3,5 @@ # Variable: ${NODE_ID} is replaced by generate-config process.roles=broker,controller -listeners=PLAINTEXT://:9092,CONTROLLER://:9093 -advertised.listeners=PLAINTEXT://node-${NODE_ID}:9092,CONTROLLER://node-${NODE_ID}:9093 +listeners=PLAINTEXT://:9092,CONTROLLER://:9093,HOST://:29092 +advertised.listeners=PLAINTEXT://node-${NODE_ID}:9092,CONTROLLER://node-${NODE_ID}:9093,HOST://localhost:${HOST_PORT} diff --git a/devkit/docker-compose.yml b/devkit/docker-compose.yml index 2cc27bd64b..4b55a0e099 100644 --- a/devkit/docker-compose.yml +++ b/devkit/docker-compose.yml @@ -75,6 +75,7 @@ services: ports: - "9092:9092" - "9093:9093" + - "9192:29092" - "5005:5005" - "9999:9999" command: @@ -95,6 +96,7 @@ services: ports: - "19092:9092" - "19093:9093" + - "19192:29092" - "5006:5005" command: - bash @@ -114,6 +116,7 @@ services: ports: - "29092:9092" - "29093:9093" + - "29192:29092" - "5007:5005" command: - bash @@ -133,6 +136,7 @@ services: ports: - "39092:9092" - "39093:9093" + - "39192:29092" - "5008:5005" command: - bash @@ -152,6 +156,7 @@ services: ports: - "49092:9092" - "49093:9093" + - "49192:29092" - "5009:5005" command: - bash diff --git a/devkit/justfile b/devkit/justfile index 7a4d3acf00..bf5652c116 100644 --- a/devkit/justfile +++ b/devkit/justfile @@ -81,6 +81,7 @@ _render-node node ctrl_count cluster_id voters bootstrap features="": export VOTERS="{{voters}}" export BOOTSTRAP="{{bootstrap}}" export NODE_ID="{{node}}" + export HOST_PORT=$(({{node}} * 10000 + 9192)) export AZ_NAME="${AZS[$(({{node}} % ${#AZS[@]}))]}" render() { envsubst < "$1" | grep -v '^#' | grep -v '^$'; } From c3080397fa2d9196d14a0f54e948f5f8f3ad9b9c Mon Sep 17 00:00:00 2001 From: Gezi-lzq Date: Fri, 3 Apr 2026 18:06:45 +0800 Subject: [PATCH 2/2] refactor(devkit): simplify host port mappings and remove unused PLAINTEXT/CONTROLLER exports Remove unused PLAINTEXT and CONTROLLER host port mappings since they are unreachable from the host (advertised.listeners uses container hostnames). Reuse the X9092 port slots for HOST listener and change the container-side HOST port from 29092 to 9192 to avoid confusing collision with node-2. --- devkit/config/role/broker.properties | 2 +- devkit/config/role/server.properties | 2 +- devkit/docker-compose.yml | 20 +++++--------------- devkit/justfile | 2 +- 4 files changed, 8 insertions(+), 18 deletions(-) diff --git a/devkit/config/role/broker.properties b/devkit/config/role/broker.properties index a2ee93943b..781d7f2fd9 100644 --- a/devkit/config/role/broker.properties +++ b/devkit/config/role/broker.properties @@ -3,5 +3,5 @@ # Variable: ${NODE_ID} is replaced by generate-config process.roles=broker -listeners=PLAINTEXT://:9092,HOST://:29092 +listeners=PLAINTEXT://:9092,HOST://:9192 advertised.listeners=PLAINTEXT://node-${NODE_ID}:9092,HOST://localhost:${HOST_PORT} diff --git a/devkit/config/role/server.properties b/devkit/config/role/server.properties index 464e50498b..ca6bf39e4d 100644 --- a/devkit/config/role/server.properties +++ b/devkit/config/role/server.properties @@ -3,5 +3,5 @@ # Variable: ${NODE_ID} is replaced by generate-config process.roles=broker,controller -listeners=PLAINTEXT://:9092,CONTROLLER://:9093,HOST://:29092 +listeners=PLAINTEXT://:9092,CONTROLLER://:9093,HOST://:9192 advertised.listeners=PLAINTEXT://node-${NODE_ID}:9092,CONTROLLER://node-${NODE_ID}:9093,HOST://localhost:${HOST_PORT} diff --git a/devkit/docker-compose.yml b/devkit/docker-compose.yml index 4b55a0e099..74e18699cc 100644 --- a/devkit/docker-compose.yml +++ b/devkit/docker-compose.yml @@ -73,9 +73,7 @@ services: hostname: node-0 profiles: [single, cluster, cluster4, cluster5] ports: - - "9092:9092" - - "9093:9093" - - "9192:29092" + - "9092:9192" - "5005:5005" - "9999:9999" command: @@ -94,9 +92,7 @@ services: hostname: node-1 profiles: [cluster, cluster4, cluster5] ports: - - "19092:9092" - - "19093:9093" - - "19192:29092" + - "19092:9192" - "5006:5005" command: - bash @@ -114,9 +110,7 @@ services: hostname: node-2 profiles: [cluster, cluster4, cluster5] ports: - - "29092:9092" - - "29093:9093" - - "29192:29092" + - "29092:9192" - "5007:5005" command: - bash @@ -134,9 +128,7 @@ services: hostname: node-3 profiles: [cluster4, cluster5] ports: - - "39092:9092" - - "39093:9093" - - "39192:29092" + - "39092:9192" - "5008:5005" command: - bash @@ -154,9 +146,7 @@ services: hostname: node-4 profiles: [cluster5] ports: - - "49092:9092" - - "49093:9093" - - "49192:29092" + - "49092:9192" - "5009:5005" command: - bash diff --git a/devkit/justfile b/devkit/justfile index bf5652c116..1b3604a1b5 100644 --- a/devkit/justfile +++ b/devkit/justfile @@ -81,7 +81,7 @@ _render-node node ctrl_count cluster_id voters bootstrap features="": export VOTERS="{{voters}}" export BOOTSTRAP="{{bootstrap}}" export NODE_ID="{{node}}" - export HOST_PORT=$(({{node}} * 10000 + 9192)) + export HOST_PORT=$(({{node}} * 10000 + 9092)) export AZ_NAME="${AZS[$(({{node}} % ${#AZS[@]}))]}" render() { envsubst < "$1" | grep -v '^#' | grep -v '^$'; }