feat(devkit): add namespace support for multi-instance isolation (#3295)

devkit/Dockerfile

@@ -2,11 +2,13 @@ ARG jdk_version=eclipse-temurin:17-jdk-jammy
 FROM $jdk_version
 
 ENV DEBIAN_FRONTEND=noninteractive
+ENV PATH="/root/.local/bin:$PATH"
 
 RUN apt-get update && apt-get install -y \
-    curl jq iproute2 iptables iputils-ping net-tools wget \
+    curl jq iproute2 iptables iputils-ping net-tools wget python3 \
     && wget -O /usr/local/bin/jmxterm.jar https://github.com/jiaqi/jmxterm/releases/download/v1.0.4/jmxterm-1.0.4-uber.jar \
     && mkdir -p /opt/arthas && wget -O /opt/arthas/arthas-boot.jar https://arthas.aliyun.com/arthas-boot.jar \
+    && curl -LsSf https://astral.sh/uv/install.sh | sh \
     && apt-get -y clean && rm -rf /var/lib/apt/lists/*
 
 WORKDIR /opt/automq

devkit/README.md

@@ -313,6 +313,29 @@ just start 5 zerozone analytics     # 5-node + zone router + query engines
 | `telemetry` | OTel metrics export via OTLP HTTP — **edit `config/features/telemetry.properties` to set collector endpoint before use** | — (config only) |
 | `analytics` | Spark + Trino for querying Iceberg tables | Spark (:8888), Trino (:8090) |
 
+## Namespace (Multi-Instance Isolation)
+
+Run multiple DevKit instances on the same machine without port conflicts. Each namespace gets a unique `COMPOSE_PROJECT_NAME` and port offset.
+
+**Requires separate git worktrees** — each worktree has its own `devkit/` directory and `.devkit/compose.env`, so each instance is fully isolated. You cannot run multiple namespaces from the same devkit directory.
+
+```bash
+# In worktree A:
+just set-ns lag-test 1    # ID=1 → ports offset by +100
+just start 3
+
+# In worktree B:
+just set-ns perf 2        # ID=2 → ports offset by +200
+just start
+
+# Revert to default ports
+just clear-ns
+```
+
+- Name: 1-8 chars, must start with a letter, only `[a-z0-9-]`
+- ID: 1-9, determines port offset (`ID * 100`)
+- Config is stored in `.devkit/compose.env` (gitignored)
+
 ## Port Allocation
 
 | Node | Kafka | Controller | JDWP | JMX |
@@ -353,6 +376,8 @@ Other services:
 | `S3_DATA_BUCKET` | `automq-data` | Data bucket |
 | `S3_OPS_BUCKET` | `automq-ops` | Ops bucket |
 | `S3_PATH_STYLE` | `true` | Path-style access (required for MinIO) |
+| `S3_ACCESS_KEY` | `admin` | S3 access key (exported as `KAFKA_S3_ACCESS_KEY` to containers) |
+| `S3_SECRET_KEY` | `password` | S3 secret key (exported as `KAFKA_S3_SECRET_KEY` to containers) |
 
 **Per-node variables** (computed per node, Layer 2/3/4):
 
@@ -373,6 +398,8 @@ S3_ENDPOINT    := "https://s3.ap-northeast-1.amazonaws.com"
 S3_DATA_BUCKET := "my-automq-data"
 S3_OPS_BUCKET  := "my-automq-ops"
 S3_PATH_STYLE  := "false"
+S3_ACCESS_KEY  := "AKIA..."
+S3_SECRET_KEY  := "..."
 ```
 
 **Change listeners (e.g. SASL)** — edit `config/role/server.properties`:

devkit/docker-compose.yml

@@ -12,10 +12,11 @@ x-automq-common: &automq-common
   labels:
     com.automq.devkit: "true"
   environment:
-    - KAFKA_S3_ACCESS_KEY=admin
-    - KAFKA_S3_SECRET_KEY=password
+    - KAFKA_S3_ACCESS_KEY=${KAFKA_S3_ACCESS_KEY}
+    - KAFKA_S3_SECRET_KEY=${KAFKA_S3_SECRET_KEY}
     - KAFKA_HEAP_OPTS=${KAFKA_HEAP_OPTS:--Xms1g -Xmx4g -XX:MetaspaceSize=96m -XX:MaxDirectMemorySize=1G}
     - KAFKA_JVM_PERFORMANCE_OPTS=-server -XX:+UseZGC -XX:ZCollectionInterval=5
+    - LOG_DIR=/tmp/kafka-logs
   depends_on:
     mc:
       condition: service_completed_successfully
@@ -35,15 +36,14 @@ services:
   # ==================== Infrastructure ====================
   minio:
     image: minio/minio:RELEASE.2025-05-24T17-08-30Z
-    container_name: minio
     hostname: warehouse.minio
     environment:
       - MINIO_ROOT_USER=admin
       - MINIO_ROOT_PASSWORD=password
       - MINIO_DOMAIN=minio
     ports:
-      - "9000:9000"
-      - "9001:9001"
+      - "${MINIO_API_PORT:-9000}:9000"
+      - "${MINIO_CONSOLE_PORT:-9001}:9001"
     command: ["server", "/data", "--console-address", ":9001"]
     healthcheck:
       test: ["CMD", "curl", "-f", "http://minio:9000/minio/health/live"]
@@ -53,7 +53,6 @@ services:
 
   mc:
     image: minio/mc:RELEASE.2025-05-21T01-59-54Z
-    container_name: mc
     depends_on:
       minio:
         condition: service_healthy
@@ -69,13 +68,12 @@ services:
   # ==================== AutoMQ Nodes ====================
   node-0:
     <<: *automq-common
-    container_name: node-0
     hostname: node-0
     profiles: [single, cluster, cluster4, cluster5]
     ports:
-      - "9092:9192"
-      - "5005:5005"
-      - "9999:9999"
+      - "${NODE0_KAFKA_PORT:-9092}:9192"
+      - "${NODE0_DEBUG_PORT:-5005}:5005"
+      - "${NODE0_JMX_PORT:-9999}:9999"
     command:
       - bash
       - -c
@@ -88,12 +86,11 @@ services:
 
   node-1:
     <<: *automq-common
-    container_name: node-1
     hostname: node-1
     profiles: [cluster, cluster4, cluster5]
     ports:
-      - "19092:9192"
-      - "5006:5005"
+      - "${NODE1_KAFKA_PORT:-19092}:9192"
+      - "${NODE1_DEBUG_PORT:-5006}:5005"
     command:
       - bash
       - -c
@@ -106,12 +103,11 @@ services:
 
   node-2:
     <<: *automq-common
-    container_name: node-2
     hostname: node-2
     profiles: [cluster, cluster4, cluster5]
     ports:
-      - "29092:9192"
-      - "5007:5005"
+      - "${NODE2_KAFKA_PORT:-29092}:9192"
+      - "${NODE2_DEBUG_PORT:-5007}:5005"
     command:
       - bash
       - -c
@@ -124,12 +120,11 @@ services:
 
   node-3:
     <<: *automq-common
-    container_name: node-3
     hostname: node-3
     profiles: [cluster4, cluster5]
     ports:
-      - "39092:9192"
-      - "5008:5005"
+      - "${NODE3_KAFKA_PORT:-39092}:9192"
+      - "${NODE3_DEBUG_PORT:-5008}:5005"
     command:
       - bash
       - -c
@@ -142,12 +137,11 @@ services:
 
   node-4:
     <<: *automq-common
-    container_name: node-4
     hostname: node-4
     profiles: [cluster5]
     ports:
-      - "49092:9192"
-      - "5009:5005"
+      - "${NODE4_KAFKA_PORT:-49092}:9192"
+      - "${NODE4_DEBUG_PORT:-5009}:5005"
     command:
       - bash
       - -c
@@ -161,11 +155,10 @@ services:
   # ==================== TableTopic ====================
   schema-registry:
     image: confluentinc/cp-schema-registry:7.7.8
-    container_name: schema-registry
     hostname: schema-registry
     profiles: [tabletopic]
     ports:
-      - "8081:8081"
+      - "${SCHEMA_REGISTRY_PORT:-8081}:8081"
     environment:
       SCHEMA_REGISTRY_HOST_NAME: schema-registry
       SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS: node-0:9092
@@ -179,11 +172,10 @@ services:
 
   rest:
     image: apache/iceberg-rest-fixture
-    container_name: iceberg-rest
     hostname: rest
     profiles: [tabletopic]
     ports:
-      - "8181:8181"
+      - "${ICEBERG_REST_PORT:-8181}:8181"
     depends_on:
       minio:
         condition: service_healthy
@@ -204,7 +196,6 @@ services:
   # ==================== Analytics ====================
   spark-iceberg:
     image: tabulario/spark-iceberg:3.5.5_1.8.1
-    container_name: spark
     hostname: spark-iceberg
     profiles: [analytics]
     depends_on:
@@ -215,18 +206,17 @@ services:
       - AWS_SECRET_ACCESS_KEY=password
       - AWS_REGION=us-east-1
     ports:
-      - "8888:8888"
-      - "8080:8080"
+      - "${SPARK_NOTEBOOK_PORT:-8888}:8888"
+      - "${SPARK_UI_PORT:-8080}:8080"
 
   trino:
     image: trinodb/trino:472
-    container_name: trino
     hostname: trino
     profiles: [analytics]
     depends_on:
       minio:
         condition: service_healthy
     ports:
-      - "8090:8080"
+      - "${TRINO_PORT:-8090}:8080"
     volumes:
       - ./config/trino-catalog:/etc/trino/catalog