diff --git a/.github/workflows/devbox-ci.yml b/.github/workflows/devkit-ci.yml similarity index 50% rename from .github/workflows/devbox-ci.yml rename to .github/workflows/devkit-ci.yml index 3c13d1b966..00a0bcfb59 100644 --- a/.github/workflows/devbox-ci.yml +++ b/.github/workflows/devkit-ci.yml @@ -1,9 +1,9 @@ -name: DevBox CI +name: DevKit CI on: pull_request: paths: - - 'devbox/**' + - 'devkit/**' workflow_dispatch: inputs: level: @@ -32,12 +32,15 @@ jobs: - name: Install just uses: extractions/setup-just@v2 + - name: Install envsubst + run: sudo apt-get install -y gettext-base + - name: Validate justfile - working-directory: devbox + working-directory: devkit run: just --list - name: Validate docker-compose - working-directory: devbox + working-directory: devkit run: | for p in single cluster cluster4 cluster5 tabletopic analytics; do echo "=== Profile: $p ===" @@ -45,43 +48,44 @@ jobs: done - name: Validate config generation - working-directory: devbox + working-directory: devkit run: | # Generate cluster-id without kafka-storage.sh - mkdir -p .devbox - echo "test-cluster-id" > .devbox/cluster-id + mkdir -p .devkit + echo "test-cluster-id" > .devkit/cluster-id # Single node, no features just generate-config 1 "" - grep -q "node.id=0" .devbox/config/node-0.properties - grep -q "controller.quorum.voters=0@node-0:9093" .devbox/config/node-0.properties - ! grep -q "broker.rack" .devbox/config/node-0.properties + grep -q "node.id=0" .devkit/config/node-0.properties + grep -q "controller.quorum.voters=0@node-0:9093" .devkit/config/node-0.properties + ! grep -q "broker.rack" .devkit/config/node-0.properties echo "✓ Single node config OK" # 3-node cluster with zerozone just generate-config 3 "zerozone" - grep -q "broker.rack=az-0" .devbox/config/node-0.properties - grep -q "broker.rack=az-1" .devbox/config/node-1.properties - grep -q "broker.rack=az-2" .devbox/config/node-2.properties - grep -q "zonerouter" .devbox/config/node-0.properties - grep -q "controller.quorum.voters=0@node-0:9093,1@node-1:9093,2@node-2:9093" .devbox/config/node-0.properties + grep -q "broker.rack=az-0" .devkit/config/node-0.properties + grep -q "broker.rack=az-1" .devkit/config/node-1.properties + grep -q "broker.rack=az-2" .devkit/config/node-2.properties + grep -q "zonerouter" .devkit/config/node-0.properties + grep -q "controller.quorum.voters=0@node-0:9093,1@node-1:9093,2@node-2:9093" .devkit/config/node-0.properties echo "✓ Cluster + zerozone config OK" # Tabletopic feature just generate-config 1 "tabletopic" - grep -q "automq.table.topic.catalog.type=rest" .devbox/config/node-0.properties + grep -q "automq.table.topic.catalog.type=rest" .devkit/config/node-0.properties echo "✓ Tabletopic config OK" # Combined features just generate-config 1 "tabletopic zerozone" - grep -q "automq.table.topic" .devbox/config/node-0.properties - grep -q "zonerouter" .devbox/config/node-0.properties - grep -q "broker.rack" .devbox/config/node-0.properties + grep -q "automq.table.topic" .devkit/config/node-0.properties + grep -q "zonerouter" .devkit/config/node-0.properties + grep -q "broker.rack" .devkit/config/node-0.properties echo "✓ Combined features config OK" smoke: name: Smoke needs: lint + if: ${{ github.event_name == 'pull_request' || inputs.level == 'smoke' || inputs.level == 'full' }} runs-on: ubuntu-latest timeout-minutes: 20 steps: @@ -94,24 +98,28 @@ jobs: - uses: extractions/setup-just@v2 + - name: Cache Gradle + uses: actions/cache@v4 + with: + path: | + ~/.gradle/caches + ~/.gradle/wrapper + key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle*', '**/gradle-wrapper.properties') }} + restore-keys: ${{ runner.os }}-gradle- + - name: Gradle build run: ./gradlew :core:build :tools:build :shell:build -x test - name: Start single node - working-directory: devbox + working-directory: devkit run: just start - name: Wait for healthy - run: | - for i in $(seq 1 30); do - S=$(docker inspect --format='{{.State.Health.Status}}' node-0 2>/dev/null) - [ "$S" = "healthy" ] && echo "✓ node-0 healthy" && exit 0 - sleep 5 - done - echo "Timeout waiting for node-0" && docker logs node-0 | tail -50 && exit 1 + working-directory: devkit + run: just wait - name: Topic operations - working-directory: devbox + working-directory: devkit run: | just topic-create ci-test just topic-list 2>&1 | grep ci-test @@ -119,35 +127,43 @@ jobs: echo "✓ Topic operations OK" - name: Produce and consume + working-directory: devkit run: | - echo -e "ci-msg-1\nci-msg-2" | docker exec -i node-0 bash -c \ - "KAFKA_HEAP_OPTS='-Xms256m -Xmx256m' KAFKA_JVM_PERFORMANCE_OPTS='' /opt/automq/bin/kafka-console-producer.sh --bootstrap-server localhost:9092 --topic ci-test" - docker exec node-0 bash -c \ - "KAFKA_HEAP_OPTS='-Xms256m -Xmx256m' KAFKA_JVM_PERFORMANCE_OPTS='' /opt/automq/bin/kafka-console-consumer.sh --bootstrap-server localhost:9092 --topic ci-test --from-beginning --max-messages 2 --timeout-ms 15000" \ - 2>&1 | grep ci-msg-1 + echo -e "ci-msg-1\nci-msg-2" | just produce ci-test + just consume ci-test --from-beginning --max-messages 2 --timeout-ms 15000 2>&1 | grep ci-msg-1 echo "✓ Produce/consume OK" + - name: Consumer group operations + working-directory: devkit + run: | + just group-list 2>&1 | grep -q "." + just group-describe ci-consumer 2>&1 || true # group may not exist yet, just verify command works + just broker-list 2>&1 | grep -q "node-0" + just broker-config 0 2>&1 | grep -q "advertised.listeners" + echo "✓ Consumer group and cluster operations OK" + - name: Bin passthrough - working-directory: devbox + working-directory: devkit run: | just bin kafka-broker-api-versions.sh --bootstrap-server localhost:9092 2>&1 | grep -q "ApiVersion" echo "✓ Bin passthrough OK" - name: JMX operations - working-directory: devbox + working-directory: devkit run: | just jmx -e domains 2>&1 | grep -q "kafka.server" - just jmx -e 'beans -d kafka.server' 2>&1 | grep -q "kafka.server" + just jmx -e 'get -b kafka.server:name=BrokerState,type=KafkaServer Value' 2>&1 | grep -q "Value = 3" echo "✓ JMX operations OK" - name: Stop if: always() - working-directory: devbox + working-directory: devkit run: just stop full: name: Full needs: lint + if: ${{ github.event_name == 'pull_request' || inputs.level == 'full' }} runs-on: ubuntu-latest timeout-minutes: 30 steps: @@ -160,60 +176,71 @@ jobs: - uses: extractions/setup-just@v2 + - name: Cache Gradle + uses: actions/cache@v4 + with: + path: | + ~/.gradle/caches + ~/.gradle/wrapper + key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle*', '**/gradle-wrapper.properties') }} + restore-keys: ${{ runner.os }}-gradle- + - name: Gradle build run: ./gradlew :core:build :tools:build :shell:build -x test - name: Test tabletopic - working-directory: devbox + working-directory: devkit run: | just start 1 tabletopic - for i in $(seq 1 30); do - S=$(docker inspect --format='{{.State.Health.Status}}' node-0 2>/dev/null) - [ "$S" = "healthy" ] && break; sleep 5 - done + just wait curl -sf http://localhost:8181/v1/config > /dev/null echo "✓ Iceberg REST OK" for i in $(seq 1 30); do curl -sf http://localhost:8081/ > /dev/null && echo "✓ Schema Registry OK" && break sleep 5 done - grep -q "automq.table.topic" .devbox/config/node-0.properties + grep -q "automq.table.topic" .devkit/config/node-0.properties echo "✓ Tabletopic config merged" just stop - name: Test 4-node cluster (3 controller + 1 broker) - working-directory: devbox + working-directory: devkit run: | just start 4 - # Verify role configs - grep -q "process.roles=broker,controller" .devbox/config/node-0.properties - grep -q "process.roles=broker,controller" .devbox/config/node-1.properties - grep -q "process.roles=broker,controller" .devbox/config/node-2.properties - grep -q "^process.roles=broker$" .devbox/config/node-3.properties - grep -q "controller.quorum.voters=0@node-0:9093,1@node-1:9093,2@node-2:9093" .devbox/config/node-0.properties + just wait + grep -q "process.roles=broker,controller" .devkit/config/node-0.properties + grep -q "process.roles=broker,controller" .devkit/config/node-1.properties + grep -q "process.roles=broker,controller" .devkit/config/node-2.properties + grep -q "^process.roles=broker$" .devkit/config/node-3.properties + grep -q "controller.quorum.voters=0@node-0:9093,1@node-1:9093,2@node-2:9093" .devkit/config/node-0.properties echo "✓ Role configs correct" - # Wait for healthy - for node in 0 1 2 3; do - for i in $(seq 1 30); do - S=$(docker inspect --format='{{.State.Health.Status}}' node-${node} 2>/dev/null) - [ "$S" = "healthy" ] && echo "✓ node-${node} healthy" && break - sleep 5 - done - done - # Test topic operations just topic-create mixed-ci-test just topic-describe mixed-ci-test 2>&1 | grep -E "Leader: [0123]" echo "✓ Mixed cluster (3 controller + 1 broker) OK" + just perf-produce mixed-ci-test --num-records 100 --record-size 512 --throughput -1 2>&1 | grep -q "records sent" + echo "✓ Perf produce OK" + just arthas-exec 0 version 2>&1 | grep -q "arthas" + echo "✓ Arthas OK" + just chaos-delay 100 node-0 + just chaos-status 2>&1 | grep -q "node-0" + just chaos-reset + just chaos-status 2>&1 | grep -q "minio" + echo "✓ Chaos OK" just stop - name: Test zerozone config - working-directory: devbox + working-directory: devkit run: | just start 5 zerozone - grep -q "broker.rack=az-0" .devbox/config/node-0.properties - grep -q "broker.rack=az-1" .devbox/config/node-1.properties - grep -q "broker.rack=az-2" .devbox/config/node-2.properties - grep -q "broker.rack=az-0" .devbox/config/node-3.properties - grep -q "broker.rack=az-1" .devbox/config/node-4.properties + grep -q "broker.rack=az-0" .devkit/config/node-0.properties + grep -q "broker.rack=az-1" .devkit/config/node-1.properties + grep -q "broker.rack=az-2" .devkit/config/node-2.properties + grep -q "broker.rack=az-0" .devkit/config/node-3.properties + grep -q "broker.rack=az-1" .devkit/config/node-4.properties echo "✓ ZeroZone broker.rack round-robin OK" just stop + + - name: Cleanup + if: always() + working-directory: devkit + run: just clean diff --git a/CONTRIBUTING_GUIDE.md b/CONTRIBUTING_GUIDE.md index 0988e5a57f..92b7e1040c 100644 --- a/CONTRIBUTING_GUIDE.md +++ b/CONTRIBUTING_GUIDE.md @@ -9,6 +9,42 @@ or [Slack](https://join.slack.com/t/automq/shared_invite/zt-29h17vye9-thf31ebIVL Before getting started, please review AutoMQ's Code of Conduct. Everyone interacting in Slack or Wechat follow [Code of Conduct](CODE_OF_CONDUCT.md). +## Suggested Onboarding Path for New Contributors + +## Quick Start for First-Time Contributors (Recommended) + +If you are new to AutoMQ, we recommend starting with the simplest path before diving into local builds. + +**Option 1: Quick exploration (recommended for beginners)** +- Run AutoMQ using Docker as described in the README +- Verify you can: + - Start the broker + - Create a topic + - Produce and consume messages +- This helps you understand AutoMQ behavior without local environment setup + +**Option 2: DevKit local development (recommended for code contributions)** +- Use the Docker Compose-based DevKit to start a local AutoMQ cluster with MinIO and JDWP debug ports +- Quick start: + - `cd devkit` + - `just start-build` (single node) or `just start-build 3` (3-node cluster) +- Useful shortcuts: `just topic-list`, `just produce `, `just logs` +- See full instructions in [devkit/README.md](devkit/README.md) + +**Option 3: Manual local development setup** +- Follow the steps below to build and run AutoMQ locally with IDEA/manual configuration +- This path is useful when you need full control over local components and configs + +> Tip: If you encounter setup issues, check [devkit/README.md](devkit/README.md), “Local Debug with IDEA”, and S3 configuration sections below. + + +If you are new to AutoMQ, it is recommended to first deploy and run AutoMQ using Docker as described in the README. +This helps you quickly understand AutoMQ’s core concepts and behavior without local environment complexity. + +After gaining familiarity, contributors who want to work on code can follow the steps in this guide to build and run AutoMQ locally. +For most contributors, we recommend starting with DevKit (`devkit/README.md`) and using the manual setup only when deeper environment customization is needed. + + ## Code Contributions Most of the issues open for contributions are tagged with 'good first issue.' To claim one, simply reply with 'pick up' in the issue and the AutoMQ maintainers will assign the issue to you. If you have any questions about the 'good first issue' please feel free to ask. We will do our best to clarify any doubts you may have. diff --git a/devbox/README.md b/devbox/README.md deleted file mode 100644 index cf0609fb6b..0000000000 --- a/devbox/README.md +++ /dev/null @@ -1,117 +0,0 @@ -# DevBox - AutoMQ Local Development Environment - -Zero-configuration local dev environment for AutoMQ. One command to start a Kafka cluster with S3 storage (MinIO), JDWP debugging, and optional features. - -## Prerequisites - -- Docker & Docker Compose -- [just](https://github.com/casey/just) command runner -- JDK 17+ - -## Quick Start - -```bash -cd devbox -just start-build # Build image & code + single node -just start-build 3 # Build image & code + 3-node cluster -``` - -After first build, iterate faster with: -```bash -just start # Single node (skip build) -just start 3 # 3-node cluster (skip build) -``` - -DevBox automatically generates a stable CLUSTER_ID, node configs, and starts MinIO + AutoMQ in Docker with JDWP enabled. - -## Commands - -### Lifecycle - -| Command | Description | -|---------|-------------| -| `just start` | Single node (skip build) | -| `just start 3` | 3-node cluster | -| `just start 1 telemetry` | Single node + OpenTelemetry metrics | -| `just start 5 tabletopic` | 5-node + TableTopic | -| `just start-build` | Build image & code, then start single node | -| `just start-build 3 tabletopic analytics` | Build + full stack | -| `just stop` | Stop all services | -| `just restart` | Rebuild and restart | -| `just status` | Show service status | -| `just logs` | Tail all logs | -| `just logs 0` | Tail node 0 logs | - -### Bin (direct passthrough) - -Run any `bin/` command inside a node container: - -```bash -just bin kafka-topics.sh --bootstrap-server localhost:9092 --list -just bin kafka-configs.sh --bootstrap-server localhost:9092 --describe --entity-type brokers -just bin --node 1 kafka-broker-api-versions.sh --bootstrap-server localhost:9092 -``` - -> **Note**: The cluster is designed for container-internal access. To interact with Kafka from your host machine, use `just bin` commands or the shortcuts below (`topic-*`, `produce`, `consume`). - -### Shortcuts - -| Command | Description | -|---------|-------------| -| `just topic-list` | List topics | -| `just topic-create my-topic --partitions 16` | Create topic with 16 partitions | -| `just topic-describe my-topic` | Describe topic | -| `just produce my-topic` | Interactive producer | -| `just consume my-topic --from-beginning` | Consumer | - -### Chaos - -| Command | Description | -|---------|-------------| -| `just chaos-delay` | 200ms delay on node-0 | -| `just chaos-delay 500 node-1` | 500ms delay on node-1 | -| `just chaos-loss` | 5% packet loss on node-0 | -| `just chaos-loss 10 node-1` | 10% packet loss on node-1 | -| `just chaos-reset` | Remove all rules | -| `just chaos-status` | Show rules | - -## Port Allocation - -| Node | Kafka | Controller | JDWP | -|------|-------|------------|------| -| 0 | 9092 | 9093 | 5005 | -| 1 | 19092 | 19093 | 5006 | -| 2 | 29092 | 29093 | 5007 | -| 3 | 39092 | 39093 | 5008 | -| 4 | 49092 | 49093 | 5009 | - -Other services: -- MinIO Console: http://localhost:9001 (admin/password) -- Schema Registry: http://localhost:8081 (with tabletopic) -- Iceberg REST: http://localhost:8181 (with tabletopic) -- Spark: http://localhost:8888 (with analytics) -- Trino: http://localhost:8090 (with analytics) - -## IDEA Remote Debugging - -1. Run → Edit Configurations → + → Remote JVM Debug -2. Host: `localhost`, Port: `5005` (node 0) -3. Start DevBox, then attach debugger - -## Configuration - -5-layer config system (low → high priority): - -1. `config/defaults.properties` — MinIO, KRaft basics -2. Role — process.roles, listeners, advertised.listeners (auto: controller for node 0-2, broker-only for 3+) -3. Node identity — node.id, cluster.id, quorum.voters (auto) -4. `config/features/*.properties` — tabletopic, zerozone, telemetry (applied via start args) -5. `config/custom.properties` — your overrides, highest priority (gitignored) - -To customize, create `config/custom.properties`: - -```properties -num.partitions=4 -log.retention.hours=24 -``` - diff --git a/devbox/config/features/telemetry.properties b/devbox/config/features/telemetry.properties deleted file mode 100644 index 21653c49c4..0000000000 --- a/devbox/config/features/telemetry.properties +++ /dev/null @@ -1,7 +0,0 @@ -# AutoMQ DevBox - Telemetry Feature -# Enables OpenTelemetry metrics export -# Change endpoint to your OTel Collector address before use - -s3.telemetry.metrics.exporter.uri=otlp://?endpoint=http://localhost:4318&protocol=http -s3.telemetry.metrics.level=INFO -s3.telemetry.exporter.report.interval.ms=30000 diff --git a/devbox/config/features/zerozone.properties b/devbox/config/features/zerozone.properties deleted file mode 100644 index d85ff1c53c..0000000000 --- a/devbox/config/features/zerozone.properties +++ /dev/null @@ -1,4 +0,0 @@ -# AutoMQ DevBox - ZeroZone Feature -# Layer 2: Enables zone router - -automq.zonerouter.channels=0@s3://automq-data?region=us-east-1&endpoint=http://minio:9000&pathStyle=true&batchInterval=250&maxBytesInBatch=8388608 diff --git a/devbox/justfile b/devbox/justfile deleted file mode 100644 index e6a36917b7..0000000000 --- a/devbox/justfile +++ /dev/null @@ -1,266 +0,0 @@ -# DevBox - AutoMQ Local Development Environment - -set dotenv-load := false - -AUTOMQ_DEV_HOME := justfile_directory() + "/.." -DEVBOX := justfile_directory() + "/.devbox" -COMPOSE := "docker compose -f " + justfile_directory() + "/docker-compose.yml" -ALL_PROFILES := "--profile single --profile cluster --profile cluster4 --profile cluster5 --profile tabletopic --profile analytics" -ALL_NODES := "node-0 node-1 node-2 node-3 node-4" -AZ_NAMES := "az-0 az-1 az-2" -KAFKA_OPTS := "KAFKA_HEAP_OPTS='-Xms256m -Xmx256m' KAFKA_JVM_PERFORMANCE_OPTS='' JMX_PORT=''" - -default: help - -help: - @echo "DevBox - AutoMQ Local Development Environment" - @echo "" - @echo "Lifecycle:" - @echo " just start [N] [features...] Start N nodes (default: 1, no build)" - @echo " just start-build [N] [features...] Build image & code, then start" - @echo "" - @echo " just start 3 zerozone tabletopic Example: 3-node + features" - @echo "" - @echo " just stop Stop all services" - @echo " just restart Rebuild and restart" - @echo " just status Show service status" - @echo " just logs [node] Tail logs" - @echo " just shell [node] Enter container shell" - @echo "" - @echo "Bin (direct passthrough):" - @echo " just bin kafka-topics.sh --bootstrap-server localhost:9092 --list" - @echo " just bin --node 1 kafka-broker-api-versions.sh --bootstrap-server localhost:9092" - @echo "" - @echo "JMX:" - @echo " just jmx -e domains List all JMX domains" - @echo " just jmx -e 'beans -d kafka.server' List beans in domain" - @echo " just jmx --node 1 -e 'get ...' Query from node-1" - @echo "" - @echo "Shortcuts:" - @echo " just topic-list List topics" - @echo " just topic-create [extra flags] Create topic" - @echo " just topic-describe Describe topic" - @echo " just produce Interactive producer" - @echo " just consume [--from-beginning] Consumer" - @echo "" - @echo "Chaos:" - @echo " just chaos-delay [ms] [node] Network delay" - @echo " just chaos-loss [%] [node] Packet loss" - @echo " just chaos-reset Remove all rules" - @echo " just chaos-status Show rules" - @echo "" - @echo "Features: tabletopic, zerozone, telemetry, analytics" - -# ==================== Internal ==================== - -[private] -ensure-cluster-id: - #!/usr/bin/env bash - mkdir -p "{{DEVBOX}}" - if [ ! -f "{{DEVBOX}}/cluster-id" ]; then - KAFKA_HEAP_OPTS="-Xms256m -Xmx256m" \ - "{{AUTOMQ_DEV_HOME}}/bin/kafka-storage.sh" random-uuid \ - > "{{DEVBOX}}/cluster-id" - echo "✓ Generated CLUSTER_ID: $(cat "{{DEVBOX}}/cluster-id")" - fi - -[private] -build: - @echo "✓ Building Docker image..." - @export AUTOMQ_DEV_HOME="{{AUTOMQ_DEV_HOME}}" && {{COMPOSE}} build - @echo "✓ Building Java code..." - cd "{{AUTOMQ_DEV_HOME}}" && ./gradlew :core:build :tools:build :shell:build -x test - -# Rebuild Docker images only -build-image *args="": - @echo "✓ Building Docker image..." - @AUTOMQ_DEV_HOME="{{AUTOMQ_DEV_HOME}}" {{COMPOSE}} build {{ if args == "" { ALL_NODES } else { args } }} - -[private] -generate-config nodes features="": - #!/usr/bin/env bash - set -e - mkdir -p "{{DEVBOX}}/config" - CLUSTER_ID=$(cat "{{DEVBOX}}/cluster-id") - MAX_CTRL=3 - CTRL_COUNT=$(({{nodes}} < MAX_CTRL ? {{nodes}} : MAX_CTRL)) - VOTERS="" ; BOOTSTRAP="" - for i in $(seq 0 $((CTRL_COUNT - 1))); do - [ -n "$VOTERS" ] && VOTERS="$VOTERS," && BOOTSTRAP="$BOOTSTRAP," - VOTERS="${VOTERS}${i}@node-${i}:9093" - BOOTSTRAP="${BOOTSTRAP}node-${i}:9093" - done - AZS=({{AZ_NAMES}}) - for i in $(seq 0 $(({{nodes}} - 1))); do - CFG="{{DEVBOX}}/config/node-${i}.properties" - - # Layer 1: defaults - cat "{{justfile_directory()}}/config/defaults.properties" > "$CFG" - - # Layer 2: role (controller+broker or broker-only) - echo "" >> "$CFG" - echo "# === Role ===" >> "$CFG" - if [ $i -lt $CTRL_COUNT ]; then - echo "process.roles=broker,controller" >> "$CFG" - echo "listeners=PLAINTEXT://:9092,CONTROLLER://:9093" >> "$CFG" - echo "advertised.listeners=PLAINTEXT://node-${i}:9092,CONTROLLER://node-${i}:9093" >> "$CFG" - else - echo "process.roles=broker" >> "$CFG" - echo "listeners=PLAINTEXT://:9092" >> "$CFG" - echo "advertised.listeners=PLAINTEXT://node-${i}:9092" >> "$CFG" - fi - - # Layer 3: node identity - echo "" >> "$CFG" - echo "# === Node ===" >> "$CFG" - echo "cluster.id=${CLUSTER_ID}" >> "$CFG" - echo "node.id=${i}" >> "$CFG" - echo "controller.quorum.voters=${VOTERS}" >> "$CFG" - echo "controller.quorum.bootstrap.servers=${BOOTSTRAP}" >> "$CFG" - - # Layer 4: features - for feat in {{features}}; do - F="{{justfile_directory()}}/config/features/${feat}.properties" - [ -f "$F" ] && echo "" >> "$CFG" && cat "$F" >> "$CFG" - [ "$feat" = "zerozone" ] && echo "broker.rack=${AZS[$((i % ${#AZS[@]}))]}" >> "$CFG" - done - - # Layer 5: custom overrides (highest priority) - C="{{justfile_directory()}}/config/custom.properties" - [ -f "$C" ] && echo "" >> "$CFG" && cat "$C" >> "$CFG" - - echo "✓ Config: node-${i}.properties" - done - -# ==================== Lifecycle ==================== - -start nodes="1" *FEATURES: ensure-cluster-id - #!/usr/bin/env bash - set -e - case {{nodes}} in - 1) PROFILE=single ;; 3) PROFILE=cluster ;; 4) PROFILE=cluster4 ;; 5) PROFILE=cluster5 ;; - *) echo "Error: nodes must be 1,3,4,5"; exit 1 ;; - esac - COMPOSE_PROFILES="--profile $PROFILE" - for feat in {{FEATURES}}; do - COMPOSE_PROFILES="$COMPOSE_PROFILES --profile $feat" - done - just generate-config {{nodes}} "{{FEATURES}}" - echo "✓ Starting {{nodes}} node(s)..." - export AUTOMQ_DEV_HOME="{{AUTOMQ_DEV_HOME}}" - {{COMPOSE}} $COMPOSE_PROFILES up -d - echo "" - echo "🎉 AutoMQ DevBox started" - echo " Kafka: localhost:9092" - echo " Debug: localhost:5005" - echo " MinIO: http://localhost:9001 (admin/password)" - -start-build nodes="1" *FEATURES: build (start nodes FEATURES) - -stop: - @echo "Stopping..." - @export AUTOMQ_DEV_HOME="{{AUTOMQ_DEV_HOME}}" && {{COMPOSE}} {{ALL_PROFILES}} down -v - -restart: - @export AUTOMQ_DEV_HOME="{{AUTOMQ_DEV_HOME}}" && {{COMPOSE}} {{ALL_PROFILES}} restart - -restart-build: build restart - -status: - @export AUTOMQ_DEV_HOME="{{AUTOMQ_DEV_HOME}}" && {{COMPOSE}} {{ALL_PROFILES}} ps - -logs node="": - #!/usr/bin/env bash - export AUTOMQ_DEV_HOME="{{AUTOMQ_DEV_HOME}}" - if [ -z "{{node}}" ]; then - {{COMPOSE}} {{ALL_PROFILES}} logs -f - else - {{COMPOSE}} {{ALL_PROFILES}} logs -f node-{{node}} - fi - -shell node="0": - @docker exec -it node-{{node}} bash - -# ==================== Bin ==================== - -bin *ARGS: - #!/usr/bin/env bash - NODE=0 - set -- {{ARGS}} - while [ $# -gt 0 ]; do - case $1 in - --node) NODE=$2; shift 2 ;; - *) break ;; - esac - done - CMD="docker exec node-${NODE} bash -c \"{{KAFKA_OPTS}} /opt/automq/bin/$*\"" - echo "→ $CMD" >&2 - eval "$CMD" - -jmx *ARGS: - #!/usr/bin/env bash - NODE=0 - set -- {{ARGS}} - while [ $# -gt 0 ]; do - case $1 in - --node) NODE=$2; shift 2 ;; - *) break ;; - esac - done - echo "$@" | docker exec -i node-${NODE} java -jar /usr/local/bin/jmxterm.jar -l localhost:9999 -n - -# ==================== Shortcuts ==================== - -topic-list node="0": - @just bin --node {{node}} kafka-topics.sh --bootstrap-server localhost:9092 --list - -topic-create name *ARGS: - @just bin kafka-topics.sh --bootstrap-server localhost:9092 --create --topic {{name}} {{ARGS}} - -topic-describe name node="0": - @just bin --node {{node}} kafka-topics.sh --bootstrap-server localhost:9092 --describe --topic {{name}} - -produce topic node="0": - @docker exec -it node-{{node}} bash -c \ - "{{KAFKA_OPTS}} /opt/automq/bin/kafka-console-producer.sh --bootstrap-server localhost:9092 --topic {{topic}}" - -consume topic *ARGS: - #!/usr/bin/env bash - NODE=0; EXTRA="" - for arg in {{ARGS}}; do - case $arg in - --from-beginning) EXTRA="--from-beginning" ;; - --node) NEXT_NODE=1 ;; - *) [ "$NEXT_NODE" = 1 ] && NODE=$arg && NEXT_NODE="" ;; - esac - done - CMD="docker exec -it node-${NODE} bash -c \"{{KAFKA_OPTS}} /opt/automq/bin/kafka-console-consumer.sh --bootstrap-server localhost:9092 --topic {{topic}} $EXTRA\"" - echo "→ $CMD" >&2 - eval "$CMD" - -# ==================== Chaos ==================== - -[private] -_node-list: - @docker ps --filter "name=node-" --format '{{ '{{' }}.Names{{ '}}' }}' - -chaos-delay ms="200" node="node-0": - docker exec {{node}} tc qdisc add dev eth0 root netem delay {{ms}}ms - @echo "✓ {{ms}}ms delay on {{node}}" - -chaos-loss percent="5" node="node-0": - docker exec {{node}} tc qdisc add dev eth0 root netem loss {{percent}}% - @echo "✓ {{percent}}% packet loss on {{node}}" - -chaos-reset: - #!/usr/bin/env bash - for n in $(just _node-list); do - docker exec $n tc qdisc del dev eth0 root 2>/dev/null || true - done - echo "✓ Reset all chaos rules" - -chaos-status: - #!/usr/bin/env bash - for n in $(just _node-list); do - echo "=== $n ===" && docker exec $n tc qdisc show dev eth0 - done diff --git a/devbox/.gitignore b/devkit/.gitignore similarity index 80% rename from devbox/.gitignore rename to devkit/.gitignore index 0b27af2f0f..ba1ba868ea 100644 --- a/devbox/.gitignore +++ b/devkit/.gitignore @@ -1,3 +1,3 @@ -.devbox/ +.devkit/ config/custom.properties __pycache__/ diff --git a/devbox/Dockerfile b/devkit/Dockerfile similarity index 68% rename from devbox/Dockerfile rename to devkit/Dockerfile index b996bbdd0b..185913a01c 100644 --- a/devbox/Dockerfile +++ b/devkit/Dockerfile @@ -4,8 +4,9 @@ FROM $jdk_version ENV DEBIAN_FRONTEND=noninteractive RUN apt-get update && apt-get install -y \ - curl jq iproute2 iputils-ping net-tools wget \ + curl jq iproute2 iptables iputils-ping net-tools wget \ && wget -O /usr/local/bin/jmxterm.jar https://github.com/jiaqi/jmxterm/releases/download/v1.0.4/jmxterm-1.0.4-uber.jar \ + && mkdir -p /opt/arthas && wget -O /opt/arthas/arthas-boot.jar https://arthas.aliyun.com/arthas-boot.jar \ && apt-get -y clean && rm -rf /var/lib/apt/lists/* WORKDIR /opt/automq diff --git a/devkit/README.md b/devkit/README.md new file mode 100644 index 0000000000..1ef8fb6f5e --- /dev/null +++ b/devkit/README.md @@ -0,0 +1,399 @@ +# DevKit - AutoMQ Local Development Environment + +## Overview + +DevKit is AutoMQ's local development environment. One-command Kafka cluster with MinIO S3 storage, JDWP debugging, chaos testing, and optional feature stacks. + +**Core principle:** All commands run via `just`. Kafka commands execute inside containers — use shortcuts or `just bin` from the host. + +**Supported node counts:** 1, 3, 4, 5. Node numbering starts at 0. + +## Prerequisites + +- Docker & Docker Compose +- [just](https://github.com/casey/just) — `brew install just` +- `envsubst` — `brew install gettext` (macOS) +- JDK 17+ (for `just start-build` only) + +## Quick Start + +```bash +# First time: build image + code, then start +just start-build # single node +just start-build 3 # 3-node cluster + +# After first build +just start # single node +just start 3 # 3-node cluster +just start 3 tabletopic # 3-node + Iceberg table topic +``` + +## Architecture + +``` +MinIO (S3 storage) + └── mc (bucket init, one-shot) + └── AutoMQ Nodes + ├── node 0-2: controller + broker (KRaft voters) + └── node 3+: broker only +``` + +Node roles by count: a 4-node cluster has nodes 0-2 as controllers, node 3 as broker-only. + +Optional feature stacks: `tabletopic` → Schema Registry + Iceberg REST Catalog, `analytics` → Spark + Trino. + +## Decision: Which start command to use? + +``` +Java code changed + Dockerfile changed → just start-build +Java code changed only → just restart-build +Config files changed only → just restart +Nothing changed → just start [N] +``` + +## Lifecycle + +```bash +just start # Start (skip build), single node +just start 3 tabletopic # Start 3-node + features +just start-build # Build image & code, then start +just stop # Stop containers (keep data) +just clean # Stop + remove all volumes +just restart # Restart containers (no rebuild) +just restart-build # Rebuild code & image, then restart +just status # Show container status +just logs # Last 100 lines of all nodes +just logs 0 # Last 100 lines of node-0 +just logs 0 500 # Last 500 lines of node-0 +just logs-follow # Follow all logs (blocks) +just logs-follow 0 # Follow node-0 logs (blocks) +just shell # Enter node-0 shell (interactive) +just shell 1 # Enter node-1 shell (interactive) +just exec 0 ls /opt/automq/bin # Run a single command in node-0 +just build-image # Rebuild Docker image only +just wait # Wait until all nodes healthy (auto-called by start) +``` + +`just logs` outputs to stdout — pipe it freely: + +```bash +just logs 0 | grep ERROR +just logs 0 500 | grep WARN +just logs 0 | head -20 # oldest entries +just logs 0 | tail -20 # newest entries +``` + +## Topic Operations + +```bash +just topic-list # List all topics +just topic-create my-topic --partitions 16 # Create topic +just topic-describe my-topic # Describe topic + +# Producer — reads from stdin, supports --node N and all Kafka producer flags +just produce my-topic # Interactive (type messages, Ctrl-D to end) +echo "hello" | just produce my-topic # Pipe a single message +just produce my-topic --node 1 # Produce via node-1 + +# Produce key:value messages +printf 'k1:v1\nk2:v2\n' | just produce my-topic \ + --property parse.key=true \ + --property "key.separator=:" + +# Specify client-id (use --producer-property, not --client-id) +echo "hello" | just produce my-topic \ + --producer-property client.id=my-producer + +# Consumer — supports --node N and all Kafka consumer flags +just consume my-topic --from-beginning # Consume from beginning +just consume my-topic --node 1 # Consume via node-1 +just consume my-topic --max-messages 10 # Stop after 10 messages + +# Print keys with custom separator (special characters passed safely) +just consume my-topic --from-beginning \ + --property print.key=true \ + --property "key.separator=:" + +# Specify client-id (use --consumer-property, not --client-id) +just consume my-topic --from-beginning \ + --consumer-property client.id=my-consumer +``` + +## Consumer Group Operations + +```bash +just group-list # List all consumer groups +just group-describe my-group # Show offsets and lag per partition +just group-members my-group # Show active members and assignments +just group-reset my-group my-topic --to-earliest # Reset to earliest offset +just group-reset my-group my-topic --to-latest # Reset to latest offset +just group-reset my-group my-topic --to-offset 100 # Reset to specific offset +``` + +## Cluster Operations + +```bash +just broker-list # List brokers and supported API versions +just broker-config # Describe all broker configs (cluster-wide) +just broker-config 0 # Describe broker 0 configs only +``` + +## Bin (direct passthrough) + +Run any Kafka `bin/` script inside a node container. Commands run via `docker exec`, so `localhost:9092` refers to the broker inside the container. + +```bash +just bin kafka-topics.sh --bootstrap-server localhost:9092 --list +just bin kafka-configs.sh --bootstrap-server localhost:9092 --describe --entity-type brokers +just bin --node 1 kafka-broker-api-versions.sh --bootstrap-server localhost:9092 + +# One-shot shell command in a container +just exec 0 ls /opt/automq/bin +just exec 0 cat /config/node-0.properties +``` + +## Perf + +```bash +# -1 = unlimited throughput +just perf-produce my-topic --num-records 100000 --record-size 1024 --throughput -1 +just perf-consume my-topic --messages 100000 +``` + +## Diagnostics + +### Arthas (Java runtime diagnostics) + +```bash +just arthas # Attach to node-0 Kafka process (interactive) +just arthas 1 # Attach to node-1 (interactive) + +# Non-interactive: run a single Arthas command and exit +just arthas-exec 0 version +just arthas-exec 0 'dashboard -n 1' # one snapshot of CPU/memory/threads +just arthas-exec 0 'thread -n 5' # top 5 busy threads +just arthas-exec 0 'jvm' # JVM info (heap, GC, classloader) +just arthas-exec 0 'sc kafka.server.*' # search loaded classes + +# Note: streaming commands like watch/trace require active traffic to produce +# output and will block until -n limit is reached. Use just arthas for those. +``` + +### JMX + +JMX exposed on node-0 at port 9999 via [jmxterm](https://github.com/jiaqi/jmxterm) (pre-installed). + +```bash +just jmx -e 'domains' # List all JMX domains +just jmx -e 'beans -d kafka.server' # List beans in domain +just jmx -e 'info -b kafka.server:name=BrokerState,type=KafkaServer' # View bean attributes +just jmx -e 'get -b kafka.server:name=BrokerState,type=KafkaServer Value' # Broker state (3 = running) +just jmx -e 'get -b kafka.server:name=MessagesInPerSec,type=BrokerTopicMetrics Count' +just jmx --node 1 -e 'get -b kafka.server:name=BrokerState,type=KafkaServer Value' +``` + +### JDK tools + +```bash +just shell # Enter node-0 shell — jstack, jmap, jcmd available +just shell 1 # Enter node-1 shell +``` + +> To change JVM heap size, edit `HEAP_OPTS` in `justfile` (default: `-Xms256m -Xmx256m`). + +## IDEA Remote Debugging + +JDWP is enabled on all nodes at startup. Port mapping: node-0 → 5005, node-1 → 5006, node-2 → 5007, ... + +```bash +# In IDEA: Run → Edit Configurations → + → Remote JVM Debug → localhost:5005 +just start +# Then attach debugger +``` + +## Chaos Testing + +Inject faults using `tc netem` (network delay/loss) and `iptables` (traffic blocking). `NET_ADMIN` capability is already set in docker-compose.yml. + +**Always run `just chaos-reset` when done** — clears all tc rules, iptables rules, and unpauses MinIO. + +### Node network + +```bash +just chaos-delay # 200ms delay on node-0 +just chaos-delay 500 node-1 # 500ms delay on node-1 +just chaos-loss # 5% packet loss on node-0 +just chaos-loss 10 node-1 # 10% packet loss on node-1 +``` + +### S3 (MinIO) faults + +```bash +# Targeted: only affects traffic between the specified node and MinIO +just chaos-s3-delay 500 node-0 # 500ms latency to S3 on node-0 +just chaos-s3-loss 10 node-0 # 10% packet loss to S3 on node-0 + +# Global +just chaos-s3-down # Pause MinIO (S3 completely unavailable) +just chaos-s3-up # Resume MinIO + +# Partition (iptables DROP) +just chaos-s3-partition node-1 # Isolate node-1 from S3 +just chaos-s3-partition-reset # Restore S3 connectivity only +``` + +### Node-to-node partition + +```bash +just chaos-partition node-0 node-1 # Bidirectional isolation (iptables DROP) +just chaos-partition-reset # Restore node connectivity only +``` + +### Reset & status + +```bash +just chaos-reset # Remove ALL chaos rules (tc + iptables + unpause MinIO) +just chaos-status # Show active rules per node and MinIO state +``` + +### Scenarios + +#### S3 slow response +```bash +just chaos-s3-delay 500 node-0 +just produce my-topic +just chaos-reset +``` + +#### S3 outage +```bash +just chaos-s3-down +just logs +just chaos-s3-up +``` + +#### Single node S3 partition +```bash +just chaos-s3-partition node-1 +just logs 1 +just chaos-s3-partition-reset +``` + +#### Controller network partition +```bash +just start 3 +just chaos-partition node-0 node-1 +just logs +just chaos-partition-reset +``` + +#### Combined faults +```bash +just start 3 +just chaos-delay 200 node-0 +just chaos-s3-loss 10 node-0 +just produce my-topic +just chaos-reset +``` + +## Features + +Append feature names to `just start`. Multiple features can be combined. + +```bash +just start 1 telemetry # Single node + OTel metrics +just start 3 tabletopic # 3-node + Iceberg table topic +just start 5 zerozone analytics # 5-node + zone router + query engines +``` + +| Feature | What it enables | Extra services | +|---------|----------------|----------------| +| `tabletopic` | Iceberg table topic, REST catalog, schema registry | Schema Registry (:8081), Iceberg REST (:8181) | +| `zerozone` | Zone router, auto AZ assignment (round-robin: az-0/1/2) | — (config only) | +| `telemetry` | OTel metrics export via OTLP HTTP — **edit `config/features/telemetry.properties` to set collector endpoint before use** | — (config only) | +| `analytics` | Spark + Trino for querying Iceberg tables | Spark (:8888), Trino (:8090) | + +## Port Allocation + +| Node | Kafka | Controller | JDWP | JMX | +|------|-------|------------|------|-----| +| 0 | 9092 | 9093 | 5005 | 9999 | +| 1 | 19092 | 19093 | 5006 | — | +| 2 | 29092 | 29093 | 5007 | — | +| 3 | 39092 | 39093 | 5008 | — | +| 4 | 49092 | 49093 | 5009 | — | + +Other services: +- MinIO Console: http://localhost:9001 (admin/password) +- Schema Registry: http://localhost:8081 (with tabletopic) +- Iceberg REST: http://localhost:8181 (with tabletopic) +- Spark: http://localhost:8888 (with analytics) +- Trino: http://localhost:8090 (with analytics) + +## Configuration + +5-layer system — later layers override earlier ones. Each layer is a template rendered via `envsubst` before Docker starts. Final merged config: `.devkit/config/node-N.properties`. + +| Layer | File | Variables | Purpose | +|-------|------|-----------|---------| +| 1 | `config/defaults.properties` | S3 vars | S3 endpoints, KRaft basics | +| 2 | `config/role/server.properties` or `broker.properties` | S3 vars, `NODE_ID` | Listeners, process roles | +| 3 | `config/role/node.properties` | `NODE_ID`, `CLUSTER_ID`, `VOTERS`, `BOOTSTRAP` | KRaft identity | +| 4 | `config/features/*.properties` | All vars + `AZ_NAME` | Feature-specific settings | +| 5 | `config/custom.properties` | verbatim (no substitution) | Highest priority overrides | + +### Variables + +**S3 variables** (defined in `justfile`, Layer 1/2/4): + +| Variable | Default | Description | +|----------|---------|-------------| +| `S3_REGION` | `us-east-1` | S3 region | +| `S3_ENDPOINT` | `http://minio:9000` | S3 endpoint URL | +| `S3_DATA_BUCKET` | `automq-data` | Data bucket | +| `S3_OPS_BUCKET` | `automq-ops` | Ops bucket | +| `S3_PATH_STYLE` | `true` | Path-style access (required for MinIO) | + +**Per-node variables** (computed per node, Layer 2/3/4): + +| Variable | Example | Description | +|----------|---------|-------------| +| `NODE_ID` | `0`, `1`, `2` | Node index (0-based) | +| `CLUSTER_ID` | `abc-123` | Stable UUID, persisted in `.devkit/cluster-id` | +| `VOTERS` | `0@node-0:9093,1@node-1:9093` | KRaft controller voter list | +| `BOOTSTRAP` | `node-0:9093,node-1:9093` | Controller bootstrap servers | +| `AZ_NAME` | `az-0`, `az-1`, `az-2` | Round-robin by `NODE_ID % 3` (used by zerozone) | + +### Customization examples + +**Use real AWS S3** — edit `justfile`: +```just +S3_REGION := "ap-northeast-1" +S3_ENDPOINT := "https://s3.ap-northeast-1.amazonaws.com" +S3_DATA_BUCKET := "my-automq-data" +S3_OPS_BUCKET := "my-automq-ops" +S3_PATH_STYLE := "false" +``` + +**Change listeners (e.g. SASL)** — edit `config/role/server.properties`: +```properties +process.roles=broker,controller +listeners=SASL_PLAINTEXT://:9092,CONTROLLER://:9093 +advertised.listeners=SASL_PLAINTEXT://node-${NODE_ID}:9092,CONTROLLER://node-${NODE_ID}:9093 +``` + +**Add broker settings** — create `config/custom.properties` (gitignored): +```properties +num.partitions=4 +log.retention.hours=24 +``` + +**Add a new feature** — create `config/features/my-feature.properties`: +```properties +my.setting=value +my.s3.path=s3://${S3_DATA_BUCKET}/my-prefix +my.node.label=node-${NODE_ID} +``` +```bash +just start 1 my-feature +``` diff --git a/devbox/config/defaults.properties b/devkit/config/defaults.properties similarity index 58% rename from devbox/config/defaults.properties rename to devkit/config/defaults.properties index 8ecb9b8736..9b3cd7f622 100644 --- a/devbox/config/defaults.properties +++ b/devkit/config/defaults.properties @@ -1,4 +1,4 @@ -# AutoMQ DevBox - Default Configuration +# AutoMQ DevKit - Default Configuration # Layer 1: Base settings for local development with MinIO # KRaft mode @@ -6,11 +6,11 @@ controller.listener.names=CONTROLLER inter.broker.listener.name=PLAINTEXT listener.security.protocol.map=CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT -# MinIO S3 storage +# S3 storage (MinIO) elasticstream.enable=true -s3.data.buckets=0@s3://automq-data?region=us-east-1&endpoint=http://minio:9000&pathStyle=true -s3.ops.buckets=1@s3://automq-ops?region=us-east-1&endpoint=http://minio:9000&pathStyle=true -s3.wal.path=0@s3://automq-data?region=us-east-1&endpoint=http://minio:9000&pathStyle=true +s3.data.buckets=0@s3://${S3_DATA_BUCKET}?region=${S3_REGION}&endpoint=${S3_ENDPOINT}&pathStyle=${S3_PATH_STYLE} +s3.ops.buckets=1@s3://${S3_OPS_BUCKET}?region=${S3_REGION}&endpoint=${S3_ENDPOINT}&pathStyle=${S3_PATH_STYLE} +s3.wal.path=0@s3://${S3_DATA_BUCKET}?region=${S3_REGION}&endpoint=${S3_ENDPOINT}&pathStyle=${S3_PATH_STYLE} # Internal topics (single-node defaults) offsets.topic.replication.factor=1 diff --git a/devbox/config/features/tabletopic.properties b/devkit/config/features/tabletopic.properties similarity index 74% rename from devbox/config/features/tabletopic.properties rename to devkit/config/features/tabletopic.properties index 3e709ce066..f0fdf8956c 100644 --- a/devbox/config/features/tabletopic.properties +++ b/devkit/config/features/tabletopic.properties @@ -1,5 +1,5 @@ -# AutoMQ DevBox - TableTopic Feature -# Layer 2: Enables Iceberg table topic support +# AutoMQ DevKit - TableTopic Feature +# Layer 4: Enables Iceberg table topic support automq.table.topic.catalog.type=rest automq.table.topic.catalog.uri=http://rest:8181 diff --git a/devkit/config/features/telemetry.properties b/devkit/config/features/telemetry.properties new file mode 100644 index 0000000000..7c83631b6c --- /dev/null +++ b/devkit/config/features/telemetry.properties @@ -0,0 +1,7 @@ +# AutoMQ DevKit - Telemetry Feature +# Layer 4: Enables OpenTelemetry metrics export +# Change endpoint to your OTel Collector address before use + +s3.telemetry.metrics.exporter.uri=otlp://?endpoint=http://host.docker.internal:4318&protocol=http +s3.telemetry.metrics.level=INFO +s3.telemetry.exporter.report.interval.ms=30000 diff --git a/devkit/config/features/zerozone.properties b/devkit/config/features/zerozone.properties new file mode 100644 index 0000000000..8d3d4ac0c3 --- /dev/null +++ b/devkit/config/features/zerozone.properties @@ -0,0 +1,5 @@ +# AutoMQ DevKit - ZeroZone Feature +# Layer 4: Enables zone router + +automq.zonerouter.channels=0@s3://${S3_DATA_BUCKET}?region=${S3_REGION}&endpoint=${S3_ENDPOINT}&pathStyle=${S3_PATH_STYLE}&batchInterval=250&maxBytesInBatch=8388608 +broker.rack=${AZ_NAME} diff --git a/devkit/config/role/broker.properties b/devkit/config/role/broker.properties new file mode 100644 index 0000000000..6f99b34884 --- /dev/null +++ b/devkit/config/role/broker.properties @@ -0,0 +1,7 @@ +# AutoMQ DevKit - Broker Role Template (broker only) +# Used for node 3+ (non-voter brokers) +# Variable: ${NODE_ID} is replaced by generate-config + +process.roles=broker +listeners=PLAINTEXT://:9092 +advertised.listeners=PLAINTEXT://node-${NODE_ID}:9092 diff --git a/devkit/config/role/node.properties b/devkit/config/role/node.properties new file mode 100644 index 0000000000..c17c968f29 --- /dev/null +++ b/devkit/config/role/node.properties @@ -0,0 +1,8 @@ +# AutoMQ DevKit - Node Identity Template +# Variables replaced by generate-config: +# ${NODE_ID}, ${CLUSTER_ID}, ${VOTERS}, ${BOOTSTRAP} + +cluster.id=${CLUSTER_ID} +node.id=${NODE_ID} +controller.quorum.voters=${VOTERS} +controller.quorum.bootstrap.servers=${BOOTSTRAP} diff --git a/devkit/config/role/server.properties b/devkit/config/role/server.properties new file mode 100644 index 0000000000..a02aac934d --- /dev/null +++ b/devkit/config/role/server.properties @@ -0,0 +1,7 @@ +# AutoMQ DevKit - Server Role Template (controller + broker) +# Used for node 0-2 (KRaft voters) +# Variable: ${NODE_ID} is replaced by generate-config + +process.roles=broker,controller +listeners=PLAINTEXT://:9092,CONTROLLER://:9093 +advertised.listeners=PLAINTEXT://node-${NODE_ID}:9092,CONTROLLER://node-${NODE_ID}:9093 diff --git a/devbox/config/trino-catalog/iceberg.properties b/devkit/config/trino-catalog/iceberg.properties similarity index 100% rename from devbox/config/trino-catalog/iceberg.properties rename to devkit/config/trino-catalog/iceberg.properties diff --git a/devbox/docker-compose.yml b/devkit/docker-compose.yml similarity index 97% rename from devbox/docker-compose.yml rename to devkit/docker-compose.yml index 4482d5b1a0..2cc27bd64b 100644 --- a/devbox/docker-compose.yml +++ b/devkit/docker-compose.yml @@ -5,10 +5,12 @@ x-automq-common: &automq-common stop_grace_period: 1m volumes: - ${AUTOMQ_DEV_HOME}:/opt/automq - - ./.devbox/config:/config + - ./.devkit/config:/config restart: unless-stopped cap_add: - NET_ADMIN + labels: + com.automq.devkit: "true" environment: - KAFKA_S3_ACCESS_KEY=admin - KAFKA_S3_SECRET_KEY=password @@ -163,7 +165,7 @@ services: # ==================== TableTopic ==================== schema-registry: - image: confluentinc/cp-schema-registry:latest + image: confluentinc/cp-schema-registry:7.7.8 container_name: schema-registry hostname: schema-registry profiles: [tabletopic] @@ -206,7 +208,7 @@ services: # ==================== Analytics ==================== spark-iceberg: - image: tabulario/spark-iceberg + image: tabulario/spark-iceberg:3.5.5_1.8.1 container_name: spark hostname: spark-iceberg profiles: [analytics] diff --git a/devkit/justfile b/devkit/justfile new file mode 100644 index 0000000000..7a4d3acf00 --- /dev/null +++ b/devkit/justfile @@ -0,0 +1,463 @@ +# DevKit - AutoMQ Local Development Environment + +set dotenv-load := false + +export AUTOMQ_DEV_HOME := justfile_directory() + "/.." + +DEVKIT := justfile_directory() + "/.devkit" +COMPOSE := "docker compose -f " + justfile_directory() + "/docker-compose.yml" +ALL_PROFILES := "--profile single --profile cluster --profile cluster4 --profile cluster5 --profile tabletopic --profile analytics" +AZ_NAMES := "az-0 az-1 az-2" +HEAP_OPTS := "-Xms256m -Xmx256m" +KAFKA_OPTS := "KAFKA_HEAP_OPTS='" + HEAP_OPTS + "' KAFKA_JVM_PERFORMANCE_OPTS='' JMX_PORT=''" + +# S3 configuration +S3_REGION := "us-east-1" +S3_ENDPOINT := "http://minio:9000" +S3_DATA_BUCKET := "automq-data" +S3_OPS_BUCKET := "automq-ops" +S3_PATH_STYLE := "true" + +default: + @just --list + +[doc("Alias for just --list")] +help: + @just --list + +# ==================== Internal ==================== + +[private] +ensure-cluster-id: + #!/usr/bin/env bash + mkdir -p "{{DEVKIT}}" + if [ ! -f "{{DEVKIT}}/cluster-id" ]; then + KAFKA_HEAP_OPTS="{{HEAP_OPTS}}" \ + "{{AUTOMQ_DEV_HOME}}/bin/kafka-storage.sh" random-uuid \ + > "{{DEVKIT}}/cluster-id" + echo "✓ Generated CLUSTER_ID: $(cat "{{DEVKIT}}/cluster-id")" + fi + +# Override to skip or replace the Java build step, e.g.: +# DEVKIT_BUILD_CMD="echo skipped" just start-build +JAVA_BUILD_CMD := env_var_or_default("DEVKIT_BUILD_CMD", "./gradlew :core:build :tools:build :shell:build -x test") + +[private] +build: + @echo "✓ Building Docker image..." + @{{COMPOSE}} build + @echo "✓ Building Java code..." + cd "{{AUTOMQ_DEV_HOME}}" && {{JAVA_BUILD_CMD}} + +[private] +_compute-cluster-vars nodes: + #!/usr/bin/env bash + MAX_CTRL=3 + CTRL_COUNT=$(({{nodes}} < MAX_CTRL ? {{nodes}} : MAX_CTRL)) + VOTERS="" ; BOOTSTRAP="" + for i in $(seq 0 $((CTRL_COUNT - 1))); do + [ -n "$VOTERS" ] && VOTERS="$VOTERS," && BOOTSTRAP="$BOOTSTRAP," + VOTERS="${VOTERS}${i}@node-${i}:9093" + BOOTSTRAP="${BOOTSTRAP}node-${i}:9093" + done + echo "CTRL_COUNT=$CTRL_COUNT" + echo "VOTERS=$VOTERS" + echo "BOOTSTRAP=$BOOTSTRAP" + +[private] +_render-node node ctrl_count cluster_id voters bootstrap features="": + #!/usr/bin/env bash + set -e + DIR="{{justfile_directory()}}/config" + CFG="{{DEVKIT}}/config/node-{{node}}.properties" + AZS=({{AZ_NAMES}}) + + export S3_REGION="{{S3_REGION}}" + export S3_ENDPOINT="{{S3_ENDPOINT}}" + export S3_DATA_BUCKET="{{S3_DATA_BUCKET}}" + export S3_OPS_BUCKET="{{S3_OPS_BUCKET}}" + export S3_PATH_STYLE="{{S3_PATH_STYLE}}" + export CLUSTER_ID="{{cluster_id}}" + export VOTERS="{{voters}}" + export BOOTSTRAP="{{bootstrap}}" + export NODE_ID="{{node}}" + export AZ_NAME="${AZS[$(({{node}} % ${#AZS[@]}))]}" + + render() { envsubst < "$1" | grep -v '^#' | grep -v '^$'; } + + # Layer 1: defaults + envsubst < "$DIR/defaults.properties" > "$CFG" + + # Layer 2: role + { echo ""; echo "# === Role ==="; \ + ROLE=$([ {{node}} -lt {{ctrl_count}} ] && echo server || echo broker); \ + render "$DIR/role/$ROLE.properties"; } >> "$CFG" + + # Layer 3: node identity + { echo ""; echo "# === Node ==="; render "$DIR/role/node.properties"; } >> "$CFG" + + # Layer 4: features + for feat in {{features}}; do + F="$DIR/features/${feat}.properties" + [ -f "$F" ] && { echo ""; render "$F"; } >> "$CFG" + done + + # Layer 5: custom overrides (verbatim) + C="$DIR/custom.properties" + [ -f "$C" ] && { echo ""; cat "$C"; } >> "$CFG" + + echo "✓ Config: node-{{node}}.properties" + +[private] +generate-config nodes features="": + #!/usr/bin/env bash + set -e + command -v envsubst >/dev/null || { echo "✗ envsubst not found: brew install gettext"; exit 1; } + mkdir -p "{{DEVKIT}}/config" + CLUSTER_ID=$(cat "{{DEVKIT}}/cluster-id") + eval "$(just _compute-cluster-vars {{nodes}})" + for i in $(seq 0 $(({{nodes}} - 1))); do + just _render-node "$i" "$CTRL_COUNT" "$CLUSTER_ID" "$VOTERS" "$BOOTSTRAP" "{{features}}" + done + +[private] +_node-list: + @docker ps --filter "label=com.automq.devkit=true" --format '{{ '{{' }}.Names{{ '}}' }}' + +[private] +_ensure-iptables-chain node: + #!/usr/bin/env bash + docker exec {{node}} iptables -N DEVKIT 2>/dev/null || true + docker exec {{node}} iptables -C OUTPUT -j DEVKIT 2>/dev/null || \ + docker exec {{node}} iptables -A OUTPUT -j DEVKIT + docker exec {{node}} iptables -C INPUT -j DEVKIT 2>/dev/null || \ + docker exec {{node}} iptables -A INPUT -j DEVKIT + +# ==================== Lifecycle ==================== + +[doc("Start N nodes (default: 1). Example: just start 3 tabletopic")] +start nodes="1" *FEATURES: ensure-cluster-id + #!/usr/bin/env bash + set -e + case {{nodes}} in + 1) PROFILE=single ;; 3) PROFILE=cluster ;; 4) PROFILE=cluster4 ;; 5) PROFILE=cluster5 ;; + *) echo "Error: nodes must be 1,3,4,5"; exit 1 ;; + esac + COMPOSE_PROFILES="--profile $PROFILE" + for feat in {{FEATURES}}; do + COMPOSE_PROFILES="$COMPOSE_PROFILES --profile $feat" + done + just generate-config {{nodes}} "{{FEATURES}}" + echo "✓ Starting {{nodes}} node(s)..." + {{COMPOSE}} $COMPOSE_PROFILES up -d + echo "" + echo "🎉 AutoMQ DevKit started" + echo " Kafka: localhost:9092" + echo " Debug: localhost:5005" + echo " MinIO: http://localhost:9001 (admin/password)" + just wait + +[doc("Build image & code, then start. Example: just start-build 3 tabletopic")] +start-build nodes="1" *FEATURES: build (start nodes FEATURES) + +[doc("Stop all services (keep data)")] +stop: + @echo "Stopping..." + @{{COMPOSE}} {{ALL_PROFILES}} stop + +[doc("Stop all services and remove volumes")] +clean: + @echo "Stopping and removing volumes..." + @{{COMPOSE}} {{ALL_PROFILES}} down -v + +[doc("Restart containers (no rebuild)")] +restart: + @{{COMPOSE}} {{ALL_PROFILES}} restart + +[doc("Rebuild code & image, then restart")] +restart-build: build restart + +[doc("Show service status")] +status: + @{{COMPOSE}} {{ALL_PROFILES}} ps + +[doc("Show logs snapshot. Example: just logs 0 200")] +logs node="" tail="100": + #!/usr/bin/env bash + if [ -z "{{node}}" ]; then + {{COMPOSE}} {{ALL_PROFILES}} logs --tail={{tail}} + else + {{COMPOSE}} {{ALL_PROFILES}} logs --tail={{tail}} node-{{node}} + fi + +[doc("Follow logs (blocks). Example: just logs-follow 0")] +logs-follow node="": + #!/usr/bin/env bash + if [ -z "{{node}}" ]; then + {{COMPOSE}} {{ALL_PROFILES}} logs -f + else + {{COMPOSE}} {{ALL_PROFILES}} logs -f node-{{node}} + fi + +[doc("Enter container shell (default: node 0)")] +shell node="0": + @docker exec -it node-{{node}} bash + +[doc("Run a single command in container. Example: just exec 0 ls /opt/automq/bin")] +exec node *CMD: + @docker exec node-{{node}} {{CMD}} + +[doc("Wait for all nodes to be healthy (auto-called by start)")] +wait timeout="180": + #!/usr/bin/env bash + echo "Waiting for nodes to be healthy..." + TIMEOUT={{timeout}}; ELAPSED=0 + while [ $ELAPSED -lt $TIMEOUT ]; do + NODES=$(just _node-list) + [ -z "$NODES" ] && sleep 5 && ELAPSED=$((ELAPSED + 5)) && continue + ALL_HEALTHY=true + for n in $NODES; do + S=$(docker inspect --format='{{ '{{' }}.State.Health.Status{{ '}}' }}' "$n" 2>/dev/null) + [ "$S" != "healthy" ] && ALL_HEALTHY=false && break + done + [ "$ALL_HEALTHY" = true ] && echo "✓ All nodes healthy" && exit 0 + sleep 5; ELAPSED=$((ELAPSED + 5)) + done + echo "✗ Timeout after ${TIMEOUT}s" && exit 1 + +[doc("Rebuild Docker image only")] +build-image *args="": + @echo "✓ Building Docker image..." + @{{COMPOSE}} build {{ if args == "" { "node-0 node-1 node-2 node-3 node-4" } else { args } }} + +# ==================== Bin ==================== + +[doc("Run bin/ command in container. Example: just bin kafka-topics.sh --bootstrap-server localhost:9092 --list")] +bin *ARGS: + #!/usr/bin/env bash + NODE=0; PARAMS=() + set -- {{ARGS}} + while [ $# -gt 0 ]; do + case $1 in + --node) NODE=$2; shift 2 ;; + *) PARAMS+=("$1"); shift ;; + esac + done + docker exec "node-${NODE}" \ + env "KAFKA_HEAP_OPTS={{HEAP_OPTS}}" KAFKA_JVM_PERFORMANCE_OPTS="" JMX_PORT="" \ + /opt/automq/bin/"${PARAMS[@]}" 2> >(grep -v '^SLF4J' >&2) + +[doc("Query JMX metrics. Example: just jmx -e domains")] +jmx *ARGS: + #!/usr/bin/env bash + NODE=0 + set -- {{ARGS}} + while [ $# -gt 0 ]; do + case $1 in + --node) NODE=$2; shift 2 ;; + *) break ;; + esac + done + echo "$@" | docker exec -i node-${NODE} java -jar /usr/local/bin/jmxterm.jar -l localhost:9999 -n + +# ==================== Shortcuts ==================== + +[doc("List topics")] +topic-list node="0": + @just bin --node {{node}} kafka-topics.sh --bootstrap-server localhost:9092 --list + +[doc("Create topic. Example: just topic-create my-topic --partitions 16")] +topic-create name *ARGS: + @just bin kafka-topics.sh --bootstrap-server localhost:9092 --create --topic {{name}} {{ARGS}} + +[doc("Describe topic")] +topic-describe name node="0": + @just bin --node {{node}} kafka-topics.sh --bootstrap-server localhost:9092 --describe --topic {{name}} + +[doc("Interactive producer. Supports --node N and all other Kafka producer flags")] +produce topic *ARGS: + #!/usr/bin/env bash + NODE=0; PARAMS=() + set -- {{ARGS}} + while [ $# -gt 0 ]; do + case $1 in + --node) NODE=$2; shift 2 ;; + *) PARAMS+=("$1"); shift ;; + esac + done + docker exec -i "node-${NODE}" \ + env "KAFKA_HEAP_OPTS={{HEAP_OPTS}}" KAFKA_JVM_PERFORMANCE_OPTS="" JMX_PORT="" \ + /opt/automq/bin/kafka-console-producer.sh \ + --bootstrap-server localhost:9092 --topic "{{topic}}" \ + "${PARAMS[@]}" 2> >(grep -v '^SLF4J' >&2) + +[doc("Consumer. Supports --node N and all other Kafka consumer flags")] +consume topic *ARGS: + #!/usr/bin/env bash + NODE=0; PARAMS=() + set -- {{ARGS}} + while [ $# -gt 0 ]; do + case $1 in + --node) NODE=$2; shift 2 ;; + *) PARAMS+=("$1"); shift ;; + esac + done + docker exec -i "node-${NODE}" \ + env "KAFKA_HEAP_OPTS={{HEAP_OPTS}}" KAFKA_JVM_PERFORMANCE_OPTS="" JMX_PORT="" \ + /opt/automq/bin/kafka-console-consumer.sh \ + --bootstrap-server localhost:9092 --topic "{{topic}}" \ + "${PARAMS[@]}" 2> >(grep -v '^SLF4J' >&2) + +[doc("List consumer groups")] +group-list: + @just bin kafka-consumer-groups.sh --bootstrap-server localhost:9092 --list + +[doc("Describe consumer group (offsets, lag). Example: just group-describe my-group")] +group-describe group: + @just bin kafka-consumer-groups.sh --bootstrap-server localhost:9092 --describe --group {{group}} + +[doc("Show active members of consumer group. Example: just group-members my-group")] +group-members group: + @just bin kafka-consumer-groups.sh --bootstrap-server localhost:9092 --describe --group {{group}} --members + +[doc("Reset consumer group offsets. Example: just group-reset my-group my-topic --to-earliest")] +group-reset group topic *ARGS: + @just bin kafka-consumer-groups.sh --bootstrap-server localhost:9092 --reset-offsets --group {{group}} --topic {{topic}} {{ARGS}} --execute + +[doc("List brokers and API versions")] +broker-list: + @just bin kafka-broker-api-versions.sh --bootstrap-server localhost:9092 + +[doc("Describe broker configs. Example: just broker-config 0")] +broker-config id="": + #!/usr/bin/env bash + if [ -z "{{id}}" ]; then + just bin kafka-configs.sh --bootstrap-server localhost:9092 --describe --entity-type brokers --all + else + just bin kafka-configs.sh --bootstrap-server localhost:9092 --describe --entity-type brokers --entity-name {{id}} --all + fi + +# ==================== Perf ==================== + +[doc("Producer perf test. Example: just perf-produce my-topic --num-records 100000 --record-size 1024 --throughput -1")] +perf-produce topic *ARGS: + @just bin kafka-producer-perf-test.sh --topic {{topic}} --producer-props bootstrap.servers=localhost:9092 {{ARGS}} + +[doc("Consumer perf test. Example: just perf-consume my-topic --messages 100000")] +perf-consume topic *ARGS: + @just bin kafka-consumer-perf-test.sh --bootstrap-server localhost:9092 --topic {{topic}} {{ARGS}} + +# ==================== Diagnostics ==================== + +[doc("Attach Arthas to Kafka process. Example: just arthas 1")] +arthas node="0": + @docker exec -it node-{{node}} java -jar /opt/arthas/arthas-boot.jar + +[doc("Run a single Arthas command non-interactively. Example: just arthas-exec 0 'dashboard -n 1'")] +arthas-exec node cmd: + #!/usr/bin/env bash + printf '%s\n' '{{cmd}}' | docker exec -i node-{{node}} bash -c \ + 'cat > /tmp/_arthas_cmd && java -jar /opt/arthas/arthas-boot.jar --select kafka.Kafka -f /tmp/_arthas_cmd' + +# ==================== Chaos: Node network ==================== + +[doc("Network delay on node. Example: just chaos-delay 500 node-1")] +chaos-delay ms="200" node="node-0": + docker exec {{node}} tc qdisc replace dev eth0 root netem delay {{ms}}ms + @echo "✓ {{ms}}ms delay on {{node}}" + +[doc("Packet loss on node. Example: just chaos-loss 10 node-1")] +chaos-loss percent="5" node="node-0": + docker exec {{node}} tc qdisc replace dev eth0 root netem loss {{percent}}% + @echo "✓ {{percent}}% packet loss on {{node}}" + +# ==================== Chaos: S3 (MinIO) ==================== + +[doc("S3 latency on a node (only S3 traffic affected). Example: just chaos-s3-delay 500 node-0")] +chaos-s3-delay ms="500" node="node-0": + #!/usr/bin/env bash + MINIO_IP=$(docker exec {{node}} getent hosts minio | awk '{print $1}') + docker exec {{node}} tc qdisc del dev eth0 root 2>/dev/null || true + docker exec {{node}} tc qdisc add dev eth0 root handle 1: prio + docker exec {{node}} tc qdisc add dev eth0 parent 1:3 handle 30: netem delay {{ms}}ms + docker exec {{node}} tc filter add dev eth0 parent 1:0 protocol ip u32 match ip dst $MINIO_IP/32 flowid 1:3 + echo "✓ {{ms}}ms delay to MinIO on {{node}} (other traffic unaffected)" + +[doc("S3 packet loss on a node (only S3 traffic affected). Example: just chaos-s3-loss 10 node-0")] +chaos-s3-loss percent="5" node="node-0": + #!/usr/bin/env bash + MINIO_IP=$(docker exec {{node}} getent hosts minio | awk '{print $1}') + docker exec {{node}} tc qdisc del dev eth0 root 2>/dev/null || true + docker exec {{node}} tc qdisc add dev eth0 root handle 1: prio + docker exec {{node}} tc qdisc add dev eth0 parent 1:3 handle 30: netem loss {{percent}}% + docker exec {{node}} tc filter add dev eth0 parent 1:0 protocol ip u32 match ip dst $MINIO_IP/32 flowid 1:3 + echo "✓ {{percent}}% packet loss to MinIO on {{node}} (other traffic unaffected)" + +[doc("Pause MinIO — S3 completely unavailable")] +chaos-s3-down: + docker pause minio + @echo "✓ MinIO (S3) paused — all nodes lost S3 access" + +[doc("Resume MinIO")] +chaos-s3-up: + docker unpause minio + @echo "✓ MinIO (S3) resumed" + +[doc("Isolate a node from S3. Example: just chaos-s3-partition node-1")] +chaos-s3-partition node="node-0": (_ensure-iptables-chain node) + docker exec {{node}} iptables -A DEVKIT -d minio -j DROP + docker exec {{node}} iptables -A DEVKIT -s minio -j DROP + @echo "✓ {{node}} isolated from MinIO (S3)" + +[doc("Restore all nodes' S3 connectivity")] +chaos-s3-partition-reset: + #!/usr/bin/env bash + for n in $(just _node-list); do + docker exec $n iptables -D DEVKIT -d minio -j DROP 2>/dev/null || true + docker exec $n iptables -D DEVKIT -s minio -j DROP 2>/dev/null || true + done + echo "✓ All S3 partitions removed" + +# ==================== Chaos: Node-to-node partition ==================== + +[doc("Isolate two nodes. Example: just chaos-partition node-0 node-1")] +chaos-partition a b: (_ensure-iptables-chain a) (_ensure-iptables-chain b) + docker exec {{a}} iptables -A DEVKIT -d {{b}} -j DROP + docker exec {{a}} iptables -A DEVKIT -s {{b}} -j DROP + docker exec {{b}} iptables -A DEVKIT -d {{a}} -j DROP + docker exec {{b}} iptables -A DEVKIT -s {{a}} -j DROP + @echo "✓ {{a}} ↔ {{b}} isolated" + +[doc("Restore all node-to-node connectivity")] +chaos-partition-reset: + #!/usr/bin/env bash + for n in $(just _node-list); do + docker exec $n iptables -F DEVKIT 2>/dev/null || true + done + echo "✓ All node partitions removed" + +# ==================== Chaos: Reset & status ==================== + +[doc("Remove ALL chaos rules (tc + iptables + unpause)")] +chaos-reset: + #!/usr/bin/env bash + for n in $(just _node-list); do + docker exec $n tc qdisc del dev eth0 root 2>/dev/null || true + docker exec $n iptables -F DEVKIT 2>/dev/null || true + done + # Unpause MinIO if paused + docker unpause minio 2>/dev/null || true + echo "✓ All chaos rules reset (tc + iptables + unpause)" + +[doc("Show all active chaos rules")] +chaos-status: + #!/usr/bin/env bash + for n in $(just _node-list); do + echo "=== $n ===" + echo " tc:" && docker exec $n tc qdisc show dev eth0 + RULES=$(docker exec $n iptables -L DEVKIT -n 2>/dev/null | grep -c DROP || true) + [ "$RULES" -gt 0 ] && echo " iptables: ${RULES} DROP rules in DEVKIT chain" + done + echo "=== minio ===" + S=$(docker inspect --format='{{ '{{' }}.State.Paused{{ '}}' }}' minio 2>/dev/null) + [ "$S" = "true" ] && echo " status: PAUSED" || echo " status: running"