diff --git a/devkit/Dockerfile b/devkit/Dockerfile index 185913a01c..33fd495f46 100644 --- a/devkit/Dockerfile +++ b/devkit/Dockerfile @@ -2,11 +2,13 @@ ARG jdk_version=eclipse-temurin:17-jdk-jammy FROM $jdk_version ENV DEBIAN_FRONTEND=noninteractive +ENV PATH="/root/.local/bin:$PATH" RUN apt-get update && apt-get install -y \ - curl jq iproute2 iptables iputils-ping net-tools wget \ + curl jq iproute2 iptables iputils-ping net-tools wget python3 \ && wget -O /usr/local/bin/jmxterm.jar https://github.com/jiaqi/jmxterm/releases/download/v1.0.4/jmxterm-1.0.4-uber.jar \ && mkdir -p /opt/arthas && wget -O /opt/arthas/arthas-boot.jar https://arthas.aliyun.com/arthas-boot.jar \ + && curl -LsSf https://astral.sh/uv/install.sh | sh \ && apt-get -y clean && rm -rf /var/lib/apt/lists/* WORKDIR /opt/automq diff --git a/devkit/README.md b/devkit/README.md index 1ef8fb6f5e..20e6d3d587 100644 --- a/devkit/README.md +++ b/devkit/README.md @@ -313,6 +313,29 @@ just start 5 zerozone analytics # 5-node + zone router + query engines | `telemetry` | OTel metrics export via OTLP HTTP — **edit `config/features/telemetry.properties` to set collector endpoint before use** | — (config only) | | `analytics` | Spark + Trino for querying Iceberg tables | Spark (:8888), Trino (:8090) | +## Namespace (Multi-Instance Isolation) + +Run multiple DevKit instances on the same machine without port conflicts. Each namespace gets a unique `COMPOSE_PROJECT_NAME` and port offset. + +**Requires separate git worktrees** — each worktree has its own `devkit/` directory and `.devkit/compose.env`, so each instance is fully isolated. You cannot run multiple namespaces from the same devkit directory. + +```bash +# In worktree A: +just set-ns lag-test 1 # ID=1 → ports offset by +100 +just start 3 + +# In worktree B: +just set-ns perf 2 # ID=2 → ports offset by +200 +just start + +# Revert to default ports +just clear-ns +``` + +- Name: 1-8 chars, must start with a letter, only `[a-z0-9-]` +- ID: 1-9, determines port offset (`ID * 100`) +- Config is stored in `.devkit/compose.env` (gitignored) + ## Port Allocation | Node | Kafka | Controller | JDWP | JMX | @@ -353,6 +376,8 @@ Other services: | `S3_DATA_BUCKET` | `automq-data` | Data bucket | | `S3_OPS_BUCKET` | `automq-ops` | Ops bucket | | `S3_PATH_STYLE` | `true` | Path-style access (required for MinIO) | +| `S3_ACCESS_KEY` | `admin` | S3 access key (exported as `KAFKA_S3_ACCESS_KEY` to containers) | +| `S3_SECRET_KEY` | `password` | S3 secret key (exported as `KAFKA_S3_SECRET_KEY` to containers) | **Per-node variables** (computed per node, Layer 2/3/4): @@ -373,6 +398,8 @@ S3_ENDPOINT := "https://s3.ap-northeast-1.amazonaws.com" S3_DATA_BUCKET := "my-automq-data" S3_OPS_BUCKET := "my-automq-ops" S3_PATH_STYLE := "false" +S3_ACCESS_KEY := "AKIA..." +S3_SECRET_KEY := "..." ``` **Change listeners (e.g. SASL)** — edit `config/role/server.properties`: diff --git a/devkit/docker-compose.yml b/devkit/docker-compose.yml index 74e18699cc..a5ea6bf629 100644 --- a/devkit/docker-compose.yml +++ b/devkit/docker-compose.yml @@ -12,10 +12,11 @@ x-automq-common: &automq-common labels: com.automq.devkit: "true" environment: - - KAFKA_S3_ACCESS_KEY=admin - - KAFKA_S3_SECRET_KEY=password + - KAFKA_S3_ACCESS_KEY=${KAFKA_S3_ACCESS_KEY} + - KAFKA_S3_SECRET_KEY=${KAFKA_S3_SECRET_KEY} - KAFKA_HEAP_OPTS=${KAFKA_HEAP_OPTS:--Xms1g -Xmx4g -XX:MetaspaceSize=96m -XX:MaxDirectMemorySize=1G} - KAFKA_JVM_PERFORMANCE_OPTS=-server -XX:+UseZGC -XX:ZCollectionInterval=5 + - LOG_DIR=/tmp/kafka-logs depends_on: mc: condition: service_completed_successfully @@ -35,15 +36,14 @@ services: # ==================== Infrastructure ==================== minio: image: minio/minio:RELEASE.2025-05-24T17-08-30Z - container_name: minio hostname: warehouse.minio environment: - MINIO_ROOT_USER=admin - MINIO_ROOT_PASSWORD=password - MINIO_DOMAIN=minio ports: - - "9000:9000" - - "9001:9001" + - "${MINIO_API_PORT:-9000}:9000" + - "${MINIO_CONSOLE_PORT:-9001}:9001" command: ["server", "/data", "--console-address", ":9001"] healthcheck: test: ["CMD", "curl", "-f", "http://minio:9000/minio/health/live"] @@ -53,7 +53,6 @@ services: mc: image: minio/mc:RELEASE.2025-05-21T01-59-54Z - container_name: mc depends_on: minio: condition: service_healthy @@ -69,13 +68,12 @@ services: # ==================== AutoMQ Nodes ==================== node-0: <<: *automq-common - container_name: node-0 hostname: node-0 profiles: [single, cluster, cluster4, cluster5] ports: - - "9092:9192" - - "5005:5005" - - "9999:9999" + - "${NODE0_KAFKA_PORT:-9092}:9192" + - "${NODE0_DEBUG_PORT:-5005}:5005" + - "${NODE0_JMX_PORT:-9999}:9999" command: - bash - -c @@ -88,12 +86,11 @@ services: node-1: <<: *automq-common - container_name: node-1 hostname: node-1 profiles: [cluster, cluster4, cluster5] ports: - - "19092:9192" - - "5006:5005" + - "${NODE1_KAFKA_PORT:-19092}:9192" + - "${NODE1_DEBUG_PORT:-5006}:5005" command: - bash - -c @@ -106,12 +103,11 @@ services: node-2: <<: *automq-common - container_name: node-2 hostname: node-2 profiles: [cluster, cluster4, cluster5] ports: - - "29092:9192" - - "5007:5005" + - "${NODE2_KAFKA_PORT:-29092}:9192" + - "${NODE2_DEBUG_PORT:-5007}:5005" command: - bash - -c @@ -124,12 +120,11 @@ services: node-3: <<: *automq-common - container_name: node-3 hostname: node-3 profiles: [cluster4, cluster5] ports: - - "39092:9192" - - "5008:5005" + - "${NODE3_KAFKA_PORT:-39092}:9192" + - "${NODE3_DEBUG_PORT:-5008}:5005" command: - bash - -c @@ -142,12 +137,11 @@ services: node-4: <<: *automq-common - container_name: node-4 hostname: node-4 profiles: [cluster5] ports: - - "49092:9192" - - "5009:5005" + - "${NODE4_KAFKA_PORT:-49092}:9192" + - "${NODE4_DEBUG_PORT:-5009}:5005" command: - bash - -c @@ -161,11 +155,10 @@ services: # ==================== TableTopic ==================== schema-registry: image: confluentinc/cp-schema-registry:7.7.8 - container_name: schema-registry hostname: schema-registry profiles: [tabletopic] ports: - - "8081:8081" + - "${SCHEMA_REGISTRY_PORT:-8081}:8081" environment: SCHEMA_REGISTRY_HOST_NAME: schema-registry SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS: node-0:9092 @@ -179,11 +172,10 @@ services: rest: image: apache/iceberg-rest-fixture - container_name: iceberg-rest hostname: rest profiles: [tabletopic] ports: - - "8181:8181" + - "${ICEBERG_REST_PORT:-8181}:8181" depends_on: minio: condition: service_healthy @@ -204,7 +196,6 @@ services: # ==================== Analytics ==================== spark-iceberg: image: tabulario/spark-iceberg:3.5.5_1.8.1 - container_name: spark hostname: spark-iceberg profiles: [analytics] depends_on: @@ -215,18 +206,17 @@ services: - AWS_SECRET_ACCESS_KEY=password - AWS_REGION=us-east-1 ports: - - "8888:8888" - - "8080:8080" + - "${SPARK_NOTEBOOK_PORT:-8888}:8888" + - "${SPARK_UI_PORT:-8080}:8080" trino: image: trinodb/trino:472 - container_name: trino hostname: trino profiles: [analytics] depends_on: minio: condition: service_healthy ports: - - "8090:8080" + - "${TRINO_PORT:-8090}:8080" volumes: - ./config/trino-catalog:/etc/trino/catalog diff --git a/devkit/justfile b/devkit/justfile index 1b3604a1b5..dcf817a06d 100644 --- a/devkit/justfile +++ b/devkit/justfile @@ -3,20 +3,25 @@ set dotenv-load := false export AUTOMQ_DEV_HOME := justfile_directory() + "/.." +export KAFKA_S3_ACCESS_KEY := S3_ACCESS_KEY +export KAFKA_S3_SECRET_KEY := S3_SECRET_KEY DEVKIT := justfile_directory() + "/.devkit" -COMPOSE := "docker compose -f " + justfile_directory() + "/docker-compose.yml" +_ENV_FILE_FLAG := if path_exists(justfile_directory() + "/.devkit/compose.env") == "true" { "--env-file " + justfile_directory() + "/.devkit/compose.env" } else { "" } +COMPOSE := "docker compose -f " + justfile_directory() + "/docker-compose.yml " + _ENV_FILE_FLAG ALL_PROFILES := "--profile single --profile cluster --profile cluster4 --profile cluster5 --profile tabletopic --profile analytics" AZ_NAMES := "az-0 az-1 az-2" HEAP_OPTS := "-Xms256m -Xmx256m" KAFKA_OPTS := "KAFKA_HEAP_OPTS='" + HEAP_OPTS + "' KAFKA_JVM_PERFORMANCE_OPTS='' JMX_PORT=''" # S3 configuration -S3_REGION := "us-east-1" -S3_ENDPOINT := "http://minio:9000" +S3_REGION := "us-east-1" +S3_ENDPOINT := "http://minio:9000" S3_DATA_BUCKET := "automq-data" S3_OPS_BUCKET := "automq-ops" S3_PATH_STYLE := "true" +S3_ACCESS_KEY := "admin" +S3_SECRET_KEY := "password" default: @just --list @@ -81,7 +86,8 @@ _render-node node ctrl_count cluster_id voters bootstrap features="": export VOTERS="{{voters}}" export BOOTSTRAP="{{bootstrap}}" export NODE_ID="{{node}}" - export HOST_PORT=$(({{node}} * 10000 + 9092)) + PORT_OFFSET=$(grep -s '^PORT_OFFSET=' "{{DEVKIT}}/compose.env" | cut -d= -f2) + export HOST_PORT=$(({{node}} * 10000 + 9092 + ${PORT_OFFSET:-0})) export AZ_NAME="${AZS[$(({{node}} % ${#AZS[@]}))]}" render() { envsubst < "$1" | grep -v '^#' | grep -v '^$'; } @@ -123,16 +129,83 @@ generate-config nodes features="": [private] _node-list: - @docker ps --filter "label=com.automq.devkit=true" --format '{{ '{{' }}.Names{{ '}}' }}' + @{{COMPOSE}} ps --format '{{ '{{' }}.Service{{ '}}' }}' 2>/dev/null | grep 'node-' || true [private] _ensure-iptables-chain node: #!/usr/bin/env bash - docker exec {{node}} iptables -N DEVKIT 2>/dev/null || true - docker exec {{node}} iptables -C OUTPUT -j DEVKIT 2>/dev/null || \ - docker exec {{node}} iptables -A OUTPUT -j DEVKIT - docker exec {{node}} iptables -C INPUT -j DEVKIT 2>/dev/null || \ - docker exec {{node}} iptables -A INPUT -j DEVKIT + {{COMPOSE}} exec {{node}} iptables -N DEVKIT 2>/dev/null || true + {{COMPOSE}} exec {{node}} iptables -C OUTPUT -j DEVKIT 2>/dev/null || \ + {{COMPOSE}} exec {{node}} iptables -A OUTPUT -j DEVKIT + {{COMPOSE}} exec {{node}} iptables -C INPUT -j DEVKIT 2>/dev/null || \ + {{COMPOSE}} exec {{node}} iptables -A INPUT -j DEVKIT + +# ==================== Namespace ==================== + +[doc("Set namespace for multi-instance isolation. Example: just set-ns lag-test 1")] +set-ns name id: + #!/usr/bin/env bash + set -e + NAME="{{name}}" + ID="{{id}}" + if ! [[ "$NAME" =~ ^[a-z][a-z0-9-]{0,7}$ ]]; then + echo "✗ Name must be 1-8 chars, start with a letter, only [a-z0-9-]" + exit 1 + fi + if ! [[ "$ID" =~ ^[1-9]$ ]]; then + echo "✗ ID must be 1-9" + exit 1 + fi + + OFFSET=$((ID * 100)) + CHECK_PORT=$((9092 + OFFSET)) + + if lsof -i ":${CHECK_PORT}" >/dev/null 2>&1; then + echo "✗ Port ${CHECK_PORT} is already in use. Try a different ID." + exit 1 + fi + + # Base ports — each gets +OFFSET in the generated env file + declare -a PORT_DEFS=( + "NODE0_KAFKA_PORT=9092" + "NODE0_DEBUG_PORT=5005" + "NODE0_JMX_PORT=9999" + "NODE1_KAFKA_PORT=19092" + "NODE1_DEBUG_PORT=5006" + "NODE2_KAFKA_PORT=29092" + "NODE2_DEBUG_PORT=5007" + "NODE3_KAFKA_PORT=39092" + "NODE3_DEBUG_PORT=5008" + "NODE4_KAFKA_PORT=49092" + "NODE4_DEBUG_PORT=5009" + "MINIO_API_PORT=9000" + "MINIO_CONSOLE_PORT=9001" + "SCHEMA_REGISTRY_PORT=8081" + "ICEBERG_REST_PORT=8181" + "SPARK_NOTEBOOK_PORT=8888" + "SPARK_UI_PORT=8080" + "TRINO_PORT=8090" + ) + + mkdir -p "{{DEVKIT}}" + { + echo "COMPOSE_PROJECT_NAME=devkit-$NAME" + echo "PORT_OFFSET=$OFFSET" + for def in "${PORT_DEFS[@]}"; do + key="${def%%=*}" + base="${def#*=}" + echo "${key}=$((base + OFFSET))" + done + } > "{{DEVKIT}}/compose.env" + + echo "✓ Namespace: ${NAME} (id=${ID})" + echo " Port offset: +${OFFSET}" + echo " Kafka will be on port ${CHECK_PORT}" + +[doc("Clear namespace, revert to default ports")] +clear-ns: + rm -f "{{DEVKIT}}/compose.env" + @echo "✓ Namespace cleared — using default ports" # ==================== Lifecycle ==================== @@ -152,10 +225,12 @@ start nodes="1" *FEATURES: ensure-cluster-id echo "✓ Starting {{nodes}} node(s)..." {{COMPOSE}} $COMPOSE_PROFILES up -d echo "" + PORT_OFFSET=$(grep -s '^PORT_OFFSET=' "{{DEVKIT}}/compose.env" | cut -d= -f2) + PORT_OFFSET=${PORT_OFFSET:-0} echo "🎉 AutoMQ DevKit started" - echo " Kafka: localhost:9092" - echo " Debug: localhost:5005" - echo " MinIO: http://localhost:9001 (admin/password)" + echo " Kafka: localhost:$((9092 + PORT_OFFSET))" + echo " Debug: localhost:$((5005 + PORT_OFFSET))" + echo " MinIO: http://localhost:$((9001 + PORT_OFFSET)) (admin/password)" just wait [doc("Build image & code, then start. Example: just start-build 3 tabletopic")] @@ -202,11 +277,11 @@ logs-follow node="": [doc("Enter container shell (default: node 0)")] shell node="0": - @docker exec -it node-{{node}} bash + @{{COMPOSE}} exec node-{{node}} bash [doc("Run a single command in container. Example: just exec 0 ls /opt/automq/bin")] exec node *CMD: - @docker exec node-{{node}} {{CMD}} + @{{COMPOSE}} exec node-{{node}} {{CMD}} [doc("Wait for all nodes to be healthy (auto-called by start)")] wait timeout="180": @@ -217,8 +292,8 @@ wait timeout="180": NODES=$(just _node-list) [ -z "$NODES" ] && sleep 5 && ELAPSED=$((ELAPSED + 5)) && continue ALL_HEALTHY=true - for n in $NODES; do - S=$(docker inspect --format='{{ '{{' }}.State.Health.Status{{ '}}' }}' "$n" 2>/dev/null) + for svc in $NODES; do + S=$({{COMPOSE}} ps --format '{{ '{{' }}.Health{{ '}}' }}' "$svc" 2>/dev/null) [ "$S" != "healthy" ] && ALL_HEALTHY=false && break done [ "$ALL_HEALTHY" = true ] && echo "✓ All nodes healthy" && exit 0 @@ -244,7 +319,7 @@ bin *ARGS: *) PARAMS+=("$1"); shift ;; esac done - docker exec "node-${NODE}" \ + {{COMPOSE}} exec "node-${NODE}" \ env "KAFKA_HEAP_OPTS={{HEAP_OPTS}}" KAFKA_JVM_PERFORMANCE_OPTS="" JMX_PORT="" \ /opt/automq/bin/"${PARAMS[@]}" 2> >(grep -v '^SLF4J' >&2) @@ -259,7 +334,7 @@ jmx *ARGS: *) break ;; esac done - echo "$@" | docker exec -i node-${NODE} java -jar /usr/local/bin/jmxterm.jar -l localhost:9999 -n + echo "$@" | {{COMPOSE}} exec -T node-${NODE} java -jar /usr/local/bin/jmxterm.jar -l localhost:9999 -n # ==================== Shortcuts ==================== @@ -286,7 +361,7 @@ produce topic *ARGS: *) PARAMS+=("$1"); shift ;; esac done - docker exec -i "node-${NODE}" \ + {{COMPOSE}} exec -T "node-${NODE}" \ env "KAFKA_HEAP_OPTS={{HEAP_OPTS}}" KAFKA_JVM_PERFORMANCE_OPTS="" JMX_PORT="" \ /opt/automq/bin/kafka-console-producer.sh \ --bootstrap-server localhost:9092 --topic "{{topic}}" \ @@ -303,7 +378,7 @@ consume topic *ARGS: *) PARAMS+=("$1"); shift ;; esac done - docker exec -i "node-${NODE}" \ + {{COMPOSE}} exec -T "node-${NODE}" \ env "KAFKA_HEAP_OPTS={{HEAP_OPTS}}" KAFKA_JVM_PERFORMANCE_OPTS="" JMX_PORT="" \ /opt/automq/bin/kafka-console-consumer.sh \ --bootstrap-server localhost:9092 --topic "{{topic}}" \ @@ -352,24 +427,24 @@ perf-consume topic *ARGS: [doc("Attach Arthas to Kafka process. Example: just arthas 1")] arthas node="0": - @docker exec -it node-{{node}} java -jar /opt/arthas/arthas-boot.jar + @{{COMPOSE}} exec node-{{node}} java -jar /opt/arthas/arthas-boot.jar [doc("Run a single Arthas command non-interactively. Example: just arthas-exec 0 'dashboard -n 1'")] arthas-exec node cmd: #!/usr/bin/env bash - printf '%s\n' '{{cmd}}' | docker exec -i node-{{node}} bash -c \ + printf '%s\n' '{{cmd}}' | {{COMPOSE}} exec -T node-{{node}} bash -c \ 'cat > /tmp/_arthas_cmd && java -jar /opt/arthas/arthas-boot.jar --select kafka.Kafka -f /tmp/_arthas_cmd' # ==================== Chaos: Node network ==================== [doc("Network delay on node. Example: just chaos-delay 500 node-1")] chaos-delay ms="200" node="node-0": - docker exec {{node}} tc qdisc replace dev eth0 root netem delay {{ms}}ms + {{COMPOSE}} exec {{node}} tc qdisc replace dev eth0 root netem delay {{ms}}ms @echo "✓ {{ms}}ms delay on {{node}}" [doc("Packet loss on node. Example: just chaos-loss 10 node-1")] chaos-loss percent="5" node="node-0": - docker exec {{node}} tc qdisc replace dev eth0 root netem loss {{percent}}% + {{COMPOSE}} exec {{node}} tc qdisc replace dev eth0 root netem loss {{percent}}% @echo "✓ {{percent}}% packet loss on {{node}}" # ==================== Chaos: S3 (MinIO) ==================== @@ -377,45 +452,45 @@ chaos-loss percent="5" node="node-0": [doc("S3 latency on a node (only S3 traffic affected). Example: just chaos-s3-delay 500 node-0")] chaos-s3-delay ms="500" node="node-0": #!/usr/bin/env bash - MINIO_IP=$(docker exec {{node}} getent hosts minio | awk '{print $1}') - docker exec {{node}} tc qdisc del dev eth0 root 2>/dev/null || true - docker exec {{node}} tc qdisc add dev eth0 root handle 1: prio - docker exec {{node}} tc qdisc add dev eth0 parent 1:3 handle 30: netem delay {{ms}}ms - docker exec {{node}} tc filter add dev eth0 parent 1:0 protocol ip u32 match ip dst $MINIO_IP/32 flowid 1:3 + MINIO_IP=$({{COMPOSE}} exec {{node}} getent hosts minio | awk '{print $1}') + {{COMPOSE}} exec {{node}} tc qdisc del dev eth0 root 2>/dev/null || true + {{COMPOSE}} exec {{node}} tc qdisc add dev eth0 root handle 1: prio + {{COMPOSE}} exec {{node}} tc qdisc add dev eth0 parent 1:3 handle 30: netem delay {{ms}}ms + {{COMPOSE}} exec {{node}} tc filter add dev eth0 parent 1:0 protocol ip u32 match ip dst $MINIO_IP/32 flowid 1:3 echo "✓ {{ms}}ms delay to MinIO on {{node}} (other traffic unaffected)" [doc("S3 packet loss on a node (only S3 traffic affected). Example: just chaos-s3-loss 10 node-0")] chaos-s3-loss percent="5" node="node-0": #!/usr/bin/env bash - MINIO_IP=$(docker exec {{node}} getent hosts minio | awk '{print $1}') - docker exec {{node}} tc qdisc del dev eth0 root 2>/dev/null || true - docker exec {{node}} tc qdisc add dev eth0 root handle 1: prio - docker exec {{node}} tc qdisc add dev eth0 parent 1:3 handle 30: netem loss {{percent}}% - docker exec {{node}} tc filter add dev eth0 parent 1:0 protocol ip u32 match ip dst $MINIO_IP/32 flowid 1:3 + MINIO_IP=$({{COMPOSE}} exec {{node}} getent hosts minio | awk '{print $1}') + {{COMPOSE}} exec {{node}} tc qdisc del dev eth0 root 2>/dev/null || true + {{COMPOSE}} exec {{node}} tc qdisc add dev eth0 root handle 1: prio + {{COMPOSE}} exec {{node}} tc qdisc add dev eth0 parent 1:3 handle 30: netem loss {{percent}}% + {{COMPOSE}} exec {{node}} tc filter add dev eth0 parent 1:0 protocol ip u32 match ip dst $MINIO_IP/32 flowid 1:3 echo "✓ {{percent}}% packet loss to MinIO on {{node}} (other traffic unaffected)" [doc("Pause MinIO — S3 completely unavailable")] chaos-s3-down: - docker pause minio + {{COMPOSE}} pause minio @echo "✓ MinIO (S3) paused — all nodes lost S3 access" [doc("Resume MinIO")] chaos-s3-up: - docker unpause minio + {{COMPOSE}} unpause minio @echo "✓ MinIO (S3) resumed" [doc("Isolate a node from S3. Example: just chaos-s3-partition node-1")] chaos-s3-partition node="node-0": (_ensure-iptables-chain node) - docker exec {{node}} iptables -A DEVKIT -d minio -j DROP - docker exec {{node}} iptables -A DEVKIT -s minio -j DROP + {{COMPOSE}} exec {{node}} iptables -A DEVKIT -d minio -j DROP + {{COMPOSE}} exec {{node}} iptables -A DEVKIT -s minio -j DROP @echo "✓ {{node}} isolated from MinIO (S3)" [doc("Restore all nodes' S3 connectivity")] chaos-s3-partition-reset: #!/usr/bin/env bash for n in $(just _node-list); do - docker exec $n iptables -D DEVKIT -d minio -j DROP 2>/dev/null || true - docker exec $n iptables -D DEVKIT -s minio -j DROP 2>/dev/null || true + {{COMPOSE}} exec $n iptables -D DEVKIT -d minio -j DROP 2>/dev/null || true + {{COMPOSE}} exec $n iptables -D DEVKIT -s minio -j DROP 2>/dev/null || true done echo "✓ All S3 partitions removed" @@ -423,17 +498,17 @@ chaos-s3-partition-reset: [doc("Isolate two nodes. Example: just chaos-partition node-0 node-1")] chaos-partition a b: (_ensure-iptables-chain a) (_ensure-iptables-chain b) - docker exec {{a}} iptables -A DEVKIT -d {{b}} -j DROP - docker exec {{a}} iptables -A DEVKIT -s {{b}} -j DROP - docker exec {{b}} iptables -A DEVKIT -d {{a}} -j DROP - docker exec {{b}} iptables -A DEVKIT -s {{a}} -j DROP + {{COMPOSE}} exec {{a}} iptables -A DEVKIT -d {{b}} -j DROP + {{COMPOSE}} exec {{a}} iptables -A DEVKIT -s {{b}} -j DROP + {{COMPOSE}} exec {{b}} iptables -A DEVKIT -d {{a}} -j DROP + {{COMPOSE}} exec {{b}} iptables -A DEVKIT -s {{a}} -j DROP @echo "✓ {{a}} ↔ {{b}} isolated" [doc("Restore all node-to-node connectivity")] chaos-partition-reset: #!/usr/bin/env bash for n in $(just _node-list); do - docker exec $n iptables -F DEVKIT 2>/dev/null || true + {{COMPOSE}} exec $n iptables -F DEVKIT 2>/dev/null || true done echo "✓ All node partitions removed" @@ -443,11 +518,11 @@ chaos-partition-reset: chaos-reset: #!/usr/bin/env bash for n in $(just _node-list); do - docker exec $n tc qdisc del dev eth0 root 2>/dev/null || true - docker exec $n iptables -F DEVKIT 2>/dev/null || true + {{COMPOSE}} exec $n tc qdisc del dev eth0 root 2>/dev/null || true + {{COMPOSE}} exec $n iptables -F DEVKIT 2>/dev/null || true done # Unpause MinIO if paused - docker unpause minio 2>/dev/null || true + {{COMPOSE}} unpause minio 2>/dev/null || true echo "✓ All chaos rules reset (tc + iptables + unpause)" [doc("Show all active chaos rules")] @@ -455,10 +530,14 @@ chaos-status: #!/usr/bin/env bash for n in $(just _node-list); do echo "=== $n ===" - echo " tc:" && docker exec $n tc qdisc show dev eth0 - RULES=$(docker exec $n iptables -L DEVKIT -n 2>/dev/null | grep -c DROP || true) + echo " tc:" && {{COMPOSE}} exec $n tc qdisc show dev eth0 + RULES=$({{COMPOSE}} exec $n iptables -L DEVKIT -n 2>/dev/null | grep -c DROP || true) [ "$RULES" -gt 0 ] && echo " iptables: ${RULES} DROP rules in DEVKIT chain" done echo "=== minio ===" - S=$(docker inspect --format='{{ '{{' }}.State.Paused{{ '}}' }}' minio 2>/dev/null) - [ "$S" = "true" ] && echo " status: PAUSED" || echo " status: running" + S=$(docker inspect --format='{{ '{{' }}.State.Paused{{ '}}' }}' "$({{COMPOSE}} ps -q minio 2>/dev/null)" 2>/dev/null) + case "$S" in + true) echo " status: PAUSED" ;; + false) echo " status: running" ;; + *) echo " status: not running" ;; + esac