From bcc49faef4049f4daac8270a66f512410d892b03 Mon Sep 17 00:00:00 2001 From: Daniil Yankouski Date: Tue, 7 Jul 2026 18:14:15 +0200 Subject: [PATCH 1/3] feat(docs): update devshard docs --- devshard/docs/devshard-update/.gitignore | 1 + devshard/docs/devshard-update/README.md | 161 +++++ devshard/docs/devshard-update/SKILL.md | 52 ++ .../devshard-update/references/admin-api.md | 62 ++ .../references/nginx-alias-switching.md | 54 ++ .../references/troubleshooting.md | 51 ++ .../scripts/update.config.sample.json | 52 ++ .../docs/devshard-update/scripts/update.sh | 622 ++++++++++++++++++ 8 files changed, 1055 insertions(+) create mode 100644 devshard/docs/devshard-update/.gitignore create mode 100644 devshard/docs/devshard-update/README.md create mode 100644 devshard/docs/devshard-update/SKILL.md create mode 100644 devshard/docs/devshard-update/references/admin-api.md create mode 100644 devshard/docs/devshard-update/references/nginx-alias-switching.md create mode 100644 devshard/docs/devshard-update/references/troubleshooting.md create mode 100644 devshard/docs/devshard-update/scripts/update.config.sample.json create mode 100755 devshard/docs/devshard-update/scripts/update.sh diff --git a/devshard/docs/devshard-update/.gitignore b/devshard/docs/devshard-update/.gitignore new file mode 100644 index 0000000000..97f3e31276 --- /dev/null +++ b/devshard/docs/devshard-update/.gitignore @@ -0,0 +1 @@ +sandbox/ diff --git a/devshard/docs/devshard-update/README.md b/devshard/docs/devshard-update/README.md new file mode 100644 index 0000000000..31ca2ed7ab --- /dev/null +++ b/devshard/docs/devshard-update/README.md @@ -0,0 +1,161 @@ +# Devshard Gateway Update + +Update a devshard gateway to a new image version behind nginx without dropping in-flight requests. Assumes multi-devshard **gateway-proxy** mode (routing prefix `/devshard-gateway/`) behind the nginx `proxy` container. Single-instance host, two ways — **Manual** (steps by hand) and **Automated** (one script). For a 2+ instance pool, see the Pool section below. + +An **escrow** and a **devshard** are the same object. The examples assume: + +```bash +KEY="$DEVSHARD_ADMIN_API_KEY" +MAIN="http://127.0.0.1:18080" +TEMP="http://127.0.0.1:18081" +REPO="ghcr.io/gonka-ai/devshard-gateway" +# / = current / target image tag; = a model id you serve +``` + +## Manual update (single instance) + +Run from the gateway host's deploy dir (e.g. `deploy/join`). A temp gateway holds traffic while main updates. + +**1. Disable escrow rotation on main** (so nothing settles on-chain mid-update): + +```bash +curl -fsS "$MAIN/v1/admin/settings" -H "Authorization: Bearer $KEY" \ + | jq '.escrow_rotation.enabled = false' \ + | curl -fsS -X POST "$MAIN/v1/admin/settings" -H "Authorization: Bearer $KEY" \ + -H 'Content-Type: application/json' -d @- +``` + +**2. Start a temp gateway** on the new image, with its own storage and no escrows; reuse main's env file (keeps secrets out of `/tmp`): + +```bash +docker run -d --name devshard-gateway-temp --restart unless-stopped \ + --network join_default --network-alias devshard-gateway-temp \ + --env-file ./config.devshard.env \ + -e DEVSHARDS_JSON='[]' -e DEVSHARD_STORAGE_DIR=/root/.devshardctl/temp -e DEVSHARD_PORT=8080 \ + -p 127.0.0.1:18081:8080 -v "$PWD/.devshardctl:/root/.devshardctl" \ + "$REPO:" +curl -fsS "$MAIN/v1/admin/settings" -H "Authorization: Bearer $KEY" \ + | jq '.escrow_rotation.enabled = false | .escrow_rotation.models = []' \ + | curl -fsS -X POST "$TEMP/v1/admin/settings" -H "Authorization: Bearer $KEY" \ + -H 'Content-Type: application/json' -d @- +``` + +**3. Mint temp escrows — for every model main serves** (a model with no temp escrows is unavailable during the update; each mint spends on-chain funds): + +```bash +curl -fsS -X POST "$TEMP/v1/admin/escrows" -H "Authorization: Bearer $KEY" \ + -H 'Content-Type: application/json' \ + -d '{"amount":5000000000,"model_id":"","private_key_env":"DEVSHARD_PRIVATE_KEY","protocol_version":"1"}' +# repeat per escrow and per model; then record the created ids: +curl -fsS "$TEMP/v1/admin/devshards" -H "Authorization: Bearer $KEY" | jq '.devshards[].id' +``` + +**4. Verify temp** — status responds and a chat smoke test passes (test each model): + +```bash +curl -fsS "$TEMP/v1/status" -H "Authorization: Bearer $KEY" | jq '[.devshards[].active_requests]|length' +curl -fsS -X POST "$TEMP/v1/chat/completions" -H 'Content-Type: application/json' \ + -d '{"model":"","max_tokens":1,"messages":[{"role":"user","content":"ok"}]}' >/dev/null && echo ok +``` + +**5. Switch nginx to temp** (graceful reload; in-flight streams finish). This matches both a direct `proxy_pass http://host:8080` and a named-upstream `server host:8080;`: + +```bash +docker exec proxy sh -lc "cp /etc/nginx/nginx.conf /tmp/nginx.conf.bak && \ + sed -E -i 's#http://devshard-gateway:8080#http://devshard-gateway-temp:8080#g; \ + s#(server[[:space:]]+)devshard-gateway:8080#\1devshard-gateway-temp:8080#g' /etc/nginx/nginx.conf && \ + nginx -t && nginx -s reload" +``` + +**6. Drain main** — repeat until this prints `0`: + +```bash +curl -fsS "$MAIN/v1/status" -H "Authorization: Bearer $KEY" | jq '[.devshards[].active_requests]|max' +``` + +**7. Bump the image tag in compose:** + +```bash +sed -i "s#$REPO:#$REPO:#g" docker-compose.devshard-gateway.yml +``` + +**8. Recreate main on the new image** (compose is the image source — not an env var): + +```bash +docker compose -f docker-compose.devshard-gateway.yml up -d --no-deps --force-recreate devshard-gateway +``` + +**9. Verify main** — status responds and a chat smoke test passes (same as step 4, against `$MAIN`). + +**10. Switch nginx back to main** — same as step 5 with `devshard-gateway-temp` → `devshard-gateway`. + +**11. Drain temp, then stop AND remove it** (`--restart unless-stopped` means a leftover container would resurrect and corrupt state): + +```bash +curl -fsS "$TEMP/v1/status" -H "Authorization: Bearer $KEY" | jq '[.devshards[].active_requests]|max' # until 0 +docker stop devshard-gateway-temp && docker rm devshard-gateway-temp +``` + +**12. Import + activate the temp escrows into main** (so their funds aren't stranded) — for each temp escrow ``: + +```bash +BODY='{"id":"","model":"","storage_path":"/root/.devshardctl/temp/escrow-/state.db","protocol_version":"1","private_key_env":"DEVSHARD_PRIVATE_KEY"}' +curl -fsS -X POST "$MAIN/v1/admin/devshards/import" -H "Authorization: Bearer $KEY" -H 'Content-Type: application/json' -d "$(jq '.active=false' <<<"$BODY")" +curl -fsS -X POST "$MAIN/v1/admin/devshards" -H "Authorization: Bearer $KEY" -H 'Content-Type: application/json' -d "$BODY" +``` + +**Rollback:** if a step fails, do not switch back to a bad main — keep the alias on temp, restore `/tmp/nginx.conf.bak` and reload, and re-pull the old image tag. If you stopped after minting temp escrows, finish steps 11–12 (or settle each via `POST /v1/admin/devshards/{id}/settle`) so their funds are recovered. + +## Automated update (single instance) + +The script runs steps 1–12 from a JSON config (models and everything else are config-driven — nothing hardcoded). + +```bash +cp scripts/update.config.sample.json update.config.json # edit: image tags, models[], nginx, compose +./scripts/update.sh --config update.config.json run # dry run — prints the full plan +./scripts/update.sh --config update.config.json run --run # execute +``` + +- `--run` executes (default is dry-run); `--yes` skips confirmations (unattended). +- Run a single step: `./scripts/update.sh --config update.config.json `. +- Recover stranded temp escrows after an aborted run: `./scripts/update.sh --config update.config.json recover` (or `recover --settle`). + +The config (see [scripts/update.config.sample.json](scripts/update.config.sample.json)) holds the image (`repository`, `from_tag`, `to_tag`), the `models[]` array (`model`, `escrow_count`, `escrow_amount`), `allow_unavailable_models`, and the nginx / compose / timeout blocks. Env vars override individual fields. + +## Restore the nginx backup + +The switch backs up the nginx config **before every change**, inside the `proxy` container next to the config: `.blue-green-backup` (e.g. `/etc/nginx/nginx.conf.blue-green-backup`). The manual step 5 above instead writes `/tmp/nginx.conf.bak`. To revert routing, restore whichever you used and reload: + +```bash +docker exec proxy sh -lc "cp /etc/nginx/nginx.conf.blue-green-backup /etc/nginx/nginx.conf && nginx -t && nginx -s reload" +# manual path: cp /tmp/nginx.conf.bak /etc/nginx/nginx.conf && nginx -t && nginx -s reload +``` + +Confirm the upstream it now points at: + +```bash +docker exec proxy sh -lc "grep -nE 'devshard.*:8080' /etc/nginx/nginx.conf" +``` + +**Caveat — the backup is overwritten on each switch.** It holds the routing as of just *before the most recent switch*: after `switch-to-main` it points back at **temp**, not the pristine original. For a guaranteed clean revert, copy the original aside before you start: + +```bash +docker exec proxy sh -lc "cp /etc/nginx/nginx.conf /etc/nginx/nginx.conf.orig" +``` + +## Pool (multi-instance) + +If 2+ gateway instances sit behind nginx, update one at a time — no temp gateway: + +1. Remove one instance from the nginx `upstream` (comment its `server` line or mark it `down`); `nginx -t && nginx -s reload`. +2. Drain it: `GET /v1/status` until `max(.devshards[].active_requests) == 0`. +3. Bump its image tag in compose; `docker compose -f up -d --no-deps --force-recreate `. +4. Verify: `/v1/status` responds and a `POST /v1/chat/completions` returns. +5. Return it to the pool; reload. Repeat for the next instance. + +## References + +- [references/admin-api.md](references/admin-api.md) — gateway admin endpoints used here. +- [references/nginx-alias-switching.md](references/nginx-alias-switching.md) — how the upstream switch keeps in-flight streams alive. +- [references/troubleshooting.md](references/troubleshooting.md) — drain stalls, unroutable escrows, rollback. +- [SKILL.md](SKILL.md) — operator/agent skill wrapping this process. diff --git a/devshard/docs/devshard-update/SKILL.md b/devshard/docs/devshard-update/SKILL.md new file mode 100644 index 0000000000..416bc72b7c --- /dev/null +++ b/devshard/docs/devshard-update/SKILL.md @@ -0,0 +1,52 @@ +--- +name: devshard-update +description: Update a Gonka devshard gateway to a new image version without dropping in-flight /v1/chat/completions requests, driven by a JSON config. Single-instance blue/green (temp gateway + nginx switch + drain + config-driven temp escrows + import); a recover command reclaims stranded temp escrows; multi-instance pools use a rolling update. +--- + +# Devshard Gateway Update + +Companion to [README.md](README.md) (manual steps + the command) and [scripts/update.sh](scripts/update.sh). + +## One command (single instance) + +```bash +cp scripts/update.config.sample.json update.config.json # edit: image tags, models[], nginx, compose +./scripts/update.sh --config update.config.json run # dry run — prints the plan +./scripts/update.sh --config update.config.json run --run # execute +``` + +Add `--yes` for unattended. Everything is config-driven (models included) — nothing is hardcoded. + +## Safety rules + +- Treat any live gateway host as production. Show the exact command and get approval before running it. +- Dry-run first (`run` without `--run`) and read the plan. +- MAIN's image comes from the compose file; the `bump-main-image` step rewrites the tag from `image.from_tag` to `image.to_tag`. +- Temp escrows must cover every model MAIN serves — `preflight` fails closed if a served model has no temp coverage and isn't in `allow_unavailable_models`. +- Public verification runs only if `nginx.public_base_url` is set — never trust loopback. +- Do not assume the nginx config path — inspect with `docker exec sh -lc 'nginx -T'`. + +## Config + +`scripts/update.config.sample.json` — blocks: `image {repository, from_tag, to_tag}`, `models[] {model, escrow_count, escrow_amount}`, `allow_unavailable_models[]`, `escrow`, `main`, `temp`, `nginx`, `compose`, `timeouts`, `rotation`. Any field is overridable by the matching env var. + +## Steps & actions + +Full-flow order (aborts on the first failed gate and names the step): +`init` → `preflight` → `disable-main-rotation` → `create-temp-gateway` → `create-temp-escrows` → `check-temp` → `switch-to-temp` → `check-alias-temp` → `drain-main` → `bump-main-image` → `update-main` → `check-main-direct` → `switch-to-main` → `check-alias-main` → `drain-temp` → `stop-temp` → `import-temp` → `activate-temp` → `restore-main-rotation` → `status`. + +- Single step: `./scripts/update.sh --config --run`. +- Resume: `... run --run --from-step drain-main`. +- Recover stranded temp escrows after an aborted run: `./scripts/update.sh --config recover --run` (add `--settle` to settle them instead of folding into main). +- `plan` / `validate` / `list-steps` for inspection (no side effects). + +## Remote execution + +```bash +ssh -p @ "$(cat <<'REMOTE' +set -euo pipefail +cd +./scripts/update.sh --config update.config.json run --run --yes +REMOTE +)" +``` diff --git a/devshard/docs/devshard-update/references/admin-api.md b/devshard/docs/devshard-update/references/admin-api.md new file mode 100644 index 0000000000..3aa8a38ac8 --- /dev/null +++ b/devshard/docs/devshard-update/references/admin-api.md @@ -0,0 +1,62 @@ +# Reference — Gateway Admin API (migration subset) + +The endpoints a broker touches during a zero-downtime update. All admin routes require `Authorization: Bearer $DEVSHARD_ADMIN_API_KEY`; `/v1/status` is public. Handlers live in `devshard/cmd/devshardctl/gateway.go`. + +## Status & draining + +| Method / path | Purpose | +|---|---| +| `GET /v1/status` | Public health + per-devshard runtime snapshot. **The drain gate.** Each devshard reports `active_requests`. | +| `GET /v1/admin/devshards` | Full admin list of persisted devshards (active + inactive) with model, protocol_version, storage, balance. | +| `GET /v1/admin/state` | Broader admin snapshot (devshards + effective settings). | + +**Drain check** — poll until zero: + +```bash +curl -fsS http://127.0.0.1:18080/v1/status \ + -H "Authorization: Bearer $DEVSHARD_ADMIN_API_KEY" \ + | jq '[.devshards[]?.active_requests // 0] | max // 0' +``` + +`active_requests` is an atomic counter per runtime, incremented on request reserve and decremented on release, then serialized under `devshards[].active_requests`. It is also exported as the Prometheus gauge `devshard_runtime_active_requests`. + +## Settings + +| Method / path | Purpose | +|---|---| +| `GET /v1/admin/settings` | Effective persisted gateway settings. | +| `POST /v1/admin/settings` | Update settings live (e.g. `escrow_rotation`, `max_concurrent_requests`). | + +During a migration, copy main's settings to the temp gateway but force rotation off so temp never auto-settles: + +```bash +curl -fsS http://127.0.0.1:18080/v1/admin/settings -H "Authorization: Bearer $KEY" \ + | jq '.escrow_rotation.enabled = false | .escrow_rotation.models = []' \ + | curl -fsS -X POST http://127.0.0.1:18081/v1/admin/settings -H "Authorization: Bearer $KEY" \ + -H 'Content-Type: application/json' -d @- +``` + +`escrow_rotation.enabled = false` is the master off switch — it stops the rotator and thus all auto-settlement. Disable it on **MAIN** directly (a running gateway keeps whatever is in `gateway.db`, not the first-boot default); `settlement_enabled` and `models` are finer controls you don't need to touch for a swap. + +## Escrow lifecycle + +| Method / path | Purpose | +|---|---| +| `POST /v1/admin/escrows` | Create a **new** escrow on-chain (`amount`, `model_id`, `private_key_env`, `protocol_version`) and optionally register it. Used to seed **temp** escrows. | +| `POST /v1/admin/devshards/import` | Import an **existing** escrow into this gateway by `id` + `storage_path` + `private_key_env` (+ optional `perf_path`), with `active:false`. Carries an escrow's sqlite state into a fresh gateway process — the storage-portability tool. | +| `POST /v1/admin/devshards` | Register / **activate** a devshard (`id`, `model`, `storage_path`, `protocol_version`, `private_key_env`). Adds it to the routing pool. | +| `POST /v1/admin/devshards/{id}/deactivate` | Set `active=false` — removes from the routing pool, keeps the runtime loaded, **does not settle**. In-flight requests finish. | +| `POST /v1/admin/devshards/{id}/settle` | Deliberately settle an escrow on-chain (drain-aware). | +| `DELETE /v1/admin/devshards/{id}` | Remove a devshard cleanly. | + +## protocol_version + +Only `"1"` (aka `""` / `"v1"`) parses; anything else is rejected. It is a stored field but effectively pinned to v1 — a gateway image "vX → vY" update is **not** a protocol change. Pass `"1"` (or leave default) when creating/importing/activating. + +## Notes for migration + +- **Import needs `storage_path`.** It is how an existing escrow's `state.db` is reattached to a new process. Point it at the temp escrow's container path (e.g. `/root/.devshardctl/temp-/escrow-/state.db`). +- **Create vs import:** `create` mints a *new* on-chain escrow; `import` re-attaches an *existing* one. Temp escrows are `create`d, then later `import`ed into main. +- **Deactivate ≠ settle.** Deactivating during a swap is safe and reversible; settlement is a separate, config-gated, drain-aware action. +- **`escrow` and `devshard` are the same object.** `create` mints it (`/v1/admin/escrows`); everything else lists/imports/activates/deactivates it (`/v1/admin/devshards`, and it appears under `/v1/status .devshards[]`). +- **Field name differs by call:** `create` takes `model_id`; `import` and `activate` take `model`. diff --git a/devshard/docs/devshard-update/references/nginx-alias-switching.md b/devshard/docs/devshard-update/references/nginx-alias-switching.md new file mode 100644 index 0000000000..c7041a62a1 --- /dev/null +++ b/devshard/docs/devshard-update/references/nginx-alias-switching.md @@ -0,0 +1,54 @@ +# Reference — nginx upstream switch + +The `proxy` container forwards `/v1/chat/completions` to the gateway. The zero-downtime switch repoints that target from the old gateway to the new one and reloads, without cutting in-flight streams. + +## Why in-flight streams survive + +`nginx -s reload` is graceful: + +1. `nginx -t` validates the new config. +2. `nginx -s reload` starts **new** workers with the new config and tells the **old** workers to stop accepting connections. +3. Old workers keep running until their in-flight requests finish (including long SSE streams), then exit. + +New requests go to the new target; existing requests drain on the old workers. Nothing is cut mid-stream — which is why the switch uses reload, never a proxy-container restart. + +## What the switch changes + +Both the manual `sed` (README step 5) and the script replace the upstream host `:PORT` → `:PORT` in the nginx config (inside the `proxy` container), then run `nginx -t && nginx -s reload`. One `sed` handles **both** config styles: + +```nginx +location /devshard-gateway/ { proxy_pass http://devshard-gateway:8080/; } # direct pass — host swapped +upstream pool { server devshard-gateway:8080; } # named upstream — host swapped +# proxy_pass http://pool; — references the block by name, left untouched (correct) +``` + +In the config these are `nginx.old_upstream`, `nginx.new_upstream`, `nginx.upstream_port`, `nginx.config_path`, `nginx.proxy_container`. + +## Pool (2+ gateways) + +A named upstream with multiple `server` lines. Rolling update: mark one `down` (or comment it), reload, drain+update that instance, restore it, reload; repeat. nginx load-balances across whoever is up, so capacity never drops to zero. + +```nginx +upstream devshard_gateway { + server gw-a:8080; + server gw-b:8080; # comment out (or `down`) while updating gw-b, then restore +} +``` + +## Finding the live config + +Do not assume the path. The effective config may be `/etc/nginx/nginx.conf`, a file under `conf.d/`, or an included fragment: + +```bash +docker exec proxy sh -lc 'nginx -T 2>&1 | grep -n "devshard\|v1/chat\|proxy_pass\|upstream"' +``` + +Set `nginx.config_path`, `nginx.proxy_container`, `nginx.old_upstream`, `nginx.new_upstream`, and `nginx.upstream_port` in the config to match what you find. + +The gateway's compose (`deploy/join/docker-compose.devshard-gateway.yml`) registers it under two network aliases — `${DEVSHARD_INSTANCE_NAME:-devshard-gateway}` and `devshard-pool` — so the proxy could reach it by either. The proxy's own config is not in this repo, so `nginx.old_upstream` (default `devshard-gateway`) is a guess: confirm the real upstream host with `nginx -T` before you switch. + +## Gotchas + +- **Verify through the public URL, not `127.0.0.1`.** Public verification runs only if `nginx.public_base_url` is set; a loopback check can pass while the public route is broken. +- **Host+port replacement.** The switch rewrites `http://:PORT` and `server :PORT;`. A `proxy_pass` that points at an upstream *name*, a variable, a map, or a non-configured port won't match — set `nginx.old_upstream` / `nginx.new_upstream` / `nginx.upstream_port`, or switch by hand. +- **Always `nginx -t` before reload.** A reload with a broken config keeps the old workers serving but blocks the switch. The switch gates the reload on `nginx -t` and backs up the config first — restore the backup to revert routing instantly. diff --git a/devshard/docs/devshard-update/references/troubleshooting.md b/devshard/docs/devshard-update/references/troubleshooting.md new file mode 100644 index 0000000000..596b7aece0 --- /dev/null +++ b/devshard/docs/devshard-update/references/troubleshooting.md @@ -0,0 +1,51 @@ +# Reference — Troubleshooting & rollback + +Symptoms, causes, and fixes during a zero-downtime gateway update. Endpoints in [admin-api.md](admin-api.md); nginx behavior in [nginx-alias-switching.md](nginx-alias-switching.md). + +## Drain never reaches zero + +`active_requests` stays above 0 past the poll window. + +- **A long stream is genuinely running.** Large `max_tokens` / slow model can hold a request for minutes. Raise `timeouts.drain_timeout_seconds` in the config (default `7200`) rather than force-killing. +- **A hung/abandoned stream.** A client that vanished mid-SSE can leave a request parked. Confirm via `GET /v1/status` per devshard; if one runtime is stuck with a client that's gone, decide whether to wait out the server-side timeout or accept dropping that single request before recreate. +- **New requests still arriving.** The nginx switch didn't take. Re-check routing through the **public** URL — you may have verified loopback only. + +## Escrows missing / not routable after update + +`GET /v1/admin/devshards` returns empty or fewer than expected. + +- **Volume not mounted.** The new container didn't get `/root/.devshardctl`. The gateway then ran the first-boot env path instead of loading `gateway.db`. Stop it, fix the `-v .../.devshardctl:/root/.devshardctl` mount, restart — nothing is lost, sqlite is still on disk. +- **Imported escrow inactive.** After `import-temp` the escrow is `active:false` by design. Run `activate-temp` (`POST /v1/admin/devshards`) to add it to the routing pool. +- **Wrong `storage_path` on import.** Import re-attaches an escrow's `state.db` by path. If the path is wrong the escrow imports but can't serve. Point it at the temp escrow's real container path (e.g. `/root/.devshardctl/temp-/escrow-/state.db`). + +## nginx reload fails + +- **`nginx -t` errors.** The switch is blocked (old workers keep serving — no outage). Fix the config or restore the backup `/nginx.conf.blue-green-backup`, then reload. +- **`sed` matched nothing.** The upstream pattern didn't match (a `proxy_pass` via upstream *name*, a variable, a different port, or already switched). Inspect with `nginx -T`; set `nginx.old_upstream` / `nginx.new_upstream` / `nginx.upstream_port` in the config. + +## Temp gateway won't start + +- **Admin port clash.** `TEMP_ADMIN_PORT` (default `18081`) is already bound. Pick a free port. +- **Same escrow as main.** Never let temp bootstrap the main escrows — it starts with `DEVSHARDS_JSON=[]` for a reason (one writer per escrow). If you see nonce/state errors, temp is fighting main over an escrow; give temp its own fresh escrows only. + +## MAIN came back on the old version + +`update-main` recreates MAIN from the compose file using the image tag pinned there. The `bump-main-image` step rewrites that tag from `image.from_tag` to `image.to_tag` before `update-main` runs. If MAIN came back on the old version, the tag wasn't bumped — confirm `bump-main-image` ran (running steps by hand, edit the compose image tag before `update-main`). + +## New image is bad + +- Do **not** `switch-to-main` (Path B) or return the instance to the pool (Path A). +- Re-pull the previous image tag and `--force-recreate` back to it; verify `check-main-direct`; then retry. +- Routing rollback is instant: restore the nginx backup and reload, or keep the alias pointed at the still-good temp/other instance. + +## Settlement fired unexpectedly + +- `escrow_rotation.enabled = false` is the master switch — it stops the rotator and all auto-settlement. The `disable-main-rotation` step sets it on MAIN; if a settle fired mid-migration, rotation was still on there (a running gateway keeps whatever is in `gateway.db`, not the first-boot default). Settle deliberately afterward via `POST /v1/admin/devshards/{id}/settle`. +- A settle that was *queued* (`settlement_queued_waiting_for_drain`) is normal and harmless — it waits for `active_requests == 0` and does not drop requests. + +## Rollback checklist + +1. Stop routing new traffic to the bad target (restore nginx backup + reload, or keep alias on the good side). +2. Confirm the good side is serving via the **public** URL smoke test. +3. Recreate the bad container on the last-known-good image; verify direct. +4. Only then resume the normal step order. diff --git a/devshard/docs/devshard-update/scripts/update.config.sample.json b/devshard/docs/devshard-update/scripts/update.config.sample.json new file mode 100644 index 0000000000..a2e9caa763 --- /dev/null +++ b/devshard/docs/devshard-update/scripts/update.config.sample.json @@ -0,0 +1,52 @@ +{ + "image": { + "repository": "ghcr.io/gonka-ai/devshard-gateway", + "from_tag": "mainnet-v0.2.13-latest", + "to_tag": "mainnet-v0.2.14-latest" + }, + "models": [ + { "model": "moonshotai/Kimi-K2.6", "escrow_count": 4, "escrow_amount": 5000000000 }, + { "model": "MiniMaxAI/MiniMax-M2.7", "escrow_count": 2, "escrow_amount": 5000000000 } + ], + "allow_unavailable_models": [], + "escrow": { + "protocol_version": "1", + "private_key_env": "DEVSHARD_PRIVATE_KEY" + }, + "main": { + "admin_url": "http://127.0.0.1:18080", + "container": "devshard-gateway", + "storage_host_dir": ".devshardctl", + "env_file": "./config.devshard.env" + }, + "temp": { + "admin_port": 18081, + "upstream_alias": "devshard-gateway-temp", + "network": "join_default" + }, + "nginx": { + "proxy_container": "proxy", + "config_path": "/etc/nginx/nginx.conf", + "old_upstream": "devshard-gateway", + "new_upstream": "devshard-gateway-temp", + "upstream_port": 8080, + "public_base_url": "", + "public_prefix": "/devshard-gateway" + }, + "compose": { + "file": "docker-compose.devshard-gateway.yml", + "service": "devshard-gateway" + }, + "timeouts": { + "ready_timeout_seconds": 180, + "ready_poll_seconds": 3, + "drain_timeout_seconds": 7200, + "drain_poll_seconds": 10 + }, + "smoke_test": { + "model": "" + }, + "rotation": { + "restore_after_update": false + } +} diff --git a/devshard/docs/devshard-update/scripts/update.sh b/devshard/docs/devshard-update/scripts/update.sh new file mode 100755 index 0000000000..e18b1b7d91 --- /dev/null +++ b/devshard/docs/devshard-update/scripts/update.sh @@ -0,0 +1,622 @@ +#!/usr/bin/env bash +set -Eeuo pipefail +# -E (errtrace): the ERR trap must be inherited by shell functions so a failed +# gate deep in a step still reports where it stopped. Requires bash >= 4. + +# Zero-downtime update for a Gonka devshard gateway (single instance, blue/green). +# One self-contained file, driven by a JSON config (models config-driven). +# +# Flow: stand up a temp gateway on its OWN fresh escrows, switch nginx to temp, +# drain main, bump the compose image tag + recreate main, switch back, drain + +# remove temp, then fold the temp escrows into main. Dry-run unless --run. +# +# Sections below: CLI -> config load/validate/resolve -> helpers (progress, +# admin API, docker/nginx) -> steps -> orchestration -> recover -> entrypoint. +# +# Usage: +# ./update.sh --config update.config.json # dry-run plan +# ./update.sh --config update.config.json run --run --yes # full flow, unattended +# ./update.sh --config update.config.json --run # one step +# ./update.sh --config update.config.json run --run --from-step drain-main +# ./update.sh --config update.config.json recover --run [--settle] +# ./update.sh --config update.config.json plan|validate|list-steps + +ORDERED_STEPS=( + init preflight disable-main-rotation create-temp-gateway create-temp-escrows + check-temp switch-to-temp check-alias-temp drain-main bump-main-image + update-main check-main-direct switch-to-main check-alias-main drain-temp + stop-temp import-temp activate-temp restore-main-rotation status +) + +# --- CLI --------------------------------------------------------------------- + +usage() { + cat <<'EOF' +Zero-downtime devshard gateway update (v2, single-file, config-driven). + + ./update.sh --config [action] [flags] + +Actions: run | plan | validate | list-steps | recover | +Flags: + --config JSON config (default $DEVSHARD_UPDATE_CONFIG or ./update.config.json) + --run execute for real (default dry-run; also RUN=1) + --dry-run force dry-run even if RUN=1 + --yes, -y auto-confirm destructive steps (unattended) + --from-step with 'run': start at this step, skip earlier ones + --settle with 'recover': settle stranded escrows instead of activating + --deploy-dir directory to operate from (default: cwd) +EOF +} + +CONFIG_ARG="${DEVSHARD_UPDATE_CONFIG:-./update.config.json}" +ACTION=""; FROM_STEP=""; DEPLOY_DIR="$(pwd)" +DRY_RUN=1; [[ "${RUN:-0}" == "1" ]] && DRY_RUN=0 +ASSUME_YES=0; RECOVER_SETTLE=0 + +while [[ $# -gt 0 ]]; do + case "$1" in + --config) CONFIG_ARG="${2:?--config needs a value}"; shift 2 ;; + --config=*) CONFIG_ARG="${1#*=}"; shift ;; + --from-step) FROM_STEP="${2:?--from-step needs a value}"; shift 2 ;; + --from-step=*) FROM_STEP="${1#*=}"; shift ;; + --deploy-dir) DEPLOY_DIR="${2:?--deploy-dir needs a value}"; shift 2 ;; + --deploy-dir=*) DEPLOY_DIR="${1#*=}"; shift ;; + --settle) RECOVER_SETTLE=1; shift ;; + --run) DRY_RUN=0; shift ;; + --dry-run) DRY_RUN=1; shift ;; + --yes|-y) ASSUME_YES=1; shift ;; + --help|-h) usage; exit 0 ;; + --*) echo "unknown flag: $1" >&2; usage >&2; exit 2 ;; + *) ACTION="$1"; shift ;; + esac +done +[[ -z "${ACTION}" ]] && ACTION="plan" + +# --- progress (linear; markers are grepped by the sandbox test) -------------- + +STEP_INDEX=0; STEP_TOTAL=0; CURRENT_STEP="" +step_begin() { CURRENT_STEP="$1"; STEP_INDEX=$(( STEP_INDEX + 1 )); printf '==> [%2d/%2d] %s\n' "${STEP_INDEX}" "${STEP_TOTAL}" "$1"; } +gate_ok() { printf ' GATE-OK %s\n' "$1"; } +gate_fail() { printf ' GATE-FAIL %s\n' "$1"; } +note() { printf ' - %s\n' "$1"; } + +# --- config: load, validate (fail fast), resolve (env overrides JSON) -------- + +cfg() { jq -r "$1 // \"\"" <<<"${CONFIG_JSON}"; } +cfgn() { jq -r "$1 // empty" <<<"${CONFIG_JSON}"; } + +config_validate() { + local errors + errors="$(jq -r ' + def req(v; name): if (v == null or v == "") then "missing/empty: \(name)" else empty end; + def num(v; name): if (v|type) != "number" then "not a number: \(name)" else empty end; + [ req(.image.repository; "image.repository"), + req(.image.from_tag; "image.from_tag"), + req(.image.to_tag; "image.to_tag"), + (if ((.models // []) | length) < 1 then "models must be a non-empty array of {model, escrow_count, escrow_amount}" else empty end), + (.models // [] | to_entries[] | + ( req(.value.model; "models[\(.key)].model"), + num(.value.escrow_count; "models[\(.key)].escrow_count"), + num(.value.escrow_amount; "models[\(.key)].escrow_amount") )), + req(.escrow.protocol_version; "escrow.protocol_version"), + req(.escrow.private_key_env; "escrow.private_key_env"), + req(.main.admin_url; "main.admin_url"), + req(.main.container; "main.container"), + req(.main.storage_host_dir; "main.storage_host_dir"), + num(.temp.admin_port; "temp.admin_port"), + req(.temp.upstream_alias; "temp.upstream_alias"), + req(.temp.network; "temp.network"), + req(.nginx.proxy_container; "nginx.proxy_container"), + req(.nginx.config_path; "nginx.config_path"), + req(.nginx.old_upstream; "nginx.old_upstream"), + req(.nginx.new_upstream; "nginx.new_upstream"), + num(.nginx.upstream_port; "nginx.upstream_port"), + req(.compose.file; "compose.file"), + req(.compose.service; "compose.service"), + num(.timeouts.ready_timeout_seconds; "timeouts.ready_timeout_seconds"), + num(.timeouts.ready_poll_seconds; "timeouts.ready_poll_seconds"), + num(.timeouts.drain_timeout_seconds; "timeouts.drain_timeout_seconds"), + num(.timeouts.drain_poll_seconds; "timeouts.drain_poll_seconds") + ] | .[]' <<<"${CONFIG_JSON}")" + if [[ -n "${errors}" ]]; then + echo "Config validation failed for ${CONFIG_PATH}:" >&2 + sed 's/^/ - /' <<<"${errors}" >&2 + return 1 + fi +} + +resolve_config() { + IMAGE_REPO="${IMAGE_REPO:-$(cfg .image.repository)}" + IMAGE_FROM_TAG="${IMAGE_FROM_TAG:-$(cfg .image.from_tag)}" + IMAGE_TO_TAG="${IMAGE_TO_TAG:-$(cfg .image.to_tag)}" + IMAGE_FROM_REF="${IMAGE_REPO}:${IMAGE_FROM_TAG}" + IMAGE_TO_REF="${IMAGE_REPO}:${IMAGE_TO_TAG}" + ESCROW_PROTOCOL_VERSION="${ESCROW_PROTOCOL_VERSION:-$(cfg .escrow.protocol_version)}" + ESCROW_PRIVATE_KEY_ENV="${ESCROW_PRIVATE_KEY_ENV:-$(cfg .escrow.private_key_env)}" + MAIN_ADMIN_URL="${MAIN_ADMIN_URL:-$(cfg .main.admin_url)}" + MAIN_CONTAINER="${MAIN_CONTAINER:-$(cfg .main.container)}" + MAIN_STORAGE_HOST_DIR="${MAIN_STORAGE_HOST_DIR:-$(cfg .main.storage_host_dir)}" + MAIN_ENV_FILE="${MAIN_ENV_FILE:-$(cfg .main.env_file)}" + TEMP_ADMIN_PORT="${TEMP_ADMIN_PORT:-$(cfgn .temp.admin_port)}" + TEMP_ADMIN_URL="${TEMP_ADMIN_URL:-http://127.0.0.1:${TEMP_ADMIN_PORT}}" + TEMP_UPSTREAM_ALIAS="${TEMP_UPSTREAM_ALIAS:-$(cfg .temp.upstream_alias)}" + TEMP_NETWORK="${TEMP_NETWORK:-$(cfg .temp.network)}" + NGINX_PROXY_CONTAINER="${NGINX_PROXY_CONTAINER:-$(cfg .nginx.proxy_container)}" + NGINX_CONFIG_PATH="${NGINX_CONFIG_PATH:-$(cfg .nginx.config_path)}" + NGINX_OLD_UPSTREAM="${NGINX_OLD_UPSTREAM:-$(cfg .nginx.old_upstream)}" + NGINX_NEW_UPSTREAM="${NGINX_NEW_UPSTREAM:-$(cfg .nginx.new_upstream)}" + [[ -n "${NGINX_NEW_UPSTREAM}" ]] || NGINX_NEW_UPSTREAM="${TEMP_UPSTREAM_ALIAS}" + NGINX_UPSTREAM_PORT="${NGINX_UPSTREAM_PORT:-$(cfgn .nginx.upstream_port)}" + NGINX_PUBLIC_BASE_URL="${NGINX_PUBLIC_BASE_URL:-$(cfg .nginx.public_base_url)}" + NGINX_PUBLIC_PREFIX="${NGINX_PUBLIC_PREFIX:-$(cfg .nginx.public_prefix)}" + COMPOSE_FILE="${COMPOSE_FILE:-$(cfg .compose.file)}" + COMPOSE_SERVICE="${COMPOSE_SERVICE:-$(cfg .compose.service)}" + READY_TIMEOUT_SECONDS="${READY_TIMEOUT_SECONDS:-$(cfgn .timeouts.ready_timeout_seconds)}" + READY_POLL_SECONDS="${READY_POLL_SECONDS:-$(cfgn .timeouts.ready_poll_seconds)}" + DRAIN_TIMEOUT_SECONDS="${DRAIN_TIMEOUT_SECONDS:-$(cfgn .timeouts.drain_timeout_seconds)}" + DRAIN_POLL_SECONDS="${DRAIN_POLL_SECONDS:-$(cfgn .timeouts.drain_poll_seconds)}" + SMOKE_MODEL="${SMOKE_MODEL:-$(cfg .smoke_test.model)}" + [[ -n "${SMOKE_MODEL}" ]] || SMOKE_MODEL="$(jq -r '.models[0].model' <<<"${CONFIG_JSON}")" + ROTATION_RESTORE="${ROTATION_RESTORE:-$(jq -r '.rotation.restore_after_update // false' <<<"${CONFIG_JSON}")}" +} + +load_config() { + local path="$1" + [[ -f "${path}" ]] || { echo "config file not found: ${path}" >&2; exit 2; } + CONFIG_PATH="${path}" + CONFIG_JSON="$(jq -c . "${path}" 2>&1)" || { echo "config is not valid JSON: ${path}"$'\n'" ${CONFIG_JSON}" >&2; exit 2; } + config_validate || exit 2 + resolve_config +} + +models_tsv() { jq -r '.models[] | [.model, (.escrow_count|tostring), (.escrow_amount|tostring)] | @tsv' <<<"${CONFIG_JSON}"; } +covered_models() { jq -r '.models[].model, ((.allow_unavailable_models // [])[])' <<<"${CONFIG_JSON}"; } + +# --- helpers: admin API, command runner, gateway ops ------------------------- + +need_key() { [[ -n "${DEVSHARD_ADMIN_API_KEY:-}" ]] || { echo "DEVSHARD_ADMIN_API_KEY is required" >&2; return 1; }; } + +admin_get() { curl -fsS "$1$2" -H "Authorization: Bearer ${DEVSHARD_ADMIN_API_KEY}"; } +admin_post() { curl -fsS -X POST "$1$2" -H "Authorization: Bearer ${DEVSHARD_ADMIN_API_KEY}" -H 'Content-Type: application/json' -d "$3"; } + +# Print a command, run it only in live mode. +run() { note "exec: $*"; [[ "${DRY_RUN}" == "1" ]] || "$@"; } +run_shell() { note "exec: $(sed -E 's/Bearer [^ ]+/Bearer /g' <<<"$1")"; [[ "${DRY_RUN}" == "1" ]] || bash -euo pipefail -c "$1"; } + +confirm() { + [[ "${DRY_RUN}" == "1" ]] && return 0 + [[ "${ASSUME_YES}" == "1" ]] && { note "auto-confirm (--yes): $1"; return 0; } + [[ -t 0 ]] || { gate_fail "confirmation required but no TTY: $1 (rerun with --yes)"; return 1; } + local reply; printf '\n?? %s [y/N] ' "$1"; read -r reply + case "${reply}" in y|Y|yes|YES) return 0 ;; *) gate_fail "declined: $1"; return 1 ;; esac +} + +wait_ready() { + local name="$1" url="$2" deadline=$(( SECONDS + READY_TIMEOUT_SECONDS )) + while :; do + admin_get "${url}" "/v1/status" >/dev/null 2>&1 && { gate_ok "${name} ready"; return 0; } + (( SECONDS >= deadline )) && { gate_fail "${name} not ready within ${READY_TIMEOUT_SECONDS}s"; return 1; } + sleep "${READY_POLL_SECONDS}" + done +} + +# The drain gate: block until max active_requests == 0 or timeout. +wait_drain() { + local name="$1" url="$2" deadline=$(( SECONDS + DRAIN_TIMEOUT_SECONDS )) active + while :; do + active="$(admin_get "${url}" "/v1/status" | jq '[.devshards[]?.active_requests // 0] | max // 0')" + note "${name} active_requests=${active}" + [[ "${active}" == "0" ]] && { gate_ok "${name} drained (active_requests=0)"; return 0; } + (( SECONDS >= deadline )) && { gate_fail "${name} did not drain within ${DRAIN_TIMEOUT_SECONDS}s (active_requests=${active})"; return 1; } + sleep "${DRAIN_POLL_SECONDS}" + done +} + +smoke_chat() { + curl -fsS -X POST "$1/v1/chat/completions" -H 'Content-Type: application/json' \ + -d "$(jq -nc --arg m "$2" '{model:$m, stream:false, max_tokens:1, messages:[{role:"user", content:"Reply with ok"}]}')" >/dev/null +} + +settings_sync_to_temp() { + local settings + settings="$(admin_get "${MAIN_ADMIN_URL}" "/v1/admin/settings" | jq '.escrow_rotation.enabled=false | .escrow_rotation.models=[]')" + admin_post "${TEMP_ADMIN_URL}" "/v1/admin/settings" "${settings}" >/dev/null +} + +assert_settings_aligned() { + local main_settings temp_settings + main_settings="$(admin_get "${MAIN_ADMIN_URL}" "/v1/admin/settings" | jq -S '.escrow_rotation.enabled=false | .escrow_rotation.models=[]')" + temp_settings="$(admin_get "${TEMP_ADMIN_URL}" "/v1/admin/settings" | jq -S '.escrow_rotation.enabled=false | .escrow_rotation.models=[]')" + [[ "${main_settings}" == "${temp_settings}" ]] || { gate_fail "temp settings do not match main"; return 1; } + gate_ok "temp settings match main (rotation disabled on temp)" +} + +# Start the temp gateway on its OWN empty escrow set. Reuses MAIN's --env-file +# so no secret material is written to a new path. --restart unless-stopped, so +# it MUST later be stopped AND removed to avoid a two-writer resurrection. +temp_start() { + run_shell "docker rm -f '${TEMP_CONTAINER}' >/dev/null 2>&1 || true" + local -a args=(docker run -d --name "${TEMP_CONTAINER}" --restart unless-stopped + --network "${TEMP_NETWORK}" --network-alias "${TEMP_UPSTREAM_ALIAS}") + [[ -n "${MAIN_ENV_FILE_ABS}" && -f "${MAIN_ENV_FILE_ABS}" ]] && args+=(--env-file "${MAIN_ENV_FILE_ABS}") + args+=(-e DEVSHARDS_JSON='[]' -e DEVSHARD_STORAGE_DIR="${TEMP_STORAGE_CONTAINER_DIR}" -e DEVSHARD_PORT=8080 + -p "127.0.0.1:${TEMP_ADMIN_PORT}:8080" -v "${STORAGE_HOST_DIR_ABS}:/root/.devshardctl" "${IMAGE_TO_REF}") + run "${args[@]}" +} + +# Rewrite the compose image tag from_ref -> to_ref. Portable (sed to a temp file +# then mv; no GNU-only -i). Idempotent; backs up first; fails loudly if absent. +compose_bump() { + local file="${COMPOSE_FILE}" from="${IMAGE_FROM_REF}" to="${IMAGE_TO_REF}" + [[ -f "${file}" ]] || { gate_fail "compose file not found: ${file}"; return 1; } + if grep -q "${to}" "${file}" && ! grep -q "${from}" "${file}"; then gate_ok "compose already pins ${to}"; return 0; fi + grep -q "${from}" "${file}" || { gate_fail "neither '${from}' nor '${to}' found in ${file}"; return 1; } + [[ "${DRY_RUN}" == "1" ]] && { note "would bump ${file}: ${from} -> ${to}"; return 0; } + cp "${file}" "${file}.blue-green-backup" + sed "s#${from}#${to}#g" "${file}" > "${file}.blue-green-tmp" + mv "${file}.blue-green-tmp" "${file}" + grep -q "${to}" "${file}" || { gate_fail "bump did not apply; restore ${file}.blue-green-backup"; return 1; } + gate_ok "compose image bumped ${from} -> ${to} (backup: ${file}.blue-green-backup)" +} + +# Switch the nginx upstream host inside the proxy container, then validate and +# gracefully reload (never restart). Handles both a direct proxy_pass +# http://host:PORT and a named-upstream `server host:PORT;`. Portable sed; +# backs up first; idempotent if already switched. +nginx_switch() { + local from="$1" to="$2" port="${NGINX_UPSTREAM_PORT}" + local cfg_path="${NGINX_CONFIG_PATH}" backup="${NGINX_CONFIG_PATH}.blue-green-backup" tmp="${NGINX_CONFIG_PATH}.blue-green-tmp" + local old_pat="(http://${from}:${port}|server[[:space:]]+${from}:${port})" + local new_pat="(http://${to}:${port}|server[[:space:]]+${to}:${port})" + note "nginx upstream switch ${from} -> ${to} in ${cfg_path}" + if [[ "${DRY_RUN}" == "1" ]]; then note "would docker exec ${NGINX_PROXY_CONTAINER} sh -lc '${to} + nginx -t + reload>'"; return 0; fi + local inner + inner="$(cat <&2; exit 3 +fi +cp '${cfg_path}' '${backup}' +sed -E 's#http://${from}:${port}#http://${to}:${port}#g; s#(server[[:space:]]+)${from}:${port}#\1${to}:${port}#g' '${cfg_path}' > '${tmp}' +mv '${tmp}' '${cfg_path}' +grep -Eq '${new_pat}' '${cfg_path}' || { echo 'ERROR: switch did not apply; restore ${backup}' >&2; exit 4; } +nginx -t +nginx -s reload +echo 'nginx upstream switched ${from} -> ${to} and reloaded' +INNER +)" + if docker exec "${NGINX_PROXY_CONTAINER}" sh -lc "${inner}"; then gate_ok "nginx switched ${from} -> ${to} (graceful reload)"; return 0; fi + gate_fail "nginx switch ${from} -> ${to} failed"; return 1 +} + +# Fold one temp escrow into main: import inactive (carries its state.db), then +# either activate (route it) or settle (drain-aware on-chain). Shared by the +# activate-temp step and the recover command. +import_escrow_into_main() { + local id="$1" model="$2" proto="$3" + local storage="${TEMP_STORAGE_CONTAINER_DIR}/escrow-${id}/state.db" perf="${TEMP_STORAGE_CONTAINER_DIR}/perf.db" + admin_post "${MAIN_ADMIN_URL}" "/v1/admin/devshards/import" \ + "$(jq -nc --arg id "${id}" --arg m "${model}" --arg s "${storage}" --arg p "${proto}" \ + --arg pk "${ESCROW_PRIVATE_KEY_ENV}" --arg perf "${perf}" \ + '{id:$id, model:$m, storage_path:$s, protocol_version:$p, private_key_env:$pk, perf_path:$perf, active:false}')" >/dev/null +} +activate_escrow_on_main() { + local id="$1" model="$2" proto="$3" + local storage="${TEMP_STORAGE_CONTAINER_DIR}/escrow-${id}/state.db" + admin_post "${MAIN_ADMIN_URL}" "/v1/admin/devshards" \ + "$(jq -nc --arg id "${id}" --arg m "${model}" --arg s "${storage}" --arg p "${proto}" \ + --arg pk "${ESCROW_PRIVATE_KEY_ENV}" \ + '{id:$id, model:$m, storage_path:$s, protocol_version:$p, private_key_env:$pk}')" >/dev/null +} + +# --- run state (survives single-step / resume / recover invocations) --------- + +runstate_init() { + RUN_ID="$(date -u +%Y%m%dT%H%M%SZ)" + TEMP_CONTAINER="${TEMP_UPSTREAM_ALIAS}-${RUN_ID}" + TEMP_STORAGE_HOST_DIR="${STORAGE_HOST_DIR_ABS}/temp-${RUN_ID}" + TEMP_STORAGE_CONTAINER_DIR="/root/.devshardctl/temp-${RUN_ID}" + TEMP_DEVSHARDS_FILE="${TEMP_STORAGE_HOST_DIR}/temp-devshards.json" +} +runstate_write() { + cat > "${RUN_STATE_FILE}" </dev/null 2>&1 || { gate_ok "dry-run: main not reachable now; coverage check runs live"; return 0; } + else + wait_ready "main gateway" "${MAIN_ADMIN_URL}" + fi + local uncovered="" served + while IFS= read -r served; do + [[ -z "${served}" ]] && continue + covered_models | grep -Fxq "${served}" || uncovered="${uncovered} ${served}" + done < <(admin_get "${MAIN_ADMIN_URL}" "/v1/admin/devshards" | jq -r '[.devshards[]? | select(.active==true) | .model] | unique | .[]') + [[ -n "${uncovered}" ]] && { gate_fail "main serves uncovered models:${uncovered} (add to models[] or allow_unavailable_models)"; return 1; } + gate_ok "every model main serves is covered by temp escrows or allow-listed" +} + +step_disable_main_rotation() { + need_key + [[ "${DRY_RUN}" == "1" ]] && { note "would disable escrow_rotation on MAIN so nothing settles on-chain mid-update"; return 0; } + local settings; settings="$(admin_get "${MAIN_ADMIN_URL}" "/v1/admin/settings" | jq '.escrow_rotation.enabled=false')" + admin_post "${MAIN_ADMIN_URL}" "/v1/admin/settings" "${settings}" >/dev/null + gate_ok "MAIN escrow_rotation disabled" +} + +step_create_temp_gateway() { + need_key + run mkdir -p "${TEMP_STORAGE_HOST_DIR}" + run docker pull "${IMAGE_TO_REF}" + temp_start + [[ "${DRY_RUN}" == "1" ]] && { note "would wait for temp readiness and sync main settings (rotation off) to temp"; return 0; } + wait_ready "temp gateway" "${TEMP_ADMIN_URL}" + settings_sync_to_temp + gate_ok "temp gateway up on ${TEMP_ADMIN_URL}; settings synced (rotation off)" +} + +step_create_temp_escrows() { + need_key + local model count amount i body + if [[ "${DRY_RUN}" == "1" ]]; then + while IFS=$'\t' read -r model count amount; do note "would mint ${count} x ${model} @ ${amount}"; done < <(models_tsv) + return 0 + fi + confirm "mint fresh temp escrows on-chain (spends real funds; irreversible)" + mkdir -p "${TEMP_STORAGE_HOST_DIR}" + ESCROWS_MINTED=1 + while IFS=$'\t' read -r model count amount; do + for (( i=1; i<=count; i++ )); do + note "mint ${i}/${count} ${model} @ ${amount}" + body="$(jq -nc --argjson amount "${amount}" --arg model "${model}" --arg pk "${ESCROW_PRIVATE_KEY_ENV}" --arg pv "${ESCROW_PROTOCOL_VERSION}" \ + '{amount:$amount, model_id:$model, private_key_env:$pk, protocol_version:$pv}')" + admin_post "${TEMP_ADMIN_URL}" "/v1/admin/escrows" "${body}" >/dev/null + done + done < <(models_tsv) + admin_get "${TEMP_ADMIN_URL}" "/v1/admin/devshards" > "${TEMP_DEVSHARDS_FILE}" + gate_ok "temp escrows minted; recorded to ${TEMP_DEVSHARDS_FILE}" +} + +step_check_temp() { + need_key + [[ "${DRY_RUN}" == "1" ]] && { note "would verify temp readiness, settings alignment, per-model routable escrows, and a chat smoke test"; return 0; } + wait_ready "temp gateway" "${TEMP_ADMIN_URL}" + assert_settings_aligned + local model count amount active + while IFS=$'\t' read -r model count amount; do + active="$(admin_get "${TEMP_ADMIN_URL}" "/v1/admin/devshards" | jq --arg m "${model}" '[.devshards[]? | select(.model==$m and .active==true)] | length')" + (( active < count )) && { gate_fail "temp has ${active} active escrows for ${model}, need ${count} (model would be unavailable during update)"; return 1; } + gate_ok "temp: ${active}/${count} active escrows for ${model}" + done < <(models_tsv) + smoke_chat "${TEMP_ADMIN_URL}" "${SMOKE_MODEL}" + gate_ok "temp chat smoke test passed (${SMOKE_MODEL})" + admin_get "${TEMP_ADMIN_URL}" "/v1/admin/devshards" > "${TEMP_DEVSHARDS_FILE}" +} + +step_switch_to_temp() { confirm "switch nginx upstream to TEMP (${NGINX_NEW_UPSTREAM})"; nginx_switch "${NGINX_OLD_UPSTREAM}" "${NGINX_NEW_UPSTREAM}"; } +step_check_alias_temp() { check_alias temp; } + +step_drain_main() { + need_key + [[ "${DRY_RUN}" == "1" ]] && { note "would wait for main active_requests to reach 0 (the drain gate)"; return 0; } + wait_drain "main gateway" "${MAIN_ADMIN_URL}" +} + +step_bump_main_image() { compose_bump; } + +step_update_main() { + confirm "recreate MAIN from ${COMPOSE_FILE} on the bumped image" + note "MAIN image comes from ${COMPOSE_FILE} (service ${COMPOSE_SERVICE}), not an env ref" + local src=""; [[ -n "${MAIN_ENV_FILE_ABS}" && -f "${MAIN_ENV_FILE_ABS}" ]] && src="source '${MAIN_ENV_FILE_ABS}' && " + run_shell "cd '${DEPLOY_DIR}' && ${src}docker compose -f '${COMPOSE_FILE}' pull ${COMPOSE_SERVICE} && docker compose -f '${COMPOSE_FILE}' up -d --no-deps --force-recreate ${COMPOSE_SERVICE} && docker compose -f '${COMPOSE_FILE}' ps ${COMPOSE_SERVICE}" + gate_ok "MAIN recreated on new image" +} + +step_check_main_direct() { + need_key + [[ "${DRY_RUN}" == "1" ]] && { note "would verify main readiness and a direct chat smoke test before switching back"; return 0; } + wait_ready "main gateway" "${MAIN_ADMIN_URL}" + smoke_chat "${MAIN_ADMIN_URL}" "${SMOKE_MODEL}" + gate_ok "main direct chat smoke test passed (${SMOKE_MODEL})" +} + +step_switch_to_main() { confirm "switch nginx upstream back to MAIN (${NGINX_OLD_UPSTREAM})"; nginx_switch "${NGINX_NEW_UPSTREAM}" "${NGINX_OLD_UPSTREAM}"; } +step_check_alias_main() { check_alias main; } + +step_drain_temp() { + need_key + [[ "${DRY_RUN}" == "1" ]] && { note "would wait for temp active_requests to reach 0 before removing it"; return 0; } + wait_drain "temp gateway" "${TEMP_ADMIN_URL}" +} + +step_stop_temp() { + confirm "stop AND remove the temp container ${TEMP_CONTAINER}" + run docker stop "${TEMP_CONTAINER}" + run docker rm "${TEMP_CONTAINER}" + gate_ok "temp container stopped and removed" +} + +step_import_temp() { + need_key + [[ "${DRY_RUN}" == "1" ]] && { note "would import temp escrows from ${TEMP_DEVSHARDS_FILE} into main (inactive)"; return 0; } + confirm "import temp escrows into MAIN (inactive)" + local row id model proto + while IFS= read -r row; do + id="$(jq -r '.id' <<<"${row}")"; model="$(jq -r '.model // ""' <<<"${row}")"; proto="$(jq -r '.protocol_version // ""' <<<"${row}")" + note "import ${id} (${model}) inactive"; import_escrow_into_main "${id}" "${model}" "${proto}" + done < <(jq -c '.devshards[]' "${TEMP_DEVSHARDS_FILE}") + gate_ok "temp escrows imported into main (inactive)" +} + +step_activate_temp() { + need_key + [[ "${DRY_RUN}" == "1" ]] && { note "would activate imported temp escrows on main"; return 0; } + confirm "activate imported temp escrows on MAIN" + local row id model proto + while IFS= read -r row; do + id="$(jq -r '.id' <<<"${row}")"; model="$(jq -r '.model // ""' <<<"${row}")"; proto="$(jq -r '.protocol_version // ""' <<<"${row}")" + note "activate ${id} (${model})"; activate_escrow_on_main "${id}" "${model}" "${proto}" + done < <(jq -c '.devshards[]' "${TEMP_DEVSHARDS_FILE}") + ESCROWS_MINTED=0 + gate_ok "temp escrows activated on main" +} + +step_restore_main_rotation() { + need_key + [[ "${ROTATION_RESTORE}" != "true" ]] && { note "leaving MAIN escrow_rotation disabled (set rotation.restore_after_update=true to re-enable)"; return 0; } + [[ "${DRY_RUN}" == "1" ]] && { note "would re-enable escrow_rotation on MAIN"; return 0; } + local settings; settings="$(admin_get "${MAIN_ADMIN_URL}" "/v1/admin/settings" | jq '.escrow_rotation.enabled=true')" + admin_post "${MAIN_ADMIN_URL}" "/v1/admin/settings" "${settings}" >/dev/null + gate_ok "MAIN escrow_rotation re-enabled" +} + +step_status() { + need_key + [[ "${DRY_RUN}" == "1" ]] && { note "would show main/temp status"; return 0; } + note "main status: $(admin_get "${MAIN_ADMIN_URL}" "/v1/status" | jq -c '{active:([.devshards[]?|select(.active==true)]|length), max_active_requests:([.devshards[]?.active_requests//0]|max//0)}')" + note "temp status: $(admin_get "${TEMP_ADMIN_URL}" "/v1/status" 2>/dev/null | jq -c '{active:([.devshards[]?|select(.active==true)]|length)}' 2>/dev/null || echo gone)" +} + +# check-alias-temp / check-alias-main: verify the public route (skipped if unset). +check_alias() { + local side="$1"; need_key + [[ -z "${NGINX_PUBLIC_BASE_URL}" ]] && { note "no nginx.public_base_url set; skipping public ${side} verification"; return 0; } + [[ "${DRY_RUN}" == "1" ]] && { note "would verify public ${side} route via ${NGINX_PUBLIC_BASE_URL}"; return 0; } + case "${NGINX_PUBLIC_BASE_URL}" in *127.0.0.1*|*localhost*) note "WARNING: public_base_url is loopback; can pass while the real route is broken" ;; esac + curl -fsS "${NGINX_PUBLIC_BASE_URL}${NGINX_PUBLIC_PREFIX}/v1/status" -H "Authorization: Bearer ${DEVSHARD_ADMIN_API_KEY}" >/dev/null + smoke_chat "${NGINX_PUBLIC_BASE_URL}${NGINX_PUBLIC_PREFIX}" "${SMOKE_MODEL}" + gate_ok "public ${side} route verified" +} + +# --- recover: fold stranded temp escrows into main, or settle them ----------- + +cmd_recover() { + need_key + [[ -f "${TEMP_DEVSHARDS_FILE:-}" ]] || { echo "recover: no temp-escrows state at ${TEMP_DEVSHARDS_FILE:-} (run init/create-temp-escrows first, or set --deploy-dir)" >&2; exit 1; } + local mode; mode="$([[ "${RECOVER_SETTLE}" == "1" ]] && echo settle || echo activate)" + echo "==> recover from ${TEMP_DEVSHARDS_FILE} (mode: import+${mode}; dry-run=${DRY_RUN})" + [[ "${DRY_RUN}" == "1" ]] || confirm "recover temp escrows into MAIN (import+${mode})" + local row id model proto count=0 + while IFS= read -r row; do + id="$(jq -r '.id' <<<"${row}")"; model="$(jq -r '.model // ""' <<<"${row}")"; proto="$(jq -r '.protocol_version // ""' <<<"${row}")" + count=$(( count + 1 )) + if [[ "${DRY_RUN}" == "1" ]]; then note "would import+${mode} ${id} (${model})"; continue; fi + import_escrow_into_main "${id}" "${model}" "${proto}" + if [[ "${RECOVER_SETTLE}" == "1" ]]; then + note "settle ${id} (${model})"; admin_post "${MAIN_ADMIN_URL}" "/v1/admin/devshards/${id}/settle" '{}' >/dev/null + else + note "activate ${id} (${model})"; activate_escrow_on_main "${id}" "${model}" "${proto}" + fi + done < <(jq -c '.devshards[]' "${TEMP_DEVSHARDS_FILE}") + ESCROWS_MINTED=0 + gate_ok "recovered ${count} temp escrow(s) into main (import+${mode})" +} + +# --- orchestration + error handling ------------------------------------------ + +run_flow() { + local -a steps=("$@"); STEP_TOTAL="${#steps[@]}"; STEP_INDEX=0 + local step + for step in "${steps[@]}"; do + step_begin "${step}" + "step_${step//-/_}" + done +} + +print_plan() { + cat < ${IMAGE_TO_REF} +Main: ${MAIN_CONTAINER} @ ${MAIN_ADMIN_URL} +Temp: ${TEMP_UPSTREAM_ALIAS} @ ${TEMP_ADMIN_URL} (network ${TEMP_NETWORK}) +Nginx: ${NGINX_PROXY_CONTAINER}:${NGINX_CONFIG_PATH} ${NGINX_OLD_UPSTREAM} <-> ${NGINX_NEW_UPSTREAM}:${NGINX_UPSTREAM_PORT} +Public: ${NGINX_PUBLIC_BASE_URL:-} +Smoke: ${SMOKE_MODEL} +Models (fresh temp escrows minted per model): +$(models_tsv | while IFS=$'\t' read -r m c a; do printf ' %s x %s @ %s\n' "${c}" "${m}" "${a}"; done) +Steps: +$(printf ' %s\n' "${ORDERED_STEPS[@]}") +EOF +} + +ESCROWS_MINTED=0; ON_ERR=0 +on_error() { + local code=$? + [[ "${ON_ERR}" == "1" ]] && return; ON_ERR=1 + { + echo "" + echo "FAILED at step: ${CURRENT_STEP:-setup} (exit ${code})" + echo "Inspect: ${0##*/} --config '${CONFIG_ARG}' status --run ; restore nginx backup (${NGINX_CONFIG_PATH:-}.blue-green-backup) to revert routing." + if [[ "${ESCROWS_MINTED}" == "1" ]]; then + echo "" + echo "WARNING: temp escrows were minted on-chain but not yet folded into main." + echo " Recorded at: ${TEMP_DEVSHARDS_FILE:-}" + echo " Recover: ${0##*/} --config '${CONFIG_ARG}' --deploy-dir '${DEPLOY_DIR}' recover --run (add --settle to settle instead of activate)" + fi + } >&2 +} +trap on_error ERR + +# --- entrypoint -------------------------------------------------------------- + +abspath() { case "$1" in /*) printf '%s\n' "$1" ;; *) printf '%s/%s\n' "$(pwd)" "$1" ;; esac; } + +main() { + DEPLOY_DIR="$(abspath "${DEPLOY_DIR}")" + local config_abs; config_abs="$(abspath "${CONFIG_ARG}")" + cd "${DEPLOY_DIR}" + load_config "${config_abs}" + + STORAGE_HOST_DIR_ABS="$(abspath "${MAIN_STORAGE_HOST_DIR}")" + MAIN_ENV_FILE_ABS=""; [[ -n "${MAIN_ENV_FILE}" ]] && MAIN_ENV_FILE_ABS="$(abspath "${MAIN_ENV_FILE}")" + RUN_STATE_FILE="${STORAGE_HOST_DIR_ABS}/blue-green-run-state.env" + # shellcheck disable=SC1090 + [[ -f "${RUN_STATE_FILE}" ]] && source "${RUN_STATE_FILE}" + + case "${ACTION}" in + validate) echo "config OK: ${CONFIG_PATH}" ;; + list-steps) printf '%s\n' "${ORDERED_STEPS[@]}" ;; + plan) print_plan ;; + recover) cmd_recover ;; + run) + local -a steps=("${ORDERED_STEPS[@]}") + if [[ -n "${FROM_STEP}" ]]; then + steps=(); local seen=0 s + for s in "${ORDERED_STEPS[@]}"; do [[ "${s}" == "${FROM_STEP}" ]] && seen=1; [[ "${seen}" == "1" ]] && steps+=("${s}"); done + [[ "${#steps[@]}" -gt 0 ]] || { echo "unknown --from-step: ${FROM_STEP}" >&2; exit 2; } + fi + run_flow "${steps[@]}" + echo "update complete: ${IMAGE_FROM_TAG} -> ${IMAGE_TO_TAG}" + ;; + *) + local ok=0 s + for s in "${ORDERED_STEPS[@]}"; do [[ "${s}" == "${ACTION}" ]] && ok=1; done + (( ok )) || { echo "unknown action/step: ${ACTION}" >&2; usage >&2; exit 2; } + run_flow "${ACTION}" + ;; + esac +} + +main From 305ab5b8e08bd69af4abf017224d20d741315e5a Mon Sep 17 00:00:00 2001 From: DimaOrekhovPS Date: Fri, 10 Jul 2026 20:27:34 -0700 Subject: [PATCH 2/3] Update devshard/docs/devshard-update/scripts/update.sh Co-authored-by: GLiberman Signed-off-by: DimaOrekhovPS --- devshard/docs/devshard-update/scripts/update.sh | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/devshard/docs/devshard-update/scripts/update.sh b/devshard/docs/devshard-update/scripts/update.sh index e18b1b7d91..87e87360c6 100755 --- a/devshard/docs/devshard-update/scripts/update.sh +++ b/devshard/docs/devshard-update/scripts/update.sh @@ -213,7 +213,14 @@ wait_drain() { } smoke_chat() { - curl -fsS -X POST "$1/v1/chat/completions" -H 'Content-Type: application/json' \ + # Prefer admin key when models are admin-only; fall back to first API key; else unauthenticated. + local auth_hdr=() + if [[ -n "${DEVSHARD_ADMIN_API_KEY:-}" ]]; then + auth_hdr=(-H "Authorization: Bearer ${DEVSHARD_ADMIN_API_KEY}") + elif [[ -n "${DEVSHARD_API_KEYS:-}" ]]; then + auth_hdr=(-H "Authorization: Bearer ${DEVSHARD_API_KEYS%%,*}") + fi + curl -fsS -X POST "$1/v1/chat/completions" "${auth_hdr[@]}" -H 'Content-Type: application/json' \ -d "$(jq -nc --arg m "$2" '{model:$m, stream:false, max_tokens:1, messages:[{role:"user", content:"Reply with ok"}]}')" >/dev/null } From 9d3d3c1fc0e17ddfed39a2ea3fd7186a90738c1f Mon Sep 17 00:00:00 2001 From: DimaOrekhovPS Date: Fri, 10 Jul 2026 20:27:41 -0700 Subject: [PATCH 3/3] Update devshard/docs/devshard-update/scripts/update.sh Co-authored-by: GLiberman Signed-off-by: DimaOrekhovPS --- devshard/docs/devshard-update/scripts/update.sh | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/devshard/docs/devshard-update/scripts/update.sh b/devshard/docs/devshard-update/scripts/update.sh index 87e87360c6..4112889333 100755 --- a/devshard/docs/devshard-update/scripts/update.sh +++ b/devshard/docs/devshard-update/scripts/update.sh @@ -287,7 +287,14 @@ if ! grep -Eq '${old_pat}' '${cfg_path}'; then fi cp '${cfg_path}' '${backup}' sed -E 's#http://${from}:${port}#http://${to}:${port}#g; s#(server[[:space:]]+)${from}:${port}#\1${to}:${port}#g' '${cfg_path}' > '${tmp}' -mv '${tmp}' '${cfg_path}' +# Prefer mv (atomic replace). Bind-mounted nginx.conf rejects inode replacement +# ("Resource busy" / "File exists") — fall back to in-place content overwrite. +if mv '${tmp}' '${cfg_path}' 2>/dev/null; then + : +else + cat '${tmp}' > '${cfg_path}' + rm -f '${tmp}' +fi grep -Eq '${new_pat}' '${cfg_path}' || { echo 'ERROR: switch did not apply; restore ${backup}' >&2; exit 4; } nginx -t nginx -s reload