diff --git a/devshard/chainoracle/server/http.go b/devshard/chainoracle/server/http.go index ec6ae5e5dd..28c90250a3 100644 --- a/devshard/chainoracle/server/http.go +++ b/devshard/chainoracle/server/http.go @@ -3,6 +3,7 @@ package server import ( + "context" "net/http" "devshard/chainoracle/blocks" @@ -24,11 +25,27 @@ type VersionConfig struct { Versions []Version `json:"versions"` } +// VersionProvider returns the currently approved versions. +type VersionProvider interface { + Versions(context.Context) ([]Version, error) +} + +// VersionProviderFunc adapts a function to VersionProvider. +type VersionProviderFunc func(context.Context) ([]Version, error) + +// Versions calls f(ctx). +func (f VersionProviderFunc) Versions(ctx context.Context) ([]Version, error) { + return f(ctx) +} + // Config wires the HTTP mux. type Config struct { Blocks blocks.BlockOracle // Versions is served at GET /versions for VERSIOND_ORACLE_URL polling. Versions []Version + // VersionProvider, when set, overrides Versions for dynamic tests and dapi + // implementations. + VersionProvider VersionProvider } // Mount registers chainoracle HTTP routes on g. @@ -40,7 +57,11 @@ func Mount(g *echo.Group, cfg Config) { panic("chainoracle/server: Blocks oracle is required") } blockserver.Mount(g, cfg.Blocks) - g.GET("/versions", handleVersions(cfg.Versions)) + if cfg.VersionProvider != nil { + g.GET("/versions", handleVersionProvider(cfg.VersionProvider)) + } else { + g.GET("/versions", handleVersions(cfg.Versions)) + } } func handleVersions(versions []Version) echo.HandlerFunc { @@ -48,3 +69,13 @@ func handleVersions(versions []Version) echo.HandlerFunc { return c.JSON(http.StatusOK, VersionConfig{Versions: versions}) } } + +func handleVersionProvider(provider VersionProvider) echo.HandlerFunc { + return func(c echo.Context) error { + versions, err := provider.Versions(c.Request().Context()) + if err != nil { + return echo.NewHTTPError(http.StatusBadGateway, err.Error()) + } + return c.JSON(http.StatusOK, VersionConfig{Versions: versions}) + } +} diff --git a/devshard/cmd/devshardd/app.go b/devshard/cmd/devshardd/app.go index 1fb29f6d20..7290487308 100644 --- a/devshard/cmd/devshardd/app.go +++ b/devshard/cmd/devshardd/app.go @@ -30,10 +30,14 @@ import ( const sessionEpochRetain = 3 type devshardApp struct { - server *echo.Echo - chainEvents *chainEventBridge - port int - close func() + server *echo.Echo + adminServer *echo.Echo + adminAddr string + chainEvents *chainEventBridge + port int + lifecycle *lifecycleState + shutdownGrace time.Duration + close func() } type chainRuntime struct { @@ -101,14 +105,24 @@ func buildApp(ctx context.Context, cfg runtimeConfig) (_ *devshardApp, err error return nil, err } - e := buildServer() + lifecycle := newLifecycleState() + e := buildServer(lifecycle) + var admin *echo.Echo + if cfg.AdminAddr != "" { + admin = buildAdminServer(lifecycle) + } manager.Register(e.Group("")) + chainRuntime.chainEvents.OnReady(lifecycle.SetReady) return &devshardApp{ - server: e, - chainEvents: chainRuntime.chainEvents, - port: cfg.Port, - close: closers.Close, + server: e, + adminServer: admin, + adminAddr: cfg.AdminAddr, + chainEvents: chainRuntime.chainEvents, + port: cfg.Port, + lifecycle: lifecycle, + shutdownGrace: cfg.ShutdownGrace, + close: closers.Close, }, nil } @@ -323,13 +337,23 @@ func (a *devshardApp) Run(ctx context.Context) error { }() addr := fmt.Sprintf(":%d", a.port) - errCh := make(chan error, 1) - go func() { - slog.Info("listening", "addr", addr) - if err := a.server.Start(addr); err != nil && err != http.ErrServerClosed { - errCh <- err - } - }() + type serverError struct { + name string + err error + } + errCh := make(chan serverError, 2) + startServer := func(name string, server *echo.Echo, addr string) { + go func() { + slog.Info("listening", "server", name, "addr", addr) + if err := server.Start(addr); err != nil && err != http.ErrServerClosed { + errCh <- serverError{name: name, err: err} + } + }() + } + startServer("public", a.server, addr) + if a.adminServer != nil { + startServer("admin", a.adminServer, a.adminAddr) + } var runErr error chainEventsStopped := false @@ -337,7 +361,7 @@ func (a *devshardApp) Run(ctx context.Context) error { case <-ctx.Done(): slog.Info("shutdown requested") case err := <-errCh: - runErr = fmt.Errorf("server error: %w", err) + runErr = fmt.Errorf("%s server error: %w", err.name, err.err) case err := <-chainEventsErrCh: chainEventsStopped = true if err != nil { @@ -347,11 +371,15 @@ func (a *devshardApp) Run(ctx context.Context) error { } } + a.lifecycle.StartDrain() cancel() - shutdownCtx, shutdownCancel := context.WithTimeout(context.Background(), 5*time.Second) + shutdownCtx, shutdownCancel := context.WithTimeout(context.Background(), a.shutdownGrace) defer shutdownCancel() _ = a.server.Shutdown(shutdownCtx) + if a.adminServer != nil { + _ = a.adminServer.Shutdown(shutdownCtx) + } if !chainEventsStopped { select { case err := <-chainEventsErrCh: diff --git a/devshard/cmd/devshardd/chain_events.go b/devshard/cmd/devshardd/chain_events.go index cee1bd1bf8..12b6583113 100644 --- a/devshard/cmd/devshardd/chain_events.go +++ b/devshard/cmd/devshardd/chain_events.go @@ -80,6 +80,10 @@ func (b *chainEventBridge) OnNewBlock(h events.NewBlockHandler) { b.listener.OnNewBlock(h) } +func (b *chainEventBridge) OnReady(h func(bool)) { + b.listener.OnReady(h) +} + func (b *chainEventBridge) Start(ctx context.Context) error { if err := b.listener.Start(ctx); err != nil && err != context.Canceled { return err diff --git a/devshard/cmd/devshardd/config.go b/devshard/cmd/devshardd/config.go index dcfba69bce..90d8a7b976 100644 --- a/devshard/cmd/devshardd/config.go +++ b/devshard/cmd/devshardd/config.go @@ -26,6 +26,7 @@ var sdkConfigOnce sync.Once type runtimeConfig struct { Port int + AdminAddr string DataDir string BinaryLogVersion string RuntimeVersion string @@ -33,6 +34,7 @@ type runtimeConfig struct { NodeManagerAddr string ValidationRetryInterval time.Duration ValidationLeaseTTL time.Duration + ShutdownGrace time.Duration Node ChainNodeConfig } @@ -128,8 +130,14 @@ func loadRuntimeConfig(args []string, protocolVersion, linkBinaryVersion string) return runtimeConfig{}, fmt.Errorf("DEVSHARD_VALIDATION_LEASE_TTL: %w", err) } + shutdownGrace, err := parseDurationEnv("DEVSHARD_SHUTDOWN_GRACE", 10*time.Minute) + if err != nil { + return runtimeConfig{}, fmt.Errorf("DEVSHARD_SHUTDOWN_GRACE: %w", err) + } + return runtimeConfig{ Port: *port, + AdminAddr: strings.TrimSpace(os.Getenv("DEVSHARD_ADMIN_ADDR")), DataDir: *dataDir, BinaryLogVersion: binaryLogVersion, RuntimeVersion: protocolVersion, @@ -137,6 +145,7 @@ func loadRuntimeConfig(args []string, protocolVersion, linkBinaryVersion string) NodeManagerAddr: envOr("NODE_MANAGER_ADDR", "localhost:9400"), ValidationRetryInterval: retryInterval, ValidationLeaseTTL: leaseTTL, + ShutdownGrace: shutdownGrace, Node: loadNodeConfigFromEnv(), }, nil } diff --git a/devshard/cmd/devshardd/events/listener.go b/devshard/cmd/devshardd/events/listener.go index b84c33a0a9..6cd2e339ad 100644 --- a/devshard/cmd/devshardd/events/listener.go +++ b/devshard/cmd/devshardd/events/listener.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "log/slog" + "sync" "time" abci "github.com/cometbft/cometbft/abci/types" @@ -67,6 +68,8 @@ type Listener struct { rpcURL string subs []subscription reconnectDelay time.Duration + readyMu sync.RWMutex + ready func(bool) } // NewListener creates a Listener that connects to the given CometBFT RPC URL. @@ -94,6 +97,23 @@ func (l *Listener) OnNewBlock(h NewBlockHandler) { Subscribe(l, "tm.event='NewBlock'", parseNewBlockEvent, h) } +// OnReady registers a callback invoked with true after all subscriptions are +// active, and false when the listener disconnects or stops. +func (l *Listener) OnReady(h func(bool)) { + l.readyMu.Lock() + l.ready = h + l.readyMu.Unlock() +} + +func (l *Listener) setReady(ready bool) { + l.readyMu.RLock() + h := l.ready + l.readyMu.RUnlock() + if h != nil { + h(ready) + } +} + // Start connects to the chain and listens for events until ctx is cancelled. // Reconnects automatically on connection failure or subscription drop. func (l *Listener) Start(ctx context.Context) error { @@ -147,6 +167,8 @@ func (l *Listener) run(ctx context.Context) error { } }() } + l.setReady(true) + defer l.setReady(false) select { case <-ctx.Done(): diff --git a/devshard/cmd/devshardd/events/listener_internal_test.go b/devshard/cmd/devshardd/events/listener_internal_test.go new file mode 100644 index 0000000000..314b753000 --- /dev/null +++ b/devshard/cmd/devshardd/events/listener_internal_test.go @@ -0,0 +1,20 @@ +package events + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestListenerReadyHandler(t *testing.T) { + l := NewListener("http://localhost:26657") + var states []bool + l.OnReady(func(ready bool) { + states = append(states, ready) + }) + + l.setReady(true) + l.setReady(false) + + require.Equal(t, []bool{true, false}, states) +} diff --git a/devshard/cmd/devshardd/lifecycle.go b/devshard/cmd/devshardd/lifecycle.go new file mode 100644 index 0000000000..a475e4857a --- /dev/null +++ b/devshard/cmd/devshardd/lifecycle.go @@ -0,0 +1,76 @@ +package main + +import ( + "net/http" + "strings" + "sync/atomic" + + "devshard/observability" + + "github.com/labstack/echo/v4" +) + +type lifecycleState struct { + ready atomic.Bool + draining atomic.Bool + inflight atomic.Int64 +} + +type drainStatus struct { + Ready bool `json:"ready"` + Draining bool `json:"draining"` + Inflight int64 `json:"inflight"` +} + +func newLifecycleState() *lifecycleState { + observability.SetLifecycleInflight(0) + return &lifecycleState{} +} + +func (s *lifecycleState) SetReady(ready bool) { + s.ready.Store(ready) +} + +func (s *lifecycleState) StartDrain() { + s.draining.Store(true) + s.ready.Store(false) +} + +func (s *lifecycleState) Status() drainStatus { + return drainStatus{ + Ready: s.ready.Load(), + Draining: s.draining.Load(), + Inflight: s.inflight.Load(), + } +} + +func (s *lifecycleState) addInflight(delta int64) { + inflight := s.inflight.Add(delta) + observability.SetLifecycleInflight(inflight) +} + +func (s *lifecycleState) middleware(next echo.HandlerFunc) echo.HandlerFunc { + return func(c echo.Context) error { + if isLifecycleBypassPath(c.Request().URL.Path) { + return next(c) + } + // Count first so drain cannot report idle after this request is admitted. + s.addInflight(1) + if s.draining.Load() { + s.addInflight(-1) + return echo.NewHTTPError(http.StatusServiceUnavailable, "devshardd is draining") + } + defer s.addInflight(-1) + return next(c) + } +} + +func isLifecycleBypassPath(path string) bool { + clean := "/" + strings.Trim(strings.TrimSpace(path), "/") + switch clean { + case "/healthz", "/metrics": + return true + default: + return false + } +} diff --git a/devshard/cmd/devshardd/lifecycle_test.go b/devshard/cmd/devshardd/lifecycle_test.go new file mode 100644 index 0000000000..0bb8a566df --- /dev/null +++ b/devshard/cmd/devshardd/lifecycle_test.go @@ -0,0 +1,92 @@ +package main + +import ( + "encoding/json" + "net/http" + "net/http/httptest" + "testing" + "time" + + "github.com/labstack/echo/v4" + "github.com/stretchr/testify/require" +) + +func TestLifecycleReadyAndDrainStatus(t *testing.T) { + lifecycle := newLifecycleState() + e := buildServer(lifecycle) + admin := buildAdminServer(lifecycle) + e.GET("/work", func(c echo.Context) error { + time.Sleep(20 * time.Millisecond) + return c.String(http.StatusOK, "done") + }) + + rec := httptest.NewRecorder() + e.ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/ready", nil)) + require.Equal(t, http.StatusNotFound, rec.Code) + + rec = httptest.NewRecorder() + admin.ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/ready", nil)) + require.Equal(t, http.StatusServiceUnavailable, rec.Code) + + lifecycle.SetReady(true) + rec = httptest.NewRecorder() + admin.ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/ready", nil)) + require.Equal(t, http.StatusOK, rec.Code) + + workDone := make(chan struct{}) + go func() { + defer close(workDone) + e.ServeHTTP(httptest.NewRecorder(), httptest.NewRequest(http.MethodGet, "/work", nil)) + }() + require.Eventually(t, func() bool { + return lifecycle.Status().Inflight == 1 + }, time.Second, 5*time.Millisecond) + + rec = httptest.NewRecorder() + admin.ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/drain/status", nil)) + require.Equal(t, http.StatusOK, rec.Code) + var status drainStatus + require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &status)) + require.EqualValues(t, 1, status.Inflight) + + rec = httptest.NewRecorder() + e.ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/metrics", nil)) + require.Equal(t, http.StatusOK, rec.Code) + require.Contains(t, rec.Body.String(), "devshardd_lifecycle_inflight_requests 1") + + <-workDone + require.EqualValues(t, 0, lifecycle.Status().Inflight) +} + +func TestLifecycleDrainRejectsNewWork(t *testing.T) { + lifecycle := newLifecycleState() + lifecycle.SetReady(true) + e := buildServer(lifecycle) + admin := buildAdminServer(lifecycle) + e.GET("/work", func(c echo.Context) error { + return c.String(http.StatusOK, "done") + }) + + rec := httptest.NewRecorder() + e.ServeHTTP(rec, httptest.NewRequest(http.MethodPost, "/drain", nil)) + require.Equal(t, http.StatusNotFound, rec.Code) + + rec = httptest.NewRecorder() + e.ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/work", nil)) + require.Equal(t, http.StatusOK, rec.Code) + + rec = httptest.NewRecorder() + admin.ServeHTTP(rec, httptest.NewRequest(http.MethodPost, "/drain", nil)) + require.Equal(t, http.StatusOK, rec.Code) + + rec = httptest.NewRecorder() + e.ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/work", nil)) + require.Equal(t, http.StatusServiceUnavailable, rec.Code) + + rec = httptest.NewRecorder() + admin.ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/drain/status", nil)) + require.Equal(t, http.StatusOK, rec.Code) + var status drainStatus + require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &status)) + require.Zero(t, status.Inflight) +} diff --git a/devshard/cmd/devshardd/main.go b/devshard/cmd/devshardd/main.go index 523f67533b..f44ad2c879 100644 --- a/devshard/cmd/devshardd/main.go +++ b/devshard/cmd/devshardd/main.go @@ -27,8 +27,8 @@ var Version = "dev" var BinaryVersion = "dev-log" func main() { - if maybePrintVersionAndExit(os.Args[1:]) { - return + if code, handled := maybePrintVersion(os.Args[1:], os.Stdout, os.Stderr); handled { + os.Exit(code) } if err := run(context.Background(), os.Args[1:], Version, BinaryVersion); err != nil { log.Fatalf("devshardd: %v", err) @@ -64,6 +64,7 @@ func run(parent context.Context, args []string, protocolVersion, binaryVersion s "binary_log_version", cfg.BinaryLogVersion, "runtime_version", cfg.RuntimeVersion, "port", cfg.Port, + "admin_addr", cfg.AdminAddr, "data-dir", cfg.DataDir) ctx, cancel := signal.NotifyContext(parent, syscall.SIGTERM, syscall.SIGINT) diff --git a/devshard/cmd/devshardd/server.go b/devshard/cmd/devshardd/server.go index ef311f7674..67d1e0aac8 100644 --- a/devshard/cmd/devshardd/server.go +++ b/devshard/cmd/devshardd/server.go @@ -11,12 +11,13 @@ import ( // buildServer creates the Echo instance for devshardd session traffic only. // Tier A read-only /v1/ routes are served by edge-api (see edge-api/). -func buildServer() *echo.Echo { +func buildServer(lifecycle *lifecycleState) *echo.Echo { e := echo.New() e.HideBanner = true e.HidePort = true e.Use(middleware.Recover()) e.Use(haStorageGuard()) + e.Use(lifecycle.middleware) observability.RegisterRuntimeCollectors() e.GET("/metrics", echo.WrapHandler(observability.MetricsHandler())) @@ -24,3 +25,27 @@ func buildServer() *echo.Echo { return e } + +func buildAdminServer(lifecycle *lifecycleState) *echo.Echo { + e := echo.New() + e.HideBanner = true + e.HidePort = true + e.Use(middleware.Recover()) + + e.GET("/ready", func(c echo.Context) error { + status := lifecycle.Status() + if !status.Ready || status.Draining { + return c.JSON(http.StatusServiceUnavailable, status) + } + return c.JSON(http.StatusOK, status) + }) + e.POST("/drain", func(c echo.Context) error { + lifecycle.StartDrain() + return c.JSON(http.StatusOK, lifecycle.Status()) + }) + e.GET("/drain/status", func(c echo.Context) error { + return c.JSON(http.StatusOK, lifecycle.Status()) + }) + + return e +} diff --git a/devshard/cmd/devshardd/version.go b/devshard/cmd/devshardd/version.go index 4e58bb71d3..7af89f8a5a 100644 --- a/devshard/cmd/devshardd/version.go +++ b/devshard/cmd/devshardd/version.go @@ -1,24 +1,42 @@ package main -import "fmt" +import ( + "fmt" + "io" + + "common/storage/mode" +) const ( printBinaryVersionFlag = "--print-binary-version" printProtocolVersionFlag = "--print-protocol-version" + printAdminAPIVersionFlag = "--print-admin-api-version" + printStorageModeFlag = "--print-storage-mode" ) -func maybePrintVersionAndExit(args []string) bool { +func maybePrintVersion(args []string, stdout, stderr io.Writer) (int, bool) { if len(args) != 1 { - return false + return 0, false } switch args[0] { case printBinaryVersionFlag: - fmt.Println(BinaryVersion) - return true + fmt.Fprintln(stdout, BinaryVersion) + return 0, true case printProtocolVersionFlag: - fmt.Println(Version) - return true + fmt.Fprintln(stdout, Version) + return 0, true + case printAdminAPIVersionFlag: + fmt.Fprintln(stdout, "1") + return 0, true + case printStorageModeFlag: + storageMode, err := mode.Resolve() + if err != nil { + fmt.Fprintln(stderr, err) + return 1, true + } + fmt.Fprintln(stdout, storageMode) + return 0, true default: - return false + return 0, false } } diff --git a/devshard/cmd/devshardd/version_test.go b/devshard/cmd/devshardd/version_test.go new file mode 100644 index 0000000000..286a45cbef --- /dev/null +++ b/devshard/cmd/devshardd/version_test.go @@ -0,0 +1,60 @@ +package main + +import ( + "bytes" + "strings" + "testing" + + "common/storage/mode" +) + +func TestMaybePrintStorageMode(t *testing.T) { + tests := []struct { + name string + envMode string + pgHost string + want string + wantCode int + wantErr string + }{ + {name: "postgres", envMode: "postgres", pgHost: "postgres", want: "postgres"}, + {name: "auto with PGHOST resolves hybrid", pgHost: "postgres", want: "hybrid"}, + {name: "empty resolves sqlite", want: "sqlite"}, + {name: "invalid fails", envMode: "bad", wantCode: 1, wantErr: mode.EnvStorageMode}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Setenv(mode.EnvStorageMode, tt.envMode) + t.Setenv("PGHOST", tt.pgHost) + + var stdout bytes.Buffer + var stderr bytes.Buffer + code, handled := maybePrintVersion([]string{printStorageModeFlag}, &stdout, &stderr) + if !handled { + t.Fatal("expected storage mode flag to be handled") + } + if code != tt.wantCode { + t.Fatalf("exit code = %d, want %d", code, tt.wantCode) + } + if tt.want != "" && strings.TrimSpace(stdout.String()) != tt.want { + t.Fatalf("stdout = %q, want %q", strings.TrimSpace(stdout.String()), tt.want) + } + if tt.wantErr != "" && !strings.Contains(stderr.String(), tt.wantErr) { + t.Fatalf("stderr = %q, want it to contain %q", stderr.String(), tt.wantErr) + } + }) + } +} + +func TestMaybePrintVersionUnknownFlag(t *testing.T) { + var stdout bytes.Buffer + var stderr bytes.Buffer + code, handled := maybePrintVersion([]string{"--unknown"}, &stdout, &stderr) + if handled { + t.Fatal("unknown flag should not be handled") + } + if code != 0 { + t.Fatalf("exit code = %d, want 0", code) + } +} diff --git a/devshard/docs/release-0.2.14-v4.md b/devshard/docs/release-0.2.14-v4.md index d321b31bbb..e92a2d1114 100644 --- a/devshard/docs/release-0.2.14-v4.md +++ b/devshard/docs/release-0.2.14-v4.md @@ -32,7 +32,7 @@ create/settle paths are gone. | **Settle confirm** | Settle waits for DeliverTx (`GetTx`) after SYNC CheckTx — same pattern as create | | **Failover** | Router retries another HA peer on first upstream 502 / connect failure | | **Status field** | Gateway `protocol_version` → `session_version` (bind / settlement tag) | -| **testenv** | Docker citest S1–S9, G1–G4, A1–A4 (see [testenv/docs/scenarios.md](../testenv/docs/scenarios.md)) | +| **testenv** | Named stack behavior tests plus G1–G4 and A1–A4 (see [testenv/docs/scenarios.md](../testenv/docs/scenarios.md)) | --- @@ -283,7 +283,7 @@ Debug headers: `X-Upstream-Addr`, `X-Versiond-Backend` 1. Add versiond-1…N with the same Postgres credentials and **same `KEY_NAME`**. 2. Keep NON_HA pin; confirm HA paths show `versiond_ha_pool` + multi upstream. -3. Exercise stickiness, kill-one-host failover (S6 / §3 plan), optional lease race (§2). +3. Exercise stickiness, kill-one-host failover (§3), and the optional lease race (§2). **Phase C — Retire pre-v4 (later)** diff --git a/devshard/docs/rolling-update.md b/devshard/docs/rolling-update.md index 671c4d1efc..073db0fe7d 100644 --- a/devshard/docs/rolling-update.md +++ b/devshard/docs/rolling-update.md @@ -161,9 +161,15 @@ wait for NEW child READINESS (HTTP /ready, not just TCP) │ ▼ atomic route swap: version → NEW port ← new requests go to NEW child - │ in-flight requests stay on OLD conn + │ + ▼ +retire OLD proxy target ← stale route lookups retry on NEW + │ accepted OLD requests keep a lease + ▼ +wait for OLD proxy leases to reach 0 OR VERSIOND_DRAIN_TIMEOUT + │ ▼ -mark OLD child "draining" (out of route table, NOT killed) +POST /drain to OLD child ← reject non-proxy late arrivals │ ▼ poll OLD child in-flight count until 0 OR VERSIOND_DRAIN_TIMEOUT @@ -172,45 +178,72 @@ poll OLD child in-flight count until 0 OR VERSIOND_DRAIN_TIMEOUT SIGTERM OLD child → wait (long grace) → SIGKILL only as last resort ``` -At the proxy layer this is already graceful: `proxy.Handler` builds a -`httputil.ReverseProxy` **per request** and dials the target named in the route -table at that moment. A request that already started keeps its own connection to -the old child until it completes; only *new* requests observe the swapped route. -The only thing we must change is: **stop killing the old child immediately**. +The proxy route value is a generation-specific `Target`, not just an address. +Before forwarding, each request acquires a lease on that target and releases it +after the complete response, including an SSE stream. The route swap publishes +the new target before retiring the old one. A request that loaded the old target +but did not acquire it before retirement retries against the new route; an +already acquired request keeps the old target non-idle until it completes. This +closes the boundary between selecting an old address and entering the old +child's own lifecycle middleware. -### 1.2 Storage prerequisite (must resolve first) +### 1.2 Storage prerequisite Rolling update means old + new `devshardd` run **concurrently**. That only -works when durable state lives in a **shared external database** (Postgres in a -separate process). **SQLite is not supported** for rolling update. - -- **Postgres** (`PGHOST` + related `PG*` env): session store (`devshard/storage`), - payloads, and validation leases are multi-writer and shared — safe for - overlap. Validation leases dedupe duplicate validation across instances. -- **SQLite** (`devshard/storage` under `cfg.DataDir`): single-writer file. - Two `devshardd` processes opening the same data dir will contend / corrupt. - Do not enable blue/green drain without Postgres. - -**Requirement:** point every child at the same external Postgres before enabling -this plan. Single-instance / local dev without overlap may keep using SQLite; -see `devshard/docs/storage-design.md` (storage-mode selection) and -`devshard/docs/release-0.2.14-v4.md` (HA ⇒ Postgres). +works when durable state lives in **Postgres-only** storage. SQLite and hybrid +fallback modes are not safe for overlap: two children can touch the same local +data dir, and a new HA child can migrate/quarantine SQLite files while an old +hybrid child is still running. + +- **Postgres-only** (`DEVSHARD_STORAGE_MODE=postgres`, with `PGHOST` / `PG*` + connection env): sessions, payloads, and validation leases are external and + shared. This is the only mode that permits blue/green overlap. +- **SQLite** or **hybrid** (`sqlite`, `hybrid`, or default `auto` with `PGHOST` + resolving to `hybrid`): local fallback can write files under the child's data + dir. Do not run old and new children concurrently in these modes. + +versiond does not reimplement storage-mode resolution. For devshard children it +probes the binary that will actually run: + +- the currently running child records its startup answer to + `--print-storage-mode`; +- the incoming binary is probed with `--print-storage-mode` before overlap; +- blue/green is allowed only when **both** answers are exactly `postgres`. + +Any uncertainty — old binary without the flag, new binary without the flag, +invalid env, a non-`postgres` mode, or a future unknown mode — fails closed to +the compatible stop-then-start path. This makes the first migration into +`DEVSHARD_STORAGE_MODE=postgres` exclusive, and only later updates of +Postgres-only binaries use blue/green. + +Point every overlapping child at the same external Postgres. Single-instance +development without overlap may keep using SQLite; see +`devshard/docs/storage-design.md` for storage-mode selection and +`devshard/docs/release-0.2.14-v4.md` for the HA deployment requirement. ### 1.3 devshardd changes -1. **Readiness endpoint** `GET /ready` (distinct from `/healthz`): +New `devshardd` exposes lifecycle controls on a loopback admin listener selected +by versiond through `DEVSHARD_ADMIN_ADDR`. These endpoints are not registered on +the public devshard traffic listener. + +1. **Readiness endpoint** `GET /ready` on the admin listener (distinct from + `/healthz`): - returns `200` only after chain runtime is connected, host manager has recovered sessions, store is started, and the listener is accepting. - returns `503` until then. - This is what versiond gates the route swap on (replaces the TCP-only `waitForPort`). -2. **Drain/idle endpoint** `GET /drain/status` (or reuse metrics): - - returns active in-flight count (sum of `devshard_inflight` stages plus open - long-lived streams). versiond polls this to decide the old child is idle. - - Optionally `POST /drain` to flip the child into "reject new, finish - existing" mode as a belt-and-braces measure (route is already swapped, so - new traffic shouldn't arrive, but this guards retries/direct hits). +2. **Drain/idle endpoint** `GET /drain/status` on the admin listener: + - returns active in-flight HTTP count from the lifecycle middleware. versiond + polls this to decide the old child is idle. + - exports the same count as Prometheus gauge + `devshardd_lifecycle_inflight_requests`. + - `POST /drain` on the same admin listener flips the child into "reject new, + finish existing" mode as a belt-and-braces measure (route is already + swapped, so new traffic shouldn't arrive, but this guards retries/direct + hits). It must not be exposed through the public versiond proxy. 3. **Honor a long shutdown grace.** Replace the hard 5s in `app.go` with a configurable `DEVSHARD_SHUTDOWN_GRACE` (default large enough for max @@ -236,19 +269,27 @@ during a swap. Minimal shape: - add `draining []*child` (or `map[string][]*child`) for children that have been taken out of the route table but are still finishing work. -`rebuildRoutes` already only emits running children; ensure draining children are -**excluded** from the route table (they keep their port but receive no new -traffic). +`rebuildRoutes` only emits running children. Each route value is the `Target` +for one concrete child generation, so same-name children with different SHA, +port, and PID never share admission state. Draining children are excluded from +the new table, but their retired targets stay alive until acquired proxy +requests release their leases. ```653:661:versioned/internal/process/manager.go func (m *Manager) rebuildRoutes() { - routes := make(map[string]string) + previous := m.routes.Load().(proxy.RouteTable) + routes := make(proxy.RouteTable) for _, c := range m.processes { if c.status == statusRunning { - routes[c.version.Name] = fmt.Sprintf("localhost:%d", c.port) + routes[c.version.Name] = c.proxyTarget } } m.routes.Store(routes) + for version, target := range previous { + if routes[version] != target { + target.Retire() + } + } } ``` @@ -271,6 +312,12 @@ func (m *Manager) assignPort(name string) int { ``` → add a swap-aware allocation (e.g. `assignSwapPort(name)`) that returns a new port even when `name` already has one, and a `releasePort` on drain completion. +The implementation uses a bounded child-port pool starting at `BasePort`, +reuses ports after child exit, and reserves versiond's own listen port so a +long-lived supervisor does not eventually allocate it to a child. Port-pool +exhaustion is returned as an explicit start error: versiond does not execute or +route a child without all required ports, and a rolling-update candidate that +cannot obtain a port leaves the current child serving traffic. #### c) Readiness gate instead of TCP-accept @@ -297,30 +344,38 @@ New flow (replaces lines 392–419): ```text 1. downloadBinary(new sha) // old child untouched in memory -2. newChild = startChild(version, NEW port) // same PGHOST / shared Postgres -3. if !waitForReady(newChild, VERSIOND_READY_TIMEOUT): +2. require old+new --print-storage-mode == postgres, else stop/start fallback +3. newChild = startChild(version, NEW port) // Postgres-only overlap +4. if admin /ready != 200 OR public /healthz != 2xx + within VERSIOND_READY_TIMEOUT: stop newChild; keep old serving; abort swap (retry next poll) -4. lock: move old child from processes -> draining[name] +5. lock: move old child from processes -> draining[name] set processes[name] = newChild (status running) rebuildRoutes() // route now points to NEW port + retire old proxy target unlock -5. go drainOld(oldChild): +6. go drainOld(oldChild): deadline = now + VERSIOND_DRAIN_TIMEOUT + wait until proxy leases on old target == 0, or deadline + POST oldChild /drain loop every VERSIOND_DRAIN_POLL_INTERVAL: if inflight(oldChild) == 0: break if now > deadline: log warn; break - oldChild.cancel() // SIGTERM, long WaitDelay - waitForChild(oldChild, VERSIOND_DRAIN_KILL_GRACE) + oldChild.Stop() // supervisor sends SIGTERM + waitForChild(oldChild) // confirmed exit and process reap release oldChild port ``` Key invariants this enforces: -- **New ready before traffic:** route is swapped only after `/ready` is `200` - (step 3–4). -- **Route new requests to new:** step 4 atomic route swap. -- **Old finishes in-flight:** step 5 waits for `inflight == 0` before any - signal; the proxy keeps existing requests on the old connection meanwhile. +- **New ready before traffic:** route is swapped only while admin `/ready` is + `200` and public `/healthz` is `2xx` (step 4–5). Both conditions are + rechecked together so logical readiness cannot hide a failed public bind. +- **Route new requests to new:** step 5 atomic route swap. +- **No boundary gap:** a stale route lookup either owns an old-target lease or + retries after retirement and uses the new target. +- **Old finishes in-flight:** step 6 first waits for proxy leases, then confirms + the child's lifecycle `inflight` count is zero before any signal. - **Don't kill until idle:** `SIGTERM` is sent only after idle or the safety `VERSIOND_DRAIN_TIMEOUT`. @@ -340,41 +395,158 @@ func (m *Manager) Status() []health.StatusEntry { ``` → also iterate `draining[...]` so draining children appear in `/healthz`. -#### f) Graceful supervisor shutdown +#### f) Removed versions drain asynchronously + +When governance removes a version from the approved set, versiond must remove +the route immediately so no new requests reach that version. The old child is +then moved from `processes` to `draining`, marked `restart=false`, and stopped +through the same `drainAndStop` path used by binary swaps. + +This keeps the reconcile poll loop responsive while the old child finishes +accepted work. New requests get `404` for the removed version, existing +requests retain their old-target proxy lease, and child drain starts only after +those leases are released. Legacy children without `/drain/status` receive the +`VERSIOND_DRAIN_KILL_GRACE` cushion before `SIGTERM`. + +If governance restores the same version name before its removed child finishes +draining, versiond defers the new start until that child exits. This prevents +two generations from opening the same version data directory. The next +reconcile poll starts the restored version normally after drain completes. -`Manager.Shutdown` waits 10s per child. When versiond itself is being stopped, -draining children should also get the long grace (or at least be `SIGTERM`'d and -waited on with `VERSIOND_DRAIN_KILL_GRACE`). +#### g) Graceful supervisor shutdown + +Each concrete child process is owned by a small process lifecycle state machine: + +```text +Running -> Terminating -> Killing -> Exited +``` + +`Stop()` moves a running process to `Terminating`, sends `SIGTERM` to its +process group, and starts its graceful-stop timer. If the process has not +exited when that timer expires, the controller moves it to `Killing` and sends +`SIGKILL`. `Done()` is closed only after `cmd.Wait()` confirms the process has +exited and has been reaped. The graceful timeout therefore controls +escalation; it is not a second, competing limit on how long callers wait. + +The timeout is exactly `VERSIOND_DRAIN_KILL_GRACE` for non-devshard binaries, +and the max of `VERSIOND_DRAIN_KILL_GRACE` and `DEVSHARD_SHUTDOWN_GRACE` for +devshardd. + +When versiond itself stops, `Manager.Shutdown` calls `Stop()` for all current +and draining children before waiting for any one of them. If the manager +shutdown context expires, it calls `ForceStop()` for the remaining children, +but still waits for every `Done()` signal so it never returns while owning an +unreaped child. ```492:509:versioned/internal/process/manager.go func (m *Manager) Shutdown(ctx context.Context) error { ... for _, c := range children { - waitForChild(c, 10*time.Second) + c.Stop() + } + select { + case <-allChildrenDone: + return nil + case <-ctx.Done(): + forceStopAll(children) + <-allChildrenDone + return ctx.Err() } - return nil } ``` +This is deliberately a process-level FSM, not the Track B host lifecycle. +The future Host FSM and router controller can use the same +`Stop`/`ForceStop`/`Done` contract after router admission and host drain have +completed, without putting host-routing policy into the child supervisor. + ### 1.5 New configuration (versiond `config.Config`) Add to `versioned/internal/config/config.go`: | Env var | Default | Meaning | |---|---|---| -| `VERSIOND_READY_PATH` | `/ready` | devshardd readiness path the supervisor probes | +| `VERSIOND_READY_PATH` | `/ready` | devshardd admin readiness path; public `/healthz` must also pass before routing | | `VERSIOND_READY_TIMEOUT` | `60s` | max wait for new child to become ready before aborting swap | -| `VERSIOND_DRAIN_TIMEOUT` | `15m` | max time to wait for old child to go idle before `SIGTERM` | -| `VERSIOND_DRAIN_POLL_INTERVAL` | `2s` | how often to poll old child in-flight count | -| `VERSIOND_DRAIN_KILL_GRACE` | `30s` | wait after `SIGTERM` before `SIGKILL` | +| `VERSIOND_DRAIN_PATH` | `/drain` | path versiond POSTs to put the old child into drain mode | +| `VERSIOND_DRAIN_STATUS_PATH` | `/drain/status` | path versiond polls for the old child's in-flight count | +| `VERSIOND_DRAIN_TIMEOUT` | `15m` | shared deadline for old proxy leases and child in-flight work before `SIGTERM` | +| `VERSIOND_DRAIN_POLL_INTERVAL` | `1s` | how often to poll old child in-flight count | +| `VERSIOND_DRAIN_KILL_GRACE` | `10m` | legacy no-status drain cushion and child stop backstop | And on the child side: `DEVSHARD_SHUTDOWN_GRACE` (default `10m`) consumed in -`app.go`. - -> Note: `cmd.WaitDelay = 5s` in `runChild` will `SIGKILL` the child 5s after -> context cancel regardless of the devshardd grace. For drained children, set -> `WaitDelay` to `VERSIOND_DRAIN_KILL_GRACE` (or `0`/disabled) so the child's own -> graceful shutdown can complete. +`app.go`. For new `devshardd` binaries, versiond also sets +`DEVSHARD_ADMIN_ADDR=127.0.0.1:` after the binary advertises admin API +support with `--print-admin-api-version`. Operators normally do not set this +manually; it is the private lifecycle channel between versiond and its child. + +> Note: the process lifecycle FSM uses `VERSIOND_DRAIN_KILL_GRACE` as the +> `SIGTERM`-to-`SIGKILL` interval for non-devshard binaries. For devshardd, +> versiond uses the max of `VERSIOND_DRAIN_KILL_GRACE` and +> `DEVSHARD_SHUTDOWN_GRACE`. +> For legacy children without `/drain/status`, the same +> `VERSIOND_DRAIN_KILL_GRACE` is also the pre-`SIGTERM` cushion because +> versiond cannot observe in-flight work. + +versiond also garbage-collects old complete per-sha install directories under +`bin///`, keeping desired/live/draining installs and a small +recent complete-install cushion. In-progress download directories without +`install.json` are never removed by GC. + +On the first upgrade from the legacy flat layout, versiond checks +`bin//` against the adjacent legacy `install.json` before +using the network. A matching install is copied atomically into the canonical +`bin///` directory, with the binary written first and +`install.json` published last as the commit marker. The promoted copy is +verified again before it starts. + +The legacy binary and metadata are retained because another versiond instance +using the shared bin mount may still run the previous release. Concurrent +promotions are idempotent because they publish the same verified content with +atomic renames. If the canonical destination cannot be written, versiond logs +a warning and starts directly from the re-verified legacy path instead of +requiring an artifact download. + +#### Legacy compatibility and oracle validation + +The first versiond upgrade can manage binaries built before these endpoints and +preflight flags existed. Compatibility is intentionally narrow: + +- `--print-binary-version` and `--print-protocol-version` preflight checks fall + back only when the child exits with a recognized "unsupported flag" usage + error. Timeouts, signals, OOM-like failures, and other execution errors fail + closed so the next poll can retry. +- `--print-admin-api-version` is an optional capability probe for devshardd + binaries. If it is supported, versiond starts the child with + `DEVSHARD_ADMIN_ADDR` and sends readiness/drain traffic to that private + listener. If it is unsupported, versiond keeps using the legacy public + lifecycle paths for that child. +- `--print-storage-mode` is a devshardd storage safety probe. A child that does + not support it can still start, but versiond treats its mode as unknown and + will not blue/green-overlap it with another devshardd process. Rolling overlap + opens only when the running child recorded `postgres` and the incoming binary + also prints `postgres`. +- readiness fallback applies only to the default `VERSIOND_READY_PATH=/ready`. + If `/ready` is missing with 404/405/501, versiond tries `/healthz`, then a TCP + connect probe. Custom readiness paths do not use this fallback. +- drain status fallback treats 404/405/501 from `/drain/status` as a legacy + child. versiond waits `VERSIOND_DRAIN_KILL_GRACE` before `SIGTERM` because + it cannot observe whether old in-flight work is still running, instead of + waiting the full `VERSIOND_DRAIN_TIMEOUT`. + +This keeps old released devshardd binaries deployable under a new versiond while +still failing closed for ambiguous failures. One consequence of the legacy drain +fallback is that an in-flight request on an old binary longer than +`VERSIOND_DRAIN_KILL_GRACE` plus that binary's own shutdown window can be cut +during the first legacy-to-new swap; stamped binaries with `/drain/status` get +the full idle wait. + +Oracle data is validated before reconciliation. Version names must be simple +path components, unique, and non-empty; sha256 values must be 64 hex characters. +An invalid oracle response fails the fetch for that poll and leaves the current +children running unchanged. This is stricter than skipping bad entries because +the oracle reflects governance-approved chain params and should be internally +consistent. ### 1.6 Single-instance limits (be honest) @@ -437,8 +609,8 @@ upgrade, scale-down, or decommission. │ → in-flight connections to versiond-N keep running ▼ 2. Poll versiond-N until idle: - GET versiond-N:8080/healthz (no draining children, or all idle) - and/or aggregate devshardd GET /drain/status on that host + GET versiond-N:8080/healthz (child status visibility) + and aggregate devshardd GET /drain/status on that host loop until inflight == 0 OR ROUTER_DRAIN_TIMEOUT ▼ 3. Graceful stop versiond-N: @@ -469,13 +641,15 @@ Key invariants: |---|---| | Upstream `down` / removal + `nginx -s reload` | Stop routing new escrows to the host being evacuated | | `ROUTER_DRAIN_TIMEOUT` | Max wait for a host to go idle before forced stop | -| `ROUTER_DRAIN_POLL_INTERVAL` | How often to poll versiond `/healthz` and devshardd `/drain/status` | +| `ROUTER_DRAIN_POLL_INTERVAL` | How often to poll versiond `/healthz`; direct devshardd `/drain/status` polling requires access to the admin listener | | `ROUTER_DRAIN_KILL_GRACE` | Wait after `SIGTERM` to versiond before `SIGKILL` / container kill | | Operator script or sidecar | Orchestrate steps 1–4; re-render `VERSIOND_HOSTS` and reload | -Re-use the devshardd endpoints from §1.3 (`/ready`, `/drain/status`) and -versiond `/healthz` draining visibility from §1.4e — implement them once; the -router track consumes the same signals. +Re-use versiond `/healthz` draining visibility from §1.4e for public host +evacuation. The devshardd endpoints from §1.3 (`/ready`, `/drain/status`, +`/drain`) are an internal admin API for versiond-managed children; consume them +directly only from a trusted sidecar or local supervisor with admin listener +access. #### When to use which layer @@ -494,13 +668,20 @@ Part 2 (K8s) maps the same host-evacuation semantics onto Service endpoints + - **Unit (`versioned/internal/process`):** extend `manager_test.go` with a swap scenario asserting: old child still routed/alive until new `/ready`; route - points to new port after readiness; old child not `SIGTERM`'d until inflight - reports 0; old killed at `VERSIOND_DRAIN_TIMEOUT`. + points to new port only after admin `/ready` and public `/healthz`; child + drain waits for old proxy leases and lifecycle inflight to reach 0; old + killed at `VERSIOND_DRAIN_TIMEOUT`. Test the process FSM separately: + graceful `SIGTERM`, timeout escalation to `SIGKILL`, and manager shutdown + returning only after process reap. +- **Unit (`versioned/internal/proxy`):** hold a request on the old target across + a route swap, assert new requests use the new target, and assert the retired + target becomes drained only after the old request completes. - **e2e (`versioned/e2e`):** drive a long request against the old child, trigger an oracle sha change for the same name, assert the long request completes with the old binary while a concurrently-started request is served by the new one. -- **devshardd:** test `/ready` flips only after init; `/drain/status` reflects - `devshard_inflight`; `SIGTERM` honors `DEVSHARD_SHUTDOWN_GRACE`. +- **devshardd:** test `/ready` flips only after init and chain subscriptions; + `/drain/status` and `devshardd_lifecycle_inflight_requests` reflect lifecycle + in-flight HTTP requests; `SIGTERM` honors `DEVSHARD_SHUTDOWN_GRACE`. ### 1.10 Rollout order @@ -510,7 +691,8 @@ Part 2 (K8s) maps the same host-evacuation semantics onto Service endpoints + 2. versiond: config flags (no behavior change yet). 3. versiond: per-name two-child model + swap-aware ports + readiness probe. 4. versiond: rewrite `downloadAndSwap` to blue/green + drain. -5. Surface draining state in `/healthz`; long `WaitDelay` for drained children. +5. Surface draining state in `/healthz`; process lifecycle FSM for graceful + stop, forced escalation, and confirmed reap. 6. Tests (§1.9), then enable by default. **Track B — versiond host removal/replacement (§1.8, HA only):** @@ -553,8 +735,9 @@ strategy: ### 2.3 Readiness + drain -- **readinessProbe** → `GET /ready` on devshardd. Endpoints only include a pod - once it is truly ready; new traffic flows to the new pod automatically. +- **readinessProbe** → `GET /ready` on a private/admin devshardd listener, not + through the public inference path. Endpoints only include a pod once it is + truly ready; new traffic flows to the new pod automatically. - **terminationGracePeriodSeconds**: large (cover max inference, e.g. minutes) so in-flight requests can finish after the pod is told to stop. - **preStop hook**: fail readiness / `sleep` so the pod is removed from Service @@ -579,8 +762,9 @@ old pod: preStop (drop from endpoints + sleep) → SIGTERM → finish in-flight ### 2.5 What carries over from Part 1 -- devshardd `/ready` + `/drain` + configurable shutdown grace are the **same** - building blocks K8s probes and hooks consume — implement them once for both. +- devshardd admin `/ready` + `/drain` + configurable shutdown grace are the + **same** building blocks K8s probes and hooks consume — implement them once + for both. - The drain/idle accounting (`devshard_inflight`) feeds the versiond binary-swap drain loop (§1.1), the versiond-router host-evacuation loop (§1.8), and K8s preStop logic / dashboards. diff --git a/devshard/docs/testenv-v2.md b/devshard/docs/testenv-v2.md index 3bded3df9b..f10215f7fd 100644 --- a/devshard/docs/testenv-v2.md +++ b/devshard/docs/testenv-v2.md @@ -61,7 +61,7 @@ make -C devshard ci-testenv-unit Covers: `common/runtimeconfig` (+ client), `common/chain`, `devshard/chainoracle`, `devshard/runtimeparams`, `devshard/testenv` unit tests, `decentralized-api/nodemanager`. -Docker stack citest (S1–S6 + A1–A4): +Docker stack behavior and adversarial citests: ```bash make -C devshard ci-testenv-integration @@ -116,6 +116,6 @@ Roadmap: [`testenv/docs/observability-plan.md`](../testenv/docs/observability-pl | Symptom | Check | |---------|-------| | `protocol mismatch` in versiond logs | Rebuild devshardd with `DEVSHARD_VERSION=v2` | -| Router 502 on sticky session | `docker compose ps`; S6 expects first-502 failover to survivor | +| Router 502 on sticky session | `docker compose ps`; `TestVersiondStickySessionFailover` expects first-502 failover to the survivor | | Long-poll stuck | mock-dapi `/healthz`; mock-chain gRPC `:9090` | | Citest port conflict | Citest uses `172.31.0.0/24` and ports `18080+` — stop dev `make up` or use harness isolation | diff --git a/devshard/docs/v4-deploy-test-plan.md b/devshard/docs/v4-deploy-test-plan.md index 83b8c2179d..65576459be 100644 --- a/devshard/docs/v4-deploy-test-plan.md +++ b/devshard/docs/v4-deploy-test-plan.md @@ -194,7 +194,7 @@ HA means **one escrow participant** is served by **N versiond/devshardd processe | Topology | Identity wiring | What it proves | | --- | --- | --- | | **Join / real HA** (`deploy/join/docker-compose.versiond.yml`) | Every HA versiond uses the **same** `KEY_NAME` and mounts the **same** keyring | Sticky routing, shared Postgres sessions, validation-lease dedup for that participant | -| **Testenv multi** (`gencompose`) | Every versiond replica uses `KEY_NAME=hosts[0]` (usually `versiond-0`); other host keys stay in the shared keyring + participants but are **not** HA replica identities; escrow slots all belong to the HA participant | Same as join for the HA participant; S7/S8 exercise that topology | +| **Testenv multi** (`gencompose`) | Every versiond replica uses `KEY_NAME=hosts[0]` (usually `versiond-0`); other host keys stay in the shared keyring + participants but are **not** HA replica identities; escrow slots all belong to the HA participant | Same as join for the HA participant; legacy routing and storage migration tests exercise that topology | For **manual** HA checks in this document (Phase B, §1.7 Phase 4, §1.8 T2/T5, §1.9, and §3), keep join/testenv multi as above: **identical `KEY_NAME` / key material on all hosts in `versiond_ha_pool`**. Validation leases (`devshard_validation_leases`) only dedupe work when those processes share one signer address (`instance_address`). @@ -268,7 +268,7 @@ migrated) sessions; `X-Versiond-Backend: versiond_legacy` for non-HA paths. 4. Confirm non-HA paths still show `X-Versiond-Backend: versiond_legacy` and `X-Upstream-Addr` always the legacy host. 5. Exercise stickiness + stop-one-host behaviour **on HA versions only** - (testenv S2 / S6 / S7 patterns). With same-key replicas, optionally confirm + (router stickiness, failover, and legacy routing tests). With same-key replicas, optionally confirm validation leases: one row per `(epoch_id, escrow_id, inference_id)` in `devshard_validation_leases`. @@ -305,13 +305,13 @@ make -C versiond-router test-render # config render only (no live nginx) | Focus | Scenarios | Covers legacy pin (`v < v4` → one host)? | | --- | --- | --- | -| HA stickiness | **S2** (`TestS2_RouterStickiness`) | **No** — probes a version **outside** `VERSIOND_NON_HA_VERSIONS` (testenv: `v2`) and asserts **distinct** upstreams | -| Validation lease exclusivity | **S9** (`TestS9_ValidationLeaseRace*`) | **No** — same-key HA + Postgres lease PASS/FAIL; see **§2 Validation race plan** (and citest S9) | -| Legacy pin | **S7** (`TestS7_LegacyVersionPinnedToSingleHost`) | **Yes** — `v1` (in non-HA list) → `versiond_legacy` / `versiond-0` only; other versions still multi-upstream | -| SQLite → HA-fail → migrate → HA | **S8** (`TestS8_SqliteHaFailMigrate`) | **Yes** — full §1.7 Phases 0–4 | -| One HA upstream down | **S6** | **No** — first-502 failover to survivor (HA pool) | -| Gateway chat / gRPC | S5, G1–G4 | No | -| Params / epoch | S3, S4 | No | +| HA stickiness | `TestRouterStickiness` | **No** — probes a version **outside** `VERSIOND_NON_HA_VERSIONS` (testenv: `v2`) and asserts **distinct** upstreams | +| Validation lease exclusivity | `TestValidationLeaseRace*` | **No** — same-key HA + Postgres lease PASS/FAIL; see **§2 Validation race plan** | +| Legacy pin | `TestLegacyVersionPinnedToSingleHost` | **Yes** — `v1` (in non-HA list) → `versiond_legacy` / `versiond-0` only; other versions still multi-upstream | +| SQLite → HA-fail → migrate → HA | `TestSQLiteToPostgresHAMigration` | **Yes** — full §1.7 Phases 0–4 | +| One HA upstream down | `TestVersiondStickySessionFailover` | **No** — first-502 failover to survivor (HA pool) | +| Gateway chat / gRPC | `TestGatewayChat`, G1–G4 | No | +| Params / epoch | `TestParamsLongPoll`, `TestEpochSwitch` | No | | Faults | A1–A4 | No | | Router template render | `versiond-router` `test-render` | **Partial** — asserts map text for mixed / all-legacy / `*`; does **not** hit a running nginx or check `X-Upstream-Addr` | @@ -319,20 +319,20 @@ make -C versiond-router test-render # config render only (no live nginx) **Goal:** with ≥2 versiond hosts in `VERSIOND_HOSTS`, traffic for a non-HA version path never leaves `VERSIOND_LEGACY_HOST`. -#### Autotest: **S7** (`TestS7_LegacyVersionPinnedToSingleHost`) — implemented +#### Autotest: `TestLegacyVersionPinnedToSingleHost` — implemented ```bash make -C devshard/testenv build-devshardd citest-images cd devshard/testenv && TESTENV_CITEST=1 go test -tags=testenvci ./citest/ \ - -run TestS7_LegacyVersionPinnedToSingleHost -count=1 -v -timeout 45m + -run TestLegacyVersionPinnedToSingleHost -count=1 -v -timeout 45m ``` -#### Autotest: **S8** (`TestS8_SqliteHaFailMigrate`) — §1.7 Phases 0–4 +#### Autotest: `TestSQLiteToPostgresHAMigration` — §1.7 Phases 0–4 ```bash make -C devshard/testenv build-devshardd citest-images cd devshard/testenv && TESTENV_CITEST=1 go test -tags=testenvci ./citest/ \ - -run TestS8_SqliteHaFailMigrate -count=1 -v -timeout 45m + -run TestSQLiteToPostgresHAMigration -count=1 -v -timeout 45m ``` | Step | Action | Expect | @@ -388,7 +388,8 @@ VERSIOND_NON_HA_VERSIONS=v1 v2 v3 # join-style; whitespace or commas | Paths in `VERSIOND_NON_HA_VERSIONS` | `X-Versiond-Backend: versiond_legacy`, always the legacy host | | Path **not** in the list (e.g. `v4`) with a **single** host | `versiond_ha_pool` but **no** `Devshard-Ha` header (header only when `len(VERSIOND_HOSTS) > 1`) | -Also covered by **S7** once multi-host is enabled (Phase 2). +Also covered by `TestLegacyVersionPinnedToSingleHost` once multi-host is enabled +(Phase 2). #### Phase 1 — Create v4 sessions on SQLite (still single host) @@ -487,13 +488,13 @@ With multi-host router + `postgres` mode still set: | Check | Expect | | --- | --- | | `/devshard/v4/…` | `versiond_ha_pool`, `Devshard-Ha: true`, **2xx** (no 503 from HA guard) | -| Stickiness | Same session id → same upstream across retries; distinct sessions can hit both hosts (S2-style) | +| Stickiness | Same session id → same upstream across retries; distinct sessions can hit both hosts (`TestRouterStickiness`) | | Migrated escrows | Readable/servable from either HA host (shared Postgres), not only the original SQLite volume | | NON_HA paths | Unchanged — still legacy host / no `Devshard-Ha` | #### Phase checklist -Covered by **S8** (`TestS8_SqliteHaFailMigrate`) in testenv (citest version name is +Covered by `TestSQLiteToPostgresHAMigration` in testenv (citest version name is `v2`, not `v4`): - [x] Separate SQLite volumes per versiond; one shared Postgres @@ -526,7 +527,7 @@ in §1.7). | T2 | Create escrow / bind session on **v4** (postgres mode); write diffs | Session lands in **shared Postgres**; sticky hash may pin either HA host | | T3 | Mix: several NON_HA + several v4 escrows active concurrently | NON_HA SQLite-bound on legacy volume; v4 Postgres-bound; no cross-host SQLite bleed | | T4 | Stop a **non-legacy** HA host while NON_HA traffic runs | NON_HA unaffected (never routed there) | -| T5 | Stop one HA host while **v4** sticky sessions exist | First-502 failover to survivor (S6); mid-stream SSE not spliced | +| T5 | Stop one HA host while **v4** sticky sessions exist | First-502 failover to survivor; mid-stream SSE not spliced | | T6 | Boot v4 child with **empty** data dir + `postgres` mode | Boot migrate finds nothing; Postgres-only | | T7 | (Negative) §1.7 Phase 2 — multi-host + sqlite for v4 | 503 from `Devshard-Ha` / `RequireConfiguredForHA` | | T8 | §1.7 Phases 3–4 — sqlite estate then `postgres` mode on that data dir | Full migrate + HA serve; row inventory matches | @@ -557,8 +558,10 @@ Checklist: - [ ] **Same `KEY_NAME` on all HA versiond replicas** of the participant under test (join + testenv multi default) - [ ] v4 chat stream + non-stream through router → Postgres-backed session -- [ ] **NON_HA path:** covered by **S7** (`X-Versiond-Backend: versiond_legacy`) -- [ ] HA path: `versiond_ha_pool` with ≥2 distinct upstreams (S2 / S7) +- [ ] **NON_HA path:** covered by `TestLegacyVersionPinnedToSingleHost` + (`X-Versiond-Backend: versiond_legacy`) +- [ ] HA path: `versiond_ha_pool` with ≥2 distinct upstreams + (`TestRouterStickiness` / `TestLegacyVersionPinnedToSingleHost`) - [ ] §1.7 walkthrough (sqlite → Devshard-Ha 503 → postgres migrate → HA OK) - [ ] `DEVSHARD_STORAGE_MODE=postgres` on HA children; join fails closed without Postgres password / `PGHOST` as documented @@ -589,7 +592,8 @@ Companion to §1.1.1 (same participant key) and §1.6 / §1.8 (HA routing). acquires a row in `devshard_validation_leases`; the loser cannot insert a second lease and must not submit a second `MsgValidation`. -This is a **manual** walkthrough (also covered by citest **S9**). Primary +This is a **manual** walkthrough (also covered by +`TestValidationLeaseRace*`). Primary evidence is the Postgres lease table; logs are corroboration. ### 2.0. Preconditions @@ -606,13 +610,13 @@ evidence is the Postgres lease table; logs are corroboration. | Chat path healthy (gateway → router / solo → mock-openai) | Generates finished inferences for the HA participant to validate | | Chain `validation_rate` = **10000** (100%) before escrow create | Every finished inference is a validation candidate → denser lease races (§2.1a) | -**Host count is a minimum, not a ceiling.** Default / citest S9 uses **3** versionds +**Host count is a minimum, not a ceiling.** The validation lease race citest uses **3** versionds (HA pair + one solo). For a manual stack you can add more hosts in `config/config.yaml` (`versiond-3`, …); gencompose keeps only the first two in `VERSIOND_HOSTS` and treats every `hosts[i≥2]` as an extra solo participant. -**Out of scope:** stickiness alone (S2), legacy pin (S7), sqlite→migrate (S8), -kill/restart routing (§3). Those do not assert lease exclusivity. +**Out of scope:** router stickiness, legacy pin, SQLite migration, and +kill/restart routing (§3). Those tests do not assert lease exclusivity. --- @@ -731,7 +735,7 @@ escrow Host so both can `Offer` → `LeaseValidator.Acquire`. ESCROW= VER=v2 -# Session routes have no /healthz — /mempool lazy-loads the Host (same as S6). +# Session routes have no /healthz; /mempool lazy-loads the Host. curl -sS -o /dev/null -w "%{http_code}\n" \ "http://versiond-0:8080/${VER}/sessions/${ESCROW}/mempool" curl -sS -o /dev/null -w "%{http_code}\n" \ @@ -753,7 +757,7 @@ reliable than direct warm. ### 2.4. Load + Postgres monitor (automated) Drive chat load and analyze `devshard_validation_leases` in parallel. Prefer the -testenv scripts (same checks as citest **S9**). +testenv scripts (the same checks as `TestValidationLeaseRace*`). #### Scripts (from `devshard/testenv/`) @@ -823,20 +827,20 @@ docker compose logs -f versiond-0 versiond-1 2>&1 | \ grep -E 'validation lease|AlreadyLeased|leased by another|mark validation submitted|submit abandoned' ``` -#### Automated citest (S9) +#### Automated validation lease race citest ```bash cd devshard/testenv make build-devshardd citest-images TESTENV_CITEST=1 go test -tags=testenvci ./citest/ \ - -run 'TestS9_' -count=1 -v -timeout 45m + -run '^TestValidationLeaseRace' -count=1 -v -timeout 45m ``` | Test | Covers | | --- | --- | -| `TestS9_ValidationLeaseRaceCore` | §2.4 load + monitor PASS/FAIL | -| `TestS9_ValidationLeaseRacePendingStretch` | §2.6a slow ML → pending | -| `TestS9_ValidationLeaseRaceStaleReclaim` | §2.6b short TTL + pause ML + stop replica | +| `TestValidationLeaseRaceCore` | §2.4 load + monitor PASS/FAIL | +| `TestValidationLeaseRacePendingStretch` | §2.6a slow ML → pending | +| `TestValidationLeaseRaceStaleReclaim` | §2.6b short TTL + pause ML + stop replica | --- @@ -855,9 +859,9 @@ Record: escrow id, monitor output, status histogram. --- -### 2.6. Stronger race (also in S9) +### 2.6. Stronger race -#### 2.6a. Stretch the pending window (S9 `…PendingStretch`) +#### 2.6a. Stretch the pending window (`TestValidationLeaseRacePendingStretch`) 1. Warm escrow on both replicas (§2.3). 2. **Slow ML** (testenv: `POST mock-openai /testenv/fault` with high @@ -876,7 +880,7 @@ ORDER BY claimed_at DESC; | Pending rows | At most **one** row per inference while both try | | After ML recovers | Row → `submitted`/`skipped`; uniqueness still PASS | -#### 2.6b. Stale reclaim (S9 `…StaleReclaim`) +#### 2.6b. Stale reclaim (`TestValidationLeaseRaceStaleReclaim`) Default `DEVSHARD_VALIDATION_LEASE_TTL` is **30m** — only with a shortened TTL. @@ -928,9 +932,9 @@ Restore any fault / TTL env overrides before other scenarios. - [ ] Chain `validation_rate` = **10000** before escrow create (§2.1a) - [ ] HA routing: `versiond_ha_pool`, multi upstream (§2.2) - [ ] Escrow warm on **both** replicas (§2.3) -- [ ] `./scripts/lease-race-run.sh` (or S9) → monitor **PASS** (§2.4–§2.5) +- [ ] `./scripts/lease-race-run.sh` (or `TestValidationLeaseRaceCore`) → monitor **PASS** (§2.4–§2.5) - [ ] One lease row per inference; `pending` → `submitted`/`skipped` (§2.5) -- [ ] (Optional / S9) pending stretch + stale reclaim with ML pause (§2.6) +- [ ] Optional pending stretch + stale reclaim with ML pause (§2.6) --- @@ -941,7 +945,7 @@ Restore any fault / TTL env overrides before other scenarios. - Stale reclaim: `devshard/cmd/devshardd/session/retry.go` (`AcquireOneStale`) - HA identity + routing: this document (§1) - Testenv scripts: `devshard/testenv/scripts/lease-race-*.sh` -- Citest S9: `devshard/testenv/citest/s9_validation_lease_race_test.go` +- Citest: `devshard/testenv/citest/validation_lease_race_test.go` - Unit coverage: `devshard/storage/leases_test.go` @@ -961,8 +965,8 @@ open a **new** request (which then lands on a survivor). **Out of scope for this plan:** validation-lease exclusivity (§2), sqlite→migrate (§1.7), legacy pin (§1.6). Those are separate. Automated coverage for stop/restart -semantics also exists as citest **S6** (`TestS6_VersiondStop`, -`TestS6_VersiondRestartPersistence`); this section is the operator walkthrough. +semantics also exists as `TestVersiondStickySessionFailover` and +`TestVersiondRestartSessionPersistence`); this section is the operator walkthrough. ### 3.0 Preconditions diff --git a/devshard/observability/metrics_lifecycle.go b/devshard/observability/metrics_lifecycle.go index 744a77ea60..3d05d6eeee 100644 --- a/devshard/observability/metrics_lifecycle.go +++ b/devshard/observability/metrics_lifecycle.go @@ -11,8 +11,9 @@ import ( ) var ( - registryOnce sync.Once - registry *prometheus.Registry + registryOnce sync.Once + runtimeCollectorsOnce sync.Once + registry *prometheus.Registry inflight *prometheus.GaugeVec requestTerminalTotal *prometheus.CounterVec @@ -31,6 +32,7 @@ var ( validationQueueDepth *prometheus.GaugeVec mempoolSize *prometheus.GaugeVec buildInfo *prometheus.GaugeVec + lifecycleInflight prometheus.Gauge ) var durationBuckets = []float64{0.001, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10} @@ -124,6 +126,10 @@ func initRegistry() { Name: "devshard_build_info", Help: "Devshard build and runtime information.", }, []string{"binary", "version", "commit"}) + lifecycleInflight = prometheus.NewGauge(prometheus.GaugeOpts{ + Name: "devshardd_lifecycle_inflight_requests", + Help: "In-flight HTTP requests counted by devshardd lifecycle drain state.", + }) registry.MustRegister( inflight, @@ -143,6 +149,7 @@ func initRegistry() { validationQueueDepth, mempoolSize, buildInfo, + lifecycleInflight, ) } @@ -155,10 +162,12 @@ func initRegistry() { // source for /metrics. func RegisterRuntimeCollectors() { ensureMetrics() - registry.MustRegister( - collectors.NewGoCollector(), - collectors.NewProcessCollector(collectors.ProcessCollectorOpts{}), - ) + runtimeCollectorsOnce.Do(func() { + registry.MustRegister( + collectors.NewGoCollector(), + collectors.NewProcessCollector(collectors.ProcessCollectorOpts{}), + ) + }) } func IncInflight(stage Stage) func() { @@ -167,6 +176,11 @@ func IncInflight(stage Stage) func() { return func() { inflight.WithLabelValues(string(stage)).Dec() } } +func SetLifecycleInflight(n int64) { + ensureMetrics() + lifecycleInflight.Set(float64(n)) +} + func IncTerminal(terminal Terminal, reason Reason) { ensureMetrics() requestTerminalTotal.WithLabelValues(string(terminal), string(reason)).Inc() diff --git a/devshard/testenv/Makefile b/devshard/testenv/Makefile index 66b64d20f6..3314483cb1 100644 --- a/devshard/testenv/Makefile +++ b/devshard/testenv/Makefile @@ -63,19 +63,27 @@ dev-clean: ## Stop dev stack and remove volumes. dev-logs: ## Follow dev overlay logs. docker compose $(DEV_OVERLAY) logs -f -.PHONY: citest-images citest-stack citest-stack-build citest-adversarial citest-observability citest-grpc-transport test-gateway-smoke obs-up obs-down +.PHONY: citest-images citest-stack citest-stack-build +.PHONY: citest-validation-lease-race +.PHONY: citest-versiond-rolling-update citest-adversarial +.PHONY: citest-observability citest-grpc-transport test-gateway-smoke +.PHONY: obs-up obs-down # versiond-0 produces devshard-versiond:latest shared by all versiond-* hosts. CITEST_BUILD_SERVICES := mock-chain mock-dapi mock-openai devshardctl versiond-router versiond-0 +STACK_CITEST_PATTERN := ^Test(StackSmoke|RouterStickiness|ParamsLongPoll|EpochSwitch|GatewayChat|Versiond.*|LegacyVersionPinnedToSingleHost|SQLiteToPostgresHAMigration|ValidationLeaseRace.*)$$ citest-images: ## Pull hub images and build all compose images required for citest. docker compose $(COMPOSE_BASE) pull devshard-postgres docker compose $(COMPOSE_BASE) build $(CITEST_BUILD_SERVICES) -citest-stack: citest-images ## Phase 8 S1–S9 citest (TESTENV_CITEST=1; needs Docker + build-devshardd). - TESTENV_CITEST=1 go test -tags=testenvci ./citest/ -run 'TestS1_|TestS2_|TestS3_|TestS4_|TestS5_|TestS6_|TestS7_|TestS8_|TestS9_' -count=1 -v -timeout 90m +citest-stack: citest-images ## Full stack behavior citests (needs Docker + build-devshardd). + TESTENV_CITEST=1 go test -tags=testenvci ./citest/ -run '$(STACK_CITEST_PATTERN)' -count=1 -v -timeout 90m -citest-s9: citest-images ## S9 validation lease race (core + 7a + 7b). - TESTENV_CITEST=1 go test -tags=testenvci ./citest/ -run 'TestS9_' -count=1 -v -timeout 45m +citest-validation-lease-race: citest-images ## Validation lease race, pending stretch, and stale reclaim. + TESTENV_CITEST=1 go test -tags=testenvci ./citest/ -run '^TestValidationLeaseRace' -count=1 -v -timeout 45m + +citest-versiond-rolling-update: citest-images ## Same-version versiond rolling update and fallback. + TESTENV_CITEST=1 go test -tags=testenvci ./citest/ -run 'TestVersiondRollingUpdate' -count=1 -v -timeout 30m citest-adversarial: citest-images ## Phase 9 adversarial citest (TESTENV_CITEST=1). TESTENV_CITEST=1 go test -tags=testenvci ./citest/ -run 'TestA1_|TestA2_|TestA3_|TestA4_' -count=1 -v -timeout 30m @@ -99,7 +107,7 @@ obs-up: ## Start base stack + observability overlay (Jaeger/Loki/Prometheus/Graf obs-down: ## Stop observability overlay (and base stack if merged). docker compose $(OBS_OVERLAY) down -citest-stack-build: dev-build ## Build dev images then run S1–S6 citest. +citest-stack-build: dev-build ## Build dev images then run full stack citests. $(MAKE) citest-stack test-gateway-smoke: ## Full-stack gateway smoke (TESTENV_GATEWAY_SMOKE=1; needs Docker + build-devshardd). diff --git a/devshard/testenv/README.md b/devshard/testenv/README.md index dfed27550a..83c36ae405 100644 --- a/devshard/testenv/README.md +++ b/devshard/testenv/README.md @@ -7,7 +7,7 @@ Config-driven via `config/config.yaml` and `cmd/gencompose`. ## Documentation -- **Stack scenarios (S1–S6):** [`docs/scenarios.md`](docs/scenarios.md) +- **Stack behavior scenarios:** [`docs/scenarios.md`](docs/scenarios.md) - **gRPC transport plan (G1–G4):** [`docs/chain-transport-consolidation.md`](docs/chain-transport-consolidation.md) - **Phase 12 index:** [`docs/phase12-followup.md`](docs/phase12-followup.md) - **Operator runbook:** [`../docs/testenv-v2.md`](../docs/testenv-v2.md) @@ -29,7 +29,7 @@ Go packages under `devshard/testenv/` — what each one is for: | **`mockopenai/`** | Fake ML node library: minimal OpenAI HTTP API used by production `devshardd` after `AcquireMLNode`. | | **`gatewayphase/`** | Tiny HTTP stubs for devshardctl’s **chain epoch phase** poller (`ChainPhaseGate`): `/v1/epochs/latest` and `/v1/epochs/current/participants`. Mounted on mock-dapi; not a mock gateway. | | **`keymaterial/`** | Builds deterministic Cosmos **file keyrings** from config host keys so devshardd can sign txs in containers (`KEYRING_DIR`, `KEY_NAME`). | -| **`citest/`** | Go integration tests: compose validation, gateway wiring, Phase 8 harness (`citest/harness/`), S1–S6 citest (`make citest-stack`), Phase 9 adversarial A1–A4 (`make citest-adversarial`, `-tags=testenvci`), optional gateway chat smoke (`TESTENV_GATEWAY_SMOKE=1`). | +| **`citest/`** | Go integration tests: compose validation, gateway wiring, full-stack behavior tests (`make citest-stack`), Phase 9 adversarial A1–A4 (`make citest-adversarial`, `-tags=testenvci`), and optional gateway chat smoke (`TESTENV_GATEWAY_SMOKE=1`). | Production binaries (`devshardd`, `devshardctl`, `versiond`) are **not** reimplemented here — testenv only fakes their external dependencies (chain, dapi, ML) and wires them in Compose. @@ -206,26 +206,40 @@ go test ./testenv/cmd/gencompose/... ./testenv/citest/... ./testenv/keymaterial/ go test ./testenv/... -count=1 ``` -### Phase 8 citest (Docker) +### Stack citest (Docker) -S1 stack smoke + S2 router stickiness + S3 params long-poll + S4 epoch switch + S5 gateway chat + S6 versiond stop fault. Uses an isolated 2× versiond stack on alternate ports (`18080` router, `18081` gateway, subnet `172.31.0.0/24`) so it can run while a dev `make up` stack is active. +The stack suite covers smoke, routing, runtime updates, gateway behavior, +versiond lifecycle, storage migration, validation leases, and rolling updates. +Tests use behavior-oriented names and isolated stacks on dedicated subnets with +Docker-assigned localhost ports, so they can run while a dev `make up` stack is +active. ```bash cd devshard/testenv make build-devshardd -make citest-stack # S1 + S2 + S3 + S4 + S5 + S6 (builds mock-chain + mock-dapi images) +make citest-stack # all core stack behavior tests +make citest-validation-lease-race # validation lease race only +make citest-versiond-rolling-update # or: ./scripts/run-stack-citest.sh ``` -S2 checks that repeated requests to `//sessions//…` through versiond-router land on the same upstream (`X-Upstream-Addr` response header). +`TestRouterStickiness` checks that repeated requests to +`//sessions//…` land on the same versiond upstream. -S3 posts `POST /testenv/params` on mock-dapi while a `GetRuntimeConfig` long-poll is blocked; asserts mock-dapi wakes with updated `max_nonce` / timeouts (the lane-C feed devshardd long-polls via `NODE_MANAGER_ADDR` while the stack runs). +`TestParamsLongPoll` updates mock-dapi params while `GetRuntimeConfig` is blocked +and requires the poll to wake with the new values. -S4 posts `POST /testenv/epoch` `{advance:true}`; mock-chain fast-forwards CometBFT blocks to `next_poc_start`, rolls `next_poc_start` forward by `epoch_length`, and `GetRuntimeConfig` long-poll wakes with a higher `current_epoch_id`. +`TestEpochSwitch` advances the mock-chain epoch and requires runtime config to +report the higher epoch. -S5 posts pooled `POST /v1/chat/completions` on devshardctl (non-stream and SSE stream) and asserts HTTP 200 through versiond-router → devshardd → mock-openai. +`TestGatewayChat` exercises non-streaming and SSE chat through the full stack. -S6 stops one versiond container and asserts sticky sessions pinned to that upstream fail over to the surviving instance on the first 502 / connect failure; sessions on the other upstream keep working. +`TestVersiondStickySessionFailover` stops one versiond and requires affected +sessions to fail over on the first upstream connection failure. + +`TestValidationLeaseRace*` exercises lease exclusivity across a same-key HA pair +and a solo executor. `TestVersiondRollingUpdate*` separately checks same-version +sha replacement with Postgres overlap and the hybrid stop-then-start fallback. **Phase 9 adversarial** (`make citest-adversarial`): A1 lost first SSE chunk, A2 ML 503, A3 stale escrow on chain gRPC, A4 bad warm-key grantees. Fault hooks: `mock-openai` `/testenv/fault`, mock-chain `/testenv/escrow` + `/testenv/grantees` (via mock-dapi). diff --git a/devshard/testenv/citest/a1_lost_first_chunk_test.go b/devshard/testenv/citest/a1_lost_first_chunk_test.go index 17d33a820d..5487708f63 100644 --- a/devshard/testenv/citest/a1_lost_first_chunk_test.go +++ b/devshard/testenv/citest/a1_lost_first_chunk_test.go @@ -20,7 +20,7 @@ func TestA1_LostFirstChunk(t *testing.T) { stack, cfg, eps := harness.BootAdversarialStack(t, "citest-a1-*") client := harness.GatewayChatClient() - mockOpenAI := harness.MockOpenAIFromConfig(cfg) + mockOpenAI := eps.MockOpenAIHTTP t.Cleanup(func() { harness.ResetMockOpenAIFault(t, client, mockOpenAI) if t.Failed() { diff --git a/devshard/testenv/citest/a2_ml_upstream_5xx_test.go b/devshard/testenv/citest/a2_ml_upstream_5xx_test.go index e01930415c..3d838d6b2f 100644 --- a/devshard/testenv/citest/a2_ml_upstream_5xx_test.go +++ b/devshard/testenv/citest/a2_ml_upstream_5xx_test.go @@ -18,7 +18,7 @@ func TestA2_MLUpstream5xx(t *testing.T) { stack, cfg, eps := harness.BootAdversarialStack(t, "citest-a2-*") client := harness.GatewayChatClient() - mockOpenAI := harness.MockOpenAIFromConfig(cfg) + mockOpenAI := eps.MockOpenAIHTTP t.Cleanup(func() { harness.ResetMockOpenAIFault(t, client, mockOpenAI) if t.Failed() { @@ -28,7 +28,7 @@ func TestA2_MLUpstream5xx(t *testing.T) { status := http.StatusServiceUnavailable harness.PatchMockOpenAIFault(t, client, mockOpenAI, mockopenai.FaultPatch{HTTPStatus: &status}) - harness.PatchAdversarialFastTimeouts(t, client, harness.MockDAPIFromConfig(cfg).HTTP) + harness.PatchAdversarialFastTimeouts(t, client, eps.MockDapiHTTP) req := harness.ChatCompletionRequest{ Model: config.PrimaryModelID(cfg), diff --git a/devshard/testenv/citest/a3_stale_escrow_test.go b/devshard/testenv/citest/a3_stale_escrow_test.go index 8f32f93942..fca21a8922 100644 --- a/devshard/testenv/citest/a3_stale_escrow_test.go +++ b/devshard/testenv/citest/a3_stale_escrow_test.go @@ -23,7 +23,7 @@ func TestA3_StaleEscrow(t *testing.T) { stack, cfg, eps := harness.BootAdversarialStack(t, "citest-a3-*") client := harness.HTTPClient() chatClient := harness.GatewayChatClient() - mockDapi := harness.MockDAPIFromConfig(cfg) + mockDapi := harness.MockDAPIFromEndpoints(eps) adminKey := harness.TestenvAdminAPIKey gatewayURL := eps.GatewayHTTP t.Cleanup(func() { @@ -51,7 +51,7 @@ func TestA3_StaleEscrow(t *testing.T) { harness.Step(t, "settle escrow %d on mock-chain while 2× versiond stack is up", id) harness.PatchTestenvEscrowSettle(t, client, mockDapi.HTTP, id) - harness.RequireEscrowSettledOnChain(t, cfg, id) + harness.RequireEscrowSettledOnChain(t, eps, id) harness.PatchAdversarialFastTimeouts(t, client, mockDapi.HTTP) staleReq := harness.ChatCompletionRequest{ @@ -72,5 +72,5 @@ func TestA3_StaleEscrow(t *testing.T) { stillSettled := harness.GetGatewayEscrowID(t, client, gatewayURL) require.Equal(t, escrowID, stillSettled, "gateway should not silently rotate to a new escrow after stale settlement") - harness.RequireEscrowSettledOnChain(t, cfg, id) + harness.RequireEscrowSettledOnChain(t, eps, id) } diff --git a/devshard/testenv/citest/a4_bad_warm_key_test.go b/devshard/testenv/citest/a4_bad_warm_key_test.go index 98ec35af88..da164b3411 100644 --- a/devshard/testenv/citest/a4_bad_warm_key_test.go +++ b/devshard/testenv/citest/a4_bad_warm_key_test.go @@ -19,7 +19,7 @@ func TestA4_BadWarmKey(t *testing.T) { stack, cfg, eps := harness.BootAdversarialStack(t, "citest-a4-*") client := harness.HTTPClient() chatClient := harness.GatewayChatClient() - mockDapi := harness.MockDAPIFromConfig(cfg) + mockDapi := harness.MockDAPIFromEndpoints(eps) t.Cleanup(func() { if t.Failed() { harness.DumpComposeLogs(t, stack, "devshardctl", "versiond-0", "versiond-1", "mock-dapi") @@ -48,7 +48,7 @@ func TestA4_BadWarmKey(t *testing.T) { GranterAddress: granter, Grantees: []string{"gonka1badwarm000000000000000000000000000"}, }) - harness.RequireWarmKeyRevoked(t, cfg, granter, warm) + harness.RequireWarmKeyRevoked(t, eps, granter, warm) harness.Step(t, "warm-key signed gossip/nonce via router should be forbidden after revocation") harness.RequireWarmKeyTransportRejected(t, client, cfg, eps, cfg.WarmGrantee.PrivateKeyHex) diff --git a/devshard/testenv/citest/s4_epoch_switch_test.go b/devshard/testenv/citest/epoch_switch_test.go similarity index 87% rename from devshard/testenv/citest/s4_epoch_switch_test.go rename to devshard/testenv/citest/epoch_switch_test.go index a02694c3d6..286a4d7879 100644 --- a/devshard/testenv/citest/s4_epoch_switch_test.go +++ b/devshard/testenv/citest/epoch_switch_test.go @@ -13,18 +13,18 @@ import ( "github.com/stretchr/testify/require" ) -// TestS4_EpochSwitch verifies POST /testenv/epoch advance fast-forwards mock-chain to the +// TestEpochSwitch verifies POST /testenv/epoch advance fast-forwards mock-chain to the // next PoC start block, rolls next_poc_start forward, and wakes GetRuntimeConfig long-poll // with a higher CurrentEpochID while devshardd is running. -func TestS4_EpochSwitch(t *testing.T) { +func TestEpochSwitch(t *testing.T) { harness.SkipUnlessEnv(t, "TESTENV_CITEST") harness.RequireDocker(t) - stack, cfg, eps := harness.BootS1Stack(t, "citest-s4-*") + stack, cfg, eps := harness.BootStack(t, "citest-epoch-switch-*") client := harness.HTTPClient() - harness.WaitS1Healthy(t, stack, eps) + harness.WaitStackHealthy(t, stack, eps) - mockDapi := harness.MockDAPIFromConfig(cfg) + mockDapi := harness.MockDAPIFromEndpoints(eps) grpcConn := harness.DialMockDAPI(t, mockDapi.GRPC) nm := gen.NewNodeManagerClient(grpcConn) ctx := context.Background() @@ -35,7 +35,7 @@ func TestS4_EpochSwitch(t *testing.T) { } harness.Step(t, "baseline mock-chain epoch snapshot") - before := harness.GetMockChainSnapshot(t, cfg, client) + before := harness.GetMockChainSnapshot(t, cfg, eps, client) require.Equal(t, uint64(1), before.EpochIndex) require.Greater(t, before.NextPocStart, before.BlockHeight) targetHeight := before.NextPocStart @@ -83,7 +83,7 @@ func TestS4_EpochSwitch(t *testing.T) { updated.Config.CurrentEpochId, updated.Config.ParamsBlockHeight) harness.Step(t, "mock-chain caught up to next PoC start and rolled next_poc forward") - after := harness.GetMockChainSnapshot(t, cfg, client) + after := harness.GetMockChainSnapshot(t, cfg, eps, client) require.Equal(t, updated.Config.CurrentEpochId, after.EpochIndex) require.GreaterOrEqual(t, after.BlockHeight, targetHeight) require.Equal(t, targetHeight, after.PocStart) diff --git a/devshard/testenv/citest/g1_grpc_escrow_create_test.go b/devshard/testenv/citest/g1_grpc_escrow_create_test.go index 2f99960acd..6a033d1a8e 100644 --- a/devshard/testenv/citest/g1_grpc_escrow_create_test.go +++ b/devshard/testenv/citest/g1_grpc_escrow_create_test.go @@ -31,7 +31,7 @@ func TestG1_GatewayEscrowCreateGRPC(t *testing.T) { }) harness.WaitGETOK(t, harness.HTTPClient(), eps.MockChainRPC+"/health", 5*time.Minute, "mock-chain RPC health") - conn := harness.DialMockChainGRPC(t, cfg) + conn := harness.DialMockChainGRPC(t, eps) txMgr, err := chaintx.New(conn, chaintx.Config{ ChainID: cfg.ChainID, PollInterval: 500 * time.Millisecond, diff --git a/devshard/testenv/citest/g2_grpc_escrow_read_test.go b/devshard/testenv/citest/g2_grpc_escrow_read_test.go index 5cc31845fc..5e3c99066e 100644 --- a/devshard/testenv/citest/g2_grpc_escrow_read_test.go +++ b/devshard/testenv/citest/g2_grpc_escrow_read_test.go @@ -18,7 +18,7 @@ func TestG2_GatewayEscrowReadGRPC(t *testing.T) { harness.SkipUnlessEnv(t, "TESTENV_CITEST") harness.RequireDocker(t) - stack, cfg, eps := harness.BootMockChainStack(t, "citest-g2-*") + stack, _, eps := harness.BootMockChainStack(t, "citest-g2-*") t.Cleanup(func() { if t.Failed() { harness.DumpComposeLogs(t, stack, "mock-chain") @@ -26,7 +26,7 @@ func TestG2_GatewayEscrowReadGRPC(t *testing.T) { }) harness.WaitGETOK(t, harness.HTTPClient(), eps.MockChainRPC+"/health", 5*time.Minute, "mock-chain RPC health") - conn := harness.DialMockChainGRPC(t, cfg) + conn := harness.DialMockChainGRPC(t, eps) br := bridge.NewGRPCBridge(chain.NewFromConn(conn)) info, err := br.GetEscrow("1") require.NoError(t, err) diff --git a/devshard/testenv/citest/g3_grpc_gateway_chat_test.go b/devshard/testenv/citest/g3_grpc_gateway_chat_test.go index f024792b8f..052e87daae 100644 --- a/devshard/testenv/citest/g3_grpc_gateway_chat_test.go +++ b/devshard/testenv/citest/g3_grpc_gateway_chat_test.go @@ -12,13 +12,13 @@ import ( "github.com/stretchr/testify/require" ) -// TestG3_GatewayChatGRPCOnly is S5-equivalent chat (non-stream + SSE) with gRPC-only gateway +// TestG3_GatewayChatGRPCOnly exercises chat with a gRPC-only gateway // chain transport — compose must not wire DEVSHARD_CHAIN_REST / DEVSHARD_TX_QUERY_REST. func TestG3_GatewayChatGRPCOnly(t *testing.T) { harness.SkipUnlessEnv(t, "TESTENV_CITEST") harness.RequireDocker(t) - stack, cfg, eps := harness.BootS1Stack(t, "citest-g3-*") + stack, cfg, eps := harness.BootStack(t, "citest-g3-*") harness.RequireGatewayGRPCOnlyCompose(t, stack.ComposePath) client := harness.GatewayChatClient() t.Cleanup(func() { @@ -26,7 +26,7 @@ func TestG3_GatewayChatGRPCOnly(t *testing.T) { harness.DumpComposeLogs(t, stack, "devshardctl", "versiond-0", "versiond-1", "mock-openai", "mock-chain") } }) - harness.WaitS1Healthy(t, stack, eps) + harness.WaitStackHealthy(t, stack, eps) harness.WaitGatewayChatReady(t, client, eps.GatewayHTTP, 3*time.Minute, stack) harness.WaitGETOK(t, client, eps.RouterHTTP+"/"+cfg.Versiond.VersionName+"/healthz", 5*time.Minute, "devshardd health via router", stack) diff --git a/devshard/testenv/citest/s5_gateway_chat_test.go b/devshard/testenv/citest/gateway_chat_test.go similarity index 82% rename from devshard/testenv/citest/s5_gateway_chat_test.go rename to devshard/testenv/citest/gateway_chat_test.go index a49dccbf79..daaeb65905 100644 --- a/devshard/testenv/citest/s5_gateway_chat_test.go +++ b/devshard/testenv/citest/gateway_chat_test.go @@ -12,20 +12,20 @@ import ( "github.com/stretchr/testify/require" ) -// TestS5_GatewayChat verifies devshardctl pooled /v1/chat/completions reaches mock-openai +// TestGatewayChat verifies devshardctl pooled /v1/chat/completions reaches mock-openai // through versiond-router and devshardd for both non-stream and SSE stream responses. -func TestS5_GatewayChat(t *testing.T) { +func TestGatewayChat(t *testing.T) { harness.SkipUnlessEnv(t, "TESTENV_CITEST") harness.RequireDocker(t) - stack, cfg, eps := harness.BootS1Stack(t, "citest-s5-*") + stack, cfg, eps := harness.BootStack(t, "citest-gateway-chat-*") client := harness.GatewayChatClient() t.Cleanup(func() { if t.Failed() { harness.DumpComposeLogs(t, stack, "devshardctl", "versiond-0", "versiond-1", "mock-openai") } }) - harness.WaitS1Healthy(t, stack, eps) + harness.WaitStackHealthy(t, stack, eps) harness.WaitGatewayChatReady(t, client, eps.GatewayHTTP, 3*time.Minute, stack) harness.WaitGETOK(t, client, eps.RouterHTTP+"/"+cfg.Versiond.VersionName+"/healthz", 5*time.Minute, "devshardd health via router", stack) @@ -36,7 +36,7 @@ func TestS5_GatewayChat(t *testing.T) { nonStreamReq := harness.ChatCompletionRequest{ Model: model, Messages: []harness.ChatMessage{ - {Role: "user", Content: "citest s5 non-stream"}, + {Role: "user", Content: "citest gateway chat non-stream"}, }, MaxTokens: 32, } @@ -48,7 +48,7 @@ func TestS5_GatewayChat(t *testing.T) { streamReq := harness.ChatCompletionRequest{ Model: model, Messages: []harness.ChatMessage{ - {Role: "user", Content: "citest s5 stream gateway"}, + {Role: "user", Content: "citest gateway chat stream"}, }, MaxTokens: 32, } diff --git a/devshard/testenv/citest/gateway_smoke_test.go b/devshard/testenv/citest/gateway_smoke_test.go index 68a14cc69e..580b867d1d 100644 --- a/devshard/testenv/citest/gateway_smoke_test.go +++ b/devshard/testenv/citest/gateway_smoke_test.go @@ -25,7 +25,7 @@ func TestGatewayPhase7_Smoke(t *testing.T) { stack.Up(t) cfg := stack.LoadConfig(t) - eps := harness.EndpointsFromConfig(cfg) + eps := stack.Endpoints(t, cfg) client := harness.HTTPClient() poll := 3 * time.Minute diff --git a/devshard/testenv/citest/harness/adversarial.go b/devshard/testenv/citest/harness/adversarial.go index 99fca10c04..ea6bc537ed 100644 --- a/devshard/testenv/citest/harness/adversarial.go +++ b/devshard/testenv/citest/harness/adversarial.go @@ -26,22 +26,17 @@ import ( "google.golang.org/grpc/credentials/insecure" ) -// BootAdversarialStack boots the 2× versiond S1 stack and waits for gateway chat readiness. +// BootAdversarialStack boots the standard stack and waits for gateway chat readiness. func BootAdversarialStack(t *testing.T, prefix string) (*Stack, *config.File, Endpoints) { t.Helper() - stack, cfg, eps := BootS1Stack(t, prefix) + stack, cfg, eps := BootStack(t, prefix) client := GatewayChatClient() - WaitS1Healthy(t, stack, eps) + WaitStackHealthy(t, stack, eps) WaitGatewayChatReady(t, client, eps.GatewayHTTP, 3*time.Minute, stack) WaitGETOK(t, client, eps.RouterHTTP+"/"+cfg.Versiond.VersionName+"/healthz", 5*time.Minute, "devshardd health via router", stack) return stack, cfg, eps } -// MockOpenAIFromConfig returns the host-published mock-openai base URL. -func MockOpenAIFromConfig(cfg *config.File) string { - return fmt.Sprintf("http://127.0.0.1:%d", cfg.MockOpenAI.HTTPPort) -} - // PatchMockOpenAIFault posts runtime fault knobs to mock-openai /testenv/fault. func PatchMockOpenAIFault(t *testing.T, client *http.Client, mockOpenAIURL string, patch mockopenai.FaultPatch) { t.Helper() @@ -86,9 +81,9 @@ func PatchTestenvGrantees(t *testing.T, client *http.Client, mockDapiHTTP string } // RequireWarmKeyRevoked asserts the warm grantee is no longer authorized for a validator granter. -func RequireWarmKeyRevoked(t *testing.T, cfg *config.File, granter, warmAddress string) { +func RequireWarmKeyRevoked(t *testing.T, eps Endpoints, granter, warmAddress string) { t.Helper() - conn := dialMockChainGRPC(t, MockChainGRPCFromConfig(cfg)) + conn := dialMockChainGRPC(t, eps.MockChainGRPC) c := chain.NewFromConn(conn) resp, err := c.InferenceQueryClient().GranteesByMessageType(context.Background(), &inferencetypes.QueryGranteesByMessageTypeRequest{ GranterAddress: granter, @@ -143,9 +138,9 @@ func RequireWarmKeyTransportRejected(t *testing.T, client *http.Client, cfg *con } // DialMockChainGRPC dials the mock-chain inference gRPC port from a citest config. -func DialMockChainGRPC(t *testing.T, cfg *config.File) *grpc.ClientConn { +func DialMockChainGRPC(t *testing.T, eps Endpoints) *grpc.ClientConn { t.Helper() - return dialMockChainGRPC(t, MockChainGRPCFromConfig(cfg)) + return dialMockChainGRPC(t, eps.MockChainGRPC) } func dialMockChainGRPC(t *testing.T, addr string) *grpc.ClientConn { @@ -157,9 +152,9 @@ func dialMockChainGRPC(t *testing.T, addr string) *grpc.ClientConn { } // RequireEscrowSettledOnChain queries mock-chain gRPC and requires the escrow is marked settled. -func RequireEscrowSettledOnChain(t *testing.T, cfg *config.File, id uint64) { +func RequireEscrowSettledOnChain(t *testing.T, eps Endpoints, id uint64) { t.Helper() - conn := dialMockChainGRPC(t, MockChainGRPCFromConfig(cfg)) + conn := dialMockChainGRPC(t, eps.MockChainGRPC) c := chain.NewFromConn(conn) resp, err := c.InferenceQueryClient().DevshardEscrow(context.Background(), &inferencetypes.QueryGetDevshardEscrowRequest{Id: id}) require.NoError(t, err) diff --git a/devshard/testenv/citest/harness/compose_patch.go b/devshard/testenv/citest/harness/compose_patch.go index ca9de725fb..265e2fdb2d 100644 --- a/devshard/testenv/citest/harness/compose_patch.go +++ b/devshard/testenv/citest/harness/compose_patch.go @@ -3,6 +3,7 @@ package harness import ( "os" "regexp" + "strings" "testing" "github.com/stretchr/testify/require" @@ -30,3 +31,51 @@ func PatchVersiondStorageMode(t *testing.T, composePath, mode string) { t.Helper() PatchComposeEnvKey(t, composePath, "DEVSHARD_STORAGE_MODE", mode) } + +// PatchComposeUseRandomHostPorts lets Docker assign localhost host ports while +// keeping the configured container ports unchanged for in-network service calls. +func PatchComposeUseRandomHostPorts(t *testing.T, composePath string) { + t.Helper() + body, err := os.ReadFile(composePath) + require.NoError(t, err) + re := regexp.MustCompile(`(?m)^(\s*-\s*")([0-9]+):([0-9]+)(".*)$`) + require.True(t, re.Match(body), "compose %s: no host-published port lines found", composePath) + updated := re.ReplaceAll(body, []byte("${1}127.0.0.1::${3}${4}")) + require.NoError(t, os.WriteFile(composePath, updated, 0o644)) +} + +// PatchComposeRemoveEnvKey removes every `KEY: ...` environment line in a compose file. +func PatchComposeRemoveEnvKey(t *testing.T, composePath, key string) { + t.Helper() + body, err := os.ReadFile(composePath) + require.NoError(t, err) + re := regexp.MustCompile(`(?m)^\s*` + regexp.QuoteMeta(key) + `:\s*.*\n?`) + require.True(t, re.Match(body), "compose %s: env key %q not found", composePath, key) + updated := re.ReplaceAll(body, nil) + require.NoError(t, os.WriteFile(composePath, updated, 0o644)) +} + +// PatchComposeInsertEnvAfter adds environment lines after the first matching env key. +func PatchComposeInsertEnvAfter(t *testing.T, composePath, afterKey string, lines ...string) { + t.Helper() + body, err := os.ReadFile(composePath) + require.NoError(t, err) + re := regexp.MustCompile(`(?m)^(\s*)` + regexp.QuoteMeta(afterKey) + `:\s*.*$`) + loc := re.FindIndex(body) + require.NotNil(t, loc, "compose %s: env key %q not found", composePath, afterKey) + lineEnd := loc[1] + if lineEnd < len(body) && body[lineEnd] == '\n' { + lineEnd++ + } + indent := string(re.FindSubmatch(body[loc[0]:loc[1]])[1]) + var insert strings.Builder + for _, line := range lines { + insert.WriteString(indent) + insert.WriteString(line) + insert.WriteByte('\n') + } + updated := append([]byte{}, body[:lineEnd]...) + updated = append(updated, []byte(insert.String())...) + updated = append(updated, body[lineEnd:]...) + require.NoError(t, os.WriteFile(composePath, updated, 0o644)) +} diff --git a/devshard/testenv/citest/harness/config.go b/devshard/testenv/citest/harness/config.go index d8c97cb9fe..170f302d17 100644 --- a/devshard/testenv/citest/harness/config.go +++ b/devshard/testenv/citest/harness/config.go @@ -19,18 +19,19 @@ type MultiConfigOpts struct { ValidationRate uint32 // 0 = default; else params + seed escrow snapshot } -// WriteS1Config writes a minimal multi-versiond (2 hosts) config skeleton for Phase 8 S1. -// Host ports are chosen at runtime so citest can run alongside local-test-net / dev stacks. -// Uses the default validation_rate (not 100%). -func WriteS1Config(t *testing.T, dir string) { +// WriteStackConfig writes the standard two-versiond stack config. +// Service ports are selected at runtime and published on Docker-assigned +// localhost ports, so citest can coexist with local-test-net and dev stacks. +// It uses the default validation_rate rather than the lease-race 100% rate. +func WriteStackConfig(t *testing.T, dir string) { t.Helper() WriteMultiConfig(t, dir, MultiConfigOpts{Hosts: 2, EscrowSlots: 2}) } -// WriteS9Config writes a 3-host multi config for validation lease races: +// WriteValidationLeaseRaceConfig writes a three-host validation lease config: // hosts[0]+hosts[1] = HA same KEY_NAME; hosts[2] = solo executor participant. // Seeds validation_rate=10000 (100%) for dense lease races. -func WriteS9Config(t *testing.T, dir string) { +func WriteValidationLeaseRaceConfig(t *testing.T, dir string) { t.Helper() WriteMultiConfig(t, dir, MultiConfigOpts{ Hosts: 3, diff --git a/devshard/testenv/citest/harness/config_test.go b/devshard/testenv/citest/harness/config_test.go index 42f89bde7b..3daf156d18 100644 --- a/devshard/testenv/citest/harness/config_test.go +++ b/devshard/testenv/citest/harness/config_test.go @@ -1,7 +1,6 @@ package harness import ( - "fmt" "os" "path/filepath" "testing" @@ -11,9 +10,9 @@ import ( "github.com/stretchr/testify/require" ) -func TestWriteS1Config_TwoHostsMultiMode(t *testing.T) { +func TestWriteStackConfig_TwoHostsMultiMode(t *testing.T) { dir := t.TempDir() - WriteS1Config(t, dir) + WriteStackConfig(t, dir) cfg, err := config.Load(filepath.Join(dir, "config.yaml")) require.NoError(t, err) @@ -22,14 +21,14 @@ func TestWriteS1Config_TwoHostsMultiMode(t *testing.T) { require.Len(t, cfg.Hosts, 2) require.Equal(t, "versiond-0", cfg.Hosts[0].ID) require.Equal(t, "versiond-1", cfg.Hosts[1].ID) - // S1 omits validation_rate; ApplyDefaults → 6000. + // The standard config omits validation_rate; ApplyDefaults sets 6000. cfg.ApplyDefaults() require.Equal(t, uint32(6000), cfg.Params.ValidationRate) } -func TestWriteS9Config_ValidationRate100(t *testing.T) { +func TestWriteValidationLeaseRaceConfig_ValidationRate100(t *testing.T) { dir := t.TempDir() - WriteS9Config(t, dir) + WriteValidationLeaseRaceConfig(t, dir) cfg, err := config.Load(filepath.Join(dir, "config.yaml")) require.NoError(t, err) @@ -48,14 +47,14 @@ func TestWriteMultiConfig_CustomValidationRate(t *testing.T) { require.Equal(t, uint32(7500), cfg.Escrows[0].ValidationRate) } -func TestWriteS1Config_GencomposeProducesTwoVersiondServices(t *testing.T) { +func TestWriteStackConfig_GencomposeProducesTwoVersiondServices(t *testing.T) { if os.Getenv("TESTENV_HARNESS_GENCOMPOSE") != "1" { t.Skip("set TESTENV_HARNESS_GENCOMPOSE=1 to run gencompose harness test") } RequireDocker(t) stack := NewStack(t, "citest-harness-gen-*") - WriteS1Config(t, stack.WorkDir) + WriteStackConfig(t, stack.WorkDir) stack.RunGencompose(t) cfg := stack.LoadConfig(t) @@ -67,5 +66,5 @@ func TestWriteS1Config_GencomposeProducesTwoVersiondServices(t *testing.T) { require.Contains(t, text, "versiond-0:") require.Contains(t, text, "versiond-1:") require.NotContains(t, text, "versiond-2:") - require.Contains(t, text, fmt.Sprintf(`"%d:8080"`, cfg.VersiondRouter.Port)) + require.Contains(t, text, `"127.0.0.1::8080"`) } diff --git a/devshard/testenv/citest/harness/epoch.go b/devshard/testenv/citest/harness/epoch.go index 13dc93a0c5..2aaafa4583 100644 --- a/devshard/testenv/citest/harness/epoch.go +++ b/devshard/testenv/citest/harness/epoch.go @@ -18,16 +18,6 @@ import ( "google.golang.org/grpc/credentials/insecure" ) -// MockChainGRPCFromConfig returns mock-chain inference gRPC address. -func MockChainGRPCFromConfig(cfg *config.File) string { - return fmt.Sprintf("127.0.0.1:%d", cfg.MockChain.GRPCPort) -} - -// MockChainAdminFromConfig returns mock-chain admin HTTP base URL. -func MockChainAdminFromConfig(cfg *config.File) string { - return fmt.Sprintf("http://127.0.0.1:%d", cfg.MockChain.TestenvPort) -} - // MockChainSnapshot holds block height and epoch from mock-chain queries. type MockChainSnapshot struct { BlockHeight int64 @@ -38,10 +28,10 @@ type MockChainSnapshot struct { } // GetMockChainSnapshot queries mock-chain gRPC EpochInfo and optional admin revision. -func GetMockChainSnapshot(t *testing.T, cfg *config.File, client *http.Client) MockChainSnapshot { +func GetMockChainSnapshot(t *testing.T, cfg *config.File, eps Endpoints, client *http.Client) MockChainSnapshot { t.Helper() - snap := queryMockChainEpochInfo(t, MockChainGRPCFromConfig(cfg)) - if rev, err := tryMockChainRevision(client, MockChainAdminFromConfig(cfg)); err == nil { + snap := queryMockChainEpochInfo(t, eps.MockChainGRPC) + if rev, err := tryMockChainRevision(client, eps.MockChainAdmin); err == nil { snap.BlockHeight = rev.BlockHeight snap.ParamsBlockHeight = rev.ParamsBlockHeight snap.NextPocStart = rev.NextPocStartBlockHeight @@ -69,9 +59,9 @@ func queryMockChainEpochInfo(t *testing.T, grpcAddr string) MockChainSnapshot { resp, err := c.InferenceQueryClient().EpochInfo(context.Background(), &inferencetypes.QueryEpochInfoRequest{}) require.NoError(t, err) return MockChainSnapshot{ - BlockHeight: resp.BlockHeight, - EpochIndex: resp.LatestEpoch.Index, - PocStart: resp.LatestEpoch.PocStartBlockHeight, + BlockHeight: resp.BlockHeight, + EpochIndex: resp.LatestEpoch.Index, + PocStart: resp.LatestEpoch.PocStartBlockHeight, } } diff --git a/devshard/testenv/citest/harness/lease_race.go b/devshard/testenv/citest/harness/lease_race.go index 66321a6f6a..1ff7419072 100644 --- a/devshard/testenv/citest/harness/lease_race.go +++ b/devshard/testenv/citest/harness/lease_race.go @@ -17,12 +17,12 @@ import ( // LeaseSnapshot is a point-in-time view of devshard_validation_leases. type LeaseSnapshot struct { - Total int - Pending int - Submitted int - Skipped int - DuplicateGroups int - Rows []LeaseRow + Total int + Pending int + Submitted int + Skipped int + DuplicateGroups int + Rows []LeaseRow } // LeaseRow is one validation lease. @@ -44,7 +44,7 @@ func (s *Stack) PostgresLeaseSnapshot(t *testing.T, cfg *config.File) LeaseSnaps // TryPostgresLeaseSnapshot is safe to call from worker goroutines (no testing.T). func (s *Stack) TryPostgresLeaseSnapshot(cfg *config.File) (LeaseSnapshot, error) { user, db, pass := postgresCreds(cfg) - dupRaw, err := s.tryComposeExec("devshard-postgres", + dupRaw, err := s.ComposeExecOutput("devshard-postgres", "env", "PGPASSWORD="+pass, "psql", "-U", user, "-d", db, "-At", "-c", `SELECT COUNT(*) FROM ( @@ -61,7 +61,7 @@ func (s *Stack) TryPostgresLeaseSnapshot(cfg *config.File) (LeaseSnapshot, error return LeaseSnapshot{}, fmt.Errorf("parse duplicate_groups %q: %w", dupRaw, err) } - countsRaw, err := s.tryComposeExec("devshard-postgres", + countsRaw, err := s.ComposeExecOutput("devshard-postgres", "env", "PGPASSWORD="+pass, "psql", "-U", user, "-d", db, "-At", "-F", ",", "-c", `SELECT @@ -82,7 +82,7 @@ func (s *Stack) TryPostgresLeaseSnapshot(cfg *config.File) (LeaseSnapshot, error submitted, _ := strconv.Atoi(parts[2]) skipped, _ := strconv.Atoi(parts[3]) - rowsRaw, err := s.tryComposeExec("devshard-postgres", + rowsRaw, err := s.ComposeExecOutput("devshard-postgres", "env", "PGPASSWORD="+pass, "psql", "-U", user, "-d", db, "-At", "-F", "|", "-c", `SELECT inference_id, instance_address, status, claimed_at @@ -161,7 +161,7 @@ func SetValidationRate(t *testing.T, client *http.Client, mockDapiHTTP string, r PatchTestenvParams(t, client, mockDapiHTTP, adminface.ParamsRequest{ValidationRate: &rate}) } -// SetValidationRate100 is SetValidationRate(10000) — 100% for dense lease races (S9). +// SetValidationRate100 configures a 100% rate for dense lease races. func SetValidationRate100(t *testing.T, client *http.Client, mockDapiHTTP string) { t.Helper() SetValidationRate(t, client, mockDapiHTTP, 10000) @@ -199,7 +199,7 @@ func DriveLeaseRaceLoad(t *testing.T, client *http.Client, gatewayURL, model str req := ChatCompletionRequest{ Model: model, Messages: []ChatMessage{ - {Role: "user", Content: fmt.Sprintf("citest s9 lease race non-stream %d", i)}, + {Role: "user", Content: fmt.Sprintf("citest validation lease race non-stream %d", i)}, }, MaxTokens: 32, } @@ -210,7 +210,7 @@ func DriveLeaseRaceLoad(t *testing.T, client *http.Client, gatewayURL, model str req := ChatCompletionRequest{ Model: model, Messages: []ChatMessage{ - {Role: "user", Content: fmt.Sprintf("citest s9 lease race stream %d", i)}, + {Role: "user", Content: fmt.Sprintf("citest validation lease race stream %d", i)}, }, MaxTokens: 32, } diff --git a/devshard/testenv/citest/harness/params_longpoll.go b/devshard/testenv/citest/harness/params_longpoll.go index 6f3166795e..3d8e8f5e40 100644 --- a/devshard/testenv/citest/harness/params_longpoll.go +++ b/devshard/testenv/citest/harness/params_longpoll.go @@ -4,13 +4,11 @@ import ( "bytes" "context" "encoding/json" - "fmt" "net/http" "testing" "time" "common/nodemanager/gen" - "devshard/testenv/config" "devshard/testenv/mockchain/adminface" "github.com/stretchr/testify/require" @@ -24,11 +22,11 @@ type MockDAPIEndpoints struct { GRPC string } -// MockDAPIFromConfig returns published mock-dapi URLs for citest. -func MockDAPIFromConfig(cfg *config.File) MockDAPIEndpoints { +// MockDAPIFromEndpoints returns published mock-dapi URLs discovered from compose. +func MockDAPIFromEndpoints(eps Endpoints) MockDAPIEndpoints { return MockDAPIEndpoints{ - HTTP: fmt.Sprintf("http://127.0.0.1:%d", cfg.MockDapi.HTTPPort), - GRPC: fmt.Sprintf("127.0.0.1:%d", cfg.MockDapi.GRPCPort), + HTTP: eps.MockDapiHTTP, + GRPC: eps.MockDapiGRPC, } } diff --git a/devshard/testenv/citest/harness/ports.go b/devshard/testenv/citest/harness/ports.go index 87b0a2c2e3..e5f9aa4e13 100644 --- a/devshard/testenv/citest/harness/ports.go +++ b/devshard/testenv/citest/harness/ports.go @@ -2,16 +2,37 @@ package harness import ( "net" + "sync" "testing" "github.com/stretchr/testify/require" ) -// pickFreePort returns an ephemeral localhost TCP port for citest host publishing. +var pickedPorts = struct { + sync.Mutex + ports map[int]struct{} +}{ports: make(map[int]struct{})} + +// pickFreePort returns an ephemeral TCP port for generated service listen ports. +// Host publishing is randomized later by Docker Compose. func pickFreePort(t *testing.T) int { t.Helper() - ln, err := net.Listen("tcp", "127.0.0.1:0") - require.NoError(t, err) - defer ln.Close() - return ln.Addr().(*net.TCPAddr).Port + for attempts := 0; attempts < 100; attempts++ { + ln, err := net.Listen("tcp", "127.0.0.1:0") + require.NoError(t, err) + port := ln.Addr().(*net.TCPAddr).Port + require.NoError(t, ln.Close()) + + pickedPorts.Lock() + _, used := pickedPorts.ports[port] + if !used { + pickedPorts.ports[port] = struct{}{} + } + pickedPorts.Unlock() + if !used { + return port + } + } + t.Fatal("could not pick a unique localhost port") + return 0 } diff --git a/devshard/testenv/citest/harness/stack_s1_test.go b/devshard/testenv/citest/harness/router_session_test.go similarity index 100% rename from devshard/testenv/citest/harness/stack_s1_test.go rename to devshard/testenv/citest/harness/router_session_test.go diff --git a/devshard/testenv/citest/harness/router_sticky.go b/devshard/testenv/citest/harness/router_sticky.go index 28eaa47ab7..8b9f1fef98 100644 --- a/devshard/testenv/citest/harness/router_sticky.go +++ b/devshard/testenv/citest/harness/router_sticky.go @@ -21,12 +21,12 @@ func FindDistinctStickySessions(t *testing.T, client *http.Client, routerHTTP, v if client == nil { client = HTTPClient() } - sessionA = "citest-s6-session-a" + sessionA = "citest-failover-session-a" urlA := RouterSessionURL(routerHTTP, version, sessionA, "/healthz") upstreamA = RequireResponseHeader(t, client, urlA, StickyUpstreamHeader) for n := 0; n < 64; n++ { - candidate := fmt.Sprintf("citest-s6-%d", n) + candidate := fmt.Sprintf("citest-failover-%d", n) if candidate == sessionA { continue } diff --git a/devshard/testenv/citest/harness/s8_migrate.go b/devshard/testenv/citest/harness/sqlite_ha_migration.go similarity index 93% rename from devshard/testenv/citest/harness/s8_migrate.go rename to devshard/testenv/citest/harness/sqlite_ha_migration.go index 067c766b83..fa828cca4f 100644 --- a/devshard/testenv/citest/harness/s8_migrate.go +++ b/devshard/testenv/citest/harness/sqlite_ha_migration.go @@ -18,14 +18,14 @@ import ( _ "modernc.org/sqlite" ) -// BootS8MigrateStack boots the 2×versiond + Postgres stack patched for §3.3 Phase 0–1: +// BootSQLiteHAMigrationStack boots the 2×versiond + Postgres stack patched for §3.3 Phase 0–1: // DEVSHARD_STORAGE_MODE=sqlite and VERSIOND_HOSTS=versiond-0 only. versiond-1 is stopped // so SQLite sessions land on the legacy host volume. -func BootS8MigrateStack(t *testing.T, prefix string) (*Stack, *config.File, Endpoints) { +func BootSQLiteHAMigrationStack(t *testing.T, prefix string) (*Stack, *config.File, Endpoints) { t.Helper() stack := NewStack(t, prefix) RequireLinuxDevshardd(t, stack.TestenvDir) - WriteS1Config(t, stack.WorkDir) + WriteStackConfig(t, stack.WorkDir) stack.RunGencompose(t) cfg := stack.LoadConfig(t) requireTwoVersiondHosts(t, cfg) @@ -35,7 +35,7 @@ func BootS8MigrateStack(t *testing.T, prefix string) (*Stack, *config.File, Endp stack.Up(t) stack.StopService(t, cfg.Hosts[1].ID) - return stack, cfg, EndpointsFromConfig(cfg) + return stack, cfg, stack.Endpoints(t, cfg) } // RecreateServices force-recreates named services (picks up compose env changes). @@ -61,12 +61,13 @@ func (s *Stack) RecreateServices(t *testing.T, services ...string) { // ComposeExec runs `docker compose exec -T service cmd...` and returns stdout+stderr. func (s *Stack) ComposeExec(t *testing.T, service string, cmdArgs ...string) string { t.Helper() - out, err := s.tryComposeExec(service, cmdArgs...) + out, err := s.ComposeExecOutput(service, cmdArgs...) require.NoError(t, err, "compose exec %s %v\n%s", service, cmdArgs, out) return out } -func (s *Stack) tryComposeExec(service string, cmdArgs ...string) (string, error) { +// ComposeExecOutput runs `docker compose exec -T service cmd...` and returns stdout+stderr. +func (s *Stack) ComposeExecOutput(service string, cmdArgs ...string) (string, error) { ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) defer cancel() args := append([]string{"compose"}, s.composeFileArgs()...) diff --git a/devshard/testenv/citest/harness/stack.go b/devshard/testenv/citest/harness/stack.go index ffe4e96a2e..3f7c57fc5f 100644 --- a/devshard/testenv/citest/harness/stack.go +++ b/devshard/testenv/citest/harness/stack.go @@ -3,9 +3,11 @@ package harness import ( "context" "fmt" + "net" "os" "os/exec" "path/filepath" + "strconv" "strings" "testing" "time" @@ -29,10 +31,14 @@ type Stack struct { // Endpoints are host-published URLs for health probes. type Endpoints struct { - MockChainRPC string - MockDapiHTTP string - RouterHTTP string - GatewayHTTP string + MockChainRPC string + MockChainGRPC string + MockChainAdmin string + MockDapiHTTP string + MockDapiGRPC string + MockOpenAIHTTP string + RouterHTTP string + GatewayHTTP string } // NewStack creates a temp workdir under testenv and registers cleanup. @@ -86,6 +92,7 @@ func (s *Stack) RunGencompose(t *testing.T) { t.Fatalf("gencompose: %v\n%s", err, out) } fixComposePaths(t, s.ComposePath, s.TestenvDir) + PatchComposeUseRandomHostPorts(t, s.ComposePath) } // LoadConfig reads the generated config.yaml after gencompose. @@ -96,14 +103,45 @@ func (s *Stack) LoadConfig(t *testing.T) *config.File { return cfg } -// EndpointsFromConfig maps published ports to localhost URLs. -func EndpointsFromConfig(cfg *config.File) Endpoints { +// Endpoints reads Docker-assigned localhost ports from a running compose stack. +func (s *Stack) Endpoints(t *testing.T, cfg *config.File) Endpoints { + t.Helper() + eps := s.MockChainEndpoints(t, cfg) + eps.MockDapiHTTP = "http://" + s.composePublishedAddr(t, "mock-dapi", cfg.MockDapi.HTTPPort) + eps.MockDapiGRPC = s.composePublishedAddr(t, "mock-dapi", cfg.MockDapi.GRPCPort) + eps.MockOpenAIHTTP = "http://" + s.composePublishedAddr(t, "mock-openai", cfg.MockOpenAI.HTTPPort) + eps.RouterHTTP = "http://" + s.composePublishedAddr(t, "versiond-router", 8080) + eps.GatewayHTTP = "http://" + s.composePublishedAddr(t, "devshardctl", cfg.Devshardctl.Port) + return eps +} + +// MockChainEndpoints reads Docker-assigned host ports for a mock-chain-only stack. +func (s *Stack) MockChainEndpoints(t *testing.T, cfg *config.File) Endpoints { + t.Helper() return Endpoints{ - MockChainRPC: fmt.Sprintf("http://127.0.0.1:%d", cfg.MockChain.RPCPort), - MockDapiHTTP: fmt.Sprintf("http://127.0.0.1:%d", cfg.MockDapi.HTTPPort), - RouterHTTP: fmt.Sprintf("http://127.0.0.1:%d", cfg.VersiondRouter.Port), - GatewayHTTP: fmt.Sprintf("http://127.0.0.1:%d", cfg.Devshardctl.Port), + MockChainRPC: "http://" + s.composePublishedAddr(t, "mock-chain", cfg.MockChain.RPCPort), + MockChainGRPC: s.composePublishedAddr(t, "mock-chain", cfg.MockChain.GRPCPort), + MockChainAdmin: "http://" + s.composePublishedAddr(t, "mock-chain", cfg.MockChain.TestenvPort), + } +} + +func (s *Stack) composePublishedAddr(t *testing.T, service string, targetPort int) string { + t.Helper() + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + args := append([]string{"compose"}, s.composeFileArgs()...) + args = append(args, "port", service, strconv.Itoa(targetPort)) + cmd := exec.CommandContext(ctx, "docker", args...) + cmd.Dir = s.WorkDir + out, err := cmd.CombinedOutput() + require.NoError(t, err, "docker compose port %s %d\n%s", service, targetPort, out) + raw := strings.TrimSpace(string(out)) + host, port, err := net.SplitHostPort(raw) + require.NoError(t, err, "parse docker compose port output %q", raw) + if host == "" || host == "0.0.0.0" || host == "::" { + host = "127.0.0.1" } + return net.JoinHostPort(host, port) } // Up starts the stack with docker compose up (expects citest-images built; pulls missing hub images). diff --git a/devshard/testenv/citest/harness/stack_s1.go b/devshard/testenv/citest/harness/stack_boot.go similarity index 70% rename from devshard/testenv/citest/harness/stack_s1.go rename to devshard/testenv/citest/harness/stack_boot.go index 305db5c7af..75f67193ab 100644 --- a/devshard/testenv/citest/harness/stack_s1.go +++ b/devshard/testenv/citest/harness/stack_boot.go @@ -11,47 +11,47 @@ import ( "github.com/stretchr/testify/require" ) -// BootS1Stack renders the 2×versiond citest config, starts compose, and returns handles. -func BootS1Stack(t *testing.T, prefix string) (*Stack, *config.File, Endpoints) { +// BootStack renders the 2×versiond citest config, starts compose, and returns handles. +func BootStack(t *testing.T, prefix string) (*Stack, *config.File, Endpoints) { t.Helper() stack := NewStack(t, prefix) RequireLinuxDevshardd(t, stack.TestenvDir) - WriteS1Config(t, stack.WorkDir) + WriteStackConfig(t, stack.WorkDir) stack.RunGencompose(t) cfg := stack.LoadConfig(t) requireTwoVersiondHosts(t, cfg) stack.Up(t) - return stack, cfg, EndpointsFromConfig(cfg) + return stack, cfg, stack.Endpoints(t, cfg) } -// BootS1StackBuild is like BootS1Stack but rebuilds compose images first (devshardctl gRPC wiring). -func BootS1StackBuild(t *testing.T, prefix string) (*Stack, *config.File, Endpoints) { +// BootStackBuild is like BootStack but rebuilds compose images first (devshardctl gRPC wiring). +func BootStackBuild(t *testing.T, prefix string) (*Stack, *config.File, Endpoints) { t.Helper() stack := NewStack(t, prefix) RequireLinuxDevshardd(t, stack.TestenvDir) - WriteS1Config(t, stack.WorkDir) + WriteStackConfig(t, stack.WorkDir) stack.RunGencompose(t) cfg := stack.LoadConfig(t) requireTwoVersiondHosts(t, cfg) RequireGatewayGRPCOnlyCompose(t, stack.ComposePath) stack.UpBuild(t) - return stack, cfg, EndpointsFromConfig(cfg) + return stack, cfg, stack.Endpoints(t, cfg) } -func BootS1ObsStack(t *testing.T, prefix string) (*Stack, *config.File, Endpoints, ObservabilityEndpoints) { +func BootObservabilityStack(t *testing.T, prefix string) (*Stack, *config.File, Endpoints, ObservabilityEndpoints) { t.Helper() stack := NewStack(t, prefix) RequireLinuxDevshardd(t, stack.TestenvDir) - WriteS1Config(t, stack.WorkDir) + WriteStackConfig(t, stack.WorkDir) stack.RunGencompose(t) cfg := stack.LoadConfig(t) requireTwoVersiondHosts(t, cfg) stack.UpWithObservability(t, cfg) - return stack, cfg, EndpointsFromConfig(cfg), DefaultObservabilityEndpoints() + return stack, cfg, stack.Endpoints(t, cfg), DefaultObservabilityEndpoints() } -// WaitS1Healthy polls the S1 boundary health endpoints (chain, dapi, router, gateway). -func WaitS1Healthy(t *testing.T, stack *Stack, eps Endpoints) { +// WaitStackHealthy polls the chain, dapi, router, and gateway boundaries. +func WaitStackHealthy(t *testing.T, stack *Stack, eps Endpoints) { t.Helper() client := HTTPClient() poll := 5 * time.Minute @@ -70,17 +70,17 @@ func requireTwoVersiondHosts(t *testing.T, cfg *config.File) { } } -// BootS9Stack renders the 3×versiond lease-race config (HA pair + solo executor). -func BootS9Stack(t *testing.T, prefix string) (*Stack, *config.File, Endpoints) { +// BootValidationLeaseRaceStack renders the 3×versiond lease-race config (HA pair + solo executor). +func BootValidationLeaseRaceStack(t *testing.T, prefix string) (*Stack, *config.File, Endpoints) { t.Helper() stack := NewStack(t, prefix) RequireLinuxDevshardd(t, stack.TestenvDir) - WriteS9Config(t, stack.WorkDir) + WriteValidationLeaseRaceConfig(t, stack.WorkDir) stack.RunGencompose(t) cfg := stack.LoadConfig(t) requireThreeVersiondHosts(t, cfg) stack.Up(t) - return stack, cfg, EndpointsFromConfig(cfg) + return stack, cfg, stack.Endpoints(t, cfg) } func requireThreeVersiondHosts(t *testing.T, cfg *config.File) { diff --git a/devshard/testenv/citest/harness/stack_mockchain.go b/devshard/testenv/citest/harness/stack_mockchain.go index 7fa2122082..2c3d213c47 100644 --- a/devshard/testenv/citest/harness/stack_mockchain.go +++ b/devshard/testenv/citest/harness/stack_mockchain.go @@ -10,9 +10,9 @@ import ( func BootMockChainStack(t *testing.T, prefix string) (*Stack, *config.File, Endpoints) { t.Helper() stack := NewStack(t, prefix) - WriteS1Config(t, stack.WorkDir) + WriteStackConfig(t, stack.WorkDir) stack.RunGencompose(t) cfg := stack.LoadConfig(t) stack.UpServices(t, false, "mock-chain") - return stack, cfg, EndpointsFromConfig(cfg) + return stack, cfg, stack.MockChainEndpoints(t, cfg) } diff --git a/devshard/testenv/citest/harness/versiond_rolling.go b/devshard/testenv/citest/harness/versiond_rolling.go new file mode 100644 index 0000000000..8c9807bfca --- /dev/null +++ b/devshard/testenv/citest/harness/versiond_rolling.go @@ -0,0 +1,243 @@ +package harness + +import ( + "archive/zip" + "bufio" + "bytes" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "fmt" + "io" + "net/http" + "os" + "path/filepath" + "strings" + "sync" + "testing" + + cosrv "devshard/chainoracle/server" + + "github.com/stretchr/testify/require" +) + +// VersiondHealthEntry mirrors versiond /healthz entries. +type VersiondHealthEntry struct { + Name string `json:"name"` + Port int `json:"port"` + Status string `json:"status"` + SHA256 string `json:"sha256,omitempty"` + BinaryVersion string `json:"binary_version,omitempty"` +} + +// WriteDevsharddZip writes a versiond-downloadable archive and returns its sha256. +func WriteDevsharddZip(t *testing.T, devsharddPath, zipPath, marker string) string { + t.Helper() + require.NoError(t, os.MkdirAll(filepath.Dir(zipPath), 0o755)) + + src, err := os.Open(devsharddPath) + require.NoError(t, err) + defer src.Close() + info, err := src.Stat() + require.NoError(t, err) + + out, err := os.Create(zipPath) + require.NoError(t, err) + zw := zip.NewWriter(out) + + hdr, err := zip.FileInfoHeader(info) + require.NoError(t, err) + hdr.Name = "devshardd" + hdr.Method = zip.Deflate + hdr.SetMode(0o755) + w, err := zw.CreateHeader(hdr) + require.NoError(t, err) + _, err = io.Copy(w, src) + require.NoError(t, err) + + markerW, err := zw.Create("testenv-marker.txt") + require.NoError(t, err) + _, err = markerW.Write([]byte(marker)) + require.NoError(t, err) + require.NoError(t, zw.Close()) + require.NoError(t, out.Close()) + + archive, err := os.ReadFile(zipPath) + require.NoError(t, err) + sum := sha256.Sum256(archive) + return hex.EncodeToString(sum[:]) +} + +// PatchTestenvVersions replaces mock-dapi /versions for versiond polling. +func PatchTestenvVersions(t *testing.T, client *http.Client, mockDapiHTTP string, versions []cosrv.Version) { + t.Helper() + if client == nil { + client = HTTPClient() + } + require.NoError(t, PostJSON(client, mockDapiHTTP+"/testenv/versions", cosrv.VersionConfig{ + Versions: versions, + }, nil)) +} + +// VersiondHealth fetches a versiond container's local /healthz. +func VersiondHealth(t *testing.T, stack *Stack, service string) []VersiondHealthEntry { + t.Helper() + entries, err := TryVersiondHealth(stack, service) + require.NoError(t, err) + return entries +} + +// TryVersiondHealth fetches a versiond container's local /healthz without failing the test. +func TryVersiondHealth(stack *Stack, service string) ([]VersiondHealthEntry, error) { + out, err := stack.ComposeExecOutput(service, "wget", "-q", "-O", "-", "http://127.0.0.1:8080/healthz") + if err != nil { + return nil, err + } + var entries []VersiondHealthEntry + if err := json.Unmarshal([]byte(out), &entries); err != nil { + return nil, fmt.Errorf("versiond health %s: %w: %s", service, err, out) + } + return entries, nil +} + +// HasVersiondHealthEntry reports whether health contains the wanted version state. +func HasVersiondHealthEntry(entries []VersiondHealthEntry, name, status, sha string) bool { + for _, e := range entries { + if e.Name != name { + continue + } + if status != "" && e.Status != status { + continue + } + if sha != "" && e.SHA256 != sha { + continue + } + return true + } + return false +} + +// GatewayStreamResult is the result of a live gateway SSE request. +type GatewayStreamResult struct { + Status int + Content string + SawDone bool + Err error + Body string +} + +// StartGatewayChatCompletionStream starts stream=true chat and closes accepted +// after HTTP 200 and the first content chunk. +func StartGatewayChatCompletionStream( + client *http.Client, + gatewayURL string, + adminAPIKey string, + req ChatCompletionRequest, +) (<-chan struct{}, <-chan GatewayStreamResult) { + if client == nil { + client = GatewayChatClient() + } + req.Stream = true + accepted := make(chan struct{}) + result := make(chan GatewayStreamResult, 1) + var acceptedOnce sync.Once + closeAccepted := func() { acceptedOnce.Do(func() { close(accepted) }) } + + go func() { + defer close(result) + var res GatewayStreamResult + data, err := json.Marshal(req) + if err != nil { + closeAccepted() + res.Err = err + result <- res + return + } + httpReq, err := http.NewRequest(http.MethodPost, gatewayURL+"/v1/chat/completions", bytes.NewReader(data)) + if err != nil { + closeAccepted() + res.Err = err + result <- res + return + } + httpReq.Header.Set("Content-Type", "application/json") + httpReq.Header.Set("Accept", "text/event-stream") + if adminAPIKey != "" { + httpReq.Header.Set("Authorization", "Bearer "+adminAPIKey) + } + + resp, err := client.Do(httpReq) + if err != nil { + closeAccepted() + res.Err = err + result <- res + return + } + defer resp.Body.Close() + res.Status = resp.StatusCode + if resp.StatusCode >= 300 { + closeAccepted() + body, _ := io.ReadAll(resp.Body) + res.Body = string(body) + result <- res + return + } + + var assembled strings.Builder + scanner := bufio.NewScanner(resp.Body) + firstChunk := false + for scanner.Scan() { + line := strings.TrimSpace(scanner.Text()) + if line == "data: [DONE]" { + res.SawDone = true + continue + } + if !strings.HasPrefix(line, "data: ") { + continue + } + payload := strings.TrimPrefix(line, "data: ") + var chunk map[string]any + if err := json.Unmarshal([]byte(payload), &chunk); err != nil { + continue + } + content := streamChunkContent(chunk) + if content == "" { + continue + } + if !firstChunk { + firstChunk = true + closeAccepted() + } + assembled.WriteString(content) + } + if err := scanner.Err(); err != nil { + res.Err = err + } + if !firstChunk { + closeAccepted() + } + res.Content = assembled.String() + result <- res + }() + + return accepted, result +} + +func streamChunkContent(chunk map[string]any) string { + choices, _ := chunk["choices"].([]any) + if len(choices) == 0 { + return "" + } + choice, _ := choices[0].(map[string]any) + delta, _ := choice["delta"].(map[string]any) + if delta == nil { + return "" + } + s, _ := delta["content"].(string) + return s +} + +// VersiondBinaryURL returns the mock-dapi URL a versiond container can download. +func VersiondBinaryURL(mockDapiPort int, fileName string) string { + return fmt.Sprintf("http://mock-dapi:%d/testenv/binaries/%s", mockDapiPort, fileName) +} diff --git a/devshard/testenv/citest/s7_legacy_version_pin_test.go b/devshard/testenv/citest/legacy_version_pin_test.go similarity index 93% rename from devshard/testenv/citest/s7_legacy_version_pin_test.go rename to devshard/testenv/citest/legacy_version_pin_test.go index 0a31283c89..e3108708c2 100644 --- a/devshard/testenv/citest/s7_legacy_version_pin_test.go +++ b/devshard/testenv/citest/legacy_version_pin_test.go @@ -20,14 +20,14 @@ const ( legacyVersionPath = "v1" ) -// TestS7_LegacyVersionPinnedToSingleHost asserts versiond-router sends +// TestLegacyVersionPinnedToSingleHost asserts versiond-router sends // VERSIOND_NON_HA_VERSIONS prefixes only to VERSIOND_LEGACY_HOST, while other // versions sticky-hash across the pool (see pr-1366-deploy-test-plan.md §3.2). -func TestS7_LegacyVersionPinnedToSingleHost(t *testing.T) { +func TestLegacyVersionPinnedToSingleHost(t *testing.T) { harness.SkipUnlessEnv(t, "TESTENV_CITEST") harness.RequireDocker(t) - stack, cfg, eps := harness.BootS1Stack(t, "citest-s7-*") + stack, cfg, eps := harness.BootStack(t, "citest-legacy-version-pin-*") client := harness.HTTPClient() t.Cleanup(func() { if t.Failed() { @@ -70,7 +70,7 @@ func TestS7_LegacyVersionPinnedToSingleHost(t *testing.T) { _, upstreamA, _, upstreamB := harness.FindDistinctStickySessions(t, client, eps.RouterHTTP, haVersion) require.NotEqual(t, upstreamA, upstreamB) - urlHA := harness.RouterSessionURL(eps.RouterHTTP, haVersion, "citest-s7-ha-check", "/healthz") + urlHA := harness.RouterSessionURL(eps.RouterHTTP, haVersion, "citest-legacy-version-pin-ha-check", "/healthz") haBackend := harness.RequireResponseHeader(t, client, urlHA, versiondBackendHeader) require.Equal(t, backendHA, haBackend, "HA path X-Versiond-Backend") diff --git a/devshard/testenv/citest/o1_observability_smoke_test.go b/devshard/testenv/citest/o1_observability_smoke_test.go index 1d82232d9d..d71528bf99 100644 --- a/devshard/testenv/citest/o1_observability_smoke_test.go +++ b/devshard/testenv/citest/o1_observability_smoke_test.go @@ -11,13 +11,13 @@ import ( "devshard/testenv/config" ) -// TestO1_ObservabilitySmoke starts the S1 stack with Jaeger/Loki/Prometheus/Grafana, +// TestO1_ObservabilitySmoke starts the standard stack with observability services, // runs a gateway chat, and asserts devshardd spans and structured logs appear. func TestO1_ObservabilitySmoke(t *testing.T) { harness.SkipUnlessEnv(t, "TESTENV_CITEST") harness.RequireDocker(t) - stack, cfg, eps, obs := harness.BootS1ObsStack(t, "citest-o1-*") + stack, cfg, eps, obs := harness.BootObservabilityStack(t, "citest-o1-*") client := harness.GatewayChatClient() t.Cleanup(func() { if t.Failed() { @@ -26,7 +26,7 @@ func TestO1_ObservabilitySmoke(t *testing.T) { }) harness.WaitObservabilityReady(t, obs, 3*time.Minute) - harness.WaitS1Healthy(t, stack, eps) + harness.WaitStackHealthy(t, stack, eps) harness.WaitGatewayChatReady(t, client, eps.GatewayHTTP, 3*time.Minute, stack) model := config.PrimaryModelID(cfg) diff --git a/devshard/testenv/citest/s3_params_longpoll_test.go b/devshard/testenv/citest/params_longpoll_test.go similarity index 92% rename from devshard/testenv/citest/s3_params_longpoll_test.go rename to devshard/testenv/citest/params_longpoll_test.go index 98ac800947..1752f01eea 100644 --- a/devshard/testenv/citest/s3_params_longpoll_test.go +++ b/devshard/testenv/citest/params_longpoll_test.go @@ -14,17 +14,17 @@ import ( "github.com/stretchr/testify/require" ) -// TestS3_ParamsLongPoll verifies mock-chain /testenv/params → mock-dapi chain poll → +// TestParamsLongPoll verifies mock-chain /testenv/params → mock-dapi chain poll → // GetRuntimeConfig long-poll wake while devshardd children are running in the stack. -func TestS3_ParamsLongPoll(t *testing.T) { +func TestParamsLongPoll(t *testing.T) { harness.SkipUnlessEnv(t, "TESTENV_CITEST") harness.RequireDocker(t) - stack, _, eps := harness.BootS1Stack(t, "citest-s3-*") + stack, _, eps := harness.BootStack(t, "citest-params-longpoll-*") client := harness.HTTPClient() - harness.WaitS1Healthy(t, stack, eps) + harness.WaitStackHealthy(t, stack, eps) - mockDapi := harness.MockDAPIFromConfig(stack.LoadConfig(t)) + mockDapi := harness.MockDAPIFromEndpoints(eps) grpcConn := harness.DialMockDAPI(t, mockDapi.GRPC) nm := gen.NewNodeManagerClient(grpcConn) ctx := context.Background() diff --git a/devshard/testenv/citest/s2_router_stickiness_test.go b/devshard/testenv/citest/router_stickiness_test.go similarity index 89% rename from devshard/testenv/citest/s2_router_stickiness_test.go rename to devshard/testenv/citest/router_stickiness_test.go index 2454f9ff21..31a6545e74 100644 --- a/devshard/testenv/citest/s2_router_stickiness_test.go +++ b/devshard/testenv/citest/router_stickiness_test.go @@ -12,13 +12,13 @@ import ( const stickyUpstreamHeader = "X-Upstream-Addr" -// TestS2_RouterStickiness asserts nginx consistent-hash routes the same session id +// TestRouterStickiness asserts nginx consistent-hash routes the same session id // to the same versiond upstream across retries, and that two backends are reachable. -func TestS2_RouterStickiness(t *testing.T) { +func TestRouterStickiness(t *testing.T) { harness.SkipUnlessEnv(t, "TESTENV_CITEST") harness.RequireDocker(t) - stack, cfg, eps := harness.BootS1Stack(t, "citest-s2-*") + stack, cfg, eps := harness.BootStack(t, "citest-router-stickiness-*") client := harness.HTTPClient() harness.WaitGETOK(t, client, eps.RouterHTTP+"/healthz", 5*time.Minute, "versiond-router healthz", stack) diff --git a/devshard/testenv/citest/s8_sqlite_ha_migrate_test.go b/devshard/testenv/citest/sqlite_ha_migration_test.go similarity index 75% rename from devshard/testenv/citest/s8_sqlite_ha_migrate_test.go rename to devshard/testenv/citest/sqlite_ha_migration_test.go index b1926fbb4d..2d73734e68 100644 --- a/devshard/testenv/citest/s8_sqlite_ha_migrate_test.go +++ b/devshard/testenv/citest/sqlite_ha_migration_test.go @@ -15,19 +15,19 @@ import ( ) const ( - s8BackendHeader = "X-Versiond-Backend" - s8BackendLegacy = "versiond_legacy" - s8BackendHA = "versiond_ha_pool" - s8NonHAVersion = "v1" + migrationBackendHeader = "X-Versiond-Backend" + migrationLegacyBackend = "versiond_legacy" + migrationHABackend = "versiond_ha_pool" + nonHAVersion = "v1" ) -// TestS8_SqliteHaFailMigrate walks pr-1366-deploy-test-plan.md §3.3 Phases 0–4: +// TestSQLiteToPostgresHAMigration walks pr-1366-deploy-test-plan.md §3.3 Phases 0–4: // single-host SQLite → multi-host Devshard-Ha 503 → postgres migrate → HA serve. -func TestS8_SqliteHaFailMigrate(t *testing.T) { +func TestSQLiteToPostgresHAMigration(t *testing.T) { harness.SkipUnlessEnv(t, "TESTENV_CITEST") harness.RequireDocker(t) - stack, cfg, eps := harness.BootS8MigrateStack(t, "citest-s8-*") + stack, cfg, eps := harness.BootSQLiteHAMigrationStack(t, "citest-sqlite-ha-migration-*") client := harness.GatewayChatClient() t.Cleanup(func() { if t.Failed() { @@ -38,7 +38,7 @@ func TestS8_SqliteHaFailMigrate(t *testing.T) { legacyHost := cfg.Hosts[0].ID secondHost := cfg.Hosts[1].ID haVersion := cfg.Versiond.VersionName - require.NotEqual(t, s8NonHAVersion, haVersion) + require.NotEqual(t, nonHAVersion, haVersion) harness.Step(t, "phase 0: single VERSIOND_HOSTS=%s, storage=sqlite", legacyHost) harness.WaitGETOK(t, client, eps.RouterHTTP+"/healthz", 5*time.Minute, "versiond-router healthz", stack) @@ -46,14 +46,14 @@ func TestS8_SqliteHaFailMigrate(t *testing.T) { harness.WaitGETOK(t, client, eps.RouterHTTP+"/"+haVersion+"/healthz", 5*time.Minute, "devshardd health (no Devshard-Ha)", stack) for i := 0; i < 8; i++ { - url := harness.RouterSessionURL(eps.RouterHTTP, s8NonHAVersion, fmt.Sprintf("s8-nonha-%d", i), "/healthz") - backend := harness.RequireResponseHeader(t, client, url, s8BackendHeader) - require.Equal(t, s8BackendLegacy, backend) + url := harness.RouterSessionURL(eps.RouterHTTP, nonHAVersion, fmt.Sprintf("sqlite-migration-nonha-%d", i), "/healthz") + backend := harness.RequireResponseHeader(t, client, url, migrationBackendHeader) + require.Equal(t, migrationLegacyBackend, backend) upstream := harness.RequireResponseHeader(t, client, url, harness.StickyUpstreamHeader) require.Equal(t, legacyHost, harness.HostIDForUpstream(cfg, upstream)) } - haURL := harness.RouterSessionURL(eps.RouterHTTP, haVersion, "s8-phase0-ha", "/healthz") - require.Equal(t, s8BackendHA, harness.RequireResponseHeader(t, client, haURL, s8BackendHeader)) + haURL := harness.RouterSessionURL(eps.RouterHTTP, haVersion, "sqlite-migration-phase0-ha", "/healthz") + require.Equal(t, migrationHABackend, harness.RequireResponseHeader(t, client, haURL, migrationBackendHeader)) harness.Step(t, "phase 1: create HA-version sessions under sqlite on %s", legacyHost) harness.WaitGatewayChatReady(t, client, eps.GatewayHTTP, 3*time.Minute, stack) @@ -63,7 +63,7 @@ func TestS8_SqliteHaFailMigrate(t *testing.T) { req := harness.ChatCompletionRequest{ Model: model, Messages: []harness.ChatMessage{ - {Role: "user", Content: fmt.Sprintf("citest s8 sqlite chat %d", i)}, + {Role: "user", Content: fmt.Sprintf("citest sqlite migration chat %d", i)}, }, MaxTokens: 32, } @@ -92,9 +92,9 @@ func TestS8_SqliteHaFailMigrate(t *testing.T) { // Confirm router still HA-pools the version and NON_HA stays pinned. // (Probe via GET that may be 503 — nginx still emits routing headers.) - require.Equal(t, s8BackendHA, harness.RequireResponseHeader(t, client, haHealth, s8BackendHeader)) - nonHAURL := harness.RouterSessionURL(eps.RouterHTTP, s8NonHAVersion, "s8-phase2-nonha", "/healthz") - require.Equal(t, s8BackendLegacy, harness.RequireResponseHeader(t, client, nonHAURL, s8BackendHeader)) + require.Equal(t, migrationHABackend, harness.RequireResponseHeader(t, client, haHealth, migrationBackendHeader)) + nonHAURL := harness.RouterSessionURL(eps.RouterHTTP, nonHAVersion, "sqlite-migration-phase2-nonha", "/healthz") + require.Equal(t, migrationLegacyBackend, harness.RequireResponseHeader(t, client, nonHAURL, migrationBackendHeader)) harness.Step(t, "phase 3: DEVSHARD_STORAGE_MODE=postgres → boot migrate") harness.PatchVersiondStorageMode(t, stack.ComposePath, "postgres") @@ -113,7 +113,7 @@ func TestS8_SqliteHaFailMigrate(t *testing.T) { okReq := harness.ChatCompletionRequest{ Model: model, Messages: []harness.ChatMessage{ - {Role: "user", Content: "citest s8 postgres ha chat"}, + {Role: "user", Content: "citest postgres ha migration chat"}, }, MaxTokens: 32, } @@ -128,7 +128,7 @@ func TestS8_SqliteHaFailMigrate(t *testing.T) { _, upstreamA, _, upstreamB := harness.FindDistinctStickySessions(t, client, eps.RouterHTTP, haVersion) require.NotEqual(t, upstreamA, upstreamB) - require.Equal(t, s8BackendHA, harness.RequireResponseHeader(t, client, haURL, s8BackendHeader)) + require.Equal(t, migrationHABackend, harness.RequireResponseHeader(t, client, haURL, migrationBackendHeader)) // Migrated escrow remains readable via gateway after HA is healthy. after := harness.GetGatewaySessionSnapshot(t, client, eps.GatewayHTTP, adminKey) @@ -136,8 +136,8 @@ func TestS8_SqliteHaFailMigrate(t *testing.T) { require.GreaterOrEqual(t, after.LatestNonce, snap.LatestNonce) for i := 0; i < 8; i++ { - url := harness.RouterSessionURL(eps.RouterHTTP, s8NonHAVersion, fmt.Sprintf("s8-final-nonha-%d", i), "/healthz") - require.Equal(t, s8BackendLegacy, harness.RequireResponseHeader(t, client, url, s8BackendHeader)) + url := harness.RouterSessionURL(eps.RouterHTTP, nonHAVersion, fmt.Sprintf("sqlite-migration-final-nonha-%d", i), "/healthz") + require.Equal(t, migrationLegacyBackend, harness.RequireResponseHeader(t, client, url, migrationBackendHeader)) upstream := harness.RequireResponseHeader(t, client, url, harness.StickyUpstreamHeader) require.Equal(t, legacyHost, harness.HostIDForUpstream(cfg, upstream)) } diff --git a/devshard/testenv/citest/s1_stack_smoke_test.go b/devshard/testenv/citest/stack_smoke_test.go similarity index 71% rename from devshard/testenv/citest/s1_stack_smoke_test.go rename to devshard/testenv/citest/stack_smoke_test.go index 3b0fde1c0b..8f1d9d9d49 100644 --- a/devshard/testenv/citest/s1_stack_smoke_test.go +++ b/devshard/testenv/citest/stack_smoke_test.go @@ -8,15 +8,15 @@ import ( "devshard/testenv/citest/harness" ) -// TestS1_StackSmoke brings up mock-chain, mock-dapi, versiond-router, two versiond +// TestStackSmoke brings up mock-chain, mock-dapi, versiond-router, two versiond // hosts, and devshardctl; asserts each boundary is healthy. -func TestS1_StackSmoke(t *testing.T) { +func TestStackSmoke(t *testing.T) { harness.SkipUnlessEnv(t, "TESTENV_CITEST") harness.RequireDocker(t) harness.Step(t, "starting stack (2 versiond + router + gateway) — versiond child boot can take 1–3 min") - stack, cfg, eps := harness.BootS1Stack(t, "citest-s1-*") - harness.WaitS1Healthy(t, stack, eps) + stack, cfg, eps := harness.BootStack(t, "citest-stack-smoke-*") + harness.WaitStackHealthy(t, stack, eps) harness.Step(t, "compose services running") stack.RequireServicesRunning(t, diff --git a/devshard/testenv/citest/s9_validation_lease_race_test.go b/devshard/testenv/citest/validation_lease_race_test.go similarity index 84% rename from devshard/testenv/citest/s9_validation_lease_race_test.go rename to devshard/testenv/citest/validation_lease_race_test.go index fdcb1b89bd..7821f54ee2 100644 --- a/devshard/testenv/citest/s9_validation_lease_race_test.go +++ b/devshard/testenv/citest/validation_lease_race_test.go @@ -15,13 +15,13 @@ import ( "github.com/stretchr/testify/require" ) -// TestS9_ValidationLeaseRaceCore drives chat load under HA + 100% validation_rate, +// TestValidationLeaseRaceCore drives chat load under HA + 100% validation_rate, // monitors Postgres leases in parallel, and PASS/FAILs on uniqueness (manual plan §§4–6). -func TestS9_ValidationLeaseRaceCore(t *testing.T) { +func TestValidationLeaseRaceCore(t *testing.T) { harness.SkipUnlessEnv(t, "TESTENV_CITEST") harness.RequireDocker(t) - stack, cfg, eps := harness.BootS9Stack(t, "citest-s9-core-*") + stack, cfg, eps := harness.BootValidationLeaseRaceStack(t, "citest-validation-lease-race-core-*") client := harness.GatewayChatClient() t.Cleanup(func() { if t.Failed() { @@ -29,11 +29,11 @@ func TestS9_ValidationLeaseRaceCore(t *testing.T) { } }) - harness.WaitS1Healthy(t, stack, eps) + harness.WaitStackHealthy(t, stack, eps) harness.WaitGatewayChatReady(t, client, eps.GatewayHTTP, 3*time.Minute, stack) harness.WaitGETOK(t, client, eps.RouterHTTP+"/"+cfg.Versiond.VersionName+"/healthz", 5*time.Minute, "devshardd health", stack) - dapi := harness.MockDAPIFromConfig(cfg) + dapi := harness.MockDAPIFromEndpoints(eps) harness.Step(t, "set validation_rate=10000 before escrow create") harness.SetValidationRate100(t, client, dapi.HTTP) @@ -42,7 +42,7 @@ func TestS9_ValidationLeaseRaceCore(t *testing.T) { seed := harness.ChatCompletionRequest{ Model: model, Messages: []harness.ChatMessage{ - {Role: "user", Content: "citest s9 seed"}, + {Role: "user", Content: "citest validation lease race seed"}, }, MaxTokens: 16, } @@ -88,15 +88,15 @@ func TestS9_ValidationLeaseRaceCore(t *testing.T) { harness.RequireLeaseExclusivityPass(t, final, 5) } -// TestS9_ValidationLeaseRacePendingStretch covers manual plan §7a: slow ML keeps +// TestValidationLeaseRacePendingStretch covers manual plan §7a: slow ML keeps // leases pending while exclusivity still holds. -func TestS9_ValidationLeaseRacePendingStretch(t *testing.T) { +func TestValidationLeaseRacePendingStretch(t *testing.T) { harness.SkipUnlessEnv(t, "TESTENV_CITEST") harness.RequireDocker(t) - stack, cfg, eps := harness.BootS9Stack(t, "citest-s9-7a-*") + stack, cfg, eps := harness.BootValidationLeaseRaceStack(t, "citest-validation-lease-race-pending-*") client := harness.GatewayChatClient() - mockOpenAI := harness.MockOpenAIFromConfig(cfg) + mockOpenAI := eps.MockOpenAIHTTP t.Cleanup(func() { harness.ResetMockOpenAIFault(t, client, mockOpenAI) if t.Failed() { @@ -104,15 +104,15 @@ func TestS9_ValidationLeaseRacePendingStretch(t *testing.T) { } }) - harness.WaitS1Healthy(t, stack, eps) + harness.WaitStackHealthy(t, stack, eps) harness.WaitGatewayChatReady(t, client, eps.GatewayHTTP, 3*time.Minute, stack) - dapi := harness.MockDAPIFromConfig(cfg) + dapi := harness.MockDAPIFromEndpoints(eps) harness.SetValidationRate100(t, client, dapi.HTTP) model := config.PrimaryModelID(cfg) seed := harness.ChatCompletionRequest{ Model: model, - Messages: []harness.ChatMessage{{Role: "user", Content: "citest s9 7a seed"}}, + Messages: []harness.ChatMessage{{Role: "user", Content: "citest validation lease race 7a seed"}}, MaxTokens: 16, } harness.PostGatewayChatCompletion(t, client, eps.GatewayHTTP, harness.TestenvAdminAPIKey, seed) @@ -131,7 +131,7 @@ func TestS9_ValidationLeaseRacePendingStretch(t *testing.T) { req := harness.ChatCompletionRequest{ Model: model, Messages: []harness.ChatMessage{ - {Role: "user", Content: fmt.Sprintf("citest s9 7a slow %d", i)}, + {Role: "user", Content: fmt.Sprintf("citest validation lease race 7a slow %d", i)}, }, MaxTokens: 16, } @@ -157,15 +157,15 @@ func TestS9_ValidationLeaseRacePendingStretch(t *testing.T) { harness.RequireLeaseExclusivityPass(t, final, 1) } -// TestS9_ValidationLeaseRaceStaleReclaim covers manual plan §7b: short TTL, pause ML +// TestValidationLeaseRaceStaleReclaim covers manual plan §7b: short TTL, pause ML // so leases stay pending, stop one replica, survivor reclaim + submit. -func TestS9_ValidationLeaseRaceStaleReclaim(t *testing.T) { +func TestValidationLeaseRaceStaleReclaim(t *testing.T) { harness.SkipUnlessEnv(t, "TESTENV_CITEST") harness.RequireDocker(t) - stack, cfg, eps := harness.BootS9Stack(t, "citest-s9-7b-*") + stack, cfg, eps := harness.BootValidationLeaseRaceStack(t, "citest-validation-lease-race-stale-reclaim-*") client := harness.GatewayChatClient() - mockOpenAI := harness.MockOpenAIFromConfig(cfg) + mockOpenAI := eps.MockOpenAIHTTP t.Cleanup(func() { harness.ResetMockOpenAIFault(t, client, mockOpenAI) if t.Failed() { @@ -177,17 +177,17 @@ func TestS9_ValidationLeaseRaceStaleReclaim(t *testing.T) { harness.PatchVersiondLeaseTTL(t, stack.ComposePath, "15s", "5s") // Recreate HA pair + solo so TTL env applies everywhere validations can run. stack.RecreateServices(t, harness.VersiondHostIDs(cfg)...) - harness.WaitS1Healthy(t, stack, eps) + harness.WaitStackHealthy(t, stack, eps) harness.WaitGatewayChatReady(t, client, eps.GatewayHTTP, 3*time.Minute, stack) harness.WaitGETOK(t, client, eps.RouterHTTP+"/"+cfg.Versiond.VersionName+"/healthz", 5*time.Minute, "devshardd after TTL patch", stack) - dapi := harness.MockDAPIFromConfig(cfg) + dapi := harness.MockDAPIFromEndpoints(eps) harness.SetValidationRate100(t, client, dapi.HTTP) model := config.PrimaryModelID(cfg) seed := harness.ChatCompletionRequest{ Model: model, - Messages: []harness.ChatMessage{{Role: "user", Content: "citest s9 7b seed"}}, + Messages: []harness.ChatMessage{{Role: "user", Content: "citest validation lease race 7b seed"}}, MaxTokens: 16, } harness.PostGatewayChatCompletion(t, client, eps.GatewayHTTP, harness.TestenvAdminAPIKey, seed) @@ -207,7 +207,7 @@ func TestS9_ValidationLeaseRaceStaleReclaim(t *testing.T) { req := harness.ChatCompletionRequest{ Model: model, Messages: []harness.ChatMessage{ - {Role: "user", Content: fmt.Sprintf("citest s9 7b inflight %d", i)}, + {Role: "user", Content: fmt.Sprintf("citest validation lease race 7b inflight %d", i)}, }, MaxTokens: 16, } diff --git a/devshard/testenv/citest/s6_versiond_stop_test.go b/devshard/testenv/citest/versiond_failover_test.go similarity index 89% rename from devshard/testenv/citest/s6_versiond_stop_test.go rename to devshard/testenv/citest/versiond_failover_test.go index 75ab02aa08..1583c50181 100644 --- a/devshard/testenv/citest/s6_versiond_stop_test.go +++ b/devshard/testenv/citest/versiond_failover_test.go @@ -11,14 +11,14 @@ import ( "github.com/stretchr/testify/require" ) -// TestS6_VersiondStop verifies versiond-router fails over a sticky session to a +// TestVersiondStickySessionFailover verifies versiond-router fails over a sticky session to a // surviving versiond on the first upstream 502 / connect failure when its preferred // peer is stopped; sessions hashed to a live upstream keep working. -func TestS6_VersiondStop(t *testing.T) { +func TestVersiondStickySessionFailover(t *testing.T) { harness.SkipUnlessEnv(t, "TESTENV_CITEST") harness.RequireDocker(t) - stack, cfg, eps := harness.BootS1Stack(t, "citest-s6-*") + stack, cfg, eps := harness.BootStack(t, "citest-versiond-failover-*") client := harness.HTTPClient() t.Cleanup(func() { if t.Failed() { diff --git a/devshard/testenv/citest/s6_versiond_restart_test.go b/devshard/testenv/citest/versiond_restart_persistence_test.go similarity index 85% rename from devshard/testenv/citest/s6_versiond_restart_test.go rename to devshard/testenv/citest/versiond_restart_persistence_test.go index cdd7e25f83..7add66e958 100644 --- a/devshard/testenv/citest/s6_versiond_restart_test.go +++ b/devshard/testenv/citest/versiond_restart_persistence_test.go @@ -10,13 +10,13 @@ import ( "devshard/testenv/config" ) -// TestS6_VersiondRestartPersistence verifies gateway chat and session nonce/state survive +// TestVersiondRestartSessionPersistence verifies gateway chat and session nonce/state survive // versiond → devshardd restarts without regression (postgres-backed host recovery). -func TestS6_VersiondRestartPersistence(t *testing.T) { +func TestVersiondRestartSessionPersistence(t *testing.T) { harness.SkipUnlessEnv(t, "TESTENV_CITEST") harness.RequireDocker(t) - stack, cfg, eps := harness.BootS1Stack(t, "citest-s6-restart-*") + stack, cfg, eps := harness.BootStack(t, "citest-versiond-restart-*") client := harness.HTTPClient() chatClient := harness.GatewayChatClient() adminKey := harness.TestenvAdminAPIKey @@ -26,7 +26,7 @@ func TestS6_VersiondRestartPersistence(t *testing.T) { } }) - harness.WaitS1Healthy(t, stack, eps) + harness.WaitStackHealthy(t, stack, eps) harness.WaitGatewayChatReady(t, chatClient, eps.GatewayHTTP, 3*time.Minute, stack) harness.WaitGETOK(t, client, eps.RouterHTTP+"/"+cfg.Versiond.VersionName+"/healthz", 5*time.Minute, "devshardd health via router", stack) @@ -37,7 +37,7 @@ func TestS6_VersiondRestartPersistence(t *testing.T) { harness.PostGatewayChatCompletion(t, chatClient, eps.GatewayHTTP, adminKey, harness.ChatCompletionRequest{ Model: model, Messages: []harness.ChatMessage{ - {Role: "user", Content: "citest s6 chat before restart"}, + {Role: "user", Content: "citest versiond chat before restart"}, }, MaxTokens: 16, }) @@ -54,7 +54,7 @@ func TestS6_VersiondRestartPersistence(t *testing.T) { harness.PostGatewayChatCompletion(t, chatClient, eps.GatewayHTTP, adminKey, harness.ChatCompletionRequest{ Model: model, Messages: []harness.ChatMessage{ - {Role: "user", Content: "citest s6 chat after one restart"}, + {Role: "user", Content: "citest versiond chat after one restart"}, }, MaxTokens: 16, }) @@ -72,7 +72,7 @@ func TestS6_VersiondRestartPersistence(t *testing.T) { harness.PostGatewayChatCompletion(t, chatClient, eps.GatewayHTTP, adminKey, harness.ChatCompletionRequest{ Model: model, Messages: []harness.ChatMessage{ - {Role: "user", Content: "citest s6 chat after all restarts"}, + {Role: "user", Content: "citest versiond chat after all restarts"}, }, MaxTokens: 16, }) diff --git a/devshard/testenv/citest/versiond_rolling_update_test.go b/devshard/testenv/citest/versiond_rolling_update_test.go new file mode 100644 index 0000000000..c56333aa6c --- /dev/null +++ b/devshard/testenv/citest/versiond_rolling_update_test.go @@ -0,0 +1,318 @@ +//go:build testenvci + +package citest + +import ( + "context" + "fmt" + "io" + "net/http" + "path/filepath" + "strings" + "testing" + "time" + + cosrv "devshard/chainoracle/server" + "devshard/testenv/citest/harness" + "devshard/testenv/config" + "devshard/testenv/mockopenai" + + "github.com/stretchr/testify/require" +) + +type versiondRollingTestStack struct { + stack *harness.Stack + cfg *config.File + eps harness.Endpoints + oldVersion cosrv.Version + newVersion cosrv.Version + hosts []string +} + +// TestVersiondRollingUpdateSameVersionSHA verifies that a same-name sha +// change through the real oracle path keeps already accepted work alive while +// versiond swaps traffic to the newly downloaded devshardd archive. +func TestVersiondRollingUpdateSameVersionSHA(t *testing.T) { + harness.SkipUnlessEnv(t, "TESTENV_CITEST") + harness.RequireDocker(t) + + for targetHostIndex := 0; targetHostIndex < 2; targetHostIndex++ { + t.Run(fmt.Sprintf("host_%d", targetHostIndex), func(t *testing.T) { + env := bootVersiondRollingStack(t, "citest-versiond-rolling-*", true, func(stack *harness.Stack, cfg *config.File) { + harness.PatchRouterVersiondHosts(t, stack.ComposePath, cfg.Hosts[targetHostIndex].ID) + }) + client := harness.GatewayChatClient() + + delayMs := 750 + harness.PatchMockOpenAIFault(t, client, env.eps.MockOpenAIHTTP, mockopenai.FaultPatch{ + StreamChunkDelay: &delayMs, + }) + + targetHost := env.hosts[targetHostIndex] + exerciseVersiondRollingFlip(t, &env, client, targetHost, env.oldVersion, env.newVersion, targetHost) + }) + } +} + +func exerciseVersiondRollingFlip( + t *testing.T, + env *versiondRollingTestStack, + client *http.Client, + overlapHost string, + fromVersion cosrv.Version, + toVersion cosrv.Version, + label string, +) { + t.Helper() + + harness.Step(t, "starting long stream on %s before %s rolling update", overlapHost, label) + accepted, gatewayStreamResult := harness.StartGatewayChatCompletionStream(client, env.eps.GatewayHTTP, harness.TestenvAdminAPIKey, + harness.ChatCompletionRequest{ + Model: "test-model", + MaxTokens: 64, + Messages: []harness.ChatMessage{{ + Role: "user", + Content: label + " rolling update long stream", + }}, + }) + requireVersiondStreamStillRunning(t, accepted, gatewayStreamResult, "gateway stream") + + probeCtx, stopProbe := context.WithCancel(context.Background()) + probeErr := startRouterContinuityProbe(probeCtx, client, env.eps.RouterHTTP+"/"+env.cfg.Versiond.VersionName+"/healthz") + defer stopProbe() + + harness.Step(t, "publishing new archive sha through mock-dapi /versions") + harness.PatchTestenvVersions(t, client, env.eps.MockDapiHTTP, []cosrv.Version{toVersion}) + requireVersiondRollingOverlap(t, env.stack, []string{overlapHost}, env.cfg.Versiond.VersionName, + fromVersion.SHA256, toVersion.SHA256) + + harness.Step(t, "new traffic succeeds after route swap") + after := harness.PostGatewayChatCompletion(t, client, env.eps.GatewayHTTP, harness.TestenvAdminAPIKey, + harness.ChatCompletionRequest{ + Model: "test-model", + MaxTokens: 64, + Messages: []harness.ChatMessage{{ + Role: "user", + Content: label + " after rolling update", + }}, + }) + harness.RequireMockOpenAIContent(t, after.Choices[0].Message.Content) + + harness.Step(t, "old in-flight stream completes cleanly") + res := <-gatewayStreamResult + require.NoError(t, res.Err) + require.Equal(t, http.StatusOK, res.Status, "stream body: %s", res.Body) + require.True(t, res.SawDone, "stream missing [DONE]") + harness.RequireMockOpenAIContent(t, res.Content) + stopProbe() + select { + case err := <-probeErr: + require.NoError(t, err) + case <-time.After(5 * time.Second): + t.Fatal("continuity probe did not stop") + } + + requireNoOldDraining(t, env.stack, env.hosts, env.cfg.Versiond.VersionName, fromVersion.SHA256) +} + +// TestVersiondRollingUpdateHybridFallback verifies that the same sha-flip +// does not overlap old and new children when devshardd storage is not Postgres. +func TestVersiondRollingUpdateHybridFallback(t *testing.T) { + harness.SkipUnlessEnv(t, "TESTENV_CITEST") + harness.RequireDocker(t) + + env := bootVersiondRollingStack(t, "citest-versiond-rolling-hybrid-*", false, func(stack *harness.Stack, _ *config.File) { + harness.PatchVersiondStorageMode(t, stack.ComposePath, "hybrid") + }) + client := harness.GatewayChatClient() + + harness.Step(t, "publishing new archive sha while storage mode is hybrid") + harness.PatchTestenvVersions(t, client, env.eps.MockDapiHTTP, []cosrv.Version{env.newVersion}) + requireVersiondFallbackWithoutOverlap(t, env.stack, env.hosts, env.cfg.Versiond.VersionName, + env.oldVersion.SHA256, env.newVersion.SHA256) +} + +func bootVersiondRollingStack(t *testing.T, namePattern string, requireRouterChild bool, patchCompose func(*harness.Stack, *config.File)) versiondRollingTestStack { + t.Helper() + + harness.Step(t, "starting stack with versiond download path (no local override)") + stack := harness.NewStack(t, namePattern) + harness.RequireLinuxDevshardd(t, stack.TestenvDir) + harness.WriteStackConfig(t, stack.WorkDir) + stack.RunGencompose(t) + cfg := stack.LoadConfig(t) + require.Len(t, cfg.Hosts, 2) + + versionName := cfg.Versiond.VersionName + devsharddPath := filepath.Join(stack.TestenvDir, "..", "..", "build", "devshardd") + binDir := filepath.Join(stack.WorkDir, "binaries") + oldFile := "devshardd-old.zip" + newFile := "devshardd-new.zip" + oldSHA := harness.WriteDevsharddZip(t, devsharddPath, filepath.Join(binDir, oldFile), "rolling-old") + newSHA := harness.WriteDevsharddZip(t, devsharddPath, filepath.Join(binDir, newFile), "rolling-new") + oldVersion := cosrv.Version{ + Name: versionName, + Binary: harness.VersiondBinaryURL(cfg.MockDapi.HTTPPort, oldFile), + SHA256: oldSHA, + } + newVersion := cosrv.Version{ + Name: versionName, + Binary: harness.VersiondBinaryURL(cfg.MockDapi.HTTPPort, newFile), + SHA256: newSHA, + } + + overrideKey := "VERSIOND_OVERRIDE_" + strings.ReplaceAll(versionName, ".", "_") + harness.PatchComposeRemoveEnvKey(t, stack.ComposePath, overrideKey) + harness.PatchComposeRemoveEnvKey(t, stack.ComposePath, "VERSIOND_FORCE") + harness.PatchComposeInsertEnvAfter(t, stack.ComposePath, "MOCK_DAPI_BINARY_DIR", + fmt.Sprintf(`MOCK_DAPI_VERSION_NAME: "%s"`, oldVersion.Name), + fmt.Sprintf(`MOCK_DAPI_VERSION_BINARY: "%s"`, oldVersion.Binary), + fmt.Sprintf(`MOCK_DAPI_VERSION_SHA256: "%s"`, oldVersion.SHA256), + ) + if patchCompose != nil { + patchCompose(stack, cfg) + } + + stack.Up(t) + eps := stack.Endpoints(t, cfg) + client := harness.GatewayChatClient() + harness.WaitStackHealthy(t, stack, eps) + harness.WaitGatewayChatReady(t, client, eps.GatewayHTTP, 3*time.Minute, stack) + if requireRouterChild { + harness.WaitGETOK(t, client, eps.RouterHTTP+"/"+versionName+"/healthz", 5*time.Minute, + "initial devshardd health via router", stack) + } + + hosts := []string{cfg.Hosts[0].ID, cfg.Hosts[1].ID} + requireAllVersiondRunningSHA(t, stack, hosts, versionName, oldSHA) + + return versiondRollingTestStack{ + stack: stack, + cfg: cfg, + eps: eps, + oldVersion: oldVersion, + newVersion: newVersion, + hosts: hosts, + } +} + +func requireVersiondStreamStillRunning(t *testing.T, accepted <-chan struct{}, result <-chan harness.GatewayStreamResult, label string) { + t.Helper() + select { + case <-accepted: + case <-time.After(30 * time.Second): + t.Fatalf("%s was not accepted before rolling update", label) + } + select { + case res := <-result: + require.NoError(t, res.Err) + require.Equal(t, http.StatusOK, res.Status, "%s ended before rolling update: %s", label, res.Body) + t.Fatalf("%s ended before rolling update", label) + default: + } +} + +func requireAllVersiondRunningSHA(t *testing.T, stack *harness.Stack, hosts []string, versionName, sha string) { + t.Helper() + ok := harness.AssertEventually(t, 2*time.Minute, time.Second, func() bool { + for _, host := range hosts { + entries, err := harness.TryVersiondHealth(stack, host) + if err != nil || !harness.HasVersiondHealthEntry(entries, versionName, "running", sha) { + return false + } + } + return true + }) + require.True(t, ok, "versiond hosts did not converge to running sha %s", sha) +} + +func requireVersiondRollingOverlap(t *testing.T, stack *harness.Stack, hosts []string, versionName, oldSHA, newSHA string) { + t.Helper() + sawOverlap := make(map[string]bool, len(hosts)) + ok := harness.AssertEventually(t, 2*time.Minute, 100*time.Millisecond, func() bool { + allNewRunning := true + allSawOverlap := true + for _, host := range hosts { + entries, err := harness.TryVersiondHealth(stack, host) + if err != nil { + allNewRunning = false + continue + } + if harness.HasVersiondHealthEntry(entries, versionName, "running", newSHA) && + harness.HasVersiondHealthEntry(entries, versionName, "draining", oldSHA) { + sawOverlap[host] = true + } + if !harness.HasVersiondHealthEntry(entries, versionName, "running", newSHA) { + allNewRunning = false + } + if !sawOverlap[host] { + allSawOverlap = false + } + } + return allSawOverlap && allNewRunning + }) + require.True(t, ok, "did not observe per-host running new sha %s with draining old sha %s", newSHA, oldSHA) +} + +func requireVersiondFallbackWithoutOverlap(t *testing.T, stack *harness.Stack, hosts []string, versionName, oldSHA, newSHA string) { + t.Helper() + ok := harness.AssertEventually(t, 2*time.Minute, 100*time.Millisecond, func() bool { + allNewRunning := true + for _, host := range hosts { + entries, err := harness.TryVersiondHealth(stack, host) + if err != nil { + allNewRunning = false + continue + } + require.False(t, harness.HasVersiondHealthEntry(entries, versionName, "draining", oldSHA), + "hybrid fallback unexpectedly drained old sha %s on host %s", oldSHA, host) + if !harness.HasVersiondHealthEntry(entries, versionName, "running", newSHA) { + allNewRunning = false + } + } + return allNewRunning + }) + require.True(t, ok, "versiond hosts did not converge to fallback running sha %s", newSHA) +} + +func requireNoOldDraining(t *testing.T, stack *harness.Stack, hosts []string, versionName, oldSHA string) { + t.Helper() + ok := harness.AssertEventually(t, 2*time.Minute, time.Second, func() bool { + for _, host := range hosts { + entries, err := harness.TryVersiondHealth(stack, host) + if err != nil || harness.HasVersiondHealthEntry(entries, versionName, "draining", oldSHA) { + return false + } + } + return true + }) + require.True(t, ok, "old sha %s still draining", oldSHA) +} + +func startRouterContinuityProbe(ctx context.Context, client *http.Client, url string) <-chan error { + errCh := make(chan error, 1) + go func() { + ticker := time.NewTicker(300 * time.Millisecond) + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + errCh <- nil + return + case <-ticker.C: + resp, err := client.Get(url) + if err != nil { + errCh <- err + return + } + _, _ = io.Copy(io.Discard, resp.Body) + _ = resp.Body.Close() + if resp.StatusCode != http.StatusOK { + errCh <- fmt.Errorf("continuity probe %s: HTTP %d", url, resp.StatusCode) + return + } + } + } + }() + return errCh +} diff --git a/devshard/testenv/cmd/gencompose/compose.go b/devshard/testenv/cmd/gencompose/compose.go index c290de1cfe..c1cb4b49ee 100644 --- a/devshard/testenv/cmd/gencompose/compose.go +++ b/devshard/testenv/cmd/gencompose/compose.go @@ -66,6 +66,9 @@ services: MOCK_CHAIN_TESTENV_URL: "http://{{ .MockChain.Host }}:{{ .MockChain.TestenvPort }}" MOCK_ML_ENDPOINT: "http://{{ .MockOpenAI.Host }}:{{ .MockOpenAI.HTTPPort }}" CHAIN_ID: "{{ .ChainID }}" + MOCK_DAPI_BINARY_DIR: /testenv-binaries + volumes: + - ./binaries:/testenv-binaries:ro ports: - "{{ .MockDapi.GRPCPort }}:{{ .MockDapi.GRPCPort }}" - "{{ .MockDapi.HTTPPort }}:{{ .MockDapi.HTTPPort }}" diff --git a/devshard/testenv/cmd/mockdapi/main.go b/devshard/testenv/cmd/mockdapi/main.go index 8c589edad3..967706f860 100644 --- a/devshard/testenv/cmd/mockdapi/main.go +++ b/devshard/testenv/cmd/mockdapi/main.go @@ -8,6 +8,7 @@ import ( "syscall" "time" + cosrv "devshard/chainoracle/server" "devshard/testenv/mockdapi" ) @@ -20,6 +21,10 @@ func main() { cfg.ChainTestenvURL = os.Getenv("MOCK_CHAIN_TESTENV_URL") cfg.MLEndpoint = envOr("MOCK_ML_ENDPOINT", "http://mock-openai:8088") cfg.ChainID = envOr("CHAIN_ID", cfg.ChainID) + cfg.BinaryDir = os.Getenv("MOCK_DAPI_BINARY_DIR") + if v := versionFromEnv(); v.Name != "" { + cfg.Versions = []cosrv.Version{v} + } if v := os.Getenv("MOCK_DAPI_CHAIN_POLL_INTERVAL"); v != "" { if d, err := time.ParseDuration(v); err == nil { cfg.ChainPollInterval = d @@ -52,3 +57,13 @@ func envOr(key, def string) string { } return def } + +func versionFromEnv() cosrv.Version { + name := os.Getenv("MOCK_DAPI_VERSION_NAME") + binary := os.Getenv("MOCK_DAPI_VERSION_BINARY") + sha := os.Getenv("MOCK_DAPI_VERSION_SHA256") + if name == "" || binary == "" || sha == "" { + return cosrv.Version{} + } + return cosrv.Version{Name: name, Binary: binary, SHA256: sha} +} diff --git a/devshard/testenv/docs/chain-transport-consolidation.md b/devshard/testenv/docs/chain-transport-consolidation.md index d4078eea41..37f6891320 100644 --- a/devshard/testenv/docs/chain-transport-consolidation.md +++ b/devshard/testenv/docs/chain-transport-consolidation.md @@ -16,7 +16,7 @@ gRPC — the same transport devshardd already uses. | Doc | Role | | ----------------------------------------------------------- | --------------------------------------------------------- | | `[phase12-followup.md](./phase12-followup.md)` | Phase 12 index | -| `[scenarios.md](./scenarios.md)` | Citest scenarios (S1–S6 shipped; **G1–G4** planned below) | +| `[scenarios.md](./scenarios.md)` | Named stack scenarios shipped; **G1–G4** planned below | | `[testenv-v2-plan.md](./testenv-v2-plan.md)` § Phase 2b, 3c | Original transport decisions | | `devshard/docs/merge-plan.md` | “Cosmos chain clients → common/” | @@ -153,7 +153,7 @@ Status key: ⬜ not started · 🟡 in progress · ✅ done | ------ | ------------------ | ------------------------------------------------------------------------------ | ---------------------------------------------- | ------ | | **G1** | gRPC escrow create | Gateway creates escrow via gRPC tx only; visible on gRPC query | `TestG1_GatewayEscrowCreateGRPC` | ✅ | | **G2** | gRPC escrow read | Gateway reads escrow via gRPC bridge (no LCD) | `TestG2_GatewayEscrowReadGRPC` | ✅ | -| **G3** | S5 without LCD | Full chat path with compose **without** `DEVSHARD_CHAIN_REST` / mock REST port | `TestG3_GatewayChatGRPCOnly` | ⬜ | +| **G3** | Chat without LCD | Full chat path with compose **without** `DEVSHARD_CHAIN_REST` / mock REST port | `TestG3_GatewayChatGRPCOnly` | ⬜ | | **G4** | REST removed gate | CI grep: no `NewRESTBridge` / `RESTChainTxClient` in devshardctl | `TestG4_NoRESTChainClientsInGatewayProduction` | ✅ | @@ -166,7 +166,7 @@ citest-grpc-transport: ## G1–G3 gRPC-only gateway citest TESTENV_CITEST=1 go test -tags=testenvci ./citest/ -run 'TestG1_|TestG2_|TestG3_' -count=1 -v -timeout 30m ``` -Fold **G3** into `make citest-stack` once green (S5 becomes gRPC-only). +Fold **G3** into `make citest-stack` once `TestGatewayChat` is gRPC-only. --- @@ -180,7 +180,8 @@ PR-4 G1–G3 citest + compose/settings cleanup (C4, E1–E4, F) PR-5 delete REST code + optional mock-chain 3c removal (C3, D6, E5) ``` -Each PR must keep **existing S1–S6** green until PR-4 switches S5 to gRPC-only. +Each PR must keep the existing stack behavior suite green until PR-4 switches +gateway chat to gRPC-only. --- @@ -219,9 +220,9 @@ Do **not** force gateway onto file keyring in v1 — adapter keeps gateway deplo - [ ] `devshardctl` has **zero** production chain `http.Client` calls (excluding `DEVSHARD_PUBLIC_API` → mock-dapi). - [ ] `grep -r 'NewRESTBridge\|RESTChainTxClient\|DEVSHARD_CHAIN_REST' devshard/cmd/devshardctl` → empty. - [ ] gencompose devshardctl service: `DEVSHARD_CHAIN_GRPC` only (no REST chain env). -- [ ] **G1, G2, G3** citest green; **S5** runs as part of G3 / `citest-stack` without LCD. +- [ ] **G1, G2, G3** citest green; gateway chat runs through G3 / `citest-stack` without LCD. - [ ] `make -C devshard ci-testenv-unit` and `ci-testenv-integration` green. -- [ ] `[scenarios.md](./scenarios.md)` updated: G1–G3 marked ✅, S5 note no longer mentions LCD. +- [ ] `[scenarios.md](./scenarios.md)` updated: G1–G3 marked ✅ and gateway chat no longer mentions LCD. --- @@ -233,7 +234,7 @@ Do **not** force gateway onto file keyring in v1 — adapter keeps gateway deplo | mock-chain gRPC tx parity with REST 3c | B4 event emission; port existing `restface` tx tests to gRPC | | Gateway signer ≠ keyring | Signer interface in `common/chain/tx` (see above) | | deploy/join still documents REST | E4 doc pass; coordinate with ops before E5 | -| S5 regression during migration | Keep LCD until G3 passes; dual-path one release optional | +| Gateway chat regression during migration | Keep LCD until G3 passes; dual-path one release optional | --- @@ -246,4 +247,3 @@ Do **not** force gateway onto file keyring in v1 — adapter keeps gateway deplo | 2026-06-24 | — | Track C + D: gateway gRPC tx/query, G2/G4 tests, gencompose REST env removed | | 2026-06-24 | — | Track A + B: `common/chain/tx`, mock-chain gRPC auth/tx, G1 citest | - diff --git a/devshard/testenv/docs/phase12-followup.md b/devshard/testenv/docs/phase12-followup.md index 000bffa068..12aa55feea 100644 --- a/devshard/testenv/docs/phase12-followup.md +++ b/devshard/testenv/docs/phase12-followup.md @@ -32,7 +32,7 @@ gRPC (same as devshardd). Testenv scenarios **G1–G3**; see [`scenarios.md`](./ ```bash make -C devshard/testenv citest-grpc-transport # G1–G3 -make -C devshard/testenv citest-stack # S1–S6 regression +make -C devshard/testenv citest-stack # stack behavior regression ``` ## Remaining (separate PRs) diff --git a/devshard/testenv/docs/scenarios.md b/devshard/testenv/docs/scenarios.md index 736be424b4..998f3f9660 100644 --- a/devshard/testenv/docs/scenarios.md +++ b/devshard/testenv/docs/scenarios.md @@ -1,4 +1,4 @@ -# Stack citest scenarios (S1–S8) +# Stack citest scenarios Implemented Go integration tests for the devshard testenv v2 stack. Each scenario boots a real Docker Compose stack (mock-chain, mock-dapi, mock-openai, versiond × 2, @@ -18,9 +18,10 @@ versiond-router, devshardctl, Postgres) and asserts production-like behaviour en | **devshardctl** | Gateway (`/v1/chat/completions`, `/v1/status`) | | **devshard-postgres** | Shared payload store (required for 2× versiond) | -Citest uses an **isolated config** (subnet `172.31.0.0/24`, router `:18080`, gateway -`:18081`, mock ports `19xxx`) so tests can run while a dev `make up` stack is active on -default ports. +Citest uses an **isolated config** (subnet `172.31.0.0/24`) and lets Docker assign +localhost host ports for router, gateway, and mock services. The harness discovers the +actual ports with `docker compose port`, so tests can run while a dev `make up` stack is +active on default ports. Harness: `citest/harness/` — temp workdir, `gencompose`, `docker compose up --wait`, HTTP/gRPC helpers, log dump on failure. @@ -32,21 +33,22 @@ HTTP/gRPC helpers, log dump on failure. ```bash cd devshard/testenv make build-devshardd -make citest-stack # S1–S9 -make citest-s9 # validation lease race only +make citest-stack # all core stack behavior tests +make citest-validation-lease-race # validation lease race only +make citest-versiond-rolling-update ``` Or run a single scenario: ```bash -TESTENV_CITEST=1 go test -tags=testenvci ./citest/ -run TestS3_ParamsLongPoll -v -timeout 30m +TESTENV_CITEST=1 go test -tags=testenvci ./citest/ -run TestParamsLongPoll -v -timeout 30m ``` | Variable / tag | Purpose | |----------------|---------| | `TESTENV_CITEST=1` | Opt-in gate (`harness.SkipUnlessEnv`) | -| `-tags=testenvci` | Build tag on `s*_*.go` tests | -| `make citest-stack` | Builds mock images + runs all S1–S8 | +| `-tags=testenvci` | Build tag on full-stack citests | +| `make citest-stack` | Builds mock images and runs all core stack behavior tests | Wrapper script: [`scripts/run-stack-citest.sh`](../scripts/run-stack-citest.sh). @@ -54,19 +56,23 @@ CI: `workflow_dispatch` with `integration: true`, or PR comment `/run-testenv` ( ## Scenario index -| ID | Name | What we validate | Test | -|----|------|------------------|------| -| **S1** | Stack smoke | Full stack boots; all boundaries healthy | `TestS1_StackSmoke` | -| **S2** | Router stickiness | Same session → same versiond upstream | `TestS2_RouterStickiness` | -| **S3** | Params long-poll | Governance patch wakes `GetRuntimeConfig` | `TestS3_ParamsLongPoll` | -| **S4** | Epoch switch | Epoch advance fast-forwards chain + bumps epoch in long-poll | `TestS4_EpochSwitch` | -| **S5** | Gateway chat | devshardctl → router → devshardd → mock-openai (stream + non-stream) | `TestS5_GatewayChat` | -| **S6** | versiond fault & restart | Stop → first-502 failover to survivor; restart with session persistence | `TestS6_VersiondStop`, `TestS6_VersiondRestartPersistence` | -| **S7** | Legacy version pin | Non-HA path → `VERSIOND_LEGACY_HOST` only; HA path still multi-upstream | `TestS7_LegacyVersionPinnedToSingleHost` | -| **S8** | SQLite → HA-fail → PG migrate | §3.3 Phases 0–4: sqlite single-host, multi-host 503, postgres migrate, HA OK | `TestS8_SqliteHaFailMigrate` | -| **S9** | Validation lease race | 3-host HA pair + solo executor; 100% `validation_rate`; Postgres lease exclusivity PASS/FAIL; §7a/§7b | `TestS9_ValidationLeaseRaceCore`, `…PendingStretch`, `…StaleReclaim` | - -Source: `devshard/testenv/citest/s{1..6}_*.go` (S6 spans `s6_versiond_stop_test.go` and `s6_versiond_restart_test.go`). +| Scenario | What we validate | Test | +|----------|------------------|------| +| **Stack smoke** | Full stack boots; all boundaries healthy | `TestStackSmoke` | +| **Router stickiness** | Same session → same versiond upstream | `TestRouterStickiness` | +| **Params long-poll** | Governance patch wakes `GetRuntimeConfig` | `TestParamsLongPoll` | +| **Epoch switch** | Epoch advance fast-forwards chain + bumps epoch in long-poll | `TestEpochSwitch` | +| **Gateway chat** | devshardctl → router → devshardd → mock-openai (stream + non-stream) | `TestGatewayChat` | +| **Versiond failover** | Stop → first-502 failover to survivor | `TestVersiondStickySessionFailover` | +| **Versiond restart persistence** | Restart preserves the gateway session | `TestVersiondRestartSessionPersistence` | +| **Legacy version pin** | Non-HA path → `VERSIOND_LEGACY_HOST`; HA path remains multi-upstream | `TestLegacyVersionPinnedToSingleHost` | +| **SQLite to Postgres HA migration** | SQLite single-host, multi-host rejection, migration, HA recovery | `TestSQLiteToPostgresHAMigration` | +| **Validation lease race** | Same-key HA lease exclusivity, pending stretch, stale reclaim | `TestValidationLeaseRaceCore`, `…PendingStretch`, `…StaleReclaim` | +| **Versiond rolling update** | Postgres blue/green drain and hybrid fallback | `TestVersiondRollingUpdateSameVersionSHA`, `…HybridFallback` | + +Source files under `devshard/testenv/citest/` use the same behavior-oriented +names. Versiond failover and restart persistence intentionally remain separate +test files because they exercise different lifecycle contracts. ### Phase 12 transport scenarios (gRPC-only gateway) @@ -76,22 +82,22 @@ Full plan: [`chain-transport-consolidation.md`](./chain-transport-consolidation. |----|------|------------------|------|--------| | **G1** | gRPC escrow create | devshardctl creates escrow via `common/chain/tx` + mock-chain gRPC; escrow visible on gRPC `DevshardEscrow` query | `TestG1_GatewayEscrowCreateGRPC` | ✅ | | **G2** | gRPC escrow read | Gateway reads escrow fields via gRPC bridge (no `RESTBridge` / LCD) | `TestG2_GatewayEscrowReadGRPC` | ✅ | -| **G3** | Chat without LCD | Same as S5 but compose omits `DEVSHARD_CHAIN_REST` and `DEVSHARD_TX_QUERY_REST` for gateway | `TestG3_GatewayChatGRPCOnly` | ✅ | +| **G3** | Chat without LCD | Gateway chat with compose omitting `DEVSHARD_CHAIN_REST` and `DEVSHARD_TX_QUERY_REST` | `TestG3_GatewayChatGRPCOnly` | ✅ | | **G4** | REST removed gate | Static test: no `NewRESTBridge` / `RESTChainTxClient` in devshardctl | `TestG4_NoRESTChainClientsInGatewayProduction` | ✅ | Run: `make citest-grpc-transport` from `devshard/testenv/`. --- -## S1 — Stack smoke +## Stack smoke **What we test:** The generated multi-versiond compose comes up cleanly and every service boundary responds. **How:** -1. `harness.BootS1Stack` — writes 2-host citest config, runs `gencompose`, `docker compose up --wait`. -2. `harness.WaitS1Healthy` — polls: +1. `harness.BootStack` — writes 2-host citest config, runs `gencompose`, `docker compose up --wait`. +2. `harness.WaitStackHealthy` — polls: - mock-chain RPC `/health` - mock-dapi `/healthz` and `/v1/epochs/latest` - versiond-router `/healthz` @@ -105,14 +111,14 @@ devshardd children started under versiond (protocol `v2`, chain dial to mock-cha --- -## S2 — Router stickiness +## Router stickiness **What we test:** versiond-router **consistent-hash** pins a session id to one upstream versiond across repeated requests, and at least two distinct upstreams are reachable. **How:** -1. Boot S1 stack; wait for router `/healthz`. +1. Boot the standard stack; wait for router `/healthz`. 2. Hit `/{version}/sessions/{sessionA}/healthz` **8 times**; read `X-Upstream-Addr` header (exposed by router nginx template). 3. Assert every retry returns the **same** upstream address. @@ -123,7 +129,7 @@ Validates deploy/join-style sticky routing before chat or long-poll scenarios de --- -## S7 — Legacy version pinned to single host +## Legacy version pinned to single host **What we test:** versiond-router sends version prefixes listed in `VERSIOND_NON_HA_VERSIONS` only to `VERSIOND_LEGACY_HOST` (`versiond_legacy`), @@ -132,11 +138,12 @@ while other versions sticky-hash across `VERSIOND_HOSTS` (and get **How:** -1. Boot S1 stack (`VERSIOND_NON_HA_VERSIONS=v1`; legacy host `versiond-0`). +1. Boot the standard stack (`VERSIOND_NON_HA_VERSIONS=v1`; legacy host + `versiond-0`). 2. Probe `/v1/sessions//healthz` for 16 distinct session ids. Assert every response has `X-Versiond-Backend: versiond_legacy` and the same `X-Upstream-Addr` mapped to `versiond-0`. -3. Reuse S2-style probes on `VersionName` (e.g. `v2`); require ≥2 distinct +3. Reuse router-stickiness probes on `VersionName` (e.g. `v2`); require ≥2 distinct upstreams and `X-Versiond-Backend: versiond_ha_pool`. 4. Stop the non-legacy versiond; repeat legacy probes — still pinned to `versiond-0`. @@ -146,7 +153,7 @@ See `devshard/docs/pr-1366-deploy-test-plan.md` §3.2. --- -## S8 — SQLite → Devshard-Ha fail → Postgres migrate → HA +## SQLite → Devshard-Ha fail → Postgres migrate → HA **What we test:** full §3.3 walkthrough from `devshard/docs/pr-1366-deploy-test-plan.md` (Phases 0–4). @@ -168,11 +175,50 @@ See `devshard/docs/pr-1366-deploy-test-plan.md` §3.2. 6. **Phase 4:** Gateway chat OK; sticky fan-out across hosts; NON_HA unchanged. **Pass criteria:** Multi-host + sqlite is rejected; migrate preserves escrow -index; HA + postgres serves. Test: `TestS8_SqliteHaFailMigrate`. +index; HA + postgres serves. Test: `TestSQLiteToPostgresHAMigration`. --- -## S9 — Validation lease race (HA exclusivity) +## Versiond rolling update + +**What we test:** A governance-style `/versions` change that keeps the same +version name but changes archive `sha256` causes `versiond` to download the new +devshardd archive. In Postgres mode, `versiond` runs a blue/green swap and +drains the old child without dropping already accepted work. In hybrid mode, the +same change falls back to stop-then-start and must not overlap old and new +children. + +**How:** + +1. Boot the standard stack with `VERSIOND_OVERRIDE_` removed, so `versiond` + downloads devshardd from mock-dapi rather than copying a local override. +2. Serve two zip archives from mock-dapi `/testenv/binaries/*`; both contain the + real linux `devshardd`, but have different archive sha values. +3. Wait both versiond hosts to report old sha `running` in `/healthz`. +4. For each versiond host, boot a fresh stack with versiond-router pinned to + that host, slow mock-openai SSE chunks, start a streaming gateway chat, and + wait for the first content chunk from the old child. +5. `POST /testenv/versions` with the same version name and new archive sha. +6. Poll the pinned versiond host `/healthz`; require a moment where a new child + is `running` while the old sha is `draining`. The positive test repeats this + for both versiond hosts. +7. Send a new gateway chat and require success while the old stream is still + finishing; continue probing router health during the swap. +8. Require the original stream to finish with `[DONE]` and the old draining child + to disappear. + +**Pass criteria:** In Postgres mode, no router-health interruption occurs during +the swap; new traffic succeeds on the new child; the old stream completes; each +versiond host is exercised in a pinned subtest and shows the expected +`running(new sha)` + `draining(old sha)` overlap. In hybrid mode, both hosts +converge to the new sha without ever reporting an old draining child. + +Tests: `TestVersiondRollingUpdateSameVersionSHA` and +`TestVersiondRollingUpdateHybridFallback`. + +--- + +## Validation lease race (HA exclusivity) **What we test:** join-style same-`KEY_NAME` HA replicas under `validation_rate=10000` do not double-validate. Postgres @@ -188,15 +234,15 @@ never validated). **How:** -1. Boot S9 stack (`WriteS9Config` / `BootS9Stack`); seed chat; warm escrow on - all versionds. +1. Boot the validation lease race stack (`WriteValidationLeaseRaceConfig` / + `BootValidationLeaseRaceStack`); seed chat; warm escrow on all versionds. 2. **Core:** parallel lease monitor + chat load; require zero duplicate groups - and ≥5 lease rows (`TestS9_ValidationLeaseRaceCore`). + and ≥5 lease rows (`TestValidationLeaseRaceCore`). 3. **7a:** slow mock-openai; observe `pending ≥ 1`; restore ML; uniqueness PASS - (`TestS9_ValidationLeaseRacePendingStretch`). + (`TestValidationLeaseRacePendingStretch`). 4. **7b:** short `DEVSHARD_VALIDATION_LEASE_TTL`; slow then **pause ML (503)**; stop one HA replica; wait TTL; restore ML; submitted grows; uniqueness PASS - (`TestS9_ValidationLeaseRaceStaleReclaim`). + (`TestValidationLeaseRaceStaleReclaim`). **Manual scripts:** `scripts/lease-race-run.sh` (monitor + load + PASS/FAIL). @@ -205,14 +251,14 @@ optional paths prove pending visibility and stale reclaim after ML pause. --- -## S3 — Params long-poll +## Params long-poll **What we test:** Lane-C governance fields flow **mock-chain → mock-dapi → GetRuntimeConfig long-poll** the way production devshardd consumes `NODE_MANAGER_ADDR`. **How:** -1. Boot S1 stack; dial mock-dapi NodeManager gRPC. +1. Boot the standard stack; dial mock-dapi NodeManager gRPC. 2. Read baseline `GetRuntimeConfig` (`max_nonce`, `refusal_timeout`, `params_block_height`). 3. Start a **blocked long-poll** at the baseline height (`max_wait` ≈ 25s). 4. `POST /testenv/params` on mock-dapi (proxied to mock-chain) with patched @@ -225,14 +271,15 @@ patch. Exercises `common/runtimeconfig` server + client path without production --- -## S4 — Epoch switch +## Epoch switch **What we test:** Epoch transition on mock-chain (block fast-forward to `next_poc_start`, roll `next_poc_start` forward) propagates into **GetRuntimeConfig** (`current_epoch_id` bump). **How:** -1. Boot S1 stack; read mock-chain epoch snapshot (`epoch_index`, `next_poc_start`, block height). +1. Boot the standard stack; read mock-chain epoch snapshot (`epoch_index`, + `next_poc_start`, block height). 2. Baseline `GetRuntimeConfig` — record `current_epoch_id` and `params_block_height`. 3. Blocked long-poll at baseline height. 4. `POST /testenv/epoch` `{advance: true}` on mock-dapi. @@ -245,14 +292,14 @@ see new epoch. Covers CometBFT RPC face (3b) + params notification path used at --- -## S5 — Gateway chat +## Gateway chat **What we test:** Full **MVP+ chat path**: devshardctl creates/uses escrow, routes through sticky versiond-router to devshardd, which calls mock-openai — **non-stream and SSE stream**. **How:** -1. Boot S1 stack; wait gateway `/v1/status` and devshardd health via router +1. Boot the standard stack; wait gateway `/v1/status` and devshardd health via router `/{version}/healthz`. 2. `POST /v1/chat/completions` on devshardctl (pooled chat, `stream=false`) with test API key. 3. Assert HTTP 200 and mock-openai deterministic assistant content/role. @@ -266,23 +313,23 @@ create/settle uses mock-chain gRPC only (see **G3** in --- -## S6 — versiond fault & restart +## Versiond fault and restart -S6 covers two production-shaped versiond lifecycle paths: **upstream stop** (router fault -semantics) and **stop/start restart** (postgres-backed devshardd recovery with the same -gateway session). +These tests cover two production-shaped versiond lifecycle paths: **upstream +stop** (router failover) and **stop/start restart** (Postgres-backed devshardd +recovery with the same gateway session). -### S6.1 — versiond stop (fault) +### Versiond sticky-session failover **What we test:** Behaviour when a **sticky upstream versiond is stopped** — nginx reroutes on the first upstream **502** / connect failure (`proxy_next_upstream`) to a surviving peer; sessions already hashed to a live upstream keep working. -**Test:** `TestS6_VersiondStop` (`citest/s6_versiond_stop_test.go`) +**Test:** `TestVersiondStickySessionFailover` (`citest/versiond_failover_test.go`) **How:** -1. Boot S1 stack; `harness.FindDistinctStickySessions` — two session ids on **different** +1. Boot the standard stack; `harness.FindDistinctStickySessions` — two session ids on **different** upstreams (`X-Upstream-Addr`). 2. `docker compose stop` the host mapped to session A's upstream. 3. Retry session A's router URL — expect **non-gateway** response with survivor in @@ -293,17 +340,17 @@ surviving peer; sessions already hashed to a live upstream keep working. surviving session keeps working. Mid-stream SSE after StartConfirm is **not** spliced (client reconnects with a new request) — out of scope for this healthz probe. -### S6.2 — versiond restart persistence +### Versiond restart persistence **What we test:** The **versiond → devshardd → router → gateway** stack survives versiond restarts without losing the active escrow session or regressing nonce/state. `devshardctl` stays up; restarted devshardd children recover from Postgres. -**Test:** `TestS6_VersiondRestartPersistence` (`citest/s6_versiond_restart_test.go`) +**Test:** `TestVersiondRestartSessionPersistence` (`citest/versiond_restart_persistence_test.go`) **How:** -1. Boot S1 stack; wait gateway chat readiness and snapshot session via `/v1/status` + +1. Boot the standard stack; wait gateway chat readiness and snapshot session via `/v1/status` + `/v1/debug/state` (`harness.GetGatewaySessionSnapshot`). 2. Gateway chat #1 — assert session nonce advances (`RequireGatewaySessionAdvanced`). 3. `docker compose stop` + `start` **one** versiond host (`harness.RestartService`); @@ -320,7 +367,7 @@ persistence across the multi-host topology, not only mock-chain or gateway in-me --- -## Related tests (not S1–S8) +## Related test suites | Suite | Command | Scenarios | |-------|---------|-----------| @@ -331,23 +378,12 @@ persistence across the multi-host topology, not only mock-chain or gateway in-me See [`README.md`](../README.md) for adversarial and observability detail. -## Not yet implemented - -| ID | Scenario | Notes | -|----|----------|-------| -| **S7** | Same-name sha swap | Phase 13 — rolling update | -| **S8** | Router host drain | Phase 13 — upstream evacuation | - -Tracked in [`testenv-v2-plan.md`](./testenv-v2-plan.md) § Phase 13. - ---- - ## G1 — gRPC escrow create ✅ **What we test:** `common/chain/tx` creates a devshard escrow via mock-chain gRPC (`BroadcastTx` + `GetTx` + auth `Account` query) — no LCD for the tx path. -**How:** `TestG1_GatewayEscrowCreateGRPC` boots S1 stack, dials mock-chain gRPC, +**How:** `TestG1_GatewayEscrowCreateGRPC` boots the standard stack, dials mock-chain gRPC, calls `chaintx.CreateDevshardEscrow`, queries `DevshardEscrow` on gRPC. **Run:** `make citest-grpc-transport` (or `-run TestG1_`). @@ -364,9 +400,11 @@ calls `chaintx.CreateDevshardEscrow`, queries `DevshardEscrow` on gRPC. ## G3 — Gateway chat without LCD ✅ -**What we test:** S5-equivalent chat (non-stream + SSE) with gRPC-only gateway chain transport. +**What we test:** Gateway chat (non-stream + SSE) with gRPC-only chain transport. -**How:** `TestG3_GatewayChatGRPCOnly` — full S1 stack with `docker compose up --build`; compose gate asserts no `DEVSHARD_CHAIN_REST` / `DEVSHARD_TX_QUERY_REST` on devshardctl. +**How:** `TestG3_GatewayChatGRPCOnly` — full standard stack with +`docker compose up --build`; the compose gate asserts that devshardctl has no +`DEVSHARD_CHAIN_REST` or `DEVSHARD_TX_QUERY_REST`. **Pass criteria:** Non-stream + stream chat return 200. diff --git a/devshard/testenv/docs/testenv-v2-plan.md b/devshard/testenv/docs/testenv-v2-plan.md index e4e59a1989..8b4497cd32 100644 --- a/devshard/testenv/docs/testenv-v2-plan.md +++ b/devshard/testenv/docs/testenv-v2-plan.md @@ -482,13 +482,14 @@ shared in-memory store and **three thin protocol faces**. Do **not** run a real `inferenced` node in testenv. Implement **only endpoints callers actually use**; add more as edge-api / gateway scenarios need them. -**Why staged:** unblocks devshardd + chainoracle (S1–S4) before gateway tx work (S5). +**Why staged:** unblocks devshardd + chainoracle behavior tests before gateway +transaction work. | Stage | Face | Port | Unblocks | Scenarios | |-------|------|------|----------|-----------| -| **3a** | Cosmos gRPC + cmtservice | `:9090` | devshardd bridge, params fetch, chainoracle | S1–S4 | -| **3b** | CometBFT RPC (ws subscribe) | `:26657` | devshardd `events.Listener`, phase bootstrap | S1–S4 | -| **3c** | Cosmos LCD REST (tx) | `:1317` | devshardctl create/settle escrow | S5 | +| **3a** | Cosmos gRPC + cmtservice | `:9090` | devshardd bridge, params fetch, chainoracle | stack smoke, params, epoch | +| **3b** | CometBFT RPC (ws subscribe) | `:26657` | devshardd `events.Listener`, phase bootstrap | stack smoke, epoch | +| **3c** | Cosmos LCD REST (tx) | `:1317` | devshardctl create/settle escrow | gateway chat | **Layout** @@ -611,7 +612,7 @@ escrow-created handler work end-to-end with 3a gRPC. --- -#### Phase 3c — LCD REST face (`:1317`) — **third (gateway / S5)** ✅ **DONE** +#### Phase 3c — LCD REST face (`:1317`) — **third (gateway chat)** ✅ **DONE** **Goal:** `devshardctl` creates/settles escrows without code changes (`chain_tx_rest.go`). @@ -645,13 +646,13 @@ Env: `MOCK_CHAIN_REST_ADDR` (default `:1317`). cd devshard && go test ./testenv/mockchain/... ./cmd/devshardctl/ -run 'MockChain|REST' -count=1 ``` -**Exit (3c):** S5 gateway chat path can create an escrow in testenv; full three-face +**Exit (3c):** the gateway chat path can create an escrow in testenv; full three-face mock-chain ready for compose (Phase 6). --- **Phase 3 overall exit:** mock-chain container healthy; `docker compose up mock-chain` serves -9090/26657/1317; integration tests for 3a+3b green before 3c; S5 blocked only until 3c +9090/26657/1317; integration tests for 3a+3b green before 3c; gateway chat blocked only until 3c lands. **Explicitly rejected (D1):** Option B (real `inferenced` node), Option C (hybrid). Testenv @@ -923,23 +924,23 @@ TESTENV_GATEWAY_SMOKE=1 go test ./testenv/citest/ -run TestGatewayPhase7_Smoke - health polling, config skeletons). Build tag `testenvci` for Docker stack scenarios. 2. Initial scenarios: - | ID | Scenario | Asserts | Status | - |----|----------|---------|--------| - | S1 | Stack smoke | mock-chain + mock-dapi + router + 2 versiond + gateway healthy | ✅ **DONE** | - | S2 | Router stickiness | same escrow/session → same versiond backend across retries | ✅ **DONE** | - | S3 | Params long-poll | `/testenv/params` change → devshardd session bind uses new timeouts | ✅ **DONE** | - | S4 | Epoch switch | `/testenv/epoch` advance → fast-forward blocks to `next_poc_start`, runtimeparams epoch bump | ✅ **DONE** | - | S5 | Gateway chat | devshardctl → router → devshardd → `mock-openai` `/v1/chat/completions` → 200 (stream + non-stream) | ✅ **DONE** | - | S6 | Fault: versiond stop | router retries another instance or fails as designed | ✅ **DONE** | + | Scenario | Asserts | Status | + |----------|---------|--------| + | Stack smoke | mock-chain + mock-dapi + router + 2 versiond + gateway healthy | ✅ **DONE** | + | Router stickiness | same escrow/session → same versiond backend across retries | ✅ **DONE** | + | Params long-poll | `/testenv/params` change → devshardd session bind uses new timeouts | ✅ **DONE** | + | Epoch switch | `/testenv/epoch` advance → fast-forward blocks to `next_poc_start`, runtimeparams epoch bump | ✅ **DONE** | + | Gateway chat | devshardctl → router → devshardd → `mock-openai` `/v1/chat/completions` → 200 (stream + non-stream) | ✅ **DONE** | + | Versiond failover | router retries the surviving instance | ✅ **DONE** | - **S4 implementation:** `citest/s4_epoch_switch_test.go` posts `POST /testenv/epoch` + **Epoch switch implementation:** `citest/epoch_switch_test.go` posts `POST /testenv/epoch` `{advance:true}` on mock-dapi; mock-chain `rpcface.AdvanceEpoch` publishes CometBFT `NewBlock` for each height up to `next_poc_start`, commits epoch index+2 with updated `next_poc_start`, and mock-dapi `RefreshRuntimeConfig` wakes `GetRuntimeConfig` long-poll with higher `current_epoch_id`. Included in `make citest-stack` (rebuild `mock-chain` image when epoch admin changes). - **S5 implementation:** `citest/s5_gateway_chat_test.go` boots the S1 stack and posts + **Gateway chat implementation:** `citest/gateway_chat_test.go` boots the standard stack and posts pooled `POST /v1/chat/completions` on devshardctl for non-stream and SSE stream; asserts HTTP 200, non-empty assistant content with `mock-openai:` prefix, and `data: [DONE]` on the stream path. Uses admin API key (`TESTENV_ADMIN_API_KEY`), @@ -948,7 +949,7 @@ TESTENV_GATEWAY_SMOKE=1 go test ./testenv/citest/ -run TestGatewayPhase7_Smoke - query, router `/devshard/` rewrite, `NODE_RPC_URL`, internal router URL `:8080`. Included in `make citest-stack`. - **S6 implementation:** `citest/s6_versiond_stop_test.go` finds two session ids + **Versiond failover implementation:** `citest/versiond_failover_test.go` finds two session ids routed to different versiond upstreams (sticky hash), stops the compose service for one upstream, asserts the pinned session either gets 502/503 (unavailable peer) or re-hashes to the surviving upstream (consistent-hash ring shrink when Docker DNS drops @@ -956,32 +957,33 @@ TESTENV_GATEWAY_SMOKE=1 go test ./testenv/citest/ -run TestGatewayPhase7_Smoke - `X-Upstream-Addr`. Helpers in `citest/harness/router_sticky.go` and `Stack.StopService`. Included in `make citest-stack`. - **S3 implementation:** `citest/s3_params_longpoll_test.go` boots the S1 stack (devshardd + **Params long-poll implementation:** `citest/params_longpoll_test.go` boots the standard stack (devshardd long-polling mock-dapi via `NODE_MANAGER_ADDR`), blocks on `GetRuntimeConfig` at the current `params_block_height`, posts `POST /testenv/params` on mock-dapi (proxied to mock-chain admin), asserts the long-poll returns higher height plus updated `max_nonce` / timeouts, then verifies caught-up clients get immediate `unchanged` and stale-height clients receive the patched snapshot — the same `GetRuntimeConfig` lane-C feed devshardd consumes via - `NODE_MANAGER_ADDR` while the S1 stack is up. Included in `make citest-stack`. + `NODE_MANAGER_ADDR` while the standard stack is up. Included in `make citest-stack`. - **S2 implementation:** `citest/s2_router_stickiness_test.go` hits + **Router stickiness implementation:** `citest/router_stickiness_test.go` hits `//sessions//healthz` through versiond-router eight times and asserts identical `X-Upstream-Addr` (nginx `$upstream_addr`); probes additional session ids until a second upstream is observed. Router image exposes the header via `versiond-router/nginx.conf.template`. Included in `make citest-stack`. - **S1 implementation:** `citest/s1_stack_smoke_test.go` (`//go:build testenvci`) uses - `harness.WriteS1Config` (2× versiond, `mode: multi`, Postgres), `docker compose up --wait`, + **Stack smoke implementation:** `citest/stack_smoke_test.go` (`//go:build testenvci`) uses + `harness.WriteStackConfig` (2× versiond, `mode: multi`, Postgres), `docker compose up --wait`, then polls mock-chain RPC `/health`, mock-dapi `/healthz` + `/v1/epochs/latest`, versiond-router `/healthz`, gateway `/v1/status`, and `docker compose ps` for `versiond-0` + `versiond-1`. Run: `make citest-stack` or - `TESTENV_CITEST=1 go test -tags=testenvci ./citest/ -run TestS1_StackSmoke`. + `TESTENV_CITEST=1 go test -tags=testenvci ./citest/ -run TestStackSmoke`. 3. **Drop:** `*HeightSync*`, cheat-anchor, force-anchor, height-sync audit scenarios. -**Scripts:** `testenv/scripts/run-stack-citest.sh` (S1); `scripts/gen-integration-testenv-config.sh` (Phase 11). +**Scripts:** `testenv/scripts/run-stack-citest.sh`; +`scripts/gen-integration-testenv-config.sh` (Phase 11). -**Exit:** `make citest-stack` green locally with Docker (S1–S6 ✅). +**Exit:** `make citest-stack` green locally with Docker. --- @@ -1014,7 +1016,8 @@ Run: `make citest-adversarial` or ### Phase 10 — Observability overlay (optional) Port `testenv/observability/` compose fragment (Jaeger/Prometheus/Loki/Grafana — same -family as `deploy/join/docker-compose.observability.yml`). Lower priority than S1–S6. +family as `deploy/join/docker-compose.observability.yml`). Lower priority than +the core stack behavior suite. **Plan (backends, profiles, Alloy/Tempo roadmap):** [`observability-plan.md`](./observability-plan.md) @@ -1090,18 +1093,19 @@ citest harness, and rolling-update **Track A** (blue/green sha swap) at minimum. 2. **mock-dapi:** extend `/versions` + `/testenv/*` to bump **sha256 for same version name** (simulates governance publishing a new binary). 3. **Binaries:** mount two devshardd builds (or tagged overrides) for swap scenarios. -4. **New citest scenarios** (after S1–S6): +4. **New citest scenarios**: | ID | Scenario | Asserts | |----|----------|---------| - | **S7** | Same-name sha swap | Long request on old child completes; concurrent new request hits new child; no 404 during swap | - | **S8** | Router host drain | Escrow pinned to `versiond-N`; mark upstream down; in-flight completes; no new traffic to N | + | **Versiond rolling update** | Same-name sha swap | Long request on old child completes; concurrent new request hits new child; no 404 during swap | + | **future** | Router host drain | Escrow pinned to `versiond-N`; mark upstream down; in-flight completes; no new traffic to N | **Tests:** align with `rolling-update.md` §1.9 (versiond unit/e2e + devshardd readiness); Phase 13 adds **stack-level** Go citest on top. -**Exit:** `make citest-stack` includes S7 (+ S8 when Track B lands); rolling-update semantics -validated before deploy/join. +**Exit:** `make citest-stack` includes `TestVersiondRollingUpdate*` (+ router +drain when Track B lands); rolling-update semantics are validated before +deploy/join. --- @@ -1122,9 +1126,9 @@ wired in Phase 11 (`.github/workflows/devshard-testenv.yml`); integration on dis | mock-dapi (Phase 5) ✅ | `go test ./testenv/mockdapi/... -count=1` | | mock-openai (Phase 5b) ✅ | `go test ./testenv/mockopenai/... -count=1` | | mock-chain gRPC face (3a) ✅ | `go test ./devshard/testenv/mockchain/... -count=1` | -| mock-chain CometBFT RPC / devshardd events (3b) ✅ | `go test ./devshard/testenv/mockchain/rpcface/... -count=1`; `make citest-stack` S1 | -| mock-chain LCD REST / gateway escrow tx (3c) ✅ | `go test ./devshard/testenv/mockchain/restface/... -count=1`; `make citest-stack` S5 | -| stack citest (Phase 8) ✅ | `make citest-stack` (S1–S6) | +| mock-chain CometBFT RPC / devshardd events (3b) ✅ | `go test ./devshard/testenv/mockchain/rpcface/... -count=1`; `make citest-stack` | +| mock-chain LCD REST / gateway escrow tx (3c) ✅ | `go test ./devshard/testenv/mockchain/restface/... -count=1`; `TestGatewayChat` | +| stack citest (Phase 8) ✅ | `make citest-stack` | | adversarial citest (Phase 9) ✅ | `make citest-adversarial` (A1–A4) | | observability smoke (Phase 10) ✅ | `make citest-observability` (O1) | | common runtimeconfig client (Phase 12) ✅ | `go test ./common/runtimeconfig/client/... -count=1` | @@ -1165,21 +1169,22 @@ Phase 4 gencompose stub + dev overlay (keys, air, dlv) ✅ DONE Phase 5 mock-dapi + mock-openai (mounts chainoracle) ✅ DONE Phase 6 gencompose versiond×N + router + devshardctl + keyrings ✅ DONE Phase 7 devshardctl gateway in compose ✅ DONE -Phase 8 Go citest harness + S1–S6 (S1–S4 ✅ stack smoke + stickiness + params + epoch switch) +Phase 8 Go citest harness + named stack behavior tests ✅ DONE Phase 9 adversarial scenarios (A1–A4 ✅) Phase 10 observability overlay (optional) ✅ DONE Phase 11 CI / Makefile / runbook ✅ DONE Phase 12 client long-poll → common (remainder tracked) ✅ client DONE -Phase 13 rolling-update citest (S7/S8) needs P6,P8 + rolling-update.md Track A/B +Phase 13 rolling-update citest needs P6,P8 + rolling-update.md Track A/B ``` -- **MVP (devshardd-only):** Phases 1–2 + **3a+3b** + 5 + 6 + S1–S4 — stack up, long-poll +- **MVP (devshardd-only):** Phases 1–2 + **3a+3b** + 5 + 6 plus stack smoke, + stickiness, params, and epoch tests — stack up, long-poll works, router routes, devshardd sees escrows/events. No gateway tx yet. -- **MVP+ (full chat):** add **3c** + Phase 7 + S5 — gateway creates escrow over REST. +- **MVP+ (full chat):** add **3c** + Phase 7 + gateway chat — gateway creates escrow over REST. - **Dev loop (early):** ✅ **available now** — Phase **3 + 4** done: `make gen-compose && make dev-up`; mock-chain hot reload with pinned keys while building Phase 5+. -- **M2:** S2, S4–S6 + CI. -- **M3 (rolling update):** Phase 13 + S7 (+ S8 when router drain lands) — after +- **M2:** router stickiness, epoch, gateway, versiond lifecycle, and CI. +- **M3 (rolling update):** Phase 13 plus legacy routing and storage migration — after `rolling-update.md` Track A/B in production code. --- diff --git a/devshard/testenv/mockdapi/config.go b/devshard/testenv/mockdapi/config.go index d61dac1a08..91747c2388 100644 --- a/devshard/testenv/mockdapi/config.go +++ b/devshard/testenv/mockdapi/config.go @@ -18,6 +18,7 @@ type Config struct { BlockInterval time.Duration ChainID string Versions []cosrv.Version + BinaryDir string // BlockSeed seeds the mock block observer (deterministic headers). BlockSeed int64 // GatewayBlockHeight / GatewayEpochIndex feed devshardctl public-API stubs. diff --git a/devshard/testenv/mockdapi/service.go b/devshard/testenv/mockdapi/service.go index 61089b6896..a7dfecc739 100644 --- a/devshard/testenv/mockdapi/service.go +++ b/devshard/testenv/mockdapi/service.go @@ -7,15 +7,16 @@ import ( "log/slog" "net" "net/http" + "path/filepath" "time" "common/chain" "common/nodemanager/gen" commonruntimeconfig "common/runtimeconfig" - cosrv "devshard/chainoracle/server" "devshard/chainoracle/blocks" "devshard/chainoracle/blocks/observer" "devshard/chainoracle/params" + cosrv "devshard/chainoracle/server" "devshard/signing" "devshard/testenv/gatewayphase" "devshard/testenv/mockchain/adminface" @@ -34,6 +35,7 @@ type Service struct { runtimeFetcher fetcher.SnapshotFetcher blockMock *observer.Mock admin *adminface.Client + versions *versionStore grpcServer *grpc.Server httpEcho *echo.Echo } @@ -99,6 +101,7 @@ func New(ctx context.Context, cfg Config) (*Service, error) { runtimeFetcher: runtimeFetcher, blockMock: blockMock, admin: adminClient, + versions: newVersionStore(cfg.Versions), } return s, nil } @@ -204,14 +207,17 @@ func (s *Service) serveHTTPOn(ctx context.Context, lis net.Listener) error { e := echo.New() e.HideBanner = true cosrv.Mount(e.Group(""), cosrv.Config{ - Blocks: s.blockMock, - Versions: s.cfg.Versions, + Blocks: s.blockMock, + VersionProvider: s.versions, }) + if s.cfg.BinaryDir != "" { + mountBinaryFiles(e.Group(""), s.cfg.BinaryDir) + } gatewayphase.Mount(e.Group(""), gatewayphase.Config{ BlockHeight: s.cfg.GatewayBlockHeight, EpochIndex: s.cfg.GatewayEpochIndex, }) - mountTestenvProxy(e.Group(""), s.admin, s.RefreshRuntimeConfig) + mountTestenvProxy(e.Group(""), s.admin, s.RefreshRuntimeConfig, s.versions) s.httpEcho = e e.Server.BaseContext = func(net.Listener) context.Context { return ctx } go func() { @@ -227,6 +233,16 @@ func (s *Service) serveHTTPOn(ctx context.Context, lis net.Listener) error { return ctx.Err() } +func mountBinaryFiles(g *echo.Group, dir string) { + g.GET("/testenv/binaries/:name", func(c echo.Context) error { + name := filepath.Base(c.Param("name")) + if name == "." || name == ".." || name == string(filepath.Separator) || name == "" { + return echo.NewHTTPError(http.StatusBadRequest, "invalid binary name") + } + return c.File(filepath.Join(dir, name)) + }) +} + // ParamsSource exposes the cached params source for tests. func (s *Service) ParamsSource() *params.CachedSource { if s == nil { diff --git a/devshard/testenv/mockdapi/service_test.go b/devshard/testenv/mockdapi/service_test.go index 85221e9324..b3e5b0dcc8 100644 --- a/devshard/testenv/mockdapi/service_test.go +++ b/devshard/testenv/mockdapi/service_test.go @@ -164,6 +164,27 @@ func TestMockDAPI_VersionsJSON(t *testing.T) { require.Len(t, cfg.Versions, 1) require.Equal(t, "v2", cfg.Versions[0].Name) require.NotEmpty(t, cfg.Versions[0].Binary) + + updated := cosrv.VersionConfig{Versions: []cosrv.Version{{ + Name: "v2", + Binary: "http://mock-dapi:9100/testenv/binaries/devshardd-new.zip", + SHA256: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + }}} + body, err := json.Marshal(updated) + require.NoError(t, err) + postResp, err := http.Post(bed.httpURL+"/testenv/versions", "application/json", strings.NewReader(string(body))) + require.NoError(t, err) + require.Equal(t, http.StatusOK, postResp.StatusCode) + _ = postResp.Body.Close() + + resp, err = http.Get(bed.httpURL + "/versions") + require.NoError(t, err) + defer resp.Body.Close() + require.Equal(t, http.StatusOK, resp.StatusCode) + + cfg = cosrv.VersionConfig{} + require.NoError(t, json.NewDecoder(resp.Body).Decode(&cfg)) + require.Equal(t, updated, cfg) } func TestMockDAPI_BlockStreamMonotonic(t *testing.T) { diff --git a/devshard/testenv/mockdapi/testenv.go b/devshard/testenv/mockdapi/testenv.go index 1eb15d6cb0..0b33d8dd11 100644 --- a/devshard/testenv/mockdapi/testenv.go +++ b/devshard/testenv/mockdapi/testenv.go @@ -4,12 +4,13 @@ import ( "context" "net/http" + cosrv "devshard/chainoracle/server" "devshard/testenv/mockchain/adminface" "github.com/labstack/echo/v4" ) -func mountTestenvProxy(g *echo.Group, admin *adminface.Client, refresh func(context.Context) error) { +func mountTestenvProxy(g *echo.Group, admin *adminface.Client, refresh func(context.Context) error, versions *versionStore) { g.POST("/testenv/params", func(c echo.Context) error { if admin == nil || admin.BaseURL() == "" { return echo.NewHTTPError(http.StatusServiceUnavailable, "mock-chain testenv admin not configured") @@ -72,4 +73,25 @@ func mountTestenvProxy(g *echo.Group, admin *adminface.Client, refresh func(cont } return c.JSON(http.StatusOK, map[string]string{"status": "ok"}) }) + g.GET("/testenv/versions", func(c echo.Context) error { + if versions == nil { + return echo.NewHTTPError(http.StatusServiceUnavailable, "mock-dapi versions not configured") + } + current, err := versions.Versions(c.Request().Context()) + if err != nil { + return echo.NewHTTPError(http.StatusBadGateway, err.Error()) + } + return c.JSON(http.StatusOK, cosrv.VersionConfig{Versions: current}) + }) + g.POST("/testenv/versions", func(c echo.Context) error { + if versions == nil { + return echo.NewHTTPError(http.StatusServiceUnavailable, "mock-dapi versions not configured") + } + var req cosrv.VersionConfig + if err := c.Bind(&req); err != nil { + return echo.NewHTTPError(http.StatusBadRequest, err.Error()) + } + versions.Set(req.Versions) + return c.JSON(http.StatusOK, map[string]string{"status": "ok"}) + }) } diff --git a/devshard/testenv/mockdapi/versions.go b/devshard/testenv/mockdapi/versions.go new file mode 100644 index 0000000000..6bd409ba6a --- /dev/null +++ b/devshard/testenv/mockdapi/versions.go @@ -0,0 +1,38 @@ +package mockdapi + +import ( + "context" + "sync" + + cosrv "devshard/chainoracle/server" +) + +type versionStore struct { + mu sync.RWMutex + versions []cosrv.Version +} + +func newVersionStore(versions []cosrv.Version) *versionStore { + return &versionStore{versions: cloneVersions(versions)} +} + +func (s *versionStore) Versions(context.Context) ([]cosrv.Version, error) { + s.mu.RLock() + defer s.mu.RUnlock() + return cloneVersions(s.versions), nil +} + +func (s *versionStore) Set(versions []cosrv.Version) { + s.mu.Lock() + defer s.mu.Unlock() + s.versions = cloneVersions(versions) +} + +func cloneVersions(in []cosrv.Version) []cosrv.Version { + if len(in) == 0 { + return nil + } + out := make([]cosrv.Version, len(in)) + copy(out, in) + return out +} diff --git a/devshard/testenv/scripts/run-stack-citest.sh b/devshard/testenv/scripts/run-stack-citest.sh index cdf4a7fa41..ead71ba2c9 100755 --- a/devshard/testenv/scripts/run-stack-citest.sh +++ b/devshard/testenv/scripts/run-stack-citest.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# Run Phase 8 stack citest (S1–S4) from devshard/testenv. +# Run core stack behavior and versiond rolling-update tests. set -euo pipefail cd "$(dirname "$0")/.." export TESTENV_CITEST=1 diff --git a/local-test-net/Dockerfile.testapp-server b/local-test-net/Dockerfile.testapp-server index 7c43ab445b..3d85b72f88 100644 --- a/local-test-net/Dockerfile.testapp-server +++ b/local-test-net/Dockerfile.testapp-server @@ -8,16 +8,23 @@ COPY versioned/ . RUN CGO_ENABLED=0 GOOS=linux go build -o testapp ./e2e/testapp && \ CGO_ENABLED=0 GOOS=linux go build \ -ldflags "-X main.Version=testapp2 -X main.BinaryVersion=testapp2" \ - -o testapp2 ./e2e/testapp + -o testapp2 ./e2e/testapp && \ + CGO_ENABLED=0 GOOS=linux go build \ + -ldflags "-X main.Version=testapp -X main.BinaryVersion=testapp-rollout" \ + -o testapp-rollout ./e2e/testapp # versiond extracts VERSIOND_BINARY_NAME ("testapp") from every zip; the slot # identity comes from --print-protocol-version inside the binary, not the zip entry name. -RUN mkdir -p /srv/files && \ - zip -j /srv/files/testapp.zip testapp && \ +RUN mkdir -p /srv/files /tmp/testapp /tmp/testapp2 /tmp/testapp-rollout && \ + cp testapp /tmp/testapp/testapp && \ + cp testapp2 /tmp/testapp2/testapp && \ + cp testapp-rollout /tmp/testapp-rollout/testapp && \ + (cd /tmp/testapp && zip -q /srv/files/testapp.zip testapp) && \ + (cd /tmp/testapp2 && zip -q /srv/files/testapp2.zip testapp) && \ + (cd /tmp/testapp-rollout && zip -q /srv/files/testapp-rollout.zip testapp) && \ sha256sum /srv/files/testapp.zip | cut -d' ' -f1 > /srv/files/testapp.zip.sha256 && \ - cp testapp2 testapp && \ - zip -j /srv/files/testapp2.zip testapp && \ - sha256sum /srv/files/testapp2.zip | cut -d' ' -f1 > /srv/files/testapp2.zip.sha256 + sha256sum /srv/files/testapp2.zip | cut -d' ' -f1 > /srv/files/testapp2.zip.sha256 && \ + sha256sum /srv/files/testapp-rollout.zip | cut -d' ' -f1 > /srv/files/testapp-rollout.zip.sha256 FROM python:3.12-alpine3.21 COPY --from=builder /srv/files /srv/files diff --git a/testermint/src/main/kotlin/LocalInferencePair.kt b/testermint/src/main/kotlin/LocalInferencePair.kt index 798e6ebe31..e7c2bc21d5 100644 --- a/testermint/src/main/kotlin/LocalInferencePair.kt +++ b/testermint/src/main/kotlin/LocalInferencePair.kt @@ -452,31 +452,83 @@ data class LocalInferencePair( api.executor.exec(listOf("sh", "-c", "curl -sf '$url'"), null).joinToString("").trim() } - fun versiondBinaryPath(versionName: String, binaryName: String = "devshardd"): String = - "/opt/versiond/bin/$versionName/$binaryName" + private fun shQuote(value: String): String = "'" + value.replace("'", "'\"'\"'") + "'" - fun versiondInstallMetadataPath(versionName: String): String = - "/opt/versiond/bin/$versionName/install.json" - - fun versiondBinaryExists(versionName: String, binaryName: String = "devshardd"): Boolean = + private fun versiondInstallDir( + versionName: String, + binaryName: String = "devshardd", + archiveSha256: String? = null, + ): String? = try { - execInVersiond( - listOf("sh", "-c", "test -x '${versiondBinaryPath(versionName, binaryName)}' && echo OK"), - null, - ).any { it.contains("OK") } + val base = "/opt/versiond/bin/$versionName" + val script = """ + base=${shQuote(base)} + binary=${shQuote(binaryName)} + expected=${shQuote(archiveSha256 ?: "")} + if [ -n "${'$'}expected" ] && [ -x "${'$'}base/${'$'}expected/${'$'}binary" ]; then + echo "${'$'}base/${'$'}expected" + exit 0 + fi + if [ -x "${'$'}base/${'$'}binary" ]; then + echo "${'$'}base" + exit 0 + fi + best="" + best_mtime=-1 + for dir in "${'$'}base"/*; do + [ -d "${'$'}dir" ] || continue + [ -x "${'$'}dir/${'$'}binary" ] || continue + mtime=${'$'}(stat -c %Y "${'$'}dir/install.json" 2>/dev/null || echo 0) + if [ "${'$'}mtime" -gt "${'$'}best_mtime" ]; then + best="${'$'}dir" + best_mtime="${'$'}mtime" + fi + done + if [ -n "${'$'}best" ]; then + echo "${'$'}best" + exit 0 + fi + exit 1 + """.trimIndent() + execInVersiond(listOf("sh", "-c", script), null) + .firstOrNull() + ?.trim() + ?.takeIf { it.isNotBlank() } } catch (_: Exception) { - false + null } - fun readVersiondInstallMetadata(versionName: String): VersiondInstallMetadata? = + fun versiondBinaryPath( + versionName: String, + binaryName: String = "devshardd", + archiveSha256: String? = null, + ): String = + "${versiondInstallDir(versionName, binaryName, archiveSha256) ?: "/opt/versiond/bin/$versionName/"}" + + "/$binaryName" + + fun versiondInstallMetadataPath(versionName: String, archiveSha256: String? = null): String = + "${versiondInstallDir(versionName, archiveSha256 = archiveSha256) ?: "/opt/versiond/bin/$versionName/"}/install.json" + + fun versiondBinaryExists( + versionName: String, + binaryName: String = "devshardd", + archiveSha256: String? = null, + ): Boolean = + versiondInstallDir(versionName, binaryName, archiveSha256) != null + + fun readVersiondInstallMetadata(versionName: String, archiveSha256: String? = null): VersiondInstallMetadata? = try { - val installPath = versiondInstallMetadataPath(versionName) - val json = execInVersiond( - listOf("sh", "-c", "test -f '$installPath' && cat '$installPath'"), - null, - ).joinToString("").trim() - json.takeIf { it.isNotBlank() }?.let { - cosmosJson.fromJson(it, VersiondInstallMetadata::class.java) + val installPath = versiondInstallMetadataPath(versionName, archiveSha256) + if (installPath.contains("")) { + null + } else { + val json = execInVersiond( + listOf("sh", "-c", "test -f '$installPath' && cat '$installPath'"), + null, + ).joinToString("").trim() + json.takeIf { it.isNotBlank() }?.let { + cosmosJson.fromJson(it, VersiondInstallMetadata::class.java) + } } } catch (_: Exception) { null diff --git a/testermint/src/test/kotlin/DevshardVersiondAdvancedTests.kt b/testermint/src/test/kotlin/DevshardVersiondAdvancedTests.kt index 282d590546..6ba9e10eee 100644 --- a/testermint/src/test/kotlin/DevshardVersiondAdvancedTests.kt +++ b/testermint/src/test/kotlin/DevshardVersiondAdvancedTests.kt @@ -133,19 +133,41 @@ class DevshardVersiondAdvancedTests : DevshardVersiondTestBase() { logSection("Waiting for every pair to download ${preparedArtifact.approvedVersion.name}") waitUntil("downloaded binary and install metadata exist on every pair", timeoutSeconds = 120) { cluster.allPairs.all { pair -> - pair.versiondBinaryExists(preparedArtifact.approvedVersion.name, "devshardd") && - pair.readVersiondInstallMetadata(preparedArtifact.approvedVersion.name)?.archiveSha256 == + pair.versiondBinaryExists( + preparedArtifact.approvedVersion.name, + "devshardd", + preparedArtifact.approvedVersion.sha256, + ) && + pair.readVersiondInstallMetadata( + preparedArtifact.approvedVersion.name, + preparedArtifact.approvedVersion.sha256, + )?.archiveSha256 == preparedArtifact.approvedVersion.sha256 } } cluster.allPairs.forEach { pair -> - assertThat(pair.versiondBinaryExists(preparedArtifact.approvedVersion.name, "devshardd")) + assertThat( + pair.versiondBinaryExists( + preparedArtifact.approvedVersion.name, + "devshardd", + preparedArtifact.approvedVersion.sha256, + ) + ) .withFailMessage( "Expected ${pair.name} versiond to download " + - pair.versiondBinaryPath(preparedArtifact.approvedVersion.name, "devshardd"), + pair.versiondBinaryPath( + preparedArtifact.approvedVersion.name, + "devshardd", + preparedArtifact.approvedVersion.sha256, + ), ) .isTrue() - val installMetadata = assertNotNull(pair.readVersiondInstallMetadata(preparedArtifact.approvedVersion.name)) + val installMetadata = assertNotNull( + pair.readVersiondInstallMetadata( + preparedArtifact.approvedVersion.name, + preparedArtifact.approvedVersion.sha256, + ) + ) assertThat(installMetadata.archiveSha256).isEqualTo(preparedArtifact.approvedVersion.sha256) assertThat(installMetadata.binarySha256).isNotBlank() } diff --git a/testermint/src/test/kotlin/VersiondTests.kt b/testermint/src/test/kotlin/VersiondTests.kt index e9612d5c64..a4f9ff277b 100644 --- a/testermint/src/test/kotlin/VersiondTests.kt +++ b/testermint/src/test/kotlin/VersiondTests.kt @@ -4,7 +4,11 @@ import com.productscience.data.* import org.assertj.core.api.Assertions.assertThat import org.junit.jupiter.api.* import org.tinylog.kotlin.Logger +import java.time.Duration +import java.util.concurrent.CompletableFuture +import java.util.concurrent.CopyOnWriteArrayList import java.util.concurrent.TimeUnit +import java.util.concurrent.atomic.AtomicBoolean /** * Full-circle E2E tests for versiond: @@ -13,6 +17,7 @@ import java.util.concurrent.TimeUnit * 3. Governance proposal adds a devshard binary version * 4. versiond downloads the binary and proxies traffic * 5. Second proposal adds another version, both route correctly + * 6. Same-version binary update drains old requests while new traffic is routed * * Approved version names must match each binary's --print-protocol-version * (see versioned/e2e/testapp: "testapp" and "testapp2"). @@ -30,11 +35,14 @@ class VersiondTests : TestermintTest() { "http://${GENESIS_KEY_NAME}-testapp-server:8080/testapp.zip" private val testapp2BinaryDockerUrl = "http://${GENESIS_KEY_NAME}-testapp-server:8080/testapp2.zip" + private val testappRolloutBinaryDockerUrl = + "http://${GENESIS_KEY_NAME}-testapp-server:8080/testapp-rollout.zip" private lateinit var cluster: LocalCluster private lateinit var genesis: LocalInferencePair private lateinit var testappSha256: String private lateinit var testapp2Sha256: String + private lateinit var testappRolloutSha256: String @BeforeAll fun setup() { @@ -53,8 +61,10 @@ class VersiondTests : TestermintTest() { logSection("Waiting for testapp-server readiness") testappSha256 = waitForTestappArtifactSha256(TESTAPP_VERSION) testapp2Sha256 = waitForTestappArtifactSha256(TESTAPP2_VERSION) + testappRolloutSha256 = waitForTestappArtifactSha256(TESTAPP_ROLLOUT_ARTIFACT) Logger.info("testapp zip sha256: $testappSha256") Logger.info("testapp2 zip sha256: $testapp2Sha256") + Logger.info("testapp-rollout zip sha256: $testappRolloutSha256") } @AfterAll @@ -190,6 +200,94 @@ class VersiondTests : TestermintTest() { logHighlight("Both $v1 and $v2 route correctly through versiond") } + @Test + @Order(5) + fun `same version binary update drains old requests and keeps serving`() { + val versionName = TESTAPP_VERSION + val oldPrefix = TESTAPP_VERSION + val newPrefix = TESTAPP_ROLLOUT_PREFIX + val slowDuration = Duration.ofSeconds(75) + + logSection("Verifying $versionName starts on the original binary") + waitForVersionedProxyPrefix(versionName, oldPrefix) + + logSection("Starting a long request before updating $versionName") + val slowRequest = CompletableFuture.supplyAsync> { + getVersiondProxySlow(versionName, slowDuration) + } + waitUntil("slow request accepted by old $versionName child", timeoutSeconds = 10) { + runCatching { statusNumber(getDrainStatus(versionName), "inflight") > 0 } + .getOrDefault(false) + } + + val stopProbe = AtomicBoolean(false) + val observedPrefixes = CopyOnWriteArrayList() + val probeFailures = CopyOnWriteArrayList() + val continuityProbe = CompletableFuture.runAsync { + while (!stopProbe.get()) { + try { + getVersiondProxy(versionName)["prefix"]?.toString()?.let(observedPrefixes::add) + } catch (e: Exception) { + probeFailures.add(e.message ?: e.toString()) + } + Thread.sleep(250) + } + } + + try { + logSection("Submitting governance proposal to update $versionName to rollout sha") + val params = genesis.getParams() + val currentVersions = params.devshardEscrowParams?.approvedVersions ?: emptyList() + assertThat(currentVersions.any { it.name == versionName && it.sha256 == testappSha256 }) + .withFailMessage("Expected $versionName to start on sha $testappSha256") + .isTrue() + + val rolloutVersion = DevshardApprovedVersion( + name = versionName, + binary = testappRolloutBinaryDockerUrl, + sha256 = testappRolloutSha256, + ) + val updatedParams = params.withApprovedVersions( + currentVersions.filterNot { it.name == versionName } + rolloutVersion + ) + genesis.runProposal(cluster, UpdateParams(params = updatedParams)) + + logSection("Waiting for dapi to expose rollout sha for $versionName") + waitUntil("dapi serves rollout sha for $versionName", timeoutSeconds = 60) { + getDapiVersions().any { it["name"] == versionName && it["sha256"] == testappRolloutSha256 } + } + + logSection("Waiting for versiond to route new requests to rollout binary") + waitForVersionedProxyPrefix(versionName, newPrefix) + + logSection("Verifying old child is draining while the long request is still running") + waitForDrainingChild(versionName, testappSha256) + + logSection("Verifying the long request completes on the old binary") + val slowResponse = slowRequest.get(slowDuration.seconds + 45, TimeUnit.SECONDS) + assertThat(slowResponse["prefix"]) + .withFailMessage("Long request should complete on old binary: $slowResponse") + .isEqualTo(oldPrefix) + + logSection("Verifying new requests stay on the rollout binary") + val newResponse = getVersiondProxy(versionName) + assertThat(newResponse["prefix"]).isEqualTo(newPrefix) + + logSection("Waiting for old draining child to exit") + waitForNoDraining(versionName) + } finally { + stopProbe.set(true) + continuityProbe.get(5, TimeUnit.SECONDS) + } + + assertThat(probeFailures) + .withFailMessage("Versioned route returned errors during rolling update: $probeFailures") + .isEmpty() + assertThat(observedPrefixes) + .withFailMessage("Continuity probe should see old and new prefixes") + .contains(oldPrefix, newPrefix) + } + // --------------------------------------------------------------------------- // Helpers // --------------------------------------------------------------------------- @@ -280,6 +378,91 @@ class VersiondTests : TestermintTest() { return getVersiondProxy(versionName) } + private fun waitForVersionedProxyPrefix( + versionName: String, + prefix: String, + timeoutSeconds: Int = 90, + ): Map { + var matched: Map? = null + waitUntil("proxy routes $versionName with prefix $prefix", timeoutSeconds = timeoutSeconds) { + runCatching { + val response = getVersiondProxy(versionName) + if (response["prefix"] == prefix) { + matched = response + true + } else { + false + } + }.getOrDefault(false) + } + return matched ?: getVersiondProxy(versionName) + } + + @Suppress("UNCHECKED_CAST") + private fun getVersiondProxySlow(versionName: String, duration: Duration): Map { + val durationParam = "${duration.seconds}s" + val (_, response, result) = + Fuel.get("${genesis.api.getPublicUrl()}/devshard/$versionName/slow?duration=$durationParam") + .timeout(5_000) + .timeoutRead((duration.toMillis() + 30_000).toInt()) + .responseString() + assertThat(response.statusCode) + .withFailMessage("GET /devshard/$versionName/slow returned ${response.statusCode}: ${result}") + .isEqualTo(200) + return cosmosJson.fromJson(result.get(), Map::class.java) as Map + } + + @Suppress("UNCHECKED_CAST") + private fun getDrainStatus(versionName: String): Map { + val (_, response, result) = Fuel.get("${genesis.api.getPublicUrl()}/devshard/$versionName/drain/status") + .timeoutRead(10_000) + .responseString() + assertThat(response.statusCode) + .withFailMessage("GET /devshard/$versionName/drain/status returned ${response.statusCode}: ${result}") + .isEqualTo(200) + return cosmosJson.fromJson(result.get(), Map::class.java) as Map + } + + @Suppress("UNCHECKED_CAST") + private fun getVersiondHealth(): List> { + val (_, response, result) = Fuel.get("${genesis.api.getPublicUrl()}/devshard/healthz") + .timeoutRead(10_000) + .responseString() + assertThat(response.statusCode) + .withFailMessage("GET /devshard/healthz returned ${response.statusCode}: ${result}") + .isEqualTo(200) + return cosmosJson.fromJson(result.get(), List::class.java) as List> + } + + private fun waitForDrainingChild(versionName: String, sha256: String, timeoutSeconds: Int = 20) { + waitUntil("old $versionName child is draining", timeoutSeconds = timeoutSeconds) { + runCatching { + getVersiondHealth().any { + it["name"] == versionName && + it["sha256"] == sha256 && + it["status"] == "draining" + } + }.getOrDefault(false) + } + } + + private fun waitForNoDraining(versionName: String, timeoutSeconds: Int = 60) { + waitUntil("no draining child for $versionName", timeoutSeconds = timeoutSeconds) { + runCatching { + getVersiondHealth().none { + it["name"] == versionName && it["status"] == "draining" + } + }.getOrDefault(false) + } + } + + private fun statusNumber(status: Map, key: String): Long = + when (val value = status[key]) { + is Number -> value.toLong() + is String -> value.toLong() + else -> 0 + } + private fun waitUntil(description: String, timeoutSeconds: Int, condition: () -> Boolean) { val deadline = System.currentTimeMillis() + timeoutSeconds * 1000L while (System.currentTimeMillis() < deadline) { @@ -296,6 +479,12 @@ class VersiondTests : TestermintTest() { /** Governance slot / --print-protocol-version for testapp2.zip */ const val TESTAPP2_VERSION = "testapp2" + /** Artifact basename for same-slot rollout binary. Protocol remains TESTAPP_VERSION. */ + const val TESTAPP_ROLLOUT_ARTIFACT = "testapp-rollout" + + /** DEVSHARD_BINARY_LOG_VERSION / HTTP prefix printed by testapp-rollout.zip. */ + const val TESTAPP_ROLLOUT_PREFIX = "testapp-rollout" + const val TESTAPP_SERVER_HOST_PORT = 7090 const val DAPI_ML_HOST_PORT = 9001 } diff --git a/versioned/Makefile b/versioned/Makefile index 8e5a4c6af1..be33ff82ad 100644 --- a/versioned/Makefile +++ b/versioned/Makefile @@ -36,6 +36,7 @@ testapp: mkdir -p build go build -o build/testapp ./e2e/testapp go build -ldflags "-X main.Version=testapp2 -X main.BinaryVersion=testapp2" -o build/testapp2 ./e2e/testapp + go build -ldflags "-X main.Version=testapp -X main.BinaryVersion=testapp-rollout" -o build/testapp-rollout ./e2e/testapp e2e: build testapp docker compose -f e2e/docker-compose.yml up --build --abort-on-container-exit --exit-code-from tests diff --git a/versioned/cmd/versiond/main.go b/versioned/cmd/versiond/main.go index 342a65121f..9024142d4b 100644 --- a/versioned/cmd/versiond/main.go +++ b/versioned/cmd/versiond/main.go @@ -101,11 +101,17 @@ func run(ctx context.Context) error { <-ctx.Done() slog.Info("shutting down") - shutdownCtx, shutdownCancel := context.WithTimeout(context.Background(), 10*time.Second) - defer shutdownCancel() + httpShutdownCtx, httpShutdownCancel := context.WithTimeout(context.Background(), 10*time.Second) + defer httpShutdownCancel() + if err := srv.Shutdown(httpShutdownCtx); err != nil { + slog.Warn("http server shutdown incomplete", "error", err) + } - srv.Shutdown(shutdownCtx) - mgr.Shutdown(shutdownCtx) + managerShutdownCtx, managerShutdownCancel := context.WithTimeout(context.Background(), mgr.ShutdownTimeout()) + defer managerShutdownCancel() + if err := mgr.Shutdown(managerShutdownCtx); err != nil { + slog.Warn("manager shutdown incomplete", "error", err) + } return nil } diff --git a/versioned/e2e/Dockerfile.tests b/versioned/e2e/Dockerfile.tests index 68a6cdaf82..3e7d54d4bb 100644 --- a/versioned/e2e/Dockerfile.tests +++ b/versioned/e2e/Dockerfile.tests @@ -10,6 +10,9 @@ RUN rm -rf /app/build && mkdir -p /app/build \ && CGO_ENABLED=0 GOOS=linux go build -o /app/build/testapp ./e2e/testapp \ && CGO_ENABLED=0 GOOS=linux go build \ -ldflags "-X main.Version=testapp2 -X main.BinaryVersion=testapp2" \ - -o /app/build/testapp2 ./e2e/testapp + -o /app/build/testapp2 ./e2e/testapp \ + && CGO_ENABLED=0 GOOS=linux go build \ + -ldflags "-X main.Version=testapp -X main.BinaryVersion=testapp-rollout" \ + -o /app/build/testapp-rollout ./e2e/testapp RUN CGO_ENABLED=0 go test -c -tags e2e -o /e2e-tests ./e2e/ CMD ["/e2e-tests", "-test.v", "-test.timeout", "300s"] diff --git a/versioned/e2e/docker-compose.yml b/versioned/e2e/docker-compose.yml index a5e934fd22..5ebbbead16 100644 --- a/versioned/e2e/docker-compose.yml +++ b/versioned/e2e/docker-compose.yml @@ -19,6 +19,9 @@ services: VERSIOND_BIN_DIR: /opt/versiond/bin VERSIOND_DATA_DIR: /opt/versiond/data VERSIOND_BINARY_NAME: testapp + VERSIOND_DRAIN_POLL_INTERVAL: "200ms" + VERSIOND_DRAIN_TIMEOUT: "30s" + VERSIOND_DRAIN_KILL_GRACE: "2s" depends_on: - oracle @@ -31,6 +34,7 @@ services: VERSIOND_URL: http://versiond:8080 TESTAPP_PATH: /app/build/testapp TESTAPP2_PATH: /app/build/testapp2 + TESTAPP_ROLLOUT_PATH: /app/build/testapp-rollout depends_on: - oracle - versiond diff --git a/versioned/e2e/e2e_test.go b/versioned/e2e/e2e_test.go index 9869ed233a..cc7946bcee 100644 --- a/versioned/e2e/e2e_test.go +++ b/versioned/e2e/e2e_test.go @@ -84,7 +84,7 @@ func TestHashMismatch(t *testing.T) { // Use testapp binary (protocol "testapp") with a mismatched slot so even a // correct hash would fail protocol checks; wrong hash fails earlier. uploadBinary(t, "bad.zip", zipData) - putVersion(t, "badslot", fmt.Sprintf("%s/binaries/bad.zip", oracleURL), "wrong_hash", 9003) + putVersion(t, "badslot", fmt.Sprintf("%s/binaries/bad.zip", oracleURL), strings.Repeat("0", 64), 9003) time.Sleep(10 * time.Second) @@ -158,3 +158,48 @@ func TestHealthEndpoint(t *testing.T) { t.Errorf("testapp not found in healthz response: %s", string(body)) } } + +func TestSameNameNewSHA_RollingUpdateDrainsOld(t *testing.T) { + oldZip, oldHash := buildTestappZip(t) + newZip, newHash := buildTestappRolloutZip(t) + + uploadBinary(t, "testapp-old.zip", oldZip) + putVersion(t, "testapp", fmt.Sprintf("%s/binaries/testapp-old.zip", oracleURL), oldHash, 9001) + waitForVersionPrefix(t, "testapp", "testapp", 90*time.Second) + + type result struct { + resp map[string]string + err error + } + slowDone := make(chan result, 1) + go func() { + var resp map[string]string + err := getJSONIfOK(fmt.Sprintf("%s/testapp/slow?duration=8s", versiondURL), &resp) + slowDone <- result{resp: resp, err: err} + }() + + time.Sleep(500 * time.Millisecond) + + uploadBinary(t, "testapp-new.zip", newZip) + putVersion(t, "testapp", fmt.Sprintf("%s/binaries/testapp-new.zip", oracleURL), newHash, 9001) + waitForVersionPrefix(t, "testapp", "testapp-rollout", 90*time.Second) + + select { + case got := <-slowDone: + if got.err != nil { + t.Fatalf("slow request failed: %v", got.err) + } + if got.resp["prefix"] != "testapp" { + t.Fatalf("slow request prefix = %q, want old prefix %q", got.resp["prefix"], "testapp") + } + case <-time.After(20 * time.Second): + t.Fatal("slow request did not finish") + } + + waitForNoDraining(t, "testapp", 30*time.Second) + var resp map[string]string + getJSON(t, fmt.Sprintf("%s/testapp/", versiondURL), &resp) + if resp["prefix"] != "testapp-rollout" { + t.Fatalf("new request prefix = %q, want %q", resp["prefix"], "testapp-rollout") + } +} diff --git a/versioned/e2e/setup_test.go b/versioned/e2e/setup_test.go index e41eae2f1e..77a9665b3b 100644 --- a/versioned/e2e/setup_test.go +++ b/versioned/e2e/setup_test.go @@ -41,6 +41,13 @@ func buildTestapp2Zip(t *testing.T) ([]byte, string) { return buildTestappZipFrom(t, envOrDefault("TESTAPP2_PATH", "/app/build/testapp2")) } +// buildTestappRolloutZip uses the same protocol slot as testapp but a different +// binary version and archive hash for same-name rolling update tests. +func buildTestappRolloutZip(t *testing.T) ([]byte, string) { + t.Helper() + return buildTestappZipFrom(t, envOrDefault("TESTAPP_ROLLOUT_PATH", "/app/build/testapp-rollout")) +} + func buildTestappZipFrom(t *testing.T, testappPath string) ([]byte, string) { t.Helper() @@ -163,19 +170,63 @@ func waitForVersionGone(t *testing.T, version string, timeout time.Duration) { t.Fatalf("version %s still available after %v", version, timeout) } +func waitForVersionPrefix(t *testing.T, version, wantPrefix string, timeout time.Duration) { + t.Helper() + deadline := time.Now().Add(timeout) + url := fmt.Sprintf("%s/%s/", versiondURL, version) + for time.Now().Before(deadline) { + var resp map[string]string + if getJSONIfOK(url, &resp) == nil && resp["prefix"] == wantPrefix { + return + } + time.Sleep(time.Second) + } + t.Fatalf("version %s did not report prefix %q after %v", version, wantPrefix, timeout) +} + +func waitForNoDraining(t *testing.T, version string, timeout time.Duration) { + t.Helper() + deadline := time.Now().Add(timeout) + url := fmt.Sprintf("%s/healthz", versiondURL) + for time.Now().Before(deadline) { + var statuses []map[string]interface{} + if getJSONIfOK(url, &statuses) == nil { + draining := false + for _, s := range statuses { + if s["name"] == version && s["status"] == "draining" { + draining = true + break + } + } + if !draining { + return + } + } + time.Sleep(200 * time.Millisecond) + } + t.Fatalf("version %s still has draining child after %v", version, timeout) +} + // getJSON does a GET and decodes the JSON response into out. func getJSON(t *testing.T, url string, out interface{}) { t.Helper() + if err := getJSONIfOK(url, out); err != nil { + t.Fatal(err) + } +} + +func getJSONIfOK(url string, out interface{}) error { resp, err := http.Get(url) if err != nil { - t.Fatalf("GET %s: %v", url, err) + return fmt.Errorf("GET %s: %w", url, err) } defer resp.Body.Close() if resp.StatusCode != http.StatusOK { body, _ := io.ReadAll(resp.Body) - t.Fatalf("GET %s: status %d, body: %s", url, resp.StatusCode, string(body)) + return fmt.Errorf("GET %s: status %d, body: %s", url, resp.StatusCode, string(body)) } if err := json.NewDecoder(resp.Body).Decode(out); err != nil { - t.Fatalf("decode response from %s: %v", url, err) + return fmt.Errorf("decode response from %s: %w", url, err) } + return nil } diff --git a/versioned/e2e/testapp/main.go b/versioned/e2e/testapp/main.go index 6393eb399f..0f7a3b18cd 100644 --- a/versioned/e2e/testapp/main.go +++ b/versioned/e2e/testapp/main.go @@ -8,6 +8,7 @@ import ( "log" "net/http" "os" + "sync/atomic" "time" pb "versioned/e2e/testapp/gen" @@ -45,21 +46,93 @@ func main() { prefix = BinaryVersion } nmAddr := os.Getenv("NODE_MANAGER_ADDR") + var ready atomic.Bool + var draining atomic.Bool + var inflight atomic.Int64 + ready.Store(true) log.Printf("[%s] starting testapp on port %d, data-dir=%s, node-manager=%s", prefix, *port, *dataDir, nmAddr) - http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { + track := func(fn http.HandlerFunc) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + if draining.Load() { + http.Error(w, "draining", http.StatusServiceUnavailable) + return + } + inflight.Add(1) + defer inflight.Add(-1) + fn(w, r) + } + } + + writeVersion := func(w http.ResponseWriter) { w.Header().Set("Content-Type", "application/json") json.NewEncoder(w).Encode(map[string]string{ "version": "testapp", "prefix": prefix, }) + } + + http.HandleFunc("/", track(func(w http.ResponseWriter, r *http.Request) { + writeVersion(w) + })) + + http.HandleFunc("/slow", track(func(w http.ResponseWriter, r *http.Request) { + delay := 8 * time.Second + if raw := r.URL.Query().Get("duration"); raw != "" { + if parsed, err := time.ParseDuration(raw); err == nil && parsed > 0 { + delay = parsed + } + } + select { + case <-r.Context().Done(): + return + case <-time.After(delay): + } + writeVersion(w) + })) + + http.HandleFunc("/ready", func(w http.ResponseWriter, r *http.Request) { + status := map[string]any{ + "ready": ready.Load(), + "draining": draining.Load(), + "inflight": inflight.Load(), + } + w.Header().Set("Content-Type", "application/json") + if !ready.Load() || draining.Load() { + w.WriteHeader(http.StatusServiceUnavailable) + } + json.NewEncoder(w).Encode(status) + }) + + http.HandleFunc("/drain", func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPost { + http.Error(w, "method not allowed", http.StatusMethodNotAllowed) + return + } + draining.Store(true) + ready.Store(false) + w.Header().Set("Content-Type", "application/json") + json.NewEncoder(w).Encode(map[string]any{ + "ready": ready.Load(), + "draining": draining.Load(), + "inflight": inflight.Load(), + }) + }) + + http.HandleFunc("/drain/status", func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + json.NewEncoder(w).Encode(map[string]any{ + "ready": ready.Load(), + "draining": draining.Load(), + "inflight": inflight.Load(), + }) }) http.HandleFunc("/health", func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusOK) }) - http.HandleFunc("/stream", func(w http.ResponseWriter, r *http.Request) { + http.HandleFunc("/stream", track(func(w http.ResponseWriter, r *http.Request) { flusher, ok := w.(http.Flusher) if !ok { http.Error(w, "streaming not supported", http.StatusInternalServerError) @@ -73,9 +146,9 @@ func main() { flusher.Flush() time.Sleep(100 * time.Millisecond) } - }) + })) - http.HandleFunc("/nodemanager-test", func(w http.ResponseWriter, r *http.Request) { + http.HandleFunc("/nodemanager-test", track(func(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "application/json") if nmAddr == "" { @@ -91,8 +164,8 @@ func main() { conn, err := grpc.NewClient(nmAddr, grpc.WithTransportCredentials(insecure.NewCredentials())) if err != nil { json.NewEncoder(w).Encode(map[string]string{ - "error": fmt.Sprintf("grpc dial failed: %v", err), - "nodemanager_addr": nmAddr, + "error": fmt.Sprintf("grpc dial failed: %v", err), + "nodemanager_addr": nmAddr, }) return } @@ -128,7 +201,7 @@ func main() { result["release_error"] = releaseErr.Error() } json.NewEncoder(w).Encode(result) - }) + })) addr := fmt.Sprintf(":%d", *port) log.Printf("[%s] listening on %s", prefix, addr) diff --git a/versioned/internal/config/config.go b/versioned/internal/config/config.go index d6f519b501..c0aeafb81b 100644 --- a/versioned/internal/config/config.go +++ b/versioned/internal/config/config.go @@ -9,15 +9,24 @@ import ( "time" ) +const DefaultDrainKillGrace = 10 * time.Minute + type Config struct { - OracleURL string - PollInterval time.Duration - BinDir string - DataDir string - BinaryName string - BasePort int - Overrides map[string]string // version name -> local binary path - ForceVersions []string // version names that must run regardless of oracle + OracleURL string + PollInterval time.Duration + BinDir string + DataDir string + BinaryName string + BasePort int + ReadyPath string + ReadyTimeout time.Duration + DrainPath string + DrainStatusPath string + DrainTimeout time.Duration + DrainPollInterval time.Duration + DrainKillGrace time.Duration + Overrides map[string]string // version name -> local binary path + ForceVersions []string // version names that must run regardless of oracle } func Load() (Config, error) { @@ -27,14 +36,21 @@ func Load() (Config, error) { } cfg := Config{ - OracleURL: oracleURL, - PollInterval: parseDuration("VERSIOND_POLL_INTERVAL", 30*time.Second), - BinDir: envOrDefault("VERSIOND_BIN_DIR", "/opt/versiond/bin"), - DataDir: envOrDefault("VERSIOND_DATA_DIR", "/opt/versiond/data"), - BinaryName: envOrDefault("VERSIOND_BINARY_NAME", "devshard"), - BasePort: 5000, - Overrides: loadOverrides(), - ForceVersions: loadForceVersions(), + OracleURL: oracleURL, + PollInterval: parseDuration("VERSIOND_POLL_INTERVAL", 30*time.Second), + BinDir: envOrDefault("VERSIOND_BIN_DIR", "/opt/versiond/bin"), + DataDir: envOrDefault("VERSIOND_DATA_DIR", "/opt/versiond/data"), + BinaryName: envOrDefault("VERSIOND_BINARY_NAME", "devshard"), + BasePort: 5000, + ReadyPath: envOrDefault("VERSIOND_READY_PATH", "/ready"), + ReadyTimeout: parseDuration("VERSIOND_READY_TIMEOUT", 60*time.Second), + DrainPath: envOrDefault("VERSIOND_DRAIN_PATH", "/drain"), + DrainStatusPath: envOrDefault("VERSIOND_DRAIN_STATUS_PATH", "/drain/status"), + DrainTimeout: parseDuration("VERSIOND_DRAIN_TIMEOUT", 15*time.Minute), + DrainPollInterval: parseDuration("VERSIOND_DRAIN_POLL_INTERVAL", time.Second), + DrainKillGrace: parseDuration("VERSIOND_DRAIN_KILL_GRACE", DefaultDrainKillGrace), + Overrides: loadOverrides(), + ForceVersions: loadForceVersions(), } slog.Info( diff --git a/versioned/internal/config/config_test.go b/versioned/internal/config/config_test.go index 59899210b8..14bf9d500d 100644 --- a/versioned/internal/config/config_test.go +++ b/versioned/internal/config/config_test.go @@ -37,6 +37,9 @@ func TestLoad_Defaults(t *testing.T) { if cfg.BasePort != 5000 { t.Errorf("BasePort = %d, want %d", cfg.BasePort, 5000) } + if cfg.DrainKillGrace != DefaultDrainKillGrace { + t.Errorf("DrainKillGrace = %v, want %v", cfg.DrainKillGrace, DefaultDrainKillGrace) + } } func TestLoad_CustomValues(t *testing.T) { diff --git a/versioned/internal/download/download.go b/versioned/internal/download/download.go index 122127f7fe..f5194c5bd2 100644 --- a/versioned/internal/download/download.go +++ b/versioned/internal/download/download.go @@ -90,7 +90,7 @@ func Download(ctx context.Context, url, expectedSHA256, destDir, binaryName stri return fmt.Errorf("hash extracted binary: %w", err) } - if err := writeInstallMetadata(destDir, InstallMetadata{ + if err := WriteInstallMetadata(destDir, InstallMetadata{ ArchiveSHA256: gotHash, BinarySHA256: binaryHash, }); err != nil { @@ -152,7 +152,9 @@ func ReadInstallMetadata(destDir string) (InstallMetadata, error) { return metadata, nil } -func writeInstallMetadata(destDir string, metadata InstallMetadata) error { +// WriteInstallMetadata atomically commits metadata for a verified install. +// Callers must write and verify the binary before publishing this marker. +func WriteInstallMetadata(destDir string, metadata InstallMetadata) error { data, err := json.Marshal(metadata) if err != nil { return fmt.Errorf("marshal install metadata: %w", err) diff --git a/versioned/internal/health/health.go b/versioned/internal/health/health.go index 2f54acccdb..b20591e919 100644 --- a/versioned/internal/health/health.go +++ b/versioned/internal/health/health.go @@ -6,9 +6,11 @@ import ( ) type StatusEntry struct { - Name string `json:"name"` - Port int `json:"port"` - Status string `json:"status"` + Name string `json:"name"` + Port int `json:"port"` + Status string `json:"status"` + SHA256 string `json:"sha256,omitempty"` + BinaryVersion string `json:"binary_version,omitempty"` } // Handler returns an http.HandlerFunc that writes the health status as JSON. diff --git a/versioned/internal/oracle/client.go b/versioned/internal/oracle/client.go index b00510573f..4bc2c410af 100644 --- a/versioned/internal/oracle/client.go +++ b/versioned/internal/oracle/client.go @@ -2,10 +2,12 @@ package oracle import ( "context" + "encoding/hex" "encoding/json" "fmt" "net/http" "net/url" + "path/filepath" "strings" "time" ) @@ -24,7 +26,7 @@ type Version struct { // Priority: sha256 field, then ?checksum=sha256:... in URL query. func (v Version) ResolvedSHA256() (string, error) { if v.SHA256 != "" { - return v.SHA256, nil + return validateSHA256(v.Name, v.SHA256) } u, err := url.Parse(v.Binary) if err != nil { @@ -36,7 +38,7 @@ func (v Version) ResolvedSHA256() (string, error) { if hash == "" { return "", fmt.Errorf("empty sha256 checksum in URL for version %s", v.Name) } - return hash, nil + return validateSHA256(v.Name, hash) } return "", fmt.Errorf("no checksum for version %s: sha256 field empty and no ?checksum=sha256: in URL", v.Name) } @@ -74,5 +76,48 @@ func (c *Client) Fetch(ctx context.Context) (VersionConfig, error) { if err := json.NewDecoder(resp.Body).Decode(&cfg); err != nil { return VersionConfig{}, fmt.Errorf("decode response: %w", err) } + if err := validateVersions(cfg.Versions); err != nil { + return VersionConfig{}, err + } return cfg, nil } + +func validateVersions(versions []Version) error { + seen := make(map[string]struct{}, len(versions)) + for i, v := range versions { + if !validVersionName(v.Name) { + return fmt.Errorf("invalid oracle version name at index %d: %q", i, v.Name) + } + if _, dup := seen[v.Name]; dup { + return fmt.Errorf("duplicate oracle version name %q", v.Name) + } + if _, err := v.ResolvedSHA256(); err != nil { + return fmt.Errorf("invalid oracle version %q sha256: %w", v.Name, err) + } + seen[v.Name] = struct{}{} + } + return nil +} + +func validateSHA256(versionName, hash string) (string, error) { + if len(hash) != 64 { + return "", fmt.Errorf("sha256 for version %s must be 64 hex characters, got %d", versionName, len(hash)) + } + if _, err := hex.DecodeString(hash); err != nil { + return "", fmt.Errorf("sha256 for version %s is not valid hex: %w", versionName, err) + } + return hash, nil +} + +func validVersionName(name string) bool { + if name == "" || name == "." || name == ".." { + return false + } + if strings.TrimSpace(name) != name { + return false + } + if filepath.IsAbs(name) || strings.ContainsAny(name, `/\`) { + return false + } + return filepath.Base(name) == name +} diff --git a/versioned/internal/oracle/client_test.go b/versioned/internal/oracle/client_test.go index eaeb85b714..2ece920e13 100644 --- a/versioned/internal/oracle/client_test.go +++ b/versioned/internal/oracle/client_test.go @@ -8,10 +8,15 @@ import ( "testing" ) +const ( + testSHA256A = "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef" + testSHA256B = "abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789" +) + func TestFetch(t *testing.T) { want := VersionConfig{ Versions: []Version{ - {Name: "v1", Binary: "http://example.com/v1.zip", SHA256: "abc123"}, + {Name: "v1", Binary: "http://example.com/v1.zip", SHA256: testSHA256A}, }, } srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -48,43 +53,115 @@ func TestFetch_ServerError(t *testing.T) { } } +func TestFetch_RejectsInvalidVersionNames(t *testing.T) { + payload := VersionConfig{ + Versions: []Version{ + {Name: "v1", Binary: "http://example.com/v1.zip", SHA256: testSHA256A}, + {Name: "", Binary: "http://example.com/empty.zip", SHA256: testSHA256B}, + {Name: " v2 ", Binary: "http://example.com/spaces.zip", SHA256: "bad"}, + {Name: "../escape", Binary: "http://example.com/escape.zip", SHA256: "bad"}, + {Name: "nested/v2", Binary: "http://example.com/nested.zip", SHA256: "bad"}, + }, + } + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + json.NewEncoder(w).Encode(payload) + })) + defer srv.Close() + + c := NewClient(srv.URL) + if _, err := c.Fetch(context.Background()); err == nil { + t.Fatal("expected invalid oracle version name to fail fetch") + } +} + +func TestFetch_RejectsDuplicateVersionNames(t *testing.T) { + payload := VersionConfig{ + Versions: []Version{ + {Name: "v1", Binary: "http://example.com/v1.zip", SHA256: testSHA256A}, + {Name: "v1", Binary: "http://example.com/v1-r2.zip", SHA256: testSHA256B}, + }, + } + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + json.NewEncoder(w).Encode(payload) + })) + defer srv.Close() + + c := NewClient(srv.URL) + if _, err := c.Fetch(context.Background()); err == nil { + t.Fatal("expected duplicate oracle version name to fail fetch") + } +} + +func TestFetch_RejectsInvalidSHA256(t *testing.T) { + payload := VersionConfig{ + Versions: []Version{ + {Name: "v1", Binary: "http://example.com/v1.zip", SHA256: "../escape"}, + }, + } + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + json.NewEncoder(w).Encode(payload) + })) + defer srv.Close() + + c := NewClient(srv.URL) + if _, err := c.Fetch(context.Background()); err == nil { + t.Fatal("expected invalid oracle sha256 to fail fetch") + } +} + func TestResolvedSHA256_FromField(t *testing.T) { - v := Version{Name: "v1", Binary: "http://example.com/v1.zip", SHA256: "abc123"} + v := Version{Name: "v1", Binary: "http://example.com/v1.zip", SHA256: testSHA256A} got, err := v.ResolvedSHA256() if err != nil { t.Fatalf("unexpected error: %v", err) } - if got != "abc123" { - t.Errorf("got %q, want %q", got, "abc123") + if got != testSHA256A { + t.Errorf("got %q, want %q", got, testSHA256A) } } func TestResolvedSHA256_FromURL(t *testing.T) { v := Version{ Name: "v1", - Binary: "http://example.com/v1.zip?checksum=sha256:def456", + Binary: "http://example.com/v1.zip?checksum=sha256:" + testSHA256B, } got, err := v.ResolvedSHA256() if err != nil { t.Fatalf("unexpected error: %v", err) } - if got != "def456" { - t.Errorf("got %q, want %q", got, "def456") + if got != testSHA256B { + t.Errorf("got %q, want %q", got, testSHA256B) } } func TestResolvedSHA256_FieldTakesPrecedence(t *testing.T) { v := Version{ Name: "v1", - Binary: "http://example.com/v1.zip?checksum=sha256:url_hash", - SHA256: "field_hash", + Binary: "http://example.com/v1.zip?checksum=sha256:" + testSHA256B, + SHA256: testSHA256A, } got, err := v.ResolvedSHA256() if err != nil { t.Fatalf("unexpected error: %v", err) } - if got != "field_hash" { - t.Errorf("got %q, want %q", got, "field_hash") + if got != testSHA256A { + t.Errorf("got %q, want %q", got, testSHA256A) + } +} + +func TestResolvedSHA256_InvalidField(t *testing.T) { + v := Version{Name: "v1", Binary: "http://example.com/v1.zip", SHA256: "../escape"} + _, err := v.ResolvedSHA256() + if err == nil { + t.Fatal("expected error for invalid sha256 field") + } +} + +func TestResolvedSHA256_InvalidURLChecksum(t *testing.T) { + v := Version{Name: "v1", Binary: "http://example.com/v1.zip?checksum=sha256:bad"} + _, err := v.ResolvedSHA256() + if err == nil { + t.Fatal("expected error for invalid URL checksum") } } diff --git a/versioned/internal/process/binary_version.go b/versioned/internal/process/binary_version.go index 256f7055ec..6ae50da30c 100644 --- a/versioned/internal/process/binary_version.go +++ b/versioned/internal/process/binary_version.go @@ -1,7 +1,9 @@ package process import ( + "bytes" "context" + "errors" "fmt" "log/slog" "os" @@ -13,6 +15,13 @@ import ( const ( printBinaryVersionFlag = "--print-binary-version" printProtocolVersionFlag = "--print-protocol-version" + printAdminAPIVersionFlag = "--print-admin-api-version" + printStorageModeFlag = "--print-storage-mode" +) + +var ( + errVersionFlagUnsupported = errors.New("version flag unsupported") + embeddedVersionProbeTimeout = 10 * time.Second ) // childPreflight holds version metadata read before starting a child binary. @@ -20,22 +29,29 @@ type childPreflight struct { // binaryLogVersion is passed to the child as DEVSHARD_BINARY_LOG_VERSION. // When --print-binary-version is unsupported, this is the governance slot // name (approved_versions.name, e.g. v2). - binaryLogVersion string + binaryLogVersion string + adminAPISupported bool + storageMode string } // preflightChild verifies a downloaded binary when --print-* flags are // available. Legacy binaries (released before the flags) fall back per flag: -// - --print-binary-version missing → use slotName for DEVSHARD_BINARY_LOG_VERSION -// - --print-protocol-version missing → trust governance slot, skip embed check +// - --print-binary-version missing: use slotName for DEVSHARD_BINARY_LOG_VERSION +// - --print-protocol-version missing: trust governance slot, skip embed check func preflightChild(binPath, slotName string) (childPreflight, error) { + return preflightChildWithAdminProbe(binPath, slotName, false) +} + +func preflightChildWithAdminProbe(binPath, slotName string, probeAdmin bool) (childPreflight, error) { if _, err := os.Stat(binPath); err != nil { return childPreflight{}, fmt.Errorf("binary not found: %w", err) } binaryLogVersion, binErr := readBinaryLogVersion(binPath) - embeddedProtocol, protoErr := readProtocolVersion(binPath) - if binErr != nil { + if !errors.Is(binErr, errVersionFlagUnsupported) { + return childPreflight{}, fmt.Errorf("read binary log version: %w", binErr) + } slog.Warn( "--print-binary-version unsupported, using slot name for DEVSHARD_BINARY_LOG_VERSION", "slot", slotName, @@ -45,7 +61,11 @@ func preflightChild(binPath, slotName string) (childPreflight, error) { binaryLogVersion = slotName } + embeddedProtocol, protoErr := readProtocolVersion(binPath) if protoErr != nil { + if !errors.Is(protoErr, errVersionFlagUnsupported) { + return childPreflight{}, fmt.Errorf("read protocol version: %w", protoErr) + } slog.Warn( "--print-protocol-version unsupported, trusting governance slot name", "slot", slotName, @@ -59,26 +79,87 @@ func preflightChild(binPath, slotName string) (childPreflight, error) { ) } - return childPreflight{binaryLogVersion: binaryLogVersion}, nil + adminSupported := false + storageMode := "" + if probeAdmin { + if _, adminErr := readAdminAPIVersion(binPath); adminErr != nil { + if !errors.Is(adminErr, errVersionFlagUnsupported) { + return childPreflight{}, fmt.Errorf("read admin api version: %w", adminErr) + } + } else { + adminSupported = true + } + + mode, modeErr := readStorageMode(binPath) + if modeErr != nil { + if !errors.Is(modeErr, errVersionFlagUnsupported) { + return childPreflight{}, fmt.Errorf("read storage mode: %w", modeErr) + } + slog.Warn( + "--print-storage-mode unsupported, treating devshard storage mode as legacy", + "slot", slotName, + "bin", binPath, + "error", modeErr, + ) + } else { + storageMode = mode + } + } + + return childPreflight{ + binaryLogVersion: binaryLogVersion, + adminAPISupported: adminSupported, + storageMode: storageMode, + }, nil } // readEmbeddedVersion runs binPath with flag and returns trimmed stdout. func readEmbeddedVersion(binPath, flag string) (string, error) { - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + ctx, cancel := context.WithTimeout(context.Background(), embeddedVersionProbeTimeout) defer cancel() cmd := exec.CommandContext(ctx, binPath, flag) - out, err := cmd.Output() + var stdout bytes.Buffer + var stderr bytes.Buffer + cmd.Stdout = &stdout + cmd.Stderr = &stderr + + err := cmd.Run() if err != nil { + if ctx.Err() != nil { + return "", fmt.Errorf("%s %s timed out: %w", binPath, flag, ctx.Err()) + } + output := strings.TrimSpace(stderr.String() + stdout.String()) + if isUnsupportedVersionFlagError(err, output) { + return "", fmt.Errorf("%w: %s %s: %v: %s", errVersionFlagUnsupported, binPath, flag, err, output) + } + if output != "" { + return "", fmt.Errorf("%s %s: %w: %s", binPath, flag, err, output) + } return "", fmt.Errorf("%s %s: %w", binPath, flag, err) } - v := strings.TrimSpace(string(out)) + v := strings.TrimSpace(stdout.String()) if v == "" { return "", fmt.Errorf("%s %s: empty output", binPath, flag) } return v, nil } +func isUnsupportedVersionFlagError(err error, output string) bool { + var exitErr *exec.ExitError + if !errors.As(err, &exitErr) { + return false + } + if exitErr.ProcessState != nil && exitErr.ProcessState.ExitCode() < 0 { + return false + } + msg := strings.ToLower(output) + return strings.Contains(msg, "unknown flag") || + strings.Contains(msg, "flag provided but not defined") || + strings.Contains(msg, "unknown shorthand flag") || + strings.Contains(msg, "unrecognized option") +} + func readBinaryLogVersion(binPath string) (string, error) { return readEmbeddedVersion(binPath, printBinaryVersionFlag) } @@ -86,3 +167,11 @@ func readBinaryLogVersion(binPath string) (string, error) { func readProtocolVersion(binPath string) (string, error) { return readEmbeddedVersion(binPath, printProtocolVersionFlag) } + +func readAdminAPIVersion(binPath string) (string, error) { + return readEmbeddedVersion(binPath, printAdminAPIVersionFlag) +} + +func readStorageMode(binPath string) (string, error) { + return readEmbeddedVersion(binPath, printStorageModeFlag) +} diff --git a/versioned/internal/process/manager.go b/versioned/internal/process/manager.go index 5b53d2522b..e433858e39 100644 --- a/versioned/internal/process/manager.go +++ b/versioned/internal/process/manager.go @@ -2,72 +2,194 @@ package process import ( "context" + "encoding/json" "errors" "fmt" + "io" "log/slog" "net" + "net/http" "os" "os/exec" "path/filepath" + "sort" + "strconv" "strings" "sync" "sync/atomic" - "syscall" "time" "versioned/internal/config" "versioned/internal/download" "versioned/internal/health" "versioned/internal/oracle" + "versioned/internal/proxy" ) const ( statusStarting = "starting" statusRunning = "running" + statusDraining = "draining" statusStopped = "stopped" + + childLoopbackHost = "127.0.0.1" + maxChildPort = 65535 + storageModePostgres = "postgres" + defaultDevshardShutdownGrace = 10 * time.Minute + installedVersionRetain = 3 +) + +var ( + errChildPortPoolExhausted = errors.New("child port pool exhausted") + errLegacyDrainStatus = errors.New("drain status endpoint unavailable") ) type child struct { - version oracle.Version - port int - cancel context.CancelFunc - done chan struct{} // closed when runChild exits - status string + version oracle.Version + archiveSHA256 string + binaryVersion string + storageMode string + binPath string + port int + adminPort int + stop context.CancelFunc + forceStopCh chan struct{} + forceStopOnce sync.Once + done chan struct{} // closed when runChild exits + ready chan struct{} // closed after readiness succeeds + readyOnce sync.Once + proxyTarget *proxy.Target + status string + restart bool +} + +func (c *child) Stop() { + if c.stop != nil { + c.stop() + } +} + +func (c *child) ForceStop() { + c.Stop() + if c.forceStopCh != nil { + c.forceStopOnce.Do(func() { close(c.forceStopCh) }) + } +} + +func (c *child) Done() <-chan struct{} { + return c.done } type Manager struct { - cfg config.Config - processes map[string]*child - downloading map[string]struct{} - assignedPorts map[string]int // version name -> assigned port (persists for manager lifetime) - nextPort int - mu sync.Mutex - routes atomic.Value // map[string]string + cfg config.Config + processes map[string]*child + draining map[string][]*child + downloading map[string]struct{} + allocatedPorts map[int]struct{} + reservedPorts map[int]struct{} + mu sync.Mutex + routes atomic.Value // proxy.RouteTable } func NewManager(cfg config.Config) *Manager { + cfg = normalizeConfig(cfg) m := &Manager{ - cfg: cfg, - processes: make(map[string]*child), - downloading: make(map[string]struct{}), - assignedPorts: make(map[string]int), - nextPort: cfg.BasePort, + cfg: cfg, + processes: make(map[string]*child), + draining: make(map[string][]*child), + downloading: make(map[string]struct{}), + allocatedPorts: make(map[int]struct{}), + reservedPorts: reservedChildPorts(), } - m.routes.Store(map[string]string{}) + m.routes.Store(proxy.RouteTable{}) return m } -// assignPort returns a stable port for the given version name. -// Once assigned, the same name always gets the same port. +func normalizeConfig(cfg config.Config) config.Config { + if cfg.BasePort <= 0 || cfg.BasePort > maxChildPort { + cfg.BasePort = 5000 + } + if cfg.ReadyPath == "" { + cfg.ReadyPath = "/ready" + } + if cfg.ReadyTimeout <= 0 { + cfg.ReadyTimeout = 60 * time.Second + } + if cfg.DrainPath == "" { + cfg.DrainPath = "/drain" + } + if cfg.DrainStatusPath == "" { + cfg.DrainStatusPath = "/drain/status" + } + if cfg.DrainTimeout <= 0 { + cfg.DrainTimeout = 15 * time.Minute + } + if cfg.DrainPollInterval <= 0 { + cfg.DrainPollInterval = time.Second + } + if cfg.DrainKillGrace <= 0 { + cfg.DrainKillGrace = config.DefaultDrainKillGrace + } + return cfg +} + +// assignPort returns a currently-free child port. // Must be called with m.mu held. -func (m *Manager) assignPort(name string) int { - if port, ok := m.assignedPorts[name]; ok { - return port +func (m *Manager) assignPort() (int, error) { + for port := m.cfg.BasePort; port <= maxChildPort; port++ { + if _, used := m.allocatedPorts[port]; used { + continue + } + if _, reserved := m.reservedPorts[port]; reserved { + continue + } + m.allocatedPorts[port] = struct{}{} + return port, nil } - port := m.nextPort - m.nextPort++ - m.assignedPorts[name] = port - return port + return 0, fmt.Errorf( + "%w in range %d-%d", + errChildPortPoolExhausted, + m.cfg.BasePort, + maxChildPort, + ) +} + +func reservedChildPorts() map[int]struct{} { + ports := make(map[int]struct{}) + if port, ok := parseListenPort(config.ListenAddr()); ok { + ports[port] = struct{}{} + } + return ports +} + +func parseListenPort(addr string) (int, bool) { + _, portStr, err := net.SplitHostPort(addr) + if err != nil { + if !strings.HasPrefix(addr, ":") { + slog.Warn("cannot parse versiond listen address for child port reservation", "addr", addr, "error", err) + return 0, false + } + portStr = strings.TrimPrefix(addr, ":") + } + port, err := strconv.Atoi(portStr) + if err != nil || port <= 0 || port > maxChildPort { + slog.Warn("cannot parse versiond listen port for child port reservation", "addr", addr, "port", portStr, "error", err) + return 0, false + } + return port, true +} + +// releasePort releases a child port after the child process exits. +// Must be called with m.mu held. +func (m *Manager) releasePort(port int) { + if port > 0 { + delete(m.allocatedPorts, port) + } +} + +func (m *Manager) devshardAdminEligible() bool { + name := strings.ToLower(m.cfg.BinaryName) + return name == "devshard" || name == "devshardd" } func (m *Manager) RouteTable() *atomic.Value { @@ -77,14 +199,27 @@ func (m *Manager) RouteTable() *atomic.Value { func (m *Manager) Status() []health.StatusEntry { m.mu.Lock() defer m.mu.Unlock() - out := make([]health.StatusEntry, 0, len(m.processes)) + out := make([]health.StatusEntry, 0, len(m.processes)+len(m.draining)) for _, c := range m.processes { out = append(out, health.StatusEntry{ - Name: c.version.Name, - Port: c.port, - Status: c.status, + Name: c.version.Name, + Port: c.port, + Status: c.status, + SHA256: c.archiveSHA256, + BinaryVersion: c.binaryVersion, }) } + for _, children := range m.draining { + for _, c := range children { + out = append(out, health.StatusEntry{ + Name: c.version.Name, + Port: c.port, + Status: c.status, + SHA256: c.archiveSHA256, + BinaryVersion: c.binaryVersion, + }) + } + } return out } @@ -118,37 +253,41 @@ func (m *Manager) Reconcile(ctx context.Context, desired []oracle.Version) error // Phase A (lock): snapshot state, identify overrides. m.mu.Lock() type overrideAction struct { - version oracle.Version - overrideSrc string - binPath string + version oracle.Version + overrideSrc string + binPath string + blockedByDrain bool } var overrides []overrideAction // Snapshot which versions are running and which are downloading. type versionSnapshot struct { version oracle.Version - versionDir string - binPath string isRunning bool + isDraining bool isDownloading bool child *child } var snapshots []versionSnapshot for _, v := range desiredSet { - versionDir := filepath.Join(m.cfg.BinDir, v.Name) - binPath := filepath.Join(versionDir, m.cfg.BinaryName) + running, isRunning := m.processes[v.Name] + isDraining := len(m.draining[v.Name]) > 0 if overrideSrc, isOverride := m.cfg.Overrides[v.Name]; isOverride { - overrides = append(overrides, overrideAction{v, overrideSrc, binPath}) + binPath := filepath.Join(m.cfg.BinDir, v.Name, m.cfg.BinaryName) + overrides = append(overrides, overrideAction{ + version: v, + overrideSrc: overrideSrc, + binPath: binPath, + blockedByDrain: !isRunning && isDraining, + }) continue } - running, isRunning := m.processes[v.Name] _, isDownloading := m.downloading[v.Name] snapshots = append(snapshots, versionSnapshot{ version: v, - versionDir: versionDir, - binPath: binPath, isRunning: isRunning, + isDraining: isDraining, isDownloading: isDownloading, child: running, }) @@ -157,12 +296,17 @@ func (m *Manager) Reconcile(ctx context.Context, desired []oracle.Version) error // Phase B (no lock): resolve hashes, do disk I/O for overrides and hash checks. for _, o := range overrides { + if o.blockedByDrain { + slog.Info("version start deferred while previous child is draining", "version", o.version.Name) + continue + } m.reconcileOverride(ctx, o.version, o.overrideSrc, o.binPath) } var toDownload []versionAction var toSwap []versionAction - var toStart []oracle.Version + var toStart []versionAction + desiredHashes := make(map[string]string) for _, snap := range snapshots { if snap.isDownloading { @@ -174,9 +318,14 @@ func (m *Manager) Reconcile(ctx context.Context, desired []oracle.Version) error slog.Error("cannot resolve sha256, skipping", "version", snap.version.Name, "error", err) continue } + desiredHashes[snap.version.Name] = desiredHash + if !snap.isRunning && snap.isDraining { + slog.Info("version start deferred while previous child is draining", "version", snap.version.Name) + continue + } if snap.isRunning { - matches, metadata, diskBinaryHash, stateErr := installedVersionMatches(snap.versionDir, snap.binPath, desiredHash) + matches, metadata, diskBinaryHash, stateErr := installedVersionMatches(filepath.Dir(snap.child.binPath), snap.child.binPath, desiredHash) if stateErr == nil && matches { continue } @@ -193,38 +342,44 @@ func (m *Manager) Reconcile(ctx context.Context, desired []oracle.Version) error } // Not running. - matches, metadata, diskBinaryHash, stateErr := installedVersionMatches(snap.versionDir, snap.binPath, desiredHash) - if stateErr == nil && matches { - toStart = append(toStart, snap.version) + if artifact, ok := m.resolveInstalledArtifact(snap.version.Name, desiredHash); ok { + toStart = append(toStart, versionAction{ + version: snap.version, + sha256: desiredHash, + binPath: artifact.binPath, + }) continue } - if stateErr == nil || !errors.Is(stateErr, os.ErrNotExist) { - logInstalledVersionMismatch( - "cached version", - snap.version.Name, - desiredHash, - metadata, - diskBinaryHash, - stateErr, - ) - } - cleanupInstalledVersionState(snap.versionDir, snap.binPath) toDownload = append(toDownload, versionAction{version: snap.version, sha256: desiredHash}) } // Phase C (lock): apply decisions -- start ready children, mark downloads, stop removed. m.mu.Lock() - for _, v := range toStart { - if _, already := m.processes[v.Name]; already { + var startErrs []error + started := 0 + for _, a := range toStart { + if _, already := m.processes[a.version.Name]; already { continue // another reconcile started it } - m.startChild(ctx, v) + if m.versionStartBlockedLocked(a.version.Name) { + continue + } + if err := m.startChild(ctx, a.version, a.sha256, a.binPath, true); err != nil { + startErrs = append(startErrs, fmt.Errorf("start cached version %s: %w", a.version.Name, err)) + continue + } + started++ } + scheduledDownloads := make([]versionAction, 0, len(toDownload)) for _, a := range toDownload { if _, already := m.downloading[a.version.Name]; already { continue } + if _, running := m.processes[a.version.Name]; running || m.versionStartBlockedLocked(a.version.Name) { + continue + } m.downloading[a.version.Name] = struct{}{} + scheduledDownloads = append(scheduledDownloads, a) } for _, a := range toSwap { if _, already := m.downloading[a.version.Name]; already { @@ -237,20 +392,34 @@ func (m *Manager) Reconcile(ctx context.Context, desired []oracle.Version) error for name, c := range m.processes { if _, wanted := desiredSet[name]; !wanted { toStop = append(toStop, c) + c.status = statusDraining + c.restart = false delete(m.processes, name) + m.draining[name] = append(m.draining[name], c) } } - changed := len(toDownload) > 0 || len(toSwap) > 0 || len(toStop) > 0 || len(toStart) > 0 + changed := len(scheduledDownloads) > 0 || len(toSwap) > 0 || len(toStop) > 0 || started > 0 if changed { m.rebuildRoutes() } + proxyDrained := make(map[*child]<-chan struct{}, len(toStop)) + for _, c := range toStop { + proxyDrained[c] = retireProxyTarget(c) + } m.mu.Unlock() + // Removed versions leave the route table immediately, then drain + // asynchronously so reconcile can continue handling other versions. + for _, c := range toStop { + slog.Info("draining removed version", "version", c.version.Name) + go m.drainAfterProxy(c, proxyDrained[c]) + } + // Downloads outside the lock (can be slow). - for _, a := range toDownload { + for _, a := range scheduledDownloads { if err := m.downloadAndStart(ctx, a.version, a.sha256); err != nil { - slog.Error("download failed, skipping", "version", a.version.Name, "error", err) + slog.Error("download or start failed, skipping", "version", a.version.Name, "error", err) } } @@ -261,24 +430,23 @@ func (m *Manager) Reconcile(ctx context.Context, desired []oracle.Version) error } } - // Stop removed versions outside the lock. - for _, c := range toStop { - slog.Info("stopping removed version", "version", c.version.Name) - c.cancel() - } - for _, c := range toStop { - waitForChild(c, 5*time.Second) - } - - return nil + m.gcInstalledVersions(desiredHashes) + return errors.Join(startErrs...) } type versionAction struct { version oracle.Version sha256 string // pre-resolved hash, avoids double resolution in downloadBinary + binPath string // non-empty for cached start actions child *child // non-nil for swap actions } +// versionStartBlockedLocked reports whether a retired generation with the same +// version name still owns the version's data directory. +func (m *Manager) versionStartBlockedLocked(name string) bool { + return len(m.draining[name]) > 0 +} + // reconcileOverride handles a version with a local override binary. // Does disk I/O outside the lock, then takes the lock to update state. func (m *Manager) reconcileOverride(ctx context.Context, v oracle.Version, overrideSrc, binPath string) { @@ -306,21 +474,24 @@ func (m *Manager) reconcileOverride(ctx context.Context, v oracle.Version, overr slog.Error("override source unreadable", "version", v.Name, "path", overrideSrc, "error", err) return } + overrideID := "override:" + srcHash // Check if already running the same binary (lock for snapshot only). m.mu.Lock() existing, isRunning := m.processes[v.Name] m.mu.Unlock() - if isRunning { + if isRunning && existing.binPath == binPath && existing.archiveSHA256 == overrideID { diskHash, hashErr := download.HashFile(binPath) if hashErr == nil && diskHash == srcHash { return // already running the same override binary } + } + if isRunning { // Override source changed: stop old, copy new, start. slog.Info("override binary changed, restarting", "version", v.Name) - existing.cancel() - waitForChild(existing, 5*time.Second) + existing.Stop() + waitForChild(existing) } // Disk I/O outside the lock. @@ -341,11 +512,25 @@ func (m *Manager) reconcileOverride(ctx context.Context, v oracle.Version, overr // Verify the process is still the one we captured before deleting. // A concurrent reconcile could have replaced it. if isRunning { - if current, ok := m.processes[v.Name]; ok && current == existing { + if current, ok := m.processes[v.Name]; ok { + if current != existing { + m.mu.Unlock() + return + } delete(m.processes, v.Name) + } else if m.versionStartBlockedLocked(v.Name) { + m.mu.Unlock() + return } + } else if _, running := m.processes[v.Name]; running || m.versionStartBlockedLocked(v.Name) { + m.mu.Unlock() + return + } + if err := m.startChild(ctx, v, overrideID, binPath, true); err != nil { + m.mu.Unlock() + slog.Error("override start failed", "version", v.Name, "error", err) + return } - m.startChild(ctx, v) m.rebuildRoutes() m.mu.Unlock() } @@ -376,21 +561,168 @@ func atomicCopy(src, dst string) error { return download.AtomicWriteFile(filepath.Dir(dst), filepath.Base(dst), in) } +func (m *Manager) installDir(versionName, sha string) string { + return filepath.Join(m.cfg.BinDir, versionName, sha) +} + +func (m *Manager) installBinPath(versionName, sha string) string { + return filepath.Join(m.installDir(versionName, sha), m.cfg.BinaryName) +} + +type installedArtifact struct { + dir string + binPath string +} + +func (m *Manager) resolveInstalledArtifact(versionName, desiredHash string) (installedArtifact, bool) { + canonical := installedArtifact{ + dir: m.installDir(versionName, desiredHash), + binPath: m.installBinPath(versionName, desiredHash), + } + matches, metadata, diskBinaryHash, stateErr := installedVersionMatches( + canonical.dir, + canonical.binPath, + desiredHash, + ) + if stateErr == nil && matches { + return canonical, true + } + if stateErr == nil || !errors.Is(stateErr, os.ErrNotExist) { + logInstalledVersionMismatch( + "cached version", + versionName, + desiredHash, + metadata, + diskBinaryHash, + stateErr, + ) + } + // Do not clean an unreadable canonical install here. Another versiond + // sharing BinDir may still be publishing it; promotion and downloads use + // atomic replacement for ordinary files. + + legacy := installedArtifact{ + dir: filepath.Join(m.cfg.BinDir, versionName), + binPath: filepath.Join(m.cfg.BinDir, versionName, m.cfg.BinaryName), + } + matches, metadata, diskBinaryHash, stateErr = installedVersionMatches( + legacy.dir, + legacy.binPath, + desiredHash, + ) + if stateErr != nil || !matches { + if stateErr == nil || !errors.Is(stateErr, os.ErrNotExist) { + logInstalledVersionMismatch( + "legacy cached version", + versionName, + desiredHash, + metadata, + diskBinaryHash, + stateErr, + ) + } + return installedArtifact{}, false + } + + if err := m.promoteLegacyInstall(legacy, canonical, metadata, desiredHash); err == nil { + slog.Info( + "promoted legacy cached install", + "version", versionName, + "sha256", desiredHash, + "source", legacy.dir, + "destination", canonical.dir, + ) + return canonical, true + } else { + slog.Warn( + "legacy install promotion failed; using verified flat install", + "version", versionName, + "sha256", desiredHash, + "source", legacy.dir, + "destination", canonical.dir, + "error", err, + ) + } + + // The source may have changed while promotion copied it. Verify it again + // before falling back to the legacy path. + matches, _, _, stateErr = installedVersionMatches(legacy.dir, legacy.binPath, desiredHash) + if stateErr != nil || !matches { + return installedArtifact{}, false + } + return legacy, true +} + +func (m *Manager) promoteLegacyInstall( + legacy installedArtifact, + canonical installedArtifact, + metadata download.InstallMetadata, + desiredHash string, +) error { + if err := os.MkdirAll(canonical.dir, 0o755); err != nil { + return fmt.Errorf("create per-sha install dir: %w", err) + } + if err := atomicCopy(legacy.binPath, canonical.binPath); err != nil { + return fmt.Errorf("copy legacy binary: %w", err) + } + promotedBinaryHash, err := download.HashFile(canonical.binPath) + if err != nil { + return fmt.Errorf("hash promoted binary: %w", err) + } + if !strings.EqualFold(promotedBinaryHash, metadata.BinarySHA256) { + return fmt.Errorf( + "verify promoted binary: got %s, want %s", + promotedBinaryHash, + metadata.BinarySHA256, + ) + } + if err := download.WriteInstallMetadata(canonical.dir, metadata); err != nil { + return fmt.Errorf("write per-sha install metadata: %w", err) + } + + matches, promotedMetadata, diskBinaryHash, err := installedVersionMatches( + canonical.dir, + canonical.binPath, + desiredHash, + ) + if err != nil { + return fmt.Errorf("verify promoted install: %w", err) + } + if !matches { + return fmt.Errorf( + "verify promoted install: archive=%s binary=%s expected_archive=%s expected_binary=%s", + promotedMetadata.ArchiveSHA256, + diskBinaryHash, + desiredHash, + promotedMetadata.BinarySHA256, + ) + } + return nil +} + // downloadAndStart downloads the binary using the pre-resolved hash, then starts the child. func (m *Manager) downloadAndStart(ctx context.Context, v oracle.Version, sha string) error { dlErr := m.downloadBinary(ctx, v, sha) m.mu.Lock() delete(m.downloading, v.Name) - if dlErr == nil && ctx.Err() == nil { - m.startChild(ctx, v) + _, running := m.processes[v.Name] + var startErr error + if dlErr == nil && ctx.Err() == nil && !running && !m.versionStartBlockedLocked(v.Name) { + startErr = m.startChild(ctx, v, sha, m.installBinPath(v.Name, sha), true) } m.mu.Unlock() - return dlErr + if dlErr != nil { + return dlErr + } + if startErr != nil { + return fmt.Errorf("start downloaded version %s: %w", v.Name, startErr) + } + return nil } -// downloadAndSwap downloads the new binary, then atomically replaces the old one. -// The old process is stopped only after the new binary is on disk. +// downloadAndSwap downloads the new binary, starts it on a fresh port, swaps the +// route after readiness, and drains the old child out of band. func (m *Manager) downloadAndSwap(ctx context.Context, v oracle.Version, sha string, old *child) error { dlErr := m.downloadBinary(ctx, v, sha) if dlErr != nil || ctx.Err() != nil { @@ -403,27 +735,77 @@ func (m *Manager) downloadAndSwap(ctx context.Context, v oracle.Version, sha str return ctx.Err() } - // Stop old process after new binary is on disk. - slog.Info("stopping old process for swap", "version", v.Name) - old.cancel() - waitForChild(old, 5*time.Second) + newBinPath := m.installBinPath(v.Name, sha) + if !m.rollingOverlapAllowed(v.Name, old, newBinPath) { + slog.Warn("rolling overlap disabled without shared storage; falling back to stop/start swap", "version", v.Name) + old.Stop() + waitForChild(old) + m.mu.Lock() + delete(m.downloading, v.Name) + if current, ok := m.processes[v.Name]; ok && current == old { + delete(m.processes, v.Name) + } + startErr := m.startChild(ctx, v, sha, newBinPath, true) + m.mu.Unlock() + if startErr != nil { + return fmt.Errorf("start replacement version %s: %w", v.Name, startErr) + } + return nil + } + + m.mu.Lock() + newChild, startErr := m.newChild(ctx, v, sha, newBinPath, false) + if startErr != nil { + delete(m.downloading, v.Name) + m.mu.Unlock() + return fmt.Errorf("create replacement version %s: %w", v.Name, startErr) + } + m.mu.Unlock() + go m.runChild(newChild.ctx, newChild.child) + if err := waitForChildReady(ctx, newChild.child); err != nil { + newChild.child.Stop() + waitForChild(newChild.child) + m.mu.Lock() + delete(m.downloading, v.Name) + m.mu.Unlock() + return err + } - // Single lock section: clear downloading, remove old, start new. m.mu.Lock() delete(m.downloading, v.Name) + if current, ok := m.processes[v.Name]; !ok || current != old { + m.mu.Unlock() + newChild.child.Stop() + waitForChild(newChild.child) + return fmt.Errorf("current child changed during swap") + } + if newChild.child.status != statusRunning || childDone(newChild.child) { + m.mu.Unlock() + newChild.child.Stop() + waitForChild(newChild.child) + return fmt.Errorf("new child stopped before swap") + } + old.status = statusDraining + old.restart = false delete(m.processes, v.Name) - m.startChild(ctx, v) + m.draining[v.Name] = append(m.draining[v.Name], old) + newChild.child.restart = true + m.processes[v.Name] = newChild.child + m.rebuildRoutes() + proxyDrained := retireProxyTarget(old) m.mu.Unlock() + slog.Info("swapped child route; old child draining", "version", v.Name, "old_port", old.port, "new_port", newChild.child.port) + go m.drainAfterProxy(old, proxyDrained) return nil } func (m *Manager) downloadBinary(ctx context.Context, v oracle.Version, sha string) error { - binDir := filepath.Join(m.cfg.BinDir, v.Name) + binDir := m.installDir(v.Name, sha) if err := download.Download(ctx, v.Binary, sha, binDir, m.cfg.BinaryName); err != nil { return err } - slog.Info("downloaded binary", "version", v.Name) + slog.Info("downloaded binary", "version", v.Name, "sha256", sha) return nil } @@ -450,6 +832,119 @@ func installedVersionMatches(versionDir, binPath, desiredArchiveHash string) (bo func cleanupInstalledVersionState(versionDir, binPath string) { _ = os.Remove(binPath) _ = os.Remove(filepath.Join(versionDir, download.InstallMetadataFilename)) + _ = os.Remove(versionDir) +} + +type installedVersionDir struct { + path string + sha string + modTime time.Time +} + +func (m *Manager) gcInstalledVersions(desiredHashes map[string]string) { + keep := make(map[string]map[string]struct{}) + addKeep := func(versionName, sha string) { + if versionName == "" || sha == "" || strings.HasPrefix(sha, "override:") { + return + } + if keep[versionName] == nil { + keep[versionName] = make(map[string]struct{}) + } + keep[versionName][sha] = struct{}{} + } + for versionName, sha := range desiredHashes { + addKeep(versionName, sha) + } + + m.mu.Lock() + for _, c := range m.processes { + addKeep(c.version.Name, c.archiveSHA256) + } + for _, children := range m.draining { + for _, c := range children { + addKeep(c.version.Name, c.archiveSHA256) + } + } + m.mu.Unlock() + + gcInstalledVersionDirs(m.cfg.BinDir, m.cfg.BinaryName, keep, installedVersionRetain) +} + +func gcInstalledVersionDirs(binDir, binaryName string, keep map[string]map[string]struct{}, retain int) { + if retain < 0 { + retain = 0 + } + versionDirs, err := os.ReadDir(binDir) + if err != nil { + if !os.IsNotExist(err) { + slog.Warn("installed version gc: read bin dir failed", "dir", binDir, "error", err) + } + return + } + for _, versionEntry := range versionDirs { + if !versionEntry.IsDir() { + continue + } + versionName := versionEntry.Name() + versionDir := filepath.Join(binDir, versionName) + shaDirs, err := os.ReadDir(versionDir) + if err != nil { + slog.Warn("installed version gc: read version dir failed", "version", versionName, "dir", versionDir, "error", err) + continue + } + var stale []installedVersionDir + for _, shaEntry := range shaDirs { + if !shaEntry.IsDir() { + continue + } + sha := shaEntry.Name() + dir := filepath.Join(versionDir, sha) + metadata, err := download.ReadInstallMetadata(dir) + if err != nil { + continue + } + if keepInstalledVersion(keep, versionName, sha, metadata.ArchiveSHA256) { + continue + } + stale = append(stale, installedVersionDir{ + path: dir, + sha: sha, + modTime: installedVersionModTime(dir), + }) + } + sort.Slice(stale, func(i, j int) bool { + if stale[i].modTime.Equal(stale[j].modTime) { + return stale[i].sha > stale[j].sha + } + return stale[i].modTime.After(stale[j].modTime) + }) + for i := retain; i < len(stale); i++ { + slog.Info("installed version gc: removing stale install", "version", versionName, "sha256", stale[i].sha, "dir", stale[i].path) + cleanupInstalledVersionState(stale[i].path, filepath.Join(stale[i].path, binaryName)) + } + } +} + +func keepInstalledVersion(keep map[string]map[string]struct{}, versionName, dirSHA, archiveSHA string) bool { + versionKeep := keep[versionName] + if versionKeep == nil { + return false + } + if _, ok := versionKeep[dirSHA]; ok { + return true + } + _, ok := versionKeep[archiveSHA] + return ok +} + +func installedVersionModTime(dir string) time.Time { + if info, err := os.Stat(filepath.Join(dir, download.InstallMetadataFilename)); err == nil { + return info.ModTime() + } + if info, err := os.Stat(dir); err == nil { + return info.ModTime() + } + return time.Time{} } func logInstalledVersionMismatch(scope, versionName, desiredArchiveHash string, metadata download.InstallMetadata, diskBinaryHash string, stateErr error) { @@ -475,49 +970,102 @@ func logInstalledVersionMismatch(scope, versionName, desiredArchiveHash string, "disk_binary", diskBinaryHash) } -// startChild must be called with m.mu held. -func (m *Manager) startChild(ctx context.Context, v oracle.Version) { +type childStart struct { + child *child + ctx context.Context +} + +func (m *Manager) newChild(ctx context.Context, v oracle.Version, sha, binPath string, restart bool) (childStart, error) { + port, err := m.assignPort() + if err != nil { + return childStart{}, fmt.Errorf("allocate public port for version %s: %w", v.Name, err) + } childCtx, childCancel := context.WithCancel(ctx) c := &child{ - version: v, - port: m.assignPort(v.Name), - cancel: childCancel, - done: make(chan struct{}), - status: statusStarting, + version: v, + archiveSHA256: sha, + binPath: binPath, + port: port, + stop: childCancel, + forceStopCh: make(chan struct{}), + done: make(chan struct{}), + ready: make(chan struct{}), + status: statusStarting, + restart: restart, + } + return childStart{child: c, ctx: childCtx}, nil +} + +// startChild must be called with m.mu held. +func (m *Manager) startChild(ctx context.Context, v oracle.Version, sha, binPath string, restart bool) error { + start, err := m.newChild(ctx, v, sha, binPath, restart) + if err != nil { + return err } + c := start.child m.processes[v.Name] = c - go m.runChild(childCtx, c) + go m.runChild(start.ctx, c) + return nil } func (m *Manager) Shutdown(ctx context.Context) error { m.mu.Lock() - children := make([]*child, 0, len(m.processes)) + children := make([]*child, 0, len(m.processes)+len(m.draining)) for _, c := range m.processes { children = append(children, c) - c.cancel() + c.Stop() + } + for _, draining := range m.draining { + for _, c := range draining { + children = append(children, c) + c.Stop() + } } m.processes = make(map[string]*child) + m.draining = make(map[string][]*child) m.downloading = make(map[string]struct{}) - m.routes.Store(map[string]string{}) + m.rebuildRoutes() m.mu.Unlock() - for _, c := range children { - slog.Info("shutting down", "version", c.version.Name) - waitForChild(c, 10*time.Second) + if len(children) == 0 { + return nil } - return nil -} -// waitForChild waits for a child's goroutine to exit within the timeout. -// The child should already have been cancelled via c.cancel(). -// exec.CommandContext sends SIGKILL when the context is cancelled, -// so the process will be killed. We just wait for runChild to finish. -func waitForChild(c *child, timeout time.Duration) { + allDone := make(chan struct{}) + go func() { + defer close(allDone) + for _, c := range children { + slog.Info("waiting for child shutdown", "version", c.version.Name) + waitForChild(c) + } + }() + select { - case <-c.done: - case <-time.After(timeout): - slog.Warn("child goroutine did not exit in time", "version", c.version.Name) + case <-allDone: + return nil + case <-ctx.Done(): + } + select { + case <-allDone: + return nil + default: } + + // The caller's deadline escalates every remaining child to SIGKILL. It does + // not waive process ownership: wait until every command has been reaped. + for _, c := range children { + c.ForceStop() + } + <-allDone + return ctx.Err() +} + +func (m *Manager) ShutdownTimeout() time.Duration { + return m.childStopTimeout() +} + +func waitForChild(c *child) { + <-c.Done() } func (m *Manager) runChild(ctx context.Context, c *child) { @@ -528,21 +1076,38 @@ func (m *Manager) runChild(ctx context.Context, c *child) { delete(m.processes, c.version.Name) m.rebuildRoutes() } + m.removeDrainingLocked(c) + m.releasePort(c.port) + m.releasePort(c.adminPort) m.mu.Unlock() }() - binPath := filepath.Join(m.cfg.BinDir, c.version.Name, m.cfg.BinaryName) dataDir := filepath.Join(m.cfg.DataDir, c.version.Name) if err := os.MkdirAll(dataDir, 0755); err != nil { slog.Error("create data dir failed", "version", c.version.Name, "error", err) return } - preflight, err := preflightChild(binPath, c.version.Name) + preflight, err := preflightChildWithAdminProbe(c.binPath, c.version.Name, m.devshardAdminEligible()) if err != nil { - slog.Error("child preflight failed", "version", c.version.Name, "bin", binPath, "error", err) + slog.Error("child preflight failed", "version", c.version.Name, "bin", c.binPath, "error", err) return } + m.mu.Lock() + c.binaryVersion = preflight.binaryLogVersion + c.storageMode = preflight.storageMode + if preflight.adminAPISupported && c.adminPort == 0 { + adminPort, portErr := m.assignPort() + if portErr != nil { + c.status = statusStopped + m.mu.Unlock() + slog.Error("allocate child admin port failed", "version", c.version.Name, "error", portErr) + return + } + c.adminPort = adminPort + } + adminAddr := c.adminAddr() + m.mu.Unlock() backoff := time.Second lastStart := time.Now() @@ -554,24 +1119,19 @@ func (m *Manager) runChild(ctx context.Context, c *child) { default: } - cmd := exec.CommandContext(ctx, binPath, + cmd := exec.Command(c.binPath, "--data-dir", dataDir, "--port", fmt.Sprintf("%d", c.port), ) - cmd.Env = childEnv(preflight.binaryLogVersion) + cmd.Env = childEnv(preflight.binaryLogVersion, adminAddr) cmd.Stdout = os.Stdout cmd.Stderr = os.Stderr - cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true} - // Cancel sends SIGKILL by default. Override to send SIGTERM for graceful shutdown. - cmd.Cancel = func() error { - return cmd.Process.Signal(syscall.SIGTERM) - } - cmd.WaitDelay = 5 * time.Second // SIGKILL after 5s if SIGTERM didn't work lastStart = time.Now() - slog.Info("starting child", "version", c.version.Name, "port", c.port) + slog.Info("starting child", "version", c.version.Name, "port", c.port, "admin_addr", adminAddr, "sha256", c.archiveSHA256) - if err := cmd.Start(); err != nil { + proc, err := startSupervisedProcess(cmd, ctx.Done(), c.forceStopCh, m.childStopTimeout()) + if err != nil { slog.Error("child start failed", "version", c.version.Name, "error", err) m.mu.Lock() c.status = statusStopped @@ -579,16 +1139,39 @@ func (m *Manager) runChild(ctx context.Context, c *child) { return } - // Wait for the child to start accepting connections before routing traffic. - if !waitForPort(ctx, c.port, 10*time.Second) { - slog.Warn("child did not start listening in time, routing anyway", "version", c.version.Name) + if !waitForChildServingReady(ctx, c, m.cfg.ReadyPath, m.cfg.ReadyTimeout) { + slog.Warn("child did not become ready in time", "version", c.version.Name, "port", c.port, "lifecycle_port", c.lifecyclePort(), "ready_path", m.cfg.ReadyPath) + proc.ForceStop() + _ = proc.Wait() + m.mu.Lock() + c.status = statusStopped + restart := c.restart + if current, ok := m.processes[c.version.Name]; ok && current == c { + m.rebuildRoutes() + } + m.mu.Unlock() + if !restart { + return + } + if !m.waitForRestartBackoff(ctx, c, backoff) { + return + } + backoff *= 2 + if backoff > 60*time.Second { + backoff = 60 * time.Second + } + continue } m.mu.Lock() c.status = statusRunning - m.rebuildRoutes() + c.proxyTarget = proxy.NewTarget(fmt.Sprintf("localhost:%d", c.port)) + c.readyOnce.Do(func() { close(c.ready) }) + if current, ok := m.processes[c.version.Name]; ok && current == c { + m.rebuildRoutes() + } m.mu.Unlock() - err := cmd.Wait() + err = proc.Wait() select { case <-ctx.Done(): @@ -600,8 +1183,14 @@ func (m *Manager) runChild(ctx context.Context, c *child) { m.mu.Lock() c.status = statusStopped - m.rebuildRoutes() + restart := c.restart + if current, ok := m.processes[c.version.Name]; ok && current == c { + m.rebuildRoutes() + } m.mu.Unlock() + if !restart { + return + } if time.Since(lastStart) > 60*time.Second { backoff = time.Second @@ -609,10 +1198,8 @@ func (m *Manager) runChild(ctx context.Context, c *child) { slog.Info("restarting child after backoff", "version", c.version.Name, "backoff", backoff) - select { - case <-ctx.Done(): + if !m.waitForRestartBackoff(ctx, c, backoff) { return - case <-time.After(backoff): } backoff *= 2 @@ -622,50 +1209,451 @@ func (m *Manager) runChild(ctx context.Context, c *child) { } } +func (m *Manager) waitForRestartBackoff(ctx context.Context, c *child, backoff time.Duration) bool { + timer := time.NewTimer(backoff) + defer timer.Stop() + + select { + case <-ctx.Done(): + return false + case <-timer.C: + } + + m.mu.Lock() + restart := c.restart + m.mu.Unlock() + return restart +} + // childEnv sets per-child env vars for devshardd (and testapp in e2e). // binaryLogVersion is normally the link-time build id from --print-binary-version // (e.g. 0.2.13-v2-r2). Legacy binaries without that flag use the governance // slot name (e.g. v2) instead. -func childEnv(binaryLogVersion string) []string { - if binaryLogVersion == "" { - return os.Environ() +func childEnv(binaryLogVersion, adminAddr string) []string { + env := make([]string, 0, len(os.Environ())+2) + for _, entry := range os.Environ() { + if strings.HasPrefix(entry, "DEVSHARD_ADMIN_ADDR=") { + continue + } + env = append(env, entry) + } + if binaryLogVersion != "" { + env = append(env, fmt.Sprintf("DEVSHARD_BINARY_LOG_VERSION=%s", binaryLogVersion)) + } + if adminAddr != "" { + env = append(env, fmt.Sprintf("DEVSHARD_ADMIN_ADDR=%s", adminAddr)) + } + return env +} + +func (m *Manager) childStopTimeout() time.Duration { + timeout := m.cfg.DrainKillGrace + name := strings.ToLower(m.cfg.BinaryName) + if name != "devshard" && name != "devshardd" { + return timeout + } + shutdownGrace := parseDevshardShutdownGrace(os.Getenv("DEVSHARD_SHUTDOWN_GRACE")) + if shutdownGrace > timeout { + return shutdownGrace + } + return timeout +} + +func parseDevshardShutdownGrace(raw string) time.Duration { + raw = strings.TrimSpace(raw) + if raw == "" { + return defaultDevshardShutdownGrace + } + d, err := time.ParseDuration(raw) + if err != nil || d <= 0 { + return defaultDevshardShutdownGrace + } + return d +} + +func (m *Manager) rollingOverlapAllowed(versionName string, old *child, newBinPath string) bool { + name := strings.ToLower(m.cfg.BinaryName) + if name != "devshard" && name != "devshardd" { + return true + } + + m.mu.Lock() + oldMode := "" + if old != nil { + oldMode = old.storageMode + } + m.mu.Unlock() + + if oldMode != storageModePostgres { + slog.Warn( + "rolling overlap disabled: running devshard storage mode is not postgres", + "version", versionName, + "storage_mode", oldMode, + ) + return false + } + + newMode, err := readStorageMode(newBinPath) + if err != nil { + slog.Warn( + "rolling overlap disabled: cannot probe new devshard storage mode", + "version", versionName, + "bin", newBinPath, + "error", err, + ) + return false + } + if newMode != storageModePostgres { + slog.Warn( + "rolling overlap disabled: new devshard storage mode is not postgres", + "version", versionName, + "bin", newBinPath, + "storage_mode", newMode, + ) + return false + } + return true +} + +func waitForChildReady(ctx context.Context, c *child) error { + select { + case <-ctx.Done(): + return ctx.Err() + case <-c.ready: + return nil + case <-c.done: + return fmt.Errorf("child exited before readiness") } - return append( - os.Environ(), - fmt.Sprintf("DEVSHARD_BINARY_LOG_VERSION=%s", binaryLogVersion), - ) } -// waitForPort polls until a TCP connection succeeds on the given port. -// Returns true if the port is reachable before the timeout or context cancellation. -func waitForPort(ctx context.Context, port int, timeout time.Duration) bool { - deadline := time.After(timeout) - addr := fmt.Sprintf("localhost:%d", port) +func childDone(c *child) bool { + select { + case <-c.done: + return true + default: + return false + } +} + +func (c *child) lifecyclePort() int { + if c.adminPort != 0 { + return c.adminPort + } + return c.port +} + +func (c *child) adminAddr() string { + if c.adminPort == 0 { + return "" + } + return fmt.Sprintf("%s:%d", childLoopbackHost, c.adminPort) +} + +func waitForReady(ctx context.Context, port int, path string, timeout time.Duration) bool { + return waitForReadiness(ctx, timeout, func(probeCtx context.Context, client *http.Client) bool { + return readyEndpointReady(probeCtx, client, port, path, true) + }) +} + +// waitForChildServingReady gates the Starting -> Running transition. Modern +// devshardd children must be logically ready on their admin listener and also +// serve health checks on the public listener that receives proxied traffic. +func waitForChildServingReady(ctx context.Context, c *child, path string, timeout time.Duration) bool { + if c.adminPort == 0 { + return waitForReady(ctx, c.port, path, timeout) + } + return waitForReadiness(ctx, timeout, func(probeCtx context.Context, client *http.Client) bool { + return readyEndpointReady(probeCtx, client, c.adminPort, path, false) && + publicEndpointReady(probeCtx, client, c.port) + }) +} + +func waitForReadiness( + ctx context.Context, + timeout time.Duration, + probe func(context.Context, *http.Client) bool, +) bool { + probeCtx, cancel := context.WithTimeout(ctx, timeout) + defer cancel() + client := &http.Client{Timeout: 500 * time.Millisecond} for { + if probe(probeCtx, client) { + return true + } + retry := time.NewTimer(100 * time.Millisecond) select { - case <-ctx.Done(): + case <-probeCtx.Done(): + retry.Stop() return false - case <-deadline: + case <-retry.C: + } + } +} + +func readyEndpointReady(ctx context.Context, client *http.Client, port int, path string, allowLegacy bool) bool { + readyPath := normalizeHTTPPath(path) + status, err := getHTTPStatus(ctx, client, port, readyPath) + if err != nil { + return false + } + if status == http.StatusOK { + return true + } + if allowLegacy && legacyReadyFallbackAllowed(readyPath, status) && legacyReady(ctx, client, port) { + slog.Warn("ready path unavailable; using legacy readiness fallback", "port", port, "ready_path", readyPath, "status", status) + return true + } + return false +} + +func publicEndpointReady(ctx context.Context, client *http.Client, port int) bool { + status, err := getHTTPStatus(ctx, client, port, "/healthz") + return err == nil && status >= http.StatusOK && status < http.StatusMultipleChoices +} + +func getHTTPStatus(ctx context.Context, client *http.Client, port int, path string) (int, error) { + url := fmt.Sprintf("http://%s:%d%s", childLoopbackHost, port, normalizeHTTPPath(path)) + req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) + if err != nil { + return 0, err + } + resp, err := client.Do(req) + if err != nil { + return 0, err + } + defer resp.Body.Close() + return resp.StatusCode, nil +} + +func legacyReadyFallbackAllowed(path string, status int) bool { + if path != "/ready" { + return false + } + switch status { + case 0, http.StatusNotFound, http.StatusMethodNotAllowed, http.StatusNotImplemented: + return true + default: + return false + } +} + +func legacyReady(ctx context.Context, client *http.Client, port int) bool { + url := fmt.Sprintf("http://%s:%d/healthz", childLoopbackHost, port) + req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) + if err != nil { + return tcpReady(ctx, port) + } + resp, err := client.Do(req) + if err == nil { + status := resp.StatusCode + resp.Body.Close() + if status >= 200 && status < 300 { + return true + } + if status != http.StatusNotFound && status != http.StatusMethodNotAllowed && status != http.StatusNotImplemented { return false + } + } + return tcpReady(ctx, port) +} + +func tcpReady(ctx context.Context, port int) bool { + dialer := net.Dialer{Timeout: 500 * time.Millisecond} + conn, err := dialer.DialContext(ctx, "tcp", fmt.Sprintf("%s:%d", childLoopbackHost, port)) + if err != nil { + return false + } + _ = conn.Close() + return true +} + +func normalizeHTTPPath(path string) string { + if path == "" { + return "/" + } + if strings.HasPrefix(path, "/") { + return path + } + return "/" + path +} + +func (m *Manager) requestDrain(c *child) { + lifecyclePort := c.lifecyclePort() + url := fmt.Sprintf("http://%s:%d%s", childLoopbackHost, lifecyclePort, normalizeHTTPPath(m.cfg.DrainPath)) + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) + defer cancel() + req, err := http.NewRequestWithContext(ctx, http.MethodPost, url, nil) + if err != nil { + return + } + resp, err := http.DefaultClient.Do(req) + if err != nil { + slog.Warn("drain request failed", "version", c.version.Name, "port", c.port, "lifecycle_port", lifecyclePort, "error", err) + return + } + _ = resp.Body.Close() + if resp.StatusCode < 200 || resp.StatusCode >= 300 { + slog.Warn("drain request returned non-success", "version", c.version.Name, "port", c.port, "lifecycle_port", lifecyclePort, "status", resp.StatusCode) + } +} + +func (m *Manager) drainAfterProxy(c *child, proxyDrained <-chan struct{}) { + // Proxy admission and child lifecycle draining share one safety deadline. + deadline := time.Now().Add(m.cfg.DrainTimeout) + timer := time.NewTimer(m.cfg.DrainTimeout) + defer timer.Stop() + select { + case <-c.done: + return + case <-proxyDrained: + slog.Info("proxy requests drained", "version", c.version.Name, "port", c.port) + case <-timer.C: + slog.Warn("proxy drain timeout reached", "version", c.version.Name, "port", c.port) + } + m.requestDrain(c) + m.drainAndStopBefore(c, deadline) +} + +func (m *Manager) drainAndStop(c *child) { + m.drainAndStopBefore(c, time.Now().Add(m.cfg.DrainTimeout)) +} + +func (m *Manager) drainAndStopBefore(c *child, deadline time.Time) { + for { + select { + case <-c.done: + return default: } - conn, err := net.DialTimeout("tcp", addr, 200*time.Millisecond) - if err == nil { - conn.Close() - return true + inflight, err := m.fetchInflight(c) + if errors.Is(err, errLegacyDrainStatus) { + slog.Warn( + "drain status endpoint unavailable; using legacy drain grace", + "version", c.version.Name, + "port", c.port, + "grace", m.cfg.DrainKillGrace, + "error", err, + ) + select { + case <-c.done: + return + case <-time.After(m.cfg.DrainKillGrace): + } + break + } + if err == nil && inflight == 0 { + slog.Info("draining child is idle", "version", c.version.Name, "port", c.port) + break + } + if time.Now().After(deadline) { + if err != nil { + slog.Warn("drain timeout reached with unreadable drain status", "version", c.version.Name, "port", c.port, "error", err) + } else { + slog.Warn("drain timeout reached with in-flight work", "version", c.version.Name, "port", c.port, "inflight", inflight) + } + break + } + if err != nil { + slog.Warn("drain status failed; retrying", "version", c.version.Name, "port", c.port, "error", err) + } + select { + case <-c.done: + return + case <-time.After(m.cfg.DrainPollInterval): } - time.Sleep(100 * time.Millisecond) } + c.Stop() + waitForChild(c) +} + +func retireProxyTarget(c *child) <-chan struct{} { + if c.proxyTarget != nil { + return c.proxyTarget.Retire() + } + drained := make(chan struct{}) + close(drained) + return drained } -// rebuildRoutes rebuilds the atomic route map. Only includes running children. +func (m *Manager) fetchInflight(c *child) (int64, error) { + url := fmt.Sprintf("http://%s:%d%s", childLoopbackHost, c.lifecyclePort(), normalizeHTTPPath(m.cfg.DrainStatusPath)) + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) + defer cancel() + req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) + if err != nil { + return 0, err + } + resp, err := http.DefaultClient.Do(req) + if err != nil { + return 0, err + } + defer resp.Body.Close() + if resp.StatusCode == http.StatusNotFound || + resp.StatusCode == http.StatusMethodNotAllowed || + resp.StatusCode == http.StatusNotImplemented { + return 0, fmt.Errorf("%w: drain status returned %d", errLegacyDrainStatus, resp.StatusCode) + } + if resp.StatusCode != http.StatusOK { + return 0, fmt.Errorf("drain status returned %d", resp.StatusCode) + } + body, err := io.ReadAll(resp.Body) + if err != nil { + return 0, err + } + var status struct { + Inflight *int64 `json:"inflight"` + Active *int64 `json:"active"` + Count *int64 `json:"count"` + } + if err := json.Unmarshal(body, &status); err != nil { + return 0, err + } + switch { + case status.Inflight != nil: + return *status.Inflight, nil + case status.Active != nil: + return *status.Active, nil + case status.Count != nil: + return *status.Count, nil + default: + return 0, fmt.Errorf("drain status missing inflight count") + } +} + +func (m *Manager) removeDrainingLocked(target *child) { + children := m.draining[target.version.Name] + for i, c := range children { + if c != target { + continue + } + children = append(children[:i], children[i+1:]...) + if len(children) == 0 { + delete(m.draining, target.version.Name) + } else { + m.draining[target.version.Name] = children + } + return + } +} + +// rebuildRoutes publishes running children, then retires replaced targets so a +// stale proxy lookup either owns a counted lease or retries against the new map. // Must be called with m.mu held. func (m *Manager) rebuildRoutes() { - routes := make(map[string]string) + previous := m.routes.Load().(proxy.RouteTable) + routes := make(proxy.RouteTable) for _, c := range m.processes { if c.status == statusRunning { - routes[c.version.Name] = fmt.Sprintf("localhost:%d", c.port) + if c.proxyTarget == nil { + c.proxyTarget = proxy.NewTarget(fmt.Sprintf("localhost:%d", c.port)) + } + routes[c.version.Name] = c.proxyTarget } } m.routes.Store(routes) + for version, target := range previous { + if routes[version] != target { + target.Retire() + } + } } diff --git a/versioned/internal/process/manager_test.go b/versioned/internal/process/manager_test.go index 522e5905e6..01628a04be 100644 --- a/versioned/internal/process/manager_test.go +++ b/versioned/internal/process/manager_test.go @@ -1,14 +1,20 @@ package process import ( + "archive/zip" + "bytes" "context" "crypto/sha256" "encoding/hex" "encoding/json" + "errors" + "net" "net/http" "net/http/httptest" "os" "path/filepath" + "strings" + "sync" "sync/atomic" "testing" "time" @@ -16,10 +22,11 @@ import ( "versioned/internal/config" "versioned/internal/download" "versioned/internal/oracle" + "versioned/internal/proxy" ) func TestChildEnvIncludesVersionLogPrefix(t *testing.T) { - env := childEnv("0.2.13-v2-r2") + env := childEnv("0.2.13-v2-r2", "") want := map[string]bool{ "DEVSHARD_BINARY_LOG_VERSION=0.2.13-v2-r2": false, } @@ -36,7 +43,7 @@ func TestChildEnvIncludesVersionLogPrefix(t *testing.T) { } func TestChildEnvSlotNameFallback(t *testing.T) { - env := childEnv("v2") + env := childEnv("v2", "") want := "DEVSHARD_BINARY_LOG_VERSION=v2" found := false for _, entry := range env { @@ -50,6 +57,35 @@ func TestChildEnvSlotNameFallback(t *testing.T) { } } +func TestChildEnvIncludesAdminAddr(t *testing.T) { + env := childEnv("v2", "127.0.0.1:6001") + want := map[string]bool{ + "DEVSHARD_BINARY_LOG_VERSION=v2": false, + "DEVSHARD_ADMIN_ADDR=127.0.0.1:6001": false, + } + for _, entry := range env { + if _, ok := want[entry]; ok { + want[entry] = true + } + } + for key, present := range want { + if !present { + t.Fatalf("childEnv missing %q", key) + } + } +} + +func TestChildEnvDoesNotLeakParentAdminAddr(t *testing.T) { + t.Setenv("DEVSHARD_ADMIN_ADDR", "127.0.0.1:9999") + + env := childEnv("v2", "") + for _, entry := range env { + if strings.HasPrefix(entry, "DEVSHARD_ADMIN_ADDR=") { + t.Fatalf("childEnv leaked parent %q", entry) + } + } +} + func TestPreflightChild_MissingBinary(t *testing.T) { _, err := preflightChild(filepath.Join(t.TempDir(), "missing"), "v2") if err == nil { @@ -60,7 +96,11 @@ func TestPreflightChild_MissingBinary(t *testing.T) { func TestPreflightChild_LegacyFallback(t *testing.T) { dir := t.TempDir() binPath := filepath.Join(dir, "legacy-bin") - if err := os.WriteFile(binPath, []byte("#!/bin/sh\nexit 0\n"), 0755); err != nil { + script := `#!/bin/sh +echo "unknown flag: $1" >&2 +exit 2 +` + if err := os.WriteFile(binPath, []byte(script), 0755); err != nil { t.Fatal(err) } @@ -79,7 +119,7 @@ func TestPreflightChild_ProtocolFlagUnsupported(t *testing.T) { script := `#!/bin/sh case "$1" in --print-binary-version) echo "0.2.13-v2-r2" ;; -*) exit 1 ;; +*) echo "unknown flag: $1" >&2; exit 2 ;; esac ` if err := os.WriteFile(binPath, []byte(script), 0755); err != nil { @@ -101,7 +141,7 @@ func TestPreflightChild_BinaryFlagUnsupported(t *testing.T) { script := `#!/bin/sh case "$1" in --print-protocol-version) echo "v2" ;; -*) exit 1 ;; +*) echo "unknown flag: $1" >&2; exit 2 ;; esac ` if err := os.WriteFile(binPath, []byte(script), 0755); err != nil { @@ -160,6 +200,104 @@ esac } } +func TestPreflightChild_AdminAPIUnsupported(t *testing.T) { + dir := t.TempDir() + binPath := filepath.Join(dir, "stamped-bin") + script := `#!/bin/sh +case "$1" in +--print-binary-version) echo "0.2.13-v2-r2" ;; +--print-protocol-version) echo "v2" ;; +*) echo "unknown flag: $1" >&2; exit 2 ;; +esac +` + if err := os.WriteFile(binPath, []byte(script), 0755); err != nil { + t.Fatal(err) + } + + preflight, err := preflightChildWithAdminProbe(binPath, "v2", true) + if err != nil { + t.Fatal(err) + } + if preflight.adminAPISupported { + t.Fatal("expected unsupported admin API flag to keep public lifecycle fallback") + } + if preflight.storageMode != "" { + t.Fatalf("storageMode = %q, want legacy empty value", preflight.storageMode) + } +} + +func TestPreflightChild_AdminAPISupported(t *testing.T) { + dir := t.TempDir() + binPath := filepath.Join(dir, "stamped-bin") + script := `#!/bin/sh +case "$1" in +--print-binary-version) echo "0.2.13-v2-r2" ;; +--print-protocol-version) echo "v2" ;; +--print-admin-api-version) echo "1" ;; +--print-storage-mode) echo "postgres" ;; +*) echo "unknown flag: $1" >&2; exit 2 ;; +esac +` + if err := os.WriteFile(binPath, []byte(script), 0755); err != nil { + t.Fatal(err) + } + + preflight, err := preflightChildWithAdminProbe(binPath, "v2", true) + if err != nil { + t.Fatal(err) + } + if !preflight.adminAPISupported { + t.Fatal("expected admin API support to be detected") + } + if preflight.storageMode != "postgres" { + t.Fatalf("storageMode = %q, want postgres", preflight.storageMode) + } +} + +func TestPreflightChild_StorageModeProbeErrorFailsClosed(t *testing.T) { + dir := t.TempDir() + binPath := filepath.Join(dir, "stamped-bin") + script := `#!/bin/sh +case "$1" in +--print-binary-version) echo "0.2.13-v2-r2" ;; +--print-protocol-version) echo "v2" ;; +--print-admin-api-version) echo "1" ;; +--print-storage-mode) echo "bad storage env" >&2; exit 1 ;; +*) echo "unknown flag: $1" >&2; exit 2 ;; +esac +` + if err := os.WriteFile(binPath, []byte(script), 0755); err != nil { + t.Fatal(err) + } + + if _, err := preflightChildWithAdminProbe(binPath, "v2", true); err == nil { + t.Fatal("expected storage mode probe error to fail preflight") + } +} + +func TestPreflightChild_ProtocolProbeTimeoutFailsClosed(t *testing.T) { + oldTimeout := embeddedVersionProbeTimeout + embeddedVersionProbeTimeout = 50 * time.Millisecond + t.Cleanup(func() { embeddedVersionProbeTimeout = oldTimeout }) + + dir := t.TempDir() + binPath := filepath.Join(dir, "slow-protocol-stamp") + script := `#!/bin/sh +case "$1" in +--print-binary-version) echo "0.2.13-v2-r2" ;; +--print-protocol-version) sleep 1; echo "v2" ;; +*) echo "unknown flag: $1" >&2; exit 2 ;; +esac +` + if err := os.WriteFile(binPath, []byte(script), 0755); err != nil { + t.Fatal(err) + } + + if _, err := preflightChild(binPath, "v2"); err == nil { + t.Fatal("expected timeout probing protocol version to fail closed") + } +} + func TestNewManager(t *testing.T) { cfg := config.Config{ BinDir: "/tmp/bin", @@ -171,7 +309,7 @@ func TestNewManager(t *testing.T) { if m == nil { t.Fatal("NewManager returned nil") } - routes := m.RouteTable().Load().(map[string]string) + routes := m.RouteTable().Load().(proxy.RouteTable) if len(routes) != 0 { t.Errorf("expected empty routes, got %v", routes) } @@ -206,12 +344,12 @@ func TestRebuildRoutes(t *testing.T) { m.rebuildRoutes() m.mu.Unlock() - routes := m.RouteTable().Load().(map[string]string) - if routes["v1"] != "localhost:9001" { - t.Errorf("v1 route = %q, want %q", routes["v1"], "localhost:9001") + routes := m.RouteTable().Load().(proxy.RouteTable) + if routes["v1"].Address() != "localhost:9001" { + t.Errorf("v1 route = %q, want %q", routes["v1"].Address(), "localhost:9001") } - if routes["v2"] != "localhost:9002" { - t.Errorf("v2 route = %q, want %q", routes["v2"], "localhost:9002") + if routes["v2"].Address() != "localhost:9002" { + t.Errorf("v2 route = %q, want %q", routes["v2"].Address(), "localhost:9002") } } @@ -246,7 +384,7 @@ func TestRebuildRoutes_ExcludesNonRunning(t *testing.T) { m.rebuildRoutes() m.mu.Unlock() - routes := m.RouteTable().Load().(map[string]string) + routes := m.RouteTable().Load().(proxy.RouteTable) if _, ok := routes["v1"]; !ok { t.Error("running v1 should be in routes") } @@ -285,6 +423,55 @@ func TestStatus(t *testing.T) { } } +func TestStatusIncludesDrainingChildrenButRoutesDoNot(t *testing.T) { + cfg := config.Config{ + BinDir: "/tmp/bin", + DataDir: "/tmp/data", + BinaryName: "testapp", + BasePort: 5000, + } + m := NewManager(cfg) + + m.mu.Lock() + m.processes["v1"] = &child{ + version: oracle.Version{Name: "v1"}, + archiveSHA256: "new-sha", + binaryVersion: "new-bin", + port: 9002, + done: make(chan struct{}), + status: statusRunning, + } + m.draining["v1"] = []*child{{ + version: oracle.Version{Name: "v1"}, + archiveSHA256: "old-sha", + binaryVersion: "old-bin", + port: 9001, + done: make(chan struct{}), + status: statusDraining, + }} + m.rebuildRoutes() + m.mu.Unlock() + + routes := m.RouteTable().Load().(proxy.RouteTable) + if routes["v1"].Address() != "localhost:9002" { + t.Fatalf("route = %q, want new child", routes["v1"].Address()) + } + + statuses := m.Status() + if len(statuses) != 2 { + t.Fatalf("expected running + draining status, got %d", len(statuses)) + } + var sawDraining bool + for _, status := range statuses { + if status.Status == statusDraining && status.Port == 9001 && status.SHA256 == "old-sha" { + sawDraining = true + } + } + if !sawDraining { + t.Fatalf("draining child not reported in status: %+v", statuses) + } +} + func TestHashFile(t *testing.T) { dir := t.TempDir() path := filepath.Join(dir, "testfile") @@ -312,14 +499,15 @@ func TestHashFile_Missing(t *testing.T) { } } -func TestAssignPort_Stable(t *testing.T) { +func TestAssignPort_ReusesReleasedPorts(t *testing.T) { cfg := config.Config{BasePort: 5000} m := NewManager(cfg) m.mu.Lock() - p1 := m.assignPort("v1") - p2 := m.assignPort("v2") - p1again := m.assignPort("v1") + p1 := mustAssignPort(t, m) + p2 := mustAssignPort(t, m) + m.releasePort(p1) + p3 := mustAssignPort(t, m) m.mu.Unlock() if p1 != 5000 { @@ -328,9 +516,131 @@ func TestAssignPort_Stable(t *testing.T) { if p2 != 5001 { t.Errorf("second port = %d, want 5001", p2) } - if p1again != p1 { - t.Errorf("repeated assignPort gave %d, want %d", p1again, p1) + if p3 != 5000 { + t.Errorf("released port should be reused; got %d, want 5000", p3) + } +} + +func TestAssignPort_SkipsVersiondListenPort(t *testing.T) { + m := NewManager(config.Config{BasePort: 8079}) + + m.mu.Lock() + p1 := mustAssignPort(t, m) + p2 := mustAssignPort(t, m) + m.mu.Unlock() + + if p1 != 8079 { + t.Errorf("first port = %d, want 8079", p1) + } + if p2 != 8081 { + t.Errorf("second port = %d, want 8081", p2) + } +} + +func TestAssignPort_NormalizesOutOfRangeBasePort(t *testing.T) { + m := NewManager(config.Config{BasePort: 70000}) + + m.mu.Lock() + port := mustAssignPort(t, m) + m.mu.Unlock() + + if port != 5000 { + t.Errorf("port = %d, want 5000", port) + } +} + +func TestAssignPort_ReturnsErrorWhenPoolExhausted(t *testing.T) { + m := NewManager(config.Config{BasePort: maxChildPort}) + m.allocatedPorts[maxChildPort] = struct{}{} + + m.mu.Lock() + port, err := m.assignPort() + m.mu.Unlock() + + if port != 0 { + t.Fatalf("port = %d, want zero value", port) + } + if !errors.Is(err, errChildPortPoolExhausted) { + t.Fatalf("assignPort error = %v, want %v", err, errChildPortPoolExhausted) + } +} + +func TestStartChild_DoesNotPublishWhenPortPoolExhausted(t *testing.T) { + m := NewManager(config.Config{BasePort: maxChildPort}) + m.allocatedPorts[maxChildPort] = struct{}{} + v := oracle.Version{Name: "v1"} + + m.mu.Lock() + err := m.startChild(context.Background(), v, "sha", "unused", true) + _, published := m.processes[v.Name] + m.mu.Unlock() + + if !errors.Is(err, errChildPortPoolExhausted) { + t.Fatalf("startChild error = %v, want %v", err, errChildPortPoolExhausted) + } + if published { + t.Fatal("child with no assigned port must not be published") + } +} + +func TestRunChild_ReleasesPublicPortWhenAdminPoolExhausted(t *testing.T) { + dir := t.TempDir() + binPath := filepath.Join(dir, "devshardd") + script := `#!/bin/sh +case "$1" in +--print-binary-version) echo "0.2.13-v2-r2" ;; +--print-protocol-version) echo "v2" ;; +--print-admin-api-version) echo "1" ;; +--print-storage-mode) echo "postgres" ;; +*) exit 99 ;; +esac +` + if err := os.WriteFile(binPath, []byte(script), 0755); err != nil { + t.Fatal(err) + } + + m := NewManager(config.Config{ + BinDir: filepath.Join(dir, "bin"), + DataDir: filepath.Join(dir, "data"), + BinaryName: "devshardd", + BasePort: maxChildPort, + }) + v := oracle.Version{Name: "v2"} + + m.mu.Lock() + if err := m.startChild(context.Background(), v, "sha", binPath, true); err != nil { + m.mu.Unlock() + t.Fatal(err) + } + c := m.processes[v.Name] + m.mu.Unlock() + + select { + case <-c.done: + case <-time.After(5 * time.Second): + t.Fatal("child did not stop after admin port allocation failed") + } + + m.mu.Lock() + _, published := m.processes[v.Name] + allocated := len(m.allocatedPorts) + m.mu.Unlock() + + if published { + t.Fatal("child with no admin port must not remain published") + } + if allocated != 0 { + t.Fatalf("allocated ports = %d, want public port released", allocated) + } +} + +func mustAssignPort(t *testing.T, m *Manager) int { + t.Helper() + port, err := m.assignPort() + if err != nil { + t.Fatal(err) } + return port } func TestAtomicCopy(t *testing.T) { @@ -364,6 +674,102 @@ func TestAtomicCopy(t *testing.T) { } } +func TestInstallBinPathUsesVersionAndSHA(t *testing.T) { + m := NewManager(config.Config{BinDir: "/opt/versiond/bin", BinaryName: "devshardd", BasePort: 5000}) + got := m.installBinPath("v2", "abc123") + want := filepath.Join("/opt/versiond/bin", "v2", "abc123", "devshardd") + if got != want { + t.Fatalf("installBinPath = %q, want %q", got, want) + } +} + +func TestRollingOverlapAllowedRequiresPostgresForDevshard(t *testing.T) { + dir := t.TempDir() + postgresBin := writeStorageModeProbeBinary(t, dir, "postgres-bin", "postgres") + hybridBin := writeStorageModeProbeBinary(t, dir, "hybrid-bin", "hybrid") + errorBin := writeStorageModeErrorBinary(t, dir, "error-bin") + legacyBin := writeStorageModeLegacyBinary(t, dir, "legacy-bin") + + devshardMgr := NewManager(config.Config{BinaryName: "devshard", BasePort: 5000}) + if devshardMgr.rollingOverlapAllowed("v1", &child{storageMode: ""}, postgresBin) { + t.Fatal("devshard overlap should be disabled when running child storage mode is unknown") + } + if devshardMgr.rollingOverlapAllowed("v1", &child{storageMode: "hybrid"}, postgresBin) { + t.Fatal("devshard overlap should be disabled when running child is not postgres-only") + } + if devshardMgr.rollingOverlapAllowed("v1", &child{storageMode: "postgres"}, legacyBin) { + t.Fatal("devshard overlap should be disabled when new binary does not expose storage mode") + } + if devshardMgr.rollingOverlapAllowed("v1", &child{storageMode: "postgres"}, hybridBin) { + t.Fatal("devshard overlap should be disabled when new binary is not postgres-only") + } + if devshardMgr.rollingOverlapAllowed("v1", &child{storageMode: "postgres"}, errorBin) { + t.Fatal("devshard overlap should be disabled when new binary storage probe fails") + } + if !devshardMgr.rollingOverlapAllowed("v1", &child{storageMode: "postgres"}, postgresBin) { + t.Fatal("devshard overlap should be allowed when both children are postgres-only") + } + + testappMgr := NewManager(config.Config{BinaryName: "testapp", BasePort: 5000}) + if !testappMgr.rollingOverlapAllowed("v1", &child{}, legacyBin) { + t.Fatal("non-devshard test binary should allow overlap without storage mode probing") + } +} + +func TestChildStopTimeoutHonorsDevshardShutdownGrace(t *testing.T) { + t.Setenv("DEVSHARD_SHUTDOWN_GRACE", "") + devshardMgr := NewManager(config.Config{BinaryName: "devshardd", DrainKillGrace: 30 * time.Second}) + if got := devshardMgr.childStopTimeout(); got != defaultDevshardShutdownGrace { + t.Fatalf("childStopTimeout = %s, want default devshard grace", got) + } + + t.Setenv("DEVSHARD_SHUTDOWN_GRACE", "2m") + devshardMgr = NewManager(config.Config{BinaryName: "devshardd", DrainKillGrace: 30 * time.Second}) + if got := devshardMgr.childStopTimeout(); got != 2*time.Minute { + t.Fatalf("childStopTimeout = %s, want DEVSHARD_SHUTDOWN_GRACE", got) + } + if got := devshardMgr.ShutdownTimeout(); got != 2*time.Minute { + t.Fatalf("ShutdownTimeout = %s, want DEVSHARD_SHUTDOWN_GRACE", got) + } + + t.Setenv("DEVSHARD_SHUTDOWN_GRACE", "5s") + devshardMgr = NewManager(config.Config{BinaryName: "devshardd", DrainKillGrace: 30 * time.Second}) + if got := devshardMgr.childStopTimeout(); got != 30*time.Second { + t.Fatalf("childStopTimeout = %s, want VERSIOND_DRAIN_KILL_GRACE", got) + } + + testappMgr := NewManager(config.Config{BinaryName: "testapp", DrainKillGrace: 30 * time.Second}) + if got := testappMgr.childStopTimeout(); got != 30*time.Second { + t.Fatalf("testapp childStopTimeout = %s, want drain kill grace", got) + } +} + +func TestGCInstalledVersionDirsRemovesOldCompleteInstallsOnly(t *testing.T) { + binDir := t.TempDir() + binaryName := "devshardd" + baseTime := time.Now().Add(-time.Hour) + + writeInstalledVersion(t, binDir, "v1", "protected", binaryName, baseTime) + writeInstalledVersion(t, binDir, "v1", "stale-newest", binaryName, baseTime.Add(4*time.Minute)) + writeInstalledVersion(t, binDir, "v1", "stale-middle", binaryName, baseTime.Add(3*time.Minute)) + writeInstalledVersion(t, binDir, "v1", "stale-old", binaryName, baseTime.Add(2*time.Minute)) + incompleteDir := filepath.Join(binDir, "v1", "incomplete") + if err := os.MkdirAll(incompleteDir, 0o755); err != nil { + t.Fatal(err) + } + + keep := map[string]map[string]struct{}{ + "v1": {"protected": {}}, + } + gcInstalledVersionDirs(binDir, binaryName, keep, 2) + + assertPathExists(t, filepath.Join(binDir, "v1", "protected")) + assertPathExists(t, filepath.Join(binDir, "v1", "stale-newest")) + assertPathExists(t, filepath.Join(binDir, "v1", "stale-middle")) + assertPathExists(t, incompleteDir) + assertPathMissing(t, filepath.Join(binDir, "v1", "stale-old")) +} + func TestReconcile_OverrideStartsChild(t *testing.T) { dir := t.TempDir() binDir := filepath.Join(dir, "bin") @@ -500,7 +906,10 @@ func TestRunChild_RemovesFromProcessesOnStartFailure(t *testing.T) { v := oracle.Version{Name: "v1"} m.mu.Lock() - m.startChild(ctx, v) + if err := m.startChild(ctx, v, "missing", filepath.Join(dir, "missing"), true); err != nil { + m.mu.Unlock() + t.Fatal(err) + } c := m.processes["v1"] m.mu.Unlock() @@ -519,55 +928,620 @@ func TestRunChild_RemovesFromProcessesOnStartFailure(t *testing.T) { } } -func TestReconcile_StopsRemovedVersions(t *testing.T) { +func TestWaitForRestartBackoffRechecksRestartAfterSleep(t *testing.T) { + m := NewManager(config.Config{BasePort: 5000}) + c := &child{restart: true} + + m.mu.Lock() + c.restart = false + m.mu.Unlock() + + if m.waitForRestartBackoff(context.Background(), c, time.Millisecond) { + t.Fatal("restart disabled during backoff should stop restart loop") + } +} + +func TestReconcile_DrainsRemovedVersionsAsync(t *testing.T) { dir := t.TempDir() + var drainHits atomic.Int32 + var statusHits atomic.Int32 + port, shutdown := startLocalHTTPServer(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch { + case r.Method == http.MethodPost && r.URL.Path == "/drain": + drainHits.Add(1) + w.WriteHeader(http.StatusOK) + case r.Method == http.MethodGet && r.URL.Path == "/drain/status": + statusHits.Add(1) + json.NewEncoder(w).Encode(map[string]int64{"inflight": 1}) + default: + http.NotFound(w, r) + } + })) + defer shutdown() + cfg := config.Config{ - BinDir: filepath.Join(dir, "bin"), - DataDir: filepath.Join(dir, "data"), - BinaryName: "devshard", - BasePort: 5000, - Overrides: map[string]string{}, + BinDir: filepath.Join(dir, "bin"), + DataDir: filepath.Join(dir, "data"), + BinaryName: "devshard", + BasePort: 5000, + DrainTimeout: time.Hour, + DrainPollInterval: time.Hour, + DrainKillGrace: 50 * time.Millisecond, + Overrides: map[string]string{}, } m := NewManager(cfg) done := make(chan struct{}) - close(done) + cancelled := make(chan struct{}) + var cancelOnce sync.Once m.mu.Lock() - cancelled := false m.processes["old"] = &child{ - version: oracle.Version{Name: "old"}, - port: 5000, - cancel: func() { cancelled = true }, + version: oracle.Version{Name: "old"}, + archiveSHA256: "old-sha", + port: port, + stop: func() { + cancelOnce.Do(func() { + close(cancelled) + close(done) + }) + }, done: done, status: statusRunning, + restart: true, } + m.rebuildRoutes() m.mu.Unlock() ctx := context.Background() - // Reconcile with empty desired list should stop "old". + start := time.Now() if err := m.Reconcile(ctx, nil); err != nil { t.Fatal(err) } - - if !cancelled { - t.Error("removed version should have been cancelled") + if elapsed := time.Since(start); elapsed > 500*time.Millisecond { + t.Fatalf("Reconcile blocked for %s while removed child was still draining", elapsed) + } + drainDeadline := time.Now().Add(500 * time.Millisecond) + for drainHits.Load() == 0 && time.Now().Before(drainDeadline) { + time.Sleep(5 * time.Millisecond) + } + if drainHits.Load() != 1 { + t.Fatalf("drain hits = %d, want 1", drainHits.Load()) + } + select { + case <-cancelled: + t.Fatal("removed child should not be cancelled while drain status reports in-flight work") + default: } m.mu.Lock() _, stillRunning := m.processes["old"] + draining := m.draining["old"] + var removed *child + if len(draining) == 1 { + removed = draining[0] + } + routes := m.RouteTable().Load().(proxy.RouteTable) m.mu.Unlock() if stillRunning { t.Error("removed version should no longer be in processes") } + if removed == nil { + t.Fatal("removed version should be tracked as draining") + } + if removed.status != statusDraining { + t.Fatalf("removed status = %q, want %q", removed.status, statusDraining) + } + if removed.restart { + t.Fatal("removed draining child should not restart") + } + if _, routed := routes["old"]; routed { + t.Fatal("removed version should not remain routed") + } + deadline := time.Now().Add(500 * time.Millisecond) + for statusHits.Load() == 0 && time.Now().Before(deadline) { + select { + case <-cancelled: + t.Fatal("removed child should not be cancelled while drain status reports in-flight work") + default: + } + time.Sleep(10 * time.Millisecond) + } + if statusHits.Load() == 0 { + t.Fatal("expected drain status to be polled") + } + + removed.Stop() + waitForChild(removed) } -func TestInstalledVersionMatches_DetectsBinaryHashMismatch(t *testing.T) { - versionDir := t.TempDir() - binPath := filepath.Join(versionDir, "devshard") - archiveHash := sha256Hex([]byte("archive")) - originalBinary := []byte("#!/bin/sh\nsleep 30\n") +func TestReconcile_ReaddedVersionWaitsForDrainingChild(t *testing.T) { + dir := t.TempDir() + binary := []byte(`#!/bin/sh +case "$1" in +--print-binary-version) echo "testapp-v1" ;; +--print-protocol-version) echo "v1" ;; +*) exec sleep 30 ;; +esac +`) + zipData, archiveHash := zipBinary(t, "testapp", binary) + var requests atomic.Int32 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requests.Add(1) + _, _ = w.Write(zipData) + })) + defer srv.Close() + + m := NewManager(config.Config{ + BinDir: filepath.Join(dir, "bin"), + DataDir: filepath.Join(dir, "data"), + BinaryName: "testapp", + BasePort: 5000, + DrainKillGrace: 100 * time.Millisecond, + }) + m.draining["v1"] = []*child{{ + version: oracle.Version{Name: "v1"}, + archiveSHA256: sha256Hex([]byte("removed-v1")), + status: statusDraining, + }} + desired := []oracle.Version{{Name: "v1", Binary: srv.URL, SHA256: archiveHash}} + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + if err := m.Reconcile(ctx, desired); err != nil { + t.Fatal(err) + } + if got := requests.Load(); got != 0 { + t.Fatalf("download requests while previous child drains = %d, want 0", got) + } + m.mu.Lock() + _, running := m.processes["v1"] + _, downloading := m.downloading["v1"] + m.mu.Unlock() + if running || downloading { + t.Fatalf("re-added version state while draining = running:%v downloading:%v, want false/false", running, downloading) + } + + m.mu.Lock() + delete(m.draining, "v1") + m.mu.Unlock() + if err := m.Reconcile(ctx, desired); err != nil { + t.Fatal(err) + } + if got := requests.Load(); got != 1 { + t.Fatalf("download requests after drain completed = %d, want 1", got) + } + m.mu.Lock() + _, running = m.processes["v1"] + m.mu.Unlock() + if !running { + t.Fatal("re-added version should start after previous child finishes draining") + } + + cancel() + if err := m.Shutdown(context.Background()); err != nil { + t.Fatal(err) + } +} + +func TestDownloadAndStart_DoesNotOverlapDrainingChild(t *testing.T) { + dir := t.TempDir() + zipData, archiveHash := zipBinary(t, "testapp", []byte("test binary")) + requestStarted := make(chan struct{}) + releaseDownload := make(chan struct{}) + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + close(requestStarted) + <-releaseDownload + _, _ = w.Write(zipData) + })) + defer srv.Close() + + m := NewManager(config.Config{ + BinDir: filepath.Join(dir, "bin"), + DataDir: filepath.Join(dir, "data"), + BinaryName: "testapp", + BasePort: 5000, + }) + m.downloading["v1"] = struct{}{} + v := oracle.Version{Name: "v1", Binary: srv.URL, SHA256: archiveHash} + errCh := make(chan error, 1) + go func() { + errCh <- m.downloadAndStart(context.Background(), v, archiveHash) + }() + + select { + case <-requestStarted: + case <-time.After(time.Second): + t.Fatal("download did not start") + } + m.mu.Lock() + m.draining["v1"] = []*child{{ + version: oracle.Version{Name: "v1"}, + archiveSHA256: sha256Hex([]byte("removed-v1")), + status: statusDraining, + }} + m.mu.Unlock() + close(releaseDownload) + if err := <-errCh; err != nil { + t.Fatal(err) + } + + m.mu.Lock() + _, running := m.processes["v1"] + _, downloading := m.downloading["v1"] + m.mu.Unlock() + if running { + t.Fatal("download completion should not start a version while its previous child drains") + } + if downloading { + t.Fatal("download marker should be cleared when start is deferred") + } +} + +func TestReconcile_ReaddedOverrideWaitsForDrainingChild(t *testing.T) { + dir := t.TempDir() + overrideBin := filepath.Join(dir, "override-binary") + script := []byte(`#!/bin/sh +case "$1" in +--print-binary-version) echo "testapp-v1" ;; +--print-protocol-version) echo "v1" ;; +*) exec sleep 30 ;; +esac +`) + if err := os.WriteFile(overrideBin, script, 0o755); err != nil { + t.Fatal(err) + } + + m := NewManager(config.Config{ + BinDir: filepath.Join(dir, "bin"), + DataDir: filepath.Join(dir, "data"), + BinaryName: "testapp", + BasePort: 5000, + DrainKillGrace: 100 * time.Millisecond, + Overrides: map[string]string{"v1": overrideBin}, + }) + m.draining["v1"] = []*child{{ + version: oracle.Version{Name: "v1"}, + archiveSHA256: "override:removed", + status: statusDraining, + }} + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + if err := m.Reconcile(ctx, []oracle.Version{{Name: "v1"}}); err != nil { + t.Fatal(err) + } + m.mu.Lock() + _, running := m.processes["v1"] + m.mu.Unlock() + if running { + t.Fatal("re-added override should wait for the previous child to finish draining") + } + if _, err := os.Stat(filepath.Join(dir, "bin", "v1", "testapp")); !os.IsNotExist(err) { + t.Fatalf("override should not be copied while previous child drains, stat error: %v", err) + } + + m.mu.Lock() + delete(m.draining, "v1") + m.mu.Unlock() + if err := m.Reconcile(ctx, []oracle.Version{{Name: "v1"}}); err != nil { + t.Fatal(err) + } + m.mu.Lock() + _, running = m.processes["v1"] + m.mu.Unlock() + if !running { + t.Fatal("re-added override should start after previous child finishes draining") + } + + cancel() + if err := m.Shutdown(context.Background()); err != nil { + t.Fatal(err) + } +} + +func TestReconcileOverride_DoesNotReplaceChildMovedToDraining(t *testing.T) { + dir := t.TempDir() + overrideSrc := filepath.Join(dir, "override-source") + binPath := filepath.Join(dir, "bin", "v1", "testapp") + if err := os.WriteFile(overrideSrc, []byte("new override"), 0o755); err != nil { + t.Fatal(err) + } + if err := os.MkdirAll(filepath.Dir(binPath), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(binPath, []byte("old override"), 0o755); err != nil { + t.Fatal(err) + } + + m := NewManager(config.Config{ + BinDir: filepath.Join(dir, "bin"), + DataDir: filepath.Join(dir, "data"), + BinaryName: "testapp", + BasePort: 5000, + DrainKillGrace: 100 * time.Millisecond, + }) + done := make(chan struct{}) + var cancelOnce sync.Once + existing := &child{ + version: oracle.Version{Name: "v1"}, + binPath: binPath, + done: done, + status: statusRunning, + restart: true, + } + existing.stop = func() { + cancelOnce.Do(func() { + m.mu.Lock() + delete(m.processes, "v1") + existing.status = statusDraining + existing.restart = false + m.draining["v1"] = append(m.draining["v1"], existing) + m.mu.Unlock() + close(done) + }) + } + m.processes["v1"] = existing + + m.reconcileOverride( + context.Background(), + oracle.Version{Name: "v1"}, + overrideSrc, + binPath, + ) + + m.mu.Lock() + _, running := m.processes["v1"] + draining := m.draining["v1"] + m.mu.Unlock() + if running { + t.Fatal("override should not restart while its previous child is draining") + } + if len(draining) != 1 || draining[0] != existing { + t.Fatalf("draining children = %v, want the original child", draining) + } +} + +func TestReconcileOverride_DoesNotTrustStaleOverrideFile(t *testing.T) { + dir := t.TempDir() + overrideSrc := filepath.Join(dir, "override-source") + binPath := filepath.Join(dir, "bin", "v1", "testapp") + downloadedPath := filepath.Join(dir, "bin", "v1", "download-sha", "testapp") + script := []byte(`#!/bin/sh +case "$1" in +--print-binary-version) echo "testapp-v1" ;; +--print-protocol-version) echo "v1" ;; +*) exec sleep 30 ;; +esac +`) + if err := os.WriteFile(overrideSrc, script, 0o755); err != nil { + t.Fatal(err) + } + if err := os.MkdirAll(filepath.Dir(binPath), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(binPath, script, 0o755); err != nil { + t.Fatal(err) + } + + m := NewManager(config.Config{ + BinDir: filepath.Join(dir, "bin"), + DataDir: filepath.Join(dir, "data"), + BinaryName: "testapp", + BasePort: 5000, + DrainKillGrace: 100 * time.Millisecond, + }) + done := make(chan struct{}) + var cancelOnce sync.Once + existing := &child{ + version: oracle.Version{Name: "v1"}, + archiveSHA256: sha256Hex([]byte("download archive")), + binPath: downloadedPath, + done: done, + status: statusRunning, + restart: true, + } + existing.stop = func() { + cancelOnce.Do(func() { + close(done) + }) + } + m.processes["v1"] = existing + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + m.reconcileOverride(ctx, oracle.Version{Name: "v1"}, overrideSrc, binPath) + + srcHash, err := download.HashFile(overrideSrc) + if err != nil { + t.Fatal(err) + } + m.mu.Lock() + current := m.processes["v1"] + m.mu.Unlock() + if current == existing { + t.Fatal("stale flat override file should not match a per-SHA child") + } + if current == nil { + t.Fatal("override child should replace the downloaded child") + } + if current.binPath != binPath { + t.Fatalf("override child path = %q, want %q", current.binPath, binPath) + } + if current.archiveSHA256 != "override:"+srcHash { + t.Fatalf("override child identity = %q, want override source hash", current.archiveSHA256) + } + + cancel() + if err := m.Shutdown(context.Background()); err != nil { + t.Fatal(err) + } +} + +func TestReconcileOverride_SameIdentityDoesNotRestart(t *testing.T) { + dir := t.TempDir() + overrideSrc := filepath.Join(dir, "override-source") + binPath := filepath.Join(dir, "bin", "v1", "testapp") + content := []byte("override binary") + if err := os.WriteFile(overrideSrc, content, 0o755); err != nil { + t.Fatal(err) + } + if err := os.MkdirAll(filepath.Dir(binPath), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(binPath, content, 0o755); err != nil { + t.Fatal(err) + } + srcHash, err := download.HashFile(overrideSrc) + if err != nil { + t.Fatal(err) + } + + m := NewManager(config.Config{ + BinDir: filepath.Join(dir, "bin"), + DataDir: filepath.Join(dir, "data"), + BinaryName: "testapp", + BasePort: 5000, + }) + cancelled := false + existing := &child{ + version: oracle.Version{Name: "v1"}, + archiveSHA256: "override:" + srcHash, + binPath: binPath, + stop: func() { cancelled = true }, + done: make(chan struct{}), + status: statusRunning, + restart: true, + } + m.processes["v1"] = existing + + m.reconcileOverride(context.Background(), oracle.Version{Name: "v1"}, overrideSrc, binPath) + + if cancelled { + t.Fatal("matching override identity should not restart the child") + } + if current := m.processes["v1"]; current != existing { + t.Fatal("matching override identity should keep the current child") + } +} + +func TestDrainAfterProxyWaitsBeforeRequestingChildDrain(t *testing.T) { + var drainHits atomic.Int32 + port, shutdown := startLocalHTTPServer(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch { + case r.Method == http.MethodPost && r.URL.Path == "/drain": + drainHits.Add(1) + w.WriteHeader(http.StatusOK) + case r.Method == http.MethodGet && r.URL.Path == "/drain/status": + json.NewEncoder(w).Encode(map[string]int64{"inflight": 0}) + default: + http.NotFound(w, r) + } + })) + defer shutdown() + + m := NewManager(config.Config{ + BasePort: 5000, + DrainTimeout: time.Second, + DrainPollInterval: 5 * time.Millisecond, + DrainKillGrace: 50 * time.Millisecond, + }) + done := make(chan struct{}) + c := &child{ + version: oracle.Version{Name: "v1"}, + port: port, + done: done, + stop: func() { close(done) }, + } + proxyDrained := make(chan struct{}) + started := make(chan struct{}) + drainedDone := make(chan struct{}) + go func() { + defer close(drainedDone) + close(started) + m.drainAfterProxy(c, proxyDrained) + }() + + <-started + if drainHits.Load() != 0 { + t.Fatal("child drain started before proxy requests drained") + } + close(proxyDrained) + + select { + case <-drainedDone: + case <-time.After(time.Second): + t.Fatal("child drain did not finish after proxy requests drained") + } + if drainHits.Load() != 1 { + t.Fatalf("drain hits = %d, want 1", drainHits.Load()) + } +} + +func TestReconcile_RemovedLegacyVersionUsesDrainGraceBeforeCancel(t *testing.T) { + dir := t.TempDir() + port, shutdown := startLocalHTTPServer(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + http.NotFound(w, r) + })) + defer shutdown() + + m := NewManager(config.Config{ + BinDir: filepath.Join(dir, "bin"), + DataDir: filepath.Join(dir, "data"), + BinaryName: "devshard", + BasePort: 5000, + DrainTimeout: time.Hour, + DrainPollInterval: time.Hour, + DrainKillGrace: 50 * time.Millisecond, + }) + + done := make(chan struct{}) + cancelled := make(chan struct{}) + var cancelOnce sync.Once + + m.mu.Lock() + m.processes["legacy"] = &child{ + version: oracle.Version{Name: "legacy"}, + port: port, + stop: func() { + cancelOnce.Do(func() { + close(cancelled) + close(done) + }) + }, + done: done, + status: statusRunning, + restart: true, + } + m.rebuildRoutes() + m.mu.Unlock() + + start := time.Now() + if err := m.Reconcile(context.Background(), nil); err != nil { + t.Fatal(err) + } + select { + case <-cancelled: + t.Fatal("legacy child should get drain grace before SIGTERM") + default: + } + + select { + case <-cancelled: + if elapsed := time.Since(start); elapsed < 40*time.Millisecond { + t.Fatalf("legacy child cancelled after %s, want drain grace first", elapsed) + } + case <-time.After(time.Second): + t.Fatal("legacy child was not cancelled after drain grace") + } +} + +func TestInstalledVersionMatches_DetectsBinaryHashMismatch(t *testing.T) { + versionDir := t.TempDir() + binPath := filepath.Join(versionDir, "devshard") + archiveHash := sha256Hex([]byte("archive")) + originalBinary := []byte("#!/bin/sh\nsleep 30\n") tamperedBinary := []byte("#!/bin/sh\necho tampered\n") if err := os.WriteFile(binPath, originalBinary, 0755); err != nil { @@ -597,13 +1571,344 @@ func TestInstalledVersionMatches_DetectsBinaryHashMismatch(t *testing.T) { } } +func TestWaitForReadyFallsBackToHealthzWhenDefaultReadyMissing(t *testing.T) { + port, shutdown := startLocalHTTPServer(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/healthz" { + w.WriteHeader(http.StatusOK) + return + } + http.NotFound(w, r) + })) + defer shutdown() + + if !waitForReady(context.Background(), port, "/ready", time.Second) { + t.Fatal("expected /ready 404 to fall back to /healthz") + } +} + +func TestWaitForReadyDoesNotFallbackForCustomReadyPath(t *testing.T) { + port, shutdown := startLocalHTTPServer(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusNotFound) + })) + defer shutdown() + + if waitForReady(context.Background(), port, "/custom-ready", 200*time.Millisecond) { + t.Fatal("custom ready path should not use legacy fallback") + } +} + +func TestWaitForChildServingReadyRequiresPublicHealth(t *testing.T) { + adminPort, adminShutdown := startLocalHTTPServer(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/ready" { + w.WriteHeader(http.StatusOK) + return + } + http.NotFound(w, r) + })) + defer adminShutdown() + + var publicReady atomic.Bool + var publicHits atomic.Int32 + publicPort, publicShutdown := startLocalHTTPServer(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + publicHits.Add(1) + if r.URL.Path == "/healthz" && publicReady.Load() { + w.WriteHeader(http.StatusOK) + return + } + http.Error(w, "public listener not ready", http.StatusServiceUnavailable) + })) + defer publicShutdown() + + c := &child{port: publicPort, adminPort: adminPort} + if waitForChildServingReady(context.Background(), c, "/ready", 200*time.Millisecond) { + t.Fatal("admin readiness must not hide an unavailable public listener") + } + if publicHits.Load() == 0 { + t.Fatal("public health endpoint was not probed") + } + + publicReady.Store(true) + if !waitForChildServingReady(context.Background(), c, "/ready", time.Second) { + t.Fatal("child should become ready when admin and public endpoints are healthy") + } +} + +func TestWaitForChildServingReadyRechecksAdminAndPublicTogether(t *testing.T) { + var adminHits atomic.Int32 + adminPort, adminShutdown := startLocalHTTPServer(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if adminHits.Add(1) == 1 { + w.WriteHeader(http.StatusOK) + return + } + http.Error(w, "admin no longer ready", http.StatusServiceUnavailable) + })) + defer adminShutdown() + + var publicHits atomic.Int32 + publicPort, publicShutdown := startLocalHTTPServer(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if publicHits.Add(1) == 1 { + http.Error(w, "public listener not ready", http.StatusServiceUnavailable) + return + } + w.WriteHeader(http.StatusOK) + })) + defer publicShutdown() + + c := &child{port: publicPort, adminPort: adminPort} + if waitForChildServingReady(context.Background(), c, "/ready", 250*time.Millisecond) { + t.Fatal("child became ready without admin and public health at the same time") + } + if adminHits.Load() < 2 { + t.Fatal("admin readiness was not rechecked after public health failed") + } +} + +func TestDrainAndStop_LegacyDrainStatusUsesShortGrace(t *testing.T) { + port, shutdown := startLocalHTTPServer(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + http.NotFound(w, r) + })) + defer shutdown() + + m := NewManager(config.Config{ + BasePort: 5000, + DrainTimeout: time.Hour, + DrainPollInterval: time.Hour, + DrainKillGrace: 20 * time.Millisecond, + }) + done := make(chan struct{}) + cancelled := false + c := &child{ + version: oracle.Version{Name: "v1"}, + port: port, + done: done, + stop: func() { + cancelled = true + close(done) + }, + } + + start := time.Now() + m.drainAndStop(c) + if !cancelled { + t.Fatal("legacy child should be cancelled after short drain grace") + } + if elapsed := time.Since(start); elapsed > 500*time.Millisecond { + t.Fatalf("legacy drain took %s, want short grace instead of full timeout", elapsed) + } +} + +func TestLifecycleRequestsUseAdminPortWhenAvailable(t *testing.T) { + var publicHits atomic.Int32 + publicPort, publicShutdown := startLocalHTTPServer(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + publicHits.Add(1) + http.Error(w, "public endpoint should not receive lifecycle traffic", http.StatusInternalServerError) + })) + defer publicShutdown() + + var drainHits atomic.Int32 + var statusHits atomic.Int32 + adminPort, adminShutdown := startLocalHTTPServer(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch { + case r.Method == http.MethodPost && r.URL.Path == "/drain": + drainHits.Add(1) + w.WriteHeader(http.StatusOK) + case r.Method == http.MethodGet && r.URL.Path == "/drain/status": + statusHits.Add(1) + json.NewEncoder(w).Encode(map[string]int64{"inflight": 0}) + default: + http.NotFound(w, r) + } + })) + defer adminShutdown() + + m := NewManager(config.Config{ + BasePort: 5000, + DrainPath: "/drain", + DrainStatusPath: "/drain/status", + }) + c := &child{ + version: oracle.Version{Name: "v1"}, + port: publicPort, + adminPort: adminPort, + } + + m.requestDrain(c) + inflight, err := m.fetchInflight(c) + if err != nil { + t.Fatal(err) + } + if inflight != 0 { + t.Fatalf("inflight = %d, want 0", inflight) + } + if drainHits.Load() != 1 { + t.Fatalf("admin drain hits = %d, want 1", drainHits.Load()) + } + if statusHits.Load() != 1 { + t.Fatalf("admin status hits = %d, want 1", statusHits.Load()) + } + if publicHits.Load() != 0 { + t.Fatalf("public lifecycle hits = %d, want 0", publicHits.Load()) + } +} + +func TestDrainAndStop_WaitsForIdleBeforeCancel(t *testing.T) { + var statusRequests int32 + var cancelled atomic.Bool + port, shutdown := startLocalHTTPServer(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + count := atomic.AddInt32(&statusRequests, 1) + if count == 1 { + if cancelled.Load() { + t.Error("child cancelled before idle drain status") + } + json.NewEncoder(w).Encode(map[string]int64{"inflight": 1}) + return + } + json.NewEncoder(w).Encode(map[string]int64{"inflight": 0}) + })) + defer shutdown() + + m := NewManager(config.Config{ + BasePort: 5000, + DrainTimeout: time.Second, + DrainPollInterval: 5 * time.Millisecond, + DrainKillGrace: 50 * time.Millisecond, + }) + done := make(chan struct{}) + c := &child{ + version: oracle.Version{Name: "v1"}, + port: port, + done: done, + stop: func() { + cancelled.Store(true) + close(done) + }, + } + + m.drainAndStop(c) + if !cancelled.Load() { + t.Fatal("idle child should be cancelled after drain") + } + if got := atomic.LoadInt32(&statusRequests); got < 2 { + t.Fatalf("drain status requests = %d, want at least 2", got) + } +} + +func TestDrainAndStop_TimesOutWithInflightWork(t *testing.T) { + port, shutdown := startLocalHTTPServer(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + json.NewEncoder(w).Encode(map[string]int64{"inflight": 1}) + })) + defer shutdown() + + m := NewManager(config.Config{ + BasePort: 5000, + DrainTimeout: 20 * time.Millisecond, + DrainPollInterval: 5 * time.Millisecond, + DrainKillGrace: 50 * time.Millisecond, + }) + done := make(chan struct{}) + cancelled := false + c := &child{ + version: oracle.Version{Name: "v1"}, + port: port, + done: done, + stop: func() { + cancelled = true + close(done) + }, + } + + start := time.Now() + m.drainAndStop(c) + if !cancelled { + t.Fatal("child should be cancelled after drain timeout") + } + if elapsed := time.Since(start); elapsed > time.Second { + t.Fatalf("drain timeout path took %s", elapsed) + } +} + +func TestDownloadAndSwap_NewChildNotReadyKeepsOldServing(t *testing.T) { + dir := t.TempDir() + binDir := filepath.Join(dir, "bin") + dataDir := filepath.Join(dir, "data") + newBinary := []byte(`#!/bin/sh +case "$1" in +--print-binary-version) echo "testapp-new" ;; +--print-protocol-version) echo "v1" ;; +*) exec sleep 60 ;; +esac +`) + zipData, archiveHash := zipBinary(t, "testapp", newBinary) + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Write(zipData) + })) + defer srv.Close() + + m := NewManager(config.Config{ + BinDir: binDir, + DataDir: dataDir, + BinaryName: "testapp", + BasePort: 6200, + ReadyTimeout: 20 * time.Millisecond, + DrainKillGrace: 100 * time.Millisecond, + }) + old := &child{ + version: oracle.Version{Name: "v1"}, + archiveSHA256: sha256Hex([]byte("old-archive")), + port: 9001, + done: make(chan struct{}), + status: statusRunning, + restart: true, + stop: func() { + t.Fatal("old child should not be cancelled when replacement is not ready") + }, + } + + m.mu.Lock() + m.processes["v1"] = old + m.downloading["v1"] = struct{}{} + m.rebuildRoutes() + m.mu.Unlock() + + err := m.downloadAndSwap(context.Background(), oracle.Version{ + Name: "v1", + Binary: srv.URL, + SHA256: archiveHash, + }, archiveHash, old) + if err == nil { + t.Fatal("expected swap error when new child is not ready") + } + + m.mu.Lock() + current := m.processes["v1"] + _, downloading := m.downloading["v1"] + draining := len(m.draining["v1"]) + m.mu.Unlock() + if current != old { + t.Fatalf("current child changed after aborted swap") + } + if old.status != statusRunning || !old.restart { + t.Fatalf("old child status/restart = %s/%v, want running/true", old.status, old.restart) + } + if downloading { + t.Fatal("downloading marker should be cleared after aborted swap") + } + if draining != 0 { + t.Fatalf("draining children = %d, want 0", draining) + } + routes := m.RouteTable().Load().(proxy.RouteTable) + if routes["v1"].Address() != "localhost:9001" { + t.Fatalf("route = %q, want old child route", routes["v1"].Address()) + } +} + func TestReconcile_DownloadedVersionDoesNotRedownloadWhenInstallStateMatches(t *testing.T) { dir := t.TempDir() binDir := filepath.Join(dir, "bin") dataDir := filepath.Join(dir, "data") - versionDir := filepath.Join(binDir, "v1") - binPath := filepath.Join(versionDir, "devshard") archiveHash := sha256Hex([]byte("archive-v1")) + versionDir := filepath.Join(binDir, "v1", archiveHash) + binPath := filepath.Join(versionDir, "devshard") binaryContent := []byte("#!/bin/sh\nsleep 30\n") if err := os.MkdirAll(versionDir, 0755); err != nil { @@ -638,11 +1943,13 @@ func TestReconcile_DownloadedVersionDoesNotRedownloadWhenInstallStateMatches(t * m.mu.Lock() m.processes["v1"] = &child{ - version: oracle.Version{Name: "v1", Binary: srv.URL, SHA256: archiveHash}, - port: 5000, - cancel: func() { cancelled = true }, - done: done, - status: statusRunning, + version: oracle.Version{Name: "v1", Binary: srv.URL, SHA256: archiveHash}, + archiveSHA256: archiveHash, + binPath: binPath, + port: 5000, + stop: func() { cancelled = true }, + done: done, + status: statusRunning, } m.mu.Unlock() @@ -669,6 +1976,306 @@ func TestReconcile_DownloadedVersionDoesNotRedownloadWhenInstallStateMatches(t * } } +func TestReconcile_PromotesLegacyCachedVersionWithoutDownload(t *testing.T) { + dir := t.TempDir() + binDir := filepath.Join(dir, "bin") + archiveHash := sha256Hex([]byte("legacy archive")) + binary := []byte(`#!/bin/sh +case "$1" in +--print-binary-version) echo "testapp-v1" ;; +--print-protocol-version) echo "v1" ;; +*) exec sleep 30 ;; +esac +`) + legacyBinPath := writeLegacyInstall(t, binDir, "v1", archiveHash, "testapp", binary) + + var requests atomic.Int32 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requests.Add(1) + w.WriteHeader(http.StatusServiceUnavailable) + })) + defer srv.Close() + + m := NewManager(config.Config{ + BinDir: binDir, + DataDir: filepath.Join(dir, "data"), + BinaryName: "testapp", + BasePort: 5000, + DrainKillGrace: 100 * time.Millisecond, + }) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + if err := m.Reconcile(ctx, []oracle.Version{{ + Name: "v1", + Binary: srv.URL, + SHA256: archiveHash, + }}); err != nil { + t.Fatal(err) + } + + if got := requests.Load(); got != 0 { + t.Fatalf("download requests = %d, want 0", got) + } + canonicalBinPath := m.installBinPath("v1", archiveHash) + matches, _, _, err := installedVersionMatches( + m.installDir("v1", archiveHash), + canonicalBinPath, + archiveHash, + ) + if err != nil { + t.Fatal(err) + } + if !matches { + t.Fatal("promoted per-SHA install should match legacy metadata") + } + assertPathExists(t, legacyBinPath) + assertPathExists(t, filepath.Join(filepath.Dir(legacyBinPath), download.InstallMetadataFilename)) + m.mu.Lock() + current := m.processes["v1"] + m.mu.Unlock() + if current == nil { + t.Fatal("promoted legacy version should start") + } + if current.binPath != canonicalBinPath { + t.Fatalf("started binary = %q, want promoted path %q", current.binPath, canonicalBinPath) + } + + cancel() + if err := m.Shutdown(context.Background()); err != nil { + t.Fatal(err) + } +} + +func TestResolveInstalledArtifact_UsesLegacyWhenPromotionFails(t *testing.T) { + dir := t.TempDir() + binDir := filepath.Join(dir, "bin") + archiveHash := sha256Hex([]byte("legacy archive")) + legacyBinPath := writeLegacyInstall( + t, + binDir, + "v1", + archiveHash, + "testapp", + []byte("legacy binary"), + ) + m := NewManager(config.Config{BinDir: binDir, BinaryName: "testapp", BasePort: 5000}) + + // Keep a non-empty directory at the canonical binary path so atomic rename + // fails without making the verified legacy source unreadable. + canonicalBinPath := m.installBinPath("v1", archiveHash) + if err := os.MkdirAll(filepath.Join(canonicalBinPath, "blocker"), 0o755); err != nil { + t.Fatal(err) + } + + artifact, ok := m.resolveInstalledArtifact("v1", archiveHash) + if !ok { + t.Fatal("verified legacy install should remain usable when promotion fails") + } + if artifact.binPath != legacyBinPath { + t.Fatalf("resolved binary = %q, want legacy path %q", artifact.binPath, legacyBinPath) + } +} + +func TestResolveInstalledArtifact_ConcurrentPromotionsConverge(t *testing.T) { + dir := t.TempDir() + binDir := filepath.Join(dir, "bin") + archiveHash := sha256Hex([]byte("legacy archive")) + legacyBinPath := writeLegacyInstall( + t, + binDir, + "v1", + archiveHash, + "testapp", + []byte("legacy binary"), + ) + cfg := config.Config{BinDir: binDir, BinaryName: "testapp", BasePort: 5000} + managers := []*Manager{NewManager(cfg), NewManager(cfg)} + type result struct { + artifact installedArtifact + ok bool + } + results := make(chan result, len(managers)) + start := make(chan struct{}) + var wg sync.WaitGroup + for _, manager := range managers { + wg.Add(1) + go func(m *Manager) { + defer wg.Done() + <-start + artifact, ok := m.resolveInstalledArtifact("v1", archiveHash) + results <- result{artifact: artifact, ok: ok} + }(manager) + } + close(start) + wg.Wait() + close(results) + + canonicalBinPath := managers[0].installBinPath("v1", archiveHash) + for result := range results { + if !result.ok { + t.Fatal("concurrent promotion did not resolve an install") + } + if result.artifact.binPath != canonicalBinPath { + t.Fatalf("resolved binary = %q, want %q", result.artifact.binPath, canonicalBinPath) + } + } + matches, _, _, err := installedVersionMatches( + managers[0].installDir("v1", archiveHash), + canonicalBinPath, + archiveHash, + ) + if err != nil { + t.Fatal(err) + } + if !matches { + t.Fatal("concurrently promoted install should be complete and verified") + } + assertPathExists(t, legacyBinPath) +} + +func TestResolveInstalledArtifact_RejectsInvalidLegacyInstall(t *testing.T) { + dir := t.TempDir() + binDir := filepath.Join(dir, "bin") + archiveHash := sha256Hex([]byte("legacy archive")) + legacyBinPath := writeLegacyInstall( + t, + binDir, + "v1", + archiveHash, + "testapp", + []byte("original binary"), + ) + if err := os.WriteFile(legacyBinPath, []byte("tampered binary"), 0o755); err != nil { + t.Fatal(err) + } + m := NewManager(config.Config{BinDir: binDir, BinaryName: "testapp", BasePort: 5000}) + + if artifact, ok := m.resolveInstalledArtifact("v1", archiveHash); ok { + t.Fatalf("invalid legacy install resolved as %+v", artifact) + } + assertPathMissing(t, m.installBinPath("v1", archiveHash)) +} + +func startLocalHTTPServer(t *testing.T, handler http.Handler) (int, func()) { + t.Helper() + ln, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + srv := &http.Server{Handler: handler} + go func() { + _ = srv.Serve(ln) + }() + shutdown := func() { + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + _ = srv.Shutdown(ctx) + } + return ln.Addr().(*net.TCPAddr).Port, shutdown +} + +func writeStorageModeProbeBinary(t *testing.T, dir, name, storageMode string) string { + t.Helper() + binPath := filepath.Join(dir, name) + script := `#!/bin/sh +case "$1" in +--print-storage-mode) echo "` + storageMode + `" ;; +*) echo "unknown flag: $1" >&2; exit 2 ;; +esac +` + if err := os.WriteFile(binPath, []byte(script), 0o755); err != nil { + t.Fatal(err) + } + return binPath +} + +func writeStorageModeLegacyBinary(t *testing.T, dir, name string) string { + t.Helper() + binPath := filepath.Join(dir, name) + script := `#!/bin/sh +echo "unknown flag: $1" >&2 +exit 2 +` + if err := os.WriteFile(binPath, []byte(script), 0o755); err != nil { + t.Fatal(err) + } + return binPath +} + +func writeStorageModeErrorBinary(t *testing.T, dir, name string) string { + t.Helper() + binPath := filepath.Join(dir, name) + script := `#!/bin/sh +case "$1" in +--print-storage-mode) echo "invalid storage env" >&2; exit 1 ;; +*) echo "unknown flag: $1" >&2; exit 2 ;; +esac +` + if err := os.WriteFile(binPath, []byte(script), 0o755); err != nil { + t.Fatal(err) + } + return binPath +} + +func writeInstalledVersion(t *testing.T, binDir, versionName, sha, binaryName string, modTime time.Time) { + t.Helper() + versionDir := filepath.Join(binDir, versionName, sha) + if err := os.MkdirAll(versionDir, 0o755); err != nil { + t.Fatal(err) + } + binPath := filepath.Join(versionDir, binaryName) + if err := os.WriteFile(binPath, []byte("binary-"+sha), 0o755); err != nil { + t.Fatal(err) + } + writeInstallMetadataFile(t, versionDir, download.InstallMetadata{ + ArchiveSHA256: sha, + BinarySHA256: "binary-" + sha, + }) + if err := os.Chtimes(filepath.Join(versionDir, download.InstallMetadataFilename), modTime, modTime); err != nil { + t.Fatal(err) + } +} + +func writeLegacyInstall( + t *testing.T, + binDir string, + versionName string, + archiveHash string, + binaryName string, + binary []byte, +) string { + t.Helper() + versionDir := filepath.Join(binDir, versionName) + if err := os.MkdirAll(versionDir, 0o755); err != nil { + t.Fatal(err) + } + binPath := filepath.Join(versionDir, binaryName) + if err := os.WriteFile(binPath, binary, 0o755); err != nil { + t.Fatal(err) + } + if err := download.WriteInstallMetadata(versionDir, download.InstallMetadata{ + ArchiveSHA256: archiveHash, + BinarySHA256: sha256Hex(binary), + }); err != nil { + t.Fatal(err) + } + return binPath +} + +func assertPathExists(t *testing.T, path string) { + t.Helper() + if _, err := os.Stat(path); err != nil { + t.Fatalf("expected path to exist %s: %v", path, err) + } +} + +func assertPathMissing(t *testing.T, path string) { + t.Helper() + if _, err := os.Stat(path); !os.IsNotExist(err) { + t.Fatalf("expected path to be missing %s, stat err %v", path, err) + } +} + func writeInstallMetadataFile(t *testing.T, versionDir string, metadata download.InstallMetadata) { t.Helper() data, err := json.Marshal(metadata) @@ -684,3 +2291,23 @@ func sha256Hex(data []byte) string { sum := sha256.Sum256(data) return hex.EncodeToString(sum[:]) } + +func zipBinary(t *testing.T, binaryName string, data []byte) ([]byte, string) { + t.Helper() + + var buf bytes.Buffer + zw := zip.NewWriter(&buf) + w, err := zw.Create(binaryName) + if err != nil { + t.Fatal(err) + } + if _, err := w.Write(data); err != nil { + t.Fatal(err) + } + if err := zw.Close(); err != nil { + t.Fatal(err) + } + + zipData := buf.Bytes() + return zipData, sha256Hex(zipData) +} diff --git a/versioned/internal/process/supervised_process.go b/versioned/internal/process/supervised_process.go new file mode 100644 index 0000000000..227f6e56ab --- /dev/null +++ b/versioned/internal/process/supervised_process.go @@ -0,0 +1,232 @@ +package process + +import ( + "errors" + "fmt" + "log/slog" + "os" + "os/exec" + "sync" + "syscall" + "time" +) + +type processState uint8 + +const ( + processStateRunning processState = iota + processStateTerminating + processStateKilling + processStateExited +) + +func (s processState) String() string { + switch s { + case processStateRunning: + return "running" + case processStateTerminating: + return "terminating" + case processStateKilling: + return "killing" + case processStateExited: + return "exited" + default: + return fmt.Sprintf("processState(%d)", s) + } +} + +// supervisedProcess owns signal delivery and reaping for one process +// incarnation. The graceful timeout controls escalation to SIGKILL; completion +// is always confirmed by cmd.Wait before Done is closed. +type supervisedProcess struct { + cmd *exec.Cmd + grace time.Duration + stop <-chan struct{} + externalForce <-chan struct{} + + force chan struct{} + forceOnce sync.Once + done chan struct{} + + mu sync.Mutex + state processState + err error + escalated bool +} + +func startSupervisedProcess( + cmd *exec.Cmd, + stop <-chan struct{}, + externalForce <-chan struct{}, + grace time.Duration, +) (*supervisedProcess, error) { + if cmd.SysProcAttr == nil { + cmd.SysProcAttr = &syscall.SysProcAttr{} + } + cmd.SysProcAttr.Setpgid = true + if err := cmd.Start(); err != nil { + return nil, err + } + + p := &supervisedProcess{ + cmd: cmd, + grace: grace, + stop: stop, + externalForce: externalForce, + force: make(chan struct{}), + done: make(chan struct{}), + state: processStateRunning, + } + go p.run() + return p, nil +} + +func (p *supervisedProcess) run() { + waitCh := make(chan error, 1) + go func() { + waitCh <- p.cmd.Wait() + }() + + var err error + select { + case err = <-waitCh: + p.complete(err) + return + case <-p.stop: + p.transition(processStateTerminating) + p.signal(syscall.SIGTERM) + case <-p.force: + err = p.killAndWait(waitCh) + p.complete(err) + return + case <-p.externalForce: + err = p.killAndWait(waitCh) + p.complete(err) + return + } + + timer := time.NewTimer(p.grace) + defer timer.Stop() + select { + case err = <-waitCh: + case <-timer.C: + err = p.killAndWait(waitCh) + case <-p.force: + err = p.killAndWait(waitCh) + case <-p.externalForce: + err = p.killAndWait(waitCh) + } + p.complete(err) +} + +func (p *supervisedProcess) killAndWait(waitCh <-chan error) error { + p.transition(processStateKilling) + p.mu.Lock() + p.escalated = true + p.mu.Unlock() + p.signal(syscall.SIGKILL) + return <-waitCh +} + +func (p *supervisedProcess) signal(signal syscall.Signal) { + if err := signalProcessGroup(p.cmd.Process, signal); err != nil { + slog.Warn( + "child process signal failed", + "pid", p.cmd.Process.Pid, + "signal", signal.String(), + "error", err, + ) + } +} + +func signalProcessGroup(process *os.Process, signal syscall.Signal) error { + if process == nil { + return errors.New("child process has not started") + } + if err := syscall.Kill(-process.Pid, signal); err != nil { + if errors.Is(err, syscall.ESRCH) || errors.Is(err, os.ErrProcessDone) { + return nil + } + fallbackErr := process.Signal(signal) + if fallbackErr == nil || errors.Is(fallbackErr, os.ErrProcessDone) { + return nil + } + return errors.Join(err, fallbackErr) + } + return nil +} + +func (p *supervisedProcess) transition(next processState) { + p.mu.Lock() + defer p.mu.Unlock() + if !validProcessTransition(p.state, next) { + slog.Error( + "invalid child process state transition", + "from", p.state.String(), + "to", next.String(), + "pid", p.cmd.Process.Pid, + ) + return + } + p.state = next +} + +func validProcessTransition(from, to processState) bool { + switch from { + case processStateRunning: + return to == processStateTerminating || to == processStateKilling || to == processStateExited + case processStateTerminating: + return to == processStateKilling || to == processStateExited + case processStateKilling: + return to == processStateExited + case processStateExited: + return false + default: + return false + } +} + +func (p *supervisedProcess) complete(err error) { + p.mu.Lock() + if validProcessTransition(p.state, processStateExited) { + p.state = processStateExited + } else { + slog.Error( + "invalid child process completion state", + "state", p.state.String(), + "pid", p.cmd.Process.Pid, + ) + } + p.err = err + p.mu.Unlock() + close(p.done) +} + +func (p *supervisedProcess) ForceStop() { + p.forceOnce.Do(func() { + close(p.force) + }) +} + +func (p *supervisedProcess) Done() <-chan struct{} { + return p.done +} + +func (p *supervisedProcess) Wait() error { + <-p.done + p.mu.Lock() + defer p.mu.Unlock() + return p.err +} + +func (p *supervisedProcess) State() processState { + p.mu.Lock() + defer p.mu.Unlock() + return p.state +} + +func (p *supervisedProcess) Escalated() bool { + p.mu.Lock() + defer p.mu.Unlock() + return p.escalated +} diff --git a/versioned/internal/process/supervised_process_test.go b/versioned/internal/process/supervised_process_test.go new file mode 100644 index 0000000000..1729b1cf2a --- /dev/null +++ b/versioned/internal/process/supervised_process_test.go @@ -0,0 +1,186 @@ +package process + +import ( + "context" + "errors" + "fmt" + "os" + "os/exec" + "os/signal" + "path/filepath" + "syscall" + "testing" + "time" + + "versioned/internal/config" + "versioned/internal/oracle" +) + +const supervisedProcessHelperEnv = "VERSIOND_SUPERVISED_PROCESS_HELPER" + +func TestSupervisedProcessHelper(t *testing.T) { + if os.Getenv(supervisedProcessHelperEnv) == "" { + return + } + + var term chan os.Signal + switch os.Getenv(supervisedProcessHelperEnv) { + case "graceful": + term = make(chan os.Signal, 1) + signal.Notify(term, syscall.SIGTERM) + case "ignore-term": + signal.Ignore(syscall.SIGTERM) + default: + os.Exit(2) + } + + readyFile := os.Getenv("VERSIOND_SUPERVISED_PROCESS_READY_FILE") + if err := os.WriteFile(readyFile, []byte("ready"), 0o600); err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(2) + } + if term != nil { + <-term + return + } + select {} +} + +func TestSupervisedProcessStopsGracefullyAndReaps(t *testing.T) { + stop := make(chan struct{}) + force := make(chan struct{}) + // The race runtime delays helper-process exit while flushing its report. + proc := startSupervisedTestProcess(t, "graceful", 3*time.Second, stop, force) + + close(stop) + if err := waitForSupervisedProcess(proc, 5*time.Second); err != nil { + t.Fatalf("graceful process wait: %v", err) + } + if proc.Escalated() { + t.Fatal("graceful process unexpectedly escalated to SIGKILL") + } + if got := proc.State(); got != processStateExited { + t.Fatalf("process state = %s, want exited", got) + } + assertProcessReaped(t, proc) +} + +func TestSupervisedProcessEscalatesAfterGraceAndReaps(t *testing.T) { + const grace = 100 * time.Millisecond + stop := make(chan struct{}) + force := make(chan struct{}) + proc := startSupervisedTestProcess(t, "ignore-term", grace, stop, force) + + started := time.Now() + close(stop) + if err := waitForSupervisedProcess(proc, 2*time.Second); err == nil { + t.Fatal("SIGKILLed process returned a nil wait error") + } + if elapsed := time.Since(started); elapsed < grace/2 { + t.Fatalf("process escalated after %s, want graceful wait first", elapsed) + } + if !proc.Escalated() { + t.Fatal("process did not record SIGKILL escalation") + } + if got := proc.State(); got != processStateExited { + t.Fatalf("process state = %s, want exited", got) + } + assertProcessReaped(t, proc) +} + +func TestManagerShutdownDeadlineForcesAndReapsChildren(t *testing.T) { + childCtx, childStop := context.WithCancel(context.Background()) + force := make(chan struct{}) + proc := startSupervisedTestProcess(t, "ignore-term", time.Hour, childCtx.Done(), force) + done := make(chan struct{}) + go func() { + _ = proc.Wait() + close(done) + }() + + c := &child{ + version: oracle.Version{Name: "v1"}, + stop: childStop, + forceStopCh: force, + done: done, + status: statusRunning, + } + m := NewManager(config.Config{BasePort: 5000}) + m.mu.Lock() + m.processes[c.version.Name] = c + m.mu.Unlock() + + shutdownCtx, cancelShutdown := context.WithTimeout(context.Background(), 50*time.Millisecond) + defer cancelShutdown() + err := m.Shutdown(shutdownCtx) + if !errors.Is(err, context.DeadlineExceeded) { + t.Fatalf("Shutdown error = %v, want context deadline exceeded", err) + } + if !proc.Escalated() { + t.Fatal("manager deadline did not escalate child to SIGKILL") + } + select { + case <-c.Done(): + default: + t.Fatal("Shutdown returned before the child was reaped") + } + assertProcessReaped(t, proc) +} + +func startSupervisedTestProcess( + t *testing.T, + mode string, + grace time.Duration, + stop <-chan struct{}, + force <-chan struct{}, +) *supervisedProcess { + t.Helper() + readyFile := filepath.Join(t.TempDir(), "ready") + cmd := exec.Command(os.Args[0], "-test.run=^TestSupervisedProcessHelper$") + cmd.Env = append(os.Environ(), + supervisedProcessHelperEnv+"="+mode, + "VERSIOND_SUPERVISED_PROCESS_READY_FILE="+readyFile, + ) + proc, err := startSupervisedProcess(cmd, stop, force, grace) + if err != nil { + t.Fatalf("start helper process: %v", err) + } + t.Cleanup(func() { + proc.ForceStop() + _ = proc.Wait() + }) + + deadline := time.Now().Add(2 * time.Second) + for { + if _, err := os.Stat(readyFile); err == nil { + return proc + } + select { + case <-proc.Done(): + t.Fatalf("helper process exited before readiness: %v", proc.Wait()) + default: + } + if time.Now().After(deadline) { + t.Fatal("helper process did not report readiness") + } + time.Sleep(5 * time.Millisecond) + } +} + +func waitForSupervisedProcess(proc *supervisedProcess, timeout time.Duration) error { + timer := time.NewTimer(timeout) + defer timer.Stop() + select { + case <-proc.Done(): + return proc.Wait() + case <-timer.C: + return fmt.Errorf("process did not exit within %s", timeout) + } +} + +func assertProcessReaped(t *testing.T, proc *supervisedProcess) { + t.Helper() + if err := proc.cmd.Process.Signal(syscall.Signal(0)); !errors.Is(err, os.ErrProcessDone) { + t.Fatalf("signal reaped process error = %v, want os.ErrProcessDone", err) + } +} diff --git a/versioned/internal/proxy/proxy.go b/versioned/internal/proxy/proxy.go index 601c3e7c2c..2affeaa345 100644 --- a/versioned/internal/proxy/proxy.go +++ b/versioned/internal/proxy/proxy.go @@ -9,6 +9,10 @@ import ( "sync/atomic" ) +type routeTableLoader interface { + Load() any +} + // Handler returns an http.Handler that routes requests by version prefix. // First path segment is the version name, stripped before forwarding. // An optional leading /devshard/ is accepted (same as versiond-router) so @@ -32,14 +36,14 @@ func Handler(routes *atomic.Value) http.Handler { rest = "/" + parts[1] } - routeMap := routes.Load().(map[string]string) - target, ok := routeMap[version] + target, ok := acquireTarget(routes, version) if !ok { http.Error(w, fmt.Sprintf("version %q not found", version), http.StatusNotFound) return } + defer target.release() - targetURL, err := url.Parse("http://" + target) + targetURL, err := url.Parse("http://" + target.Address()) if err != nil { http.Error(w, "internal error", http.StatusInternalServerError) return @@ -60,3 +64,15 @@ func Handler(routes *atomic.Value) http.Handler { p.ServeHTTP(w, r) }) } + +func acquireTarget(routes routeTableLoader, version string) (*Target, bool) { + for { + target, ok := routes.Load().(RouteTable)[version] + if !ok { + return nil, false + } + if target.acquire() { + return target, true + } + } +} diff --git a/versioned/internal/proxy/proxy_test.go b/versioned/internal/proxy/proxy_test.go index 57ad48c8ab..934c91991c 100644 --- a/versioned/internal/proxy/proxy_test.go +++ b/versioned/internal/proxy/proxy_test.go @@ -7,17 +7,156 @@ import ( "net/http" "net/http/httptest" "strings" + "sync" "sync/atomic" "testing" "time" ) func newRoutes(m map[string]string) *atomic.Value { + targets := make(RouteTable, len(m)) + for version, address := range m { + targets[version] = NewTarget(address) + } v := &atomic.Value{} - v.Store(m) + v.Store(targets) return v } +type sequenceRouteLoader struct { + tables []RouteTable + calls int +} + +func (l *sequenceRouteLoader) Load() any { + index := l.calls + if index >= len(l.tables) { + index = len(l.tables) - 1 + } + l.calls++ + return l.tables[index] +} + +func TestAcquireTarget_RetriesRetiredRoute(t *testing.T) { + oldTarget := NewTarget("localhost:9001") + oldTarget.Retire() + newTarget := NewTarget("localhost:9002") + loader := &sequenceRouteLoader{tables: []RouteTable{ + {"v1": oldTarget}, + {"v1": newTarget}, + }} + + target, ok := acquireTarget(loader, "v1") + if !ok { + t.Fatal("new target should be acquired after retired route retry") + } + defer target.release() + if target != newTarget { + t.Fatal("acquired target is not the replacement target") + } + if loader.calls != 2 { + t.Fatalf("route loads = %d, want retired route plus replacement", loader.calls) + } +} + +func TestTargetRetireWaitsForAcquiredRequest(t *testing.T) { + target := NewTarget("localhost:9001") + if !target.acquire() { + t.Fatal("active target should accept a request") + } + + drained := target.Retire() + select { + case <-drained: + t.Fatal("target drained before its acquired request was released") + default: + } + if target.acquire() { + t.Fatal("retired target should reject new requests") + } + + target.release() + select { + case <-drained: + case <-time.After(time.Second): + t.Fatal("target did not drain after its acquired request was released") + } +} + +func TestProxy_RouteSwapKeepsAcquiredRequestOnRetiredTarget(t *testing.T) { + oldStarted := make(chan struct{}) + releaseOld := make(chan struct{}) + var oldStartedOnce sync.Once + var releaseOldOnce sync.Once + releaseOldRequest := func() { releaseOldOnce.Do(func() { close(releaseOld) }) } + defer releaseOldRequest() + oldBackend := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + oldStartedOnce.Do(func() { close(oldStarted) }) + <-releaseOld + fmt.Fprint(w, "old") + })) + defer oldBackend.Close() + newBackend := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + fmt.Fprint(w, "new") + })) + defer newBackend.Close() + + oldTarget := NewTarget(strings.TrimPrefix(oldBackend.URL, "http://")) + newTarget := NewTarget(strings.TrimPrefix(newBackend.URL, "http://")) + routes := &atomic.Value{} + routes.Store(RouteTable{"v1": oldTarget}) + server := httptest.NewServer(Handler(routes)) + defer server.Close() + + oldResult := make(chan string, 1) + go func() { + resp, err := http.Get(server.URL + "/v1/work") + if err != nil { + oldResult <- "request error: " + err.Error() + return + } + defer resp.Body.Close() + body, err := io.ReadAll(resp.Body) + if err != nil { + oldResult <- "read error: " + err.Error() + return + } + oldResult <- string(body) + }() + <-oldStarted + + routes.Store(RouteTable{"v1": newTarget}) + drained := oldTarget.Retire() + + resp, err := http.Get(server.URL + "/v1/work") + if err != nil { + t.Fatal(err) + } + body, err := io.ReadAll(resp.Body) + resp.Body.Close() + if err != nil { + t.Fatal(err) + } + if string(body) != "new" { + t.Fatalf("new request body = %q, want new target", body) + } + select { + case <-drained: + t.Fatal("old target drained while its request was still running") + default: + } + + releaseOldRequest() + if body := <-oldResult; body != "old" { + t.Fatalf("old request body = %q, want old target", body) + } + select { + case <-drained: + case <-time.After(time.Second): + t.Fatal("old target did not drain after its request completed") + } +} + func TestProxy_BasicForwarding(t *testing.T) { backend := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { fmt.Fprintf(w, "path=%s", r.URL.Path) diff --git a/versioned/internal/proxy/target.go b/versioned/internal/proxy/target.go new file mode 100644 index 0000000000..463c9e537b --- /dev/null +++ b/versioned/internal/proxy/target.go @@ -0,0 +1,62 @@ +package proxy + +import "sync" + +// RouteTable maps a public version name to one concrete child generation. +type RouteTable map[string]*Target + +// Target tracks requests pinned by the proxy to one child process. +type Target struct { + address string + + mu sync.Mutex + retired bool + inflight int + drained chan struct{} + closed bool +} + +func NewTarget(address string) *Target { + return &Target{ + address: address, + drained: make(chan struct{}), + } +} + +func (t *Target) Address() string { + return t.address +} + +func (t *Target) acquire() bool { + t.mu.Lock() + defer t.mu.Unlock() + if t.retired { + return false + } + t.inflight++ + return true +} + +func (t *Target) release() { + t.mu.Lock() + defer t.mu.Unlock() + t.inflight-- + t.closeDrainedLocked() +} + +// Retire prevents new proxy requests from using the target. The returned +// channel closes after every request that already acquired the target exits. +func (t *Target) Retire() <-chan struct{} { + t.mu.Lock() + defer t.mu.Unlock() + t.retired = true + t.closeDrainedLocked() + return t.drained +} + +func (t *Target) closeDrainedLocked() { + if t.retired && t.inflight == 0 && !t.closed { + close(t.drained) + t.closed = true + } +}