From 00069ecfbac06b69aeaecfa52ff8b04dbe2bd948 Mon Sep 17 00:00:00 2001 From: Daniil Yankouski Date: Mon, 29 Jun 2026 17:24:06 +0200 Subject: [PATCH 01/13] feat(devshard): settle depleted escrow after in-flight requests drain (#1314) * feat(devshard): settle depleted escrow when last active request drains * fix(devshard): skip settlement reconcile on restart when settlement disabled --- devshard/cmd/devshardctl/gateway.go | 125 +++++++++++++++++- devshard/cmd/devshardctl/gateway_store.go | 53 ++++++-- .../cmd/devshardctl/gateway_store_test.go | 43 ++++++ devshard/cmd/devshardctl/gateway_test.go | 87 ++++++++++++ 4 files changed, 294 insertions(+), 14 deletions(-) diff --git a/devshard/cmd/devshardctl/gateway.go b/devshard/cmd/devshardctl/gateway.go index d1c62eca91..3a5bf75a99 100644 --- a/devshard/cmd/devshardctl/gateway.go +++ b/devshard/cmd/devshardctl/gateway.go @@ -84,6 +84,13 @@ type devshardRuntime struct { activeRequests atomic.Int64 reservedTokens atomic.Int64 + // settlementPending marks an escrow that has been deactivated and must + // be settled once its in-flight requests drain. settlementReason is + // written before the flag and read after it in the lock-free drain hook; + // the atomic Store→Load pair supplies the happens-before. + settlementPending atomic.Bool + settlementReason string + activeConfigured bool } @@ -97,6 +104,7 @@ type runtimeStatus struct { ProtocolVersion string `json:"protocol_version,omitempty"` ActiveRequests int64 `json:"active_requests"` ReservedTokens int64 `json:"reserved_tokens"` + SettlementPending bool `json:"settlement_pending,omitempty"` ChainPhase string `json:"chain_phase,omitempty"` ConfirmationPoCPhase string `json:"confirmation_poc_phase,omitempty"` RequestsBlocked bool `json:"requests_blocked"` @@ -406,11 +414,12 @@ func sessionPhaseLabel(phase types.SessionPhase) string { func (rt *devshardRuntime) snapshot() runtimeStatus { status := runtimeStatus{ - ID: rt.id, - Model: rt.model, - Active: rt.active.Load(), - ActiveRequests: rt.activeRequests.Load(), - ReservedTokens: rt.reservedTokens.Load(), + ID: rt.id, + Model: rt.model, + Active: rt.active.Load(), + ActiveRequests: rt.activeRequests.Load(), + ReservedTokens: rt.reservedTokens.Load(), + SettlementPending: rt.settlementPending.Load(), } if rt.proxy != nil && rt.proxy.sm != nil && rt.proxy.session != nil { phase := rt.proxy.sm.Phase() @@ -536,6 +545,10 @@ func NewManagedGateway(runtimes []*devshardRuntime, limiter *GatewayLimiter, set g.attachEscrowChecker(rt) } g.startEscrowRotatorIfEnabled() + // Settle escrows left pending by a pre-restart drain. Runs synchronously + // so the store read completes before the gateway serves traffic; the + // settlements it schedules run in their own goroutines. + g.reconcilePendingSettlements() go g.balanceCheckLoop() return g } @@ -1591,8 +1604,15 @@ func (g *Gateway) reserveRuntimeLocked(rt *devshardRuntime, inputTokens int64) { } func (g *Gateway) releaseRuntime(rt *devshardRuntime, inputTokens int64) { - rt.activeRequests.Add(-1) + remaining := rt.activeRequests.Add(-1) rt.reservedTokens.Add(-inputTokens) + // active=false (set during enqueue) blocks new reservations, so the count + // only drains downward — remaining==0 is the exact "last request finished" + // edge. scheduleAutoSettlement dedups, so a double-fire is harmless. + if remaining == 0 && rt.settlementPending.Load() { + log.Printf("settlement_drain_complete escrow=%s reason=%s", rt.id, rt.settlementReason) + g.scheduleAutoSettlement(rt.id, rt.settlementReason) + } } func (rt *devshardRuntime) validateRequestedModel(requestModel string) error { @@ -3388,13 +3408,105 @@ func (g *Gateway) deactivateDevshardByIDWithReason(id, reason string) bool { return true } +// deactivateAndSettleDevshardByID stops new traffic to an escrow and settles +// it. If requests are still in flight it marks the escrow settlement-pending +// and returns; the drain hook in releaseRuntime settles once the last request +// finishes. Otherwise it settles immediately. func (g *Gateway) deactivateAndSettleDevshardByID(id, reason string) { if !g.deactivateDevshardByIDWithReason(id, reason) { return } + g.markSettlementPending(id, reason) + + g.mu.Lock() + rt, ok := g.runtimes[id] + g.mu.Unlock() + if ok && rt.activeRequests.Load() > 0 { + log.Printf("settlement_queued_waiting_for_drain escrow=%s reason=%s active_requests=%d", + id, reason, rt.activeRequests.Load()) + return + } g.scheduleAutoSettlement(id, reason) } +// markSettlementPending records that an escrow must be settled once its +// in-flight requests drain. The reason is stored before the flag so the +// lock-free drain hook in releaseRuntime reads a consistent value. +func (g *Gateway) markSettlementPending(id, reason string) { + g.mu.Lock() + rt, ok := g.runtimes[id] + if ok { + rt.settlementReason = reason + rt.settlementPending.Store(true) + } + g.mu.Unlock() + if g.store != nil { + if err := g.store.SetDevshardSettlementPending(id, true); err != nil { + log.Printf("settlement_pending_persist_failed escrow=%s error=%v", id, err) + } + } +} + +// reconcilePendingSettlements settles escrows that were marked pending before +// a restart. After a restart no requests are in flight, so each such escrow +// can settle immediately. Hydrates the in-memory marker too. +func (g *Gateway) reconcilePendingSettlements() { + if g.store == nil { + return + } + state, ok, err := g.store.LoadState() + if err != nil || !ok { + if err != nil { + log.Printf("settlement_reconcile_load_failed error=%v", err) + } + return + } + // Honor the operator's config: when settlement is disabled, never settle on + // startup. Leave the marker intact so a later re-enable still settles it. + if !state.Settings.EscrowRotation.SettlementEnabled { + for _, devshard := range state.Devshards { + if !devshard.Active && devshard.SettlementPending { + log.Printf("settlement_reconcile_skipped escrow=%s reason=settlement_disabled", devshard.ID) + } + } + return + } + for _, devshard := range state.Devshards { + if devshard.Active || !devshard.SettlementPending { + continue + } + g.mu.Lock() + rt, exists := g.runtimes[devshard.ID] + if exists { + rt.settlementReason = "startup_reconcile" + rt.settlementPending.Store(true) + } + g.mu.Unlock() + if !exists { + log.Printf("settlement_reconcile_skipped escrow=%s reason=runtime_not_loaded", devshard.ID) + continue + } + log.Printf("settlement_reconcile_queued escrow=%s", devshard.ID) + g.scheduleAutoSettlement(devshard.ID, "startup_reconcile") + } +} + +// clearSettlementPending is called after a successful settlement so a +// restart-time reconcile does not re-settle the escrow. +func (g *Gateway) clearSettlementPending(id string) { + g.mu.Lock() + rt, ok := g.runtimes[id] + if ok { + rt.settlementPending.Store(false) + } + g.mu.Unlock() + if g.store != nil { + if err := g.store.SetDevshardSettlementPending(id, false); err != nil { + log.Printf("settlement_pending_clear_failed escrow=%s error=%v", id, err) + } + } +} + func (g *Gateway) retireRotatedDevshard(ctx context.Context, id, reason string, settings GatewaySettings) (bool, error) { if !settings.EscrowRotation.SettlementEnabled { if g.deactivateDevshardByIDWithReason(id, reason) { @@ -3518,6 +3630,7 @@ func (g *Gateway) scheduleAutoSettlement(id, reason string) { result, err := gatewaySettleDevshardOnChain(g, ctx, id, adminSettleEscrowRequest{}) cancel() if err == nil { + g.clearSettlementPending(id) log.Printf("auto_settle_submitted escrow=%s reason=%s tx_hash=%s settler=%s", id, reason, result.TxHash, result.Settler) return diff --git a/devshard/cmd/devshardctl/gateway_store.go b/devshard/cmd/devshardctl/gateway_store.go index f8fb2e05b3..f7544402d5 100644 --- a/devshard/cmd/devshardctl/gateway_store.go +++ b/devshard/cmd/devshardctl/gateway_store.go @@ -285,11 +285,12 @@ func normalizeGatewayModelAccess(access []GatewayModelAccessSettings) []GatewayM type GatewayDevshardState struct { RuntimeConfig - Active bool `json:"active"` - RotationRole string `json:"rotation_role,omitempty"` - RotationEpoch uint64 `json:"rotation_epoch,omitempty"` - CreatedAt string `json:"created_at,omitempty"` - UpdatedAt string `json:"updated_at,omitempty"` + Active bool `json:"active"` + SettlementPending bool `json:"settlement_pending,omitempty"` + RotationRole string `json:"rotation_role,omitempty"` + RotationEpoch uint64 `json:"rotation_epoch,omitempty"` + CreatedAt string `json:"created_at,omitempty"` + UpdatedAt string `json:"updated_at,omitempty"` } type GatewaySuspiciousHost struct { @@ -375,6 +376,7 @@ func NewGatewayStore(path string) (*GatewayStore, error) { active INTEGER NOT NULL DEFAULT 1, rotation_role TEXT NOT NULL DEFAULT '', rotation_epoch INTEGER NOT NULL DEFAULT 0, + settlement_pending INTEGER NOT NULL DEFAULT 0, created_at TEXT NOT NULL, updated_at TEXT NOT NULL )`, @@ -437,6 +439,10 @@ func NewGatewayStore(path string) (*GatewayStore, error) { db.Close() return nil, fmt.Errorf("migrate gateway devshard epoch: %w", err) } + if err := ensureGatewayDevshardsColumn(db, "settlement_pending", "INTEGER NOT NULL DEFAULT 0"); err != nil { + db.Close() + return nil, fmt.Errorf("migrate gateway devshard settlement_pending: %w", err) + } if _, err := db.Exec(`CREATE TABLE IF NOT EXISTS participant_throttle_state ( participant_key TEXT PRIMARY KEY, tokens REAL NOT NULL DEFAULT 0, @@ -626,7 +632,7 @@ func (s *GatewayStore) LoadState() (GatewayState, bool, error) { rows, err := s.db.Query(` SELECT id, private_key_hex, private_key_env, model, storage_path, active, created_at, updated_at, protocol_version, - rotation_role, rotation_epoch + rotation_role, rotation_epoch, settlement_pending FROM gateway_devshards ORDER BY id`) if err != nil { @@ -636,6 +642,7 @@ func (s *GatewayStore) LoadState() (GatewayState, bool, error) { for rows.Next() { var devshard GatewayDevshardState var active int + var settlementPending int if err := rows.Scan( &devshard.ID, &devshard.PrivateKeyHex, @@ -648,10 +655,12 @@ func (s *GatewayStore) LoadState() (GatewayState, bool, error) { &devshard.ProtocolVersion, &devshard.RotationRole, &devshard.RotationEpoch, + &settlementPending, ); err != nil { return GatewayState{}, false, fmt.Errorf("scan gateway devshard: %w", err) } devshard.Active = active != 0 + devshard.SettlementPending = settlementPending != 0 state.Devshards = append(state.Devshards, devshard) } if err := rows.Err(); err != nil { @@ -1081,11 +1090,16 @@ func (s *GatewayStore) UpsertDevshard(devshard GatewayDevshardState) error { func (s *GatewayStore) upsertDevshardTx(tx *sql.Tx, devshard GatewayDevshardState, now string) error { createdAt := now _ = tx.QueryRow(`SELECT created_at FROM gateway_devshards WHERE id = ?`, devshard.ID).Scan(&createdAt) + // Preserve the existing settlement_pending marker so an unrelated upsert + // never silently clears a queued settlement; a brand-new row falls back + // to the value carried on devshard. + settlementPending := gatewayBoolToInt(devshard.SettlementPending) + _ = tx.QueryRow(`SELECT settlement_pending FROM gateway_devshards WHERE id = ?`, devshard.ID).Scan(&settlementPending) if _, err := tx.Exec(` INSERT OR REPLACE INTO gateway_devshards ( id, private_key_hex, private_key_env, model, storage_path, active, created_at, updated_at, protocol_version, - rotation_role, rotation_epoch - ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, + rotation_role, rotation_epoch, settlement_pending + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, strings.TrimSpace(devshard.ID), strings.TrimSpace(devshard.PrivateKeyHex), strings.TrimSpace(devshard.PrivateKeyEnv), @@ -1097,6 +1111,7 @@ func (s *GatewayStore) upsertDevshardTx(tx *sql.Tx, devshard GatewayDevshardStat strings.TrimSpace(devshard.ProtocolVersion), strings.TrimSpace(devshard.RotationRole), devshard.RotationEpoch, + settlementPending, ); err != nil { return fmt.Errorf("upsert gateway devshard %s: %w", devshard.ID, err) } @@ -1125,6 +1140,28 @@ func (s *GatewayStore) SetDevshardActive(id string, active bool) error { return nil } +func (s *GatewayStore) SetDevshardSettlementPending(id string, pending bool) error { + res, err := s.db.Exec(` + UPDATE gateway_devshards + SET settlement_pending = ?, updated_at = ? + WHERE id = ?`, + gatewayBoolToInt(pending), + time.Now().UTC().Format(time.RFC3339Nano), + strings.TrimSpace(id), + ) + if err != nil { + return fmt.Errorf("update devshard %s settlement_pending=%t: %w", id, pending, err) + } + n, err := res.RowsAffected() + if err != nil { + return fmt.Errorf("rows affected for devshard %s: %w", id, err) + } + if n == 0 { + return fmt.Errorf("devshard %s not found", id) + } + return nil +} + func (s *GatewayStore) DeleteDevshard(id string) error { res, err := s.db.Exec(`DELETE FROM gateway_devshards WHERE id = ?`, strings.TrimSpace(id)) if err != nil { diff --git a/devshard/cmd/devshardctl/gateway_store_test.go b/devshard/cmd/devshardctl/gateway_store_test.go index c61371e1e1..9b150ba8a3 100644 --- a/devshard/cmd/devshardctl/gateway_store_test.go +++ b/devshard/cmd/devshardctl/gateway_store_test.go @@ -789,6 +789,49 @@ func TestEscrowRotationUsesEpochSwitchHeightDuringPoC(t *testing.T) { require.Equal(t, 1, settleAttempts) } +func TestGatewayStoreSetDevshardSettlementPending(t *testing.T) { + store, err := NewGatewayStore(filepath.Join(t.TempDir(), "gateway.db")) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, store.Close()) }) + + require.NoError(t, store.Initialize(GatewaySettings{ + ChainREST: "http://node:1317", DefaultModel: "m", DefaultRequestMaxTokens: 1000, + }.WithTuningDefaults(), []GatewayDevshardState{{ + RuntimeConfig: RuntimeConfig{ID: "12", PrivateKeyHex: "secret", Model: "m"}, + Active: true, + }})) + + // Default is not pending. + state, ok, err := store.LoadState() + require.NoError(t, err) + require.True(t, ok) + require.False(t, gatewayDevshardsByID(state.Devshards)["12"].SettlementPending) + + // Set pending → persisted and survives reload. + require.NoError(t, store.SetDevshardSettlementPending("12", true)) + state, _, err = store.LoadState() + require.NoError(t, err) + require.True(t, gatewayDevshardsByID(state.Devshards)["12"].SettlementPending) + + // An unrelated upsert must NOT wipe the pending marker. + require.NoError(t, store.UpsertDevshard(GatewayDevshardState{ + RuntimeConfig: RuntimeConfig{ID: "12", PrivateKeyHex: "secret", Model: "m"}, + Active: false, + })) + state, _, err = store.LoadState() + require.NoError(t, err) + require.True(t, gatewayDevshardsByID(state.Devshards)["12"].SettlementPending) + + // Clear pending. + require.NoError(t, store.SetDevshardSettlementPending("12", false)) + state, _, err = store.LoadState() + require.NoError(t, err) + require.False(t, gatewayDevshardsByID(state.Devshards)["12"].SettlementPending) + + // Unknown id errors. + require.Error(t, store.SetDevshardSettlementPending("nope", true)) +} + func gatewayDevshardsByID(devshards []GatewayDevshardState) map[string]GatewayDevshardState { byID := make(map[string]GatewayDevshardState, len(devshards)) for _, devshard := range devshards { diff --git a/devshard/cmd/devshardctl/gateway_test.go b/devshard/cmd/devshardctl/gateway_test.go index d488acd1cd..d5bc7c71f3 100644 --- a/devshard/cmd/devshardctl/gateway_test.go +++ b/devshard/cmd/devshardctl/gateway_test.go @@ -238,6 +238,93 @@ func TestGatewayCheckBalancesKeepsRuntimeBelowLimits(t *testing.T) { require.True(t, rt.active.Load()) } +func TestEnqueueSettlementWaitsForActiveRequests(t *testing.T) { + rt := gatewayTestRuntimeForLimits(t, "12", balanceMinimumThreshold-1, nonceDeactivationLimit-1) + g, _, settled := gatewayTestDepletionGateway(t, rt) + + // One request in flight → settlement must NOT fire yet, but escrow is + // deactivated and marked pending (in-memory + persisted). + g.reserveRuntime(rt, 1) + g.deactivateAndSettleDevshardByID("12", "low_balance") + + require.False(t, rt.active.Load()) + require.True(t, rt.settlementPending.Load()) + state, ok, err := g.store.LoadState() + require.NoError(t, err) + require.True(t, ok) + require.True(t, gatewayDevshardsByID(state.Devshards)["12"].SettlementPending) + require.EqualValues(t, 0, settled.Load()) + + // Draining the last request triggers exactly one settlement, which + // clears the marker. + g.releaseRuntime(rt, 1) + require.Eventually(t, func() bool { + return settled.Load() == 1 && !rt.settlementPending.Load() + }, time.Second, 10*time.Millisecond) + + state, _, err = g.store.LoadState() + require.NoError(t, err) + require.False(t, gatewayDevshardsByID(state.Devshards)["12"].SettlementPending) +} + +func TestEnqueueSettlementSettlesImmediatelyWhenDrained(t *testing.T) { + rt := gatewayTestRuntimeForLimits(t, "12", balanceMinimumThreshold-1, nonceDeactivationLimit-1) + g, _, settled := gatewayTestDepletionGateway(t, rt) + + // No active requests → settle right away. + g.deactivateAndSettleDevshardByID("12", "low_balance") + + require.Eventually(t, func() bool { + return settled.Load() == 1 && !rt.active.Load() + }, time.Second, 10*time.Millisecond) +} + +func TestReconcilePendingSettlementsSettlesDrainedEscrow(t *testing.T) { + rt := gatewayTestRuntimeForLimits(t, "12", balanceMinimumThreshold, nonceDeactivationLimit-1) + g, _, settled := gatewayTestDepletionGateway(t, rt) + + // Simulate a marker left behind by a pre-restart drain. + rt.active.Store(false) + require.NoError(t, g.store.SetDevshardActive("12", false)) + require.NoError(t, g.store.SetDevshardSettlementPending("12", true)) + + g.reconcilePendingSettlements() + + require.Eventually(t, func() bool { + return settled.Load() == 1 && !rt.settlementPending.Load() + }, time.Second, 10*time.Millisecond) +} + +func TestReconcilePendingSettlementsSkipsActiveOrUnflagged(t *testing.T) { + rt := gatewayTestRuntimeForLimits(t, "12", balanceMinimumThreshold, nonceDeactivationLimit-1) + g, _, settled := gatewayTestDepletionGateway(t, rt) + + // Active escrow, no pending marker → nothing to do. + g.reconcilePendingSettlements() + require.Never(t, func() bool { return settled.Load() > 0 }, 200*time.Millisecond, 20*time.Millisecond) +} + +func TestReconcilePendingSettlementsSkipsWhenSettlementDisabled(t *testing.T) { + rt := gatewayTestRuntimeForLimits(t, "12", balanceMinimumThreshold, nonceDeactivationLimit-1) + g, _, settled := gatewayTestDepletionGateway(t, rt, func(settings *GatewaySettings) { + settings.EscrowRotation.SettlementEnabled = false + }) + + // Inactive escrow flagged pending, but settlement is disabled → reconcile + // must not settle, and the marker is preserved for a later re-enable. + rt.active.Store(false) + require.NoError(t, g.store.SetDevshardActive("12", false)) + require.NoError(t, g.store.SetDevshardSettlementPending("12", true)) + + g.reconcilePendingSettlements() + + require.Never(t, func() bool { return settled.Load() > 0 }, 200*time.Millisecond, 20*time.Millisecond) + state, ok, err := g.store.LoadState() + require.NoError(t, err) + require.True(t, ok) + require.True(t, gatewayDevshardsByID(state.Devshards)["12"].SettlementPending) +} + func TestParseDevshardPath(t *testing.T) { id, inner, ok := parseDevshardPath("/devshard/12/v1/debug/perf") require.True(t, ok) From 18669d884fe82696ed8c26fe02441286f52334fd Mon Sep 17 00:00:00 2001 From: Daniil Yankouski Date: Tue, 30 Jun 2026 09:40:11 +0200 Subject: [PATCH 02/13] fix(devshard): retire runtime escrow (#1349) * fix(devshard): retire runtime escrow * review(devshard): copilot suggestions & todo for future task implementation * fix(devshard): retire runtime escrow when in-fligh request completed --- devshard/cmd/devshardctl/gateway.go | 70 +++++++- .../gateway_runtime_retire_test.go | 149 ++++++++++++++++++ devshard/user/session_close_test.go | 58 +++++++ 3 files changed, 276 insertions(+), 1 deletion(-) create mode 100644 devshard/cmd/devshardctl/gateway_runtime_retire_test.go create mode 100644 devshard/user/session_close_test.go diff --git a/devshard/cmd/devshardctl/gateway.go b/devshard/cmd/devshardctl/gateway.go index 3a5bf75a99..7a6b93db45 100644 --- a/devshard/cmd/devshardctl/gateway.go +++ b/devshard/cmd/devshardctl/gateway.go @@ -91,6 +91,11 @@ type devshardRuntime struct { settlementPending atomic.Bool settlementReason string + // retirePending marks a runtime whose retirement was deferred because a + // request was still in flight; + retirePending atomic.Bool + retireReason string + activeConfigured bool } @@ -1609,9 +1614,19 @@ func (g *Gateway) releaseRuntime(rt *devshardRuntime, inputTokens int64) { // active=false (set during enqueue) blocks new reservations, so the count // only drains downward — remaining==0 is the exact "last request finished" // edge. scheduleAutoSettlement dedups, so a double-fire is harmless. - if remaining == 0 && rt.settlementPending.Load() { + if remaining != 0 { + return + } + + if rt.settlementPending.Load() { log.Printf("settlement_drain_complete escrow=%s reason=%s", rt.id, rt.settlementReason) g.scheduleAutoSettlement(rt.id, rt.settlementReason) + return + } + + if rt.retirePending.Load() { + log.Printf("runtime_retire_drain_complete escrow=%s reason=%s", rt.id, rt.retireReason) + g.retireRuntime(rt.id, rt.retireReason) } } @@ -3338,6 +3353,48 @@ func removeRuntime(runtimes []*devshardRuntime, id string) []*devshardRuntime { return out } +// retireRuntime drops a runtime from the in-memory registry and closes it, +// releasing its user session and the per-runtime SQLite handles that session owns. +func (g *Gateway) retireRuntime(id, reason string) bool { + g.mu.Lock() + rt := g.retireRuntimeLocked(id, reason) + g.mu.Unlock() + if rt == nil { + return false + } + // Close the per-runtime SQLite store outside g.mu: it is disk I/O and must + // not block other gateway operations that contend for the lock. + if err := rt.close(); err != nil { + log.Printf("runtime_retire_close_error escrow=%s reason=%q error=%v", id, reason, err) + } + log.Printf("runtime_retired escrow=%s reason=%q", id, reason) + return true +} + +// retireRuntimeLocked removes the runtime from the registry and returns it so +// the caller can close it outside the lock. It returns nil when nothing was +// retired: the runtime is unknown, or its retirement was deferred because +// requests are still in flight. Callers must hold g.mu. +func (g *Gateway) retireRuntimeLocked(id, reason string) *devshardRuntime { + rt, ok := g.runtimes[id] + if !ok { + log.Printf("runtime_retire_skipped escrow=%s reason=%q cause=not_registered", id, reason) + return nil + } + if inFlight := rt.activeRequests.Load(); inFlight > 0 { + rt.retireReason = reason + rt.retirePending.Store(true) + log.Printf("runtime_retire_deferred escrow=%s reason=%q active_requests=%d", id, reason, inFlight) + return nil + } + delete(g.runtimes, id) + g.runtimeOrder = removeRuntime(g.runtimeOrder, id) + if g.capacity != nil { + g.capacity.RemoveEscrow(id) + } + return rt +} + func (g *Gateway) sortRuntimeOrderLocked() { slices.SortFunc(g.runtimeOrder, func(a, b *devshardRuntime) int { return strings.Compare(a.id, b.id) @@ -3362,12 +3419,15 @@ func (g *Gateway) attachEscrowChecker(rt *devshardRuntime) { rt.proxy.redundancy.onEscrowMissing = func() { go g.escrowChecker.TriggerCheck(escrowID, func() { g.deactivateDevshardByID(escrowID) + // Escrow no longer exists on chain -- nothing to settle. + g.retireRuntime(escrowID, "escrow confirmed missing on chain") }) } } rt.proxy.redundancy.onBalanceExhausted = func() { if !g.escrowRotationEnabled() { g.deactivateDevshardByIDWithReason(escrowID, "escrow balance exhausted") + g.retireRuntime(escrowID, "escrow balance exhausted") return } log.Printf("gateway_replacing_exhausted_escrow escrow=%s", escrowID) @@ -3512,12 +3572,14 @@ func (g *Gateway) retireRotatedDevshard(ctx context.Context, id, reason string, if g.deactivateDevshardByIDWithReason(id, reason) { log.Printf("escrow_rotation_deactivated_without_settlement escrow=%s reason=%q", id, reason) } + g.retireRuntime(id, reason) return false, nil } log.Printf("escrow_rotation_settling escrow=%s reason=%q", id, reason) if _, err := gatewaySettleDevshardOnChain(g, ctx, id, adminSettleEscrowRequest{}); err != nil { return false, err } + g.retireRuntime(id, reason) return true, nil } @@ -3582,6 +3644,7 @@ func (g *Gateway) replaceDepletedEscrow(ctx context.Context, id, modelID, reason id, result.EscrowID, model.ModelID, reason, result.TxHash) if !settings.EscrowRotation.SettlementEnabled { g.deactivateDevshardByIDWithReason(id, reason) + g.retireRuntime(id, reason) } else { g.deactivateAndSettleDevshardByID(id, reason) } @@ -3633,11 +3696,16 @@ func (g *Gateway) scheduleAutoSettlement(id, reason string) { g.clearSettlementPending(id) log.Printf("auto_settle_submitted escrow=%s reason=%s tx_hash=%s settler=%s", id, reason, result.TxHash, result.Settler) + g.retireRuntime(id, reason) return } log.Printf("auto_settle_failed escrow=%s reason=%s attempt=%d/%d error=%v", id, reason, attempt, autoSettlementMaxAttempts, err) if attempt == autoSettlementMaxAttempts { + // Settlement exhausted its retries; free the in-memory runtime + // anyway so a permanently-unsettleable escrow cannot leak its + // SQLite store. On-disk state is preserved for manual recovery. + g.retireRuntime(id, reason) return } time.Sleep(autoSettlementRetryInterval) diff --git a/devshard/cmd/devshardctl/gateway_runtime_retire_test.go b/devshard/cmd/devshardctl/gateway_runtime_retire_test.go new file mode 100644 index 0000000000..680706b11c --- /dev/null +++ b/devshard/cmd/devshardctl/gateway_runtime_retire_test.go @@ -0,0 +1,149 @@ +package main + +import ( + "context" + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +// newRetireTestGateway builds a minimal Gateway holding a single active runtime +// registered in both the lookup map and the ordered slice, mirroring how the +// real registry is populated. +func newRetireTestGateway(id string) (*Gateway, *devshardRuntime) { + rt := &devshardRuntime{id: id} + rt.active.Store(true) + g := &Gateway{ + runtimes: map[string]*devshardRuntime{id: rt}, + runtimeOrder: []*devshardRuntime{rt}, + rotationFailures: make(map[string]struct{}), + } + return g, rt +} + +// TestRetireRuntimeRemovesRuntimeFromRegistry pins the core leak fix: retiring a +// runtime must drop it from both g.runtimes and g.runtimeOrder so its +// user.Session (and the per-runtime SQLite handles it owns) can be released. +func TestRetireRuntimeRemovesRuntimeFromRegistry(t *testing.T) { + g, _ := newRetireTestGateway("12") + + require.True(t, g.retireRuntime("12", "test")) + + _, stillRegistered := g.runtimes["12"] + require.False(t, stillRegistered, "runtime must be removed from g.runtimes") + require.Empty(t, g.runtimeOrder, "runtime must be removed from g.runtimeOrder") + + // Idempotent: retiring an already-gone runtime is a no-op, not a panic. + require.False(t, g.retireRuntime("12", "test")) +} + +// TestRetireRuntimeDefersWhileRequestsInFlight guards against closing a SQLite +// store out from under an in-flight request: retirement must defer (and leave +// the runtime registered) until the request count drains to zero. +func TestRetireRuntimeDefersWhileRequestsInFlight(t *testing.T) { + g, rt := newRetireTestGateway("12") + rt.activeRequests.Store(1) + + require.False(t, g.retireRuntime("12", "busy")) + _, stillRegistered := g.runtimes["12"] + require.True(t, stillRegistered, "busy runtime must stay registered") + require.True(t, rt.retirePending.Load(), "deferred retire must record its intent") + require.Equal(t, "busy", rt.retireReason) + + rt.activeRequests.Store(0) + require.True(t, g.retireRuntime("12", "drained")) + _, stillRegistered = g.runtimes["12"] + require.False(t, stillRegistered) +} + +// TestReleaseRuntimeRetiresAfterDrain: a retire deferred while busy fires once +// the last request drains through releaseRuntime. +func TestReleaseRuntimeRetiresAfterDrain(t *testing.T) { + g, rt := newRetireTestGateway("12") + rt.activeRequests.Store(1) + + require.False(t, g.retireRuntime("12", "balance exhausted")) + _, stillRegistered := g.runtimes["12"] + require.True(t, stillRegistered, "busy runtime must stay registered") + + g.releaseRuntime(rt, 0) + + _, stillRegistered = g.runtimes["12"] + require.False(t, stillRegistered, "drained runtime must be retired by releaseRuntime") + require.Empty(t, g.runtimeOrder) +} + +// TestReleaseRuntimeRetiresWithOnlyRetirePending exercises the retire branch in +// isolation: only retirePending set (no settlement), drain → retire. +func TestReleaseRuntimeRetiresWithOnlyRetirePending(t *testing.T) { + g, rt := newRetireTestGateway("12") + rt.activeRequests.Store(1) + rt.retireReason = "balance exhausted" + rt.retirePending.Store(true) + + g.releaseRuntime(rt, 0) + + _, stillRegistered := g.runtimes["12"] + require.False(t, stillRegistered, "retire branch must fire on drain") + require.Empty(t, g.runtimeOrder) +} + +// TestReleaseRuntimeDefersWhileRequestsRemain: while remaining != 0 nothing +// fires; settled stays 0 until the last request drains. Uses settlementPending +// because scheduleAutoSettlement fires regardless of the live count, so a +// broken guard leaks as settled>0 (retire would self-defer and hide it). +func TestReleaseRuntimeDefersWhileRequestsRemain(t *testing.T) { + rt := gatewayTestRuntimeForLimits(t, "12", balanceMinimumThreshold-1, nonceDeactivationLimit-1) + g, _, settled := gatewayTestDepletionGateway(t, rt) + + g.reserveRuntime(rt, 1) + g.reserveRuntime(rt, 1) + rt.settlementReason = "low_balance" + rt.settlementPending.Store(true) + + g.releaseRuntime(rt, 1) // remaining == 1 → quiet + require.Never(t, func() bool { return settled.Load() > 0 }, 200*time.Millisecond, 20*time.Millisecond) + + g.releaseRuntime(rt, 1) // remaining == 0 → settles once + require.Eventually(t, func() bool { return settled.Load() == 1 }, time.Second, 10*time.Millisecond) +} + +// TestRetireRotatedDevshardRetiresWithoutSettlement covers the no-settle +// terminal path: when settlement is disabled, the rotated-out runtime is +// deactivated AND retired in the same step. +func TestRetireRotatedDevshardRetiresWithoutSettlement(t *testing.T) { + g, _ := newRetireTestGateway("12") + settings := GatewaySettings{EscrowRotation: EscrowRotationSettings{SettlementEnabled: false}} + + settled, err := g.retireRotatedDevshard(context.Background(), "12", "rotated", settings) + require.NoError(t, err) + require.False(t, settled) + + _, stillRegistered := g.runtimes["12"] + require.False(t, stillRegistered, "no-settle rotation must retire the runtime") +} + +// TestRetireRotatedDevshardRetiresAfterSettlement covers the settle terminal +// path: the runtime stays alive through settlement (which reads its session) +// and is retired only once settlement succeeds. +func TestRetireRotatedDevshardRetiresAfterSettlement(t *testing.T) { + g, _ := newRetireTestGateway("12") + settings := GatewaySettings{EscrowRotation: EscrowRotationSettings{SettlementEnabled: true}} + + oldSettle := gatewaySettleDevshardOnChain + gatewaySettleDevshardOnChain = func(g *Gateway, _ context.Context, id string, _ adminSettleEscrowRequest) (*SettleDevshardEscrowResult, error) { + // The session must still be reachable at settlement time. + _, ok := g.runtimes[id] + require.True(t, ok, "runtime must still be registered during settlement") + return &SettleDevshardEscrowResult{TxHash: "OK"}, nil + } + t.Cleanup(func() { gatewaySettleDevshardOnChain = oldSettle }) + + settled, err := g.retireRotatedDevshard(context.Background(), "12", "rotated", settings) + require.NoError(t, err) + require.True(t, settled) + + _, stillRegistered := g.runtimes["12"] + require.False(t, stillRegistered, "settled rotation must retire the runtime") +} diff --git a/devshard/user/session_close_test.go b/devshard/user/session_close_test.go new file mode 100644 index 0000000000..b939b69986 --- /dev/null +++ b/devshard/user/session_close_test.go @@ -0,0 +1,58 @@ +package user + +import ( + "testing" + + "github.com/stretchr/testify/require" + + "devshard/storage" + "devshard/types" +) + +// closeCountingStore is a storage.Storage that records how many times Close is +// called. Every other method is an inert stub: this fake exists only to prove +// that Session.Close releases the underlying store, which is the resource the +// per-runtime memory leak was failing to free. +type closeCountingStore struct { + closeCalls int +} + +func (s *closeCountingStore) CreateSession(storage.CreateSessionParams) error { return nil } +func (s *closeCountingStore) MarkSettled(string) error { return nil } +func (s *closeCountingStore) ListActiveSessions() ([]storage.ActiveSession, error) { + return nil, nil +} +func (s *closeCountingStore) AppendDiff(string, types.DiffRecord) error { return nil } +func (s *closeCountingStore) GetDiffs(string, uint64, uint64) ([]types.DiffRecord, error) { + return nil, nil +} +func (s *closeCountingStore) AddSignature(string, uint64, uint32, []byte) error { return nil } +func (s *closeCountingStore) GetSignatures(string, uint64) (map[uint32][]byte, error) { + return nil, nil +} +func (s *closeCountingStore) GetSessionMeta(string) (*storage.SessionMeta, error) { + return nil, storage.ErrSessionNotFound +} +func (s *closeCountingStore) MarkFinalized(string, uint64) error { return nil } +func (s *closeCountingStore) LastFinalized(string) (uint64, error) { return 0, nil } +func (s *closeCountingStore) SaveSnapshot(string, uint64, []byte) error { return nil } +func (s *closeCountingStore) LoadSnapshot(string) (uint64, []byte, error) { + return 0, nil, storage.ErrSnapshotNotFound +} +func (s *closeCountingStore) PruneEpoch(uint64) error { return nil } +func (s *closeCountingStore) Close() error { + s.closeCalls++ + return nil +} + +// TestSession_Close_ClosesUnderlyingStore proves the resource-release leg of the +// leak fix: closing a Session must close the storage it owns. The gateway-side +// tests prove rt.close() is now invoked on every automatic deactivation path; +// this proves that invocation actually frees the SQLite store the session holds. +func TestSession_Close_ClosesUnderlyingStore(t *testing.T) { + store := &closeCountingStore{} + session, _, _ := setupSessionWithOptions(t, 1, 1_000_000, 0, WithStorage(store)) + + require.NoError(t, session.Close()) + require.Equal(t, 1, store.closeCalls, "Session.Close must close the injected storage exactly once") +} From d7d6a5e8dbb480191a6a6a586b6aa553b13913a9 Mon Sep 17 00:00:00 2001 From: Daniil Yankouski Date: Tue, 30 Jun 2026 10:52:59 +0200 Subject: [PATCH 03/13] =?UTF-8?q?MiniMax-M2.7=20route=20=E2=80=94=20per-mo?= =?UTF-8?q?del=20dispatch=20+=20tool-message=20shape=20+=20reasoning=5Fspl?= =?UTF-8?q?it=20(#1226)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat(devshard): MiniMax-M2.7 route — per-model dispatch + tool-message shape + reasoning_split * fix(devshard): reject structural_tag string form and whitespace-only names that hang/crash the engine * refactor(devshard): address review nits — clearer validator names + shared testutil --- .../cmd/devshardctl/filtercore/scoping.go | 13 + .../devshardctl/filtercore/scoping_test.go | 25 + .../devshardctl/messagevalidators/content.go | 124 ++++ .../messagevalidators/content_test.go | 261 ++++++++ .../devshardctl/messagevalidators/fields.go | 44 ++ .../messagevalidators/fields_test.go | 120 ++++ .../messagevalidators/minimax_tool_message.go | 76 +++ .../minimax_tool_message_test.go | 112 ++++ .../messagevalidators/normalizers.go | 249 ++++++++ .../messagevalidators/normalizers_test.go | 379 ++++++++++++ .../messagevalidators/toolcalls.go | 82 +++ .../messagevalidators/toolcalls_test.go | 202 +++++++ .../devshardctl/messagevalidators/types.go | 14 + .../devshardctl/paramvalidators/context.go | 15 + .../devshardctl/paramvalidators/handlers.go | 237 ++++++++ .../paramvalidators/handlers_test.go | 450 ++++++++++++++ .../devshardctl/paramvalidators/numeric.go | 14 + .../paramvalidators/response_format.go | 2 +- .../paramvalidators/response_format_test.go | 1 + .../paramvalidators/structured_outputs.go | 25 +- .../structured_outputs_test.go | 17 +- .../devshardctl/paramvalidators/thinking.go | 9 +- .../paramvalidators/tool_choice.go | 3 +- .../paramvalidators/tool_choice_test.go | 1 + .../cmd/devshardctl/paramvalidators/tools.go | 3 +- .../devshardctl/paramvalidators/tools_test.go | 1 + devshard/cmd/devshardctl/request_filters.go | 4 +- .../cmd/devshardctl/request_filters_config.go | 16 +- .../devshardctl/request_filters_messages.go | 566 +++++------------- .../devshardctl/request_filters_parameters.go | 358 +++-------- .../cmd/devshardctl/request_filters_test.go | 329 ++++++++++ devshard/cmd/devshardctl/testutil/testutil.go | 20 + devshard/json.go | 27 + devshard/json_test.go | 38 ++ docs/chat-api/README.md | 6 +- docs/chat-api/agents.md | 8 + docs/chat-api/minimax-m2.7.md | 88 +++ docs/chat-api/references.md | 28 +- docs/chat-api/troubleshooting.md | 91 +++ 39 files changed, 3340 insertions(+), 718 deletions(-) create mode 100644 devshard/cmd/devshardctl/filtercore/scoping.go create mode 100644 devshard/cmd/devshardctl/filtercore/scoping_test.go create mode 100644 devshard/cmd/devshardctl/messagevalidators/content.go create mode 100644 devshard/cmd/devshardctl/messagevalidators/content_test.go create mode 100644 devshard/cmd/devshardctl/messagevalidators/fields.go create mode 100644 devshard/cmd/devshardctl/messagevalidators/fields_test.go create mode 100644 devshard/cmd/devshardctl/messagevalidators/minimax_tool_message.go create mode 100644 devshard/cmd/devshardctl/messagevalidators/minimax_tool_message_test.go create mode 100644 devshard/cmd/devshardctl/messagevalidators/normalizers.go create mode 100644 devshard/cmd/devshardctl/messagevalidators/normalizers_test.go create mode 100644 devshard/cmd/devshardctl/messagevalidators/toolcalls.go create mode 100644 devshard/cmd/devshardctl/messagevalidators/toolcalls_test.go create mode 100644 devshard/cmd/devshardctl/messagevalidators/types.go create mode 100644 devshard/cmd/devshardctl/paramvalidators/handlers.go create mode 100644 devshard/cmd/devshardctl/paramvalidators/handlers_test.go create mode 100644 devshard/cmd/devshardctl/paramvalidators/numeric.go create mode 100644 devshard/cmd/devshardctl/testutil/testutil.go create mode 100644 docs/chat-api/minimax-m2.7.md diff --git a/devshard/cmd/devshardctl/filtercore/scoping.go b/devshard/cmd/devshardctl/filtercore/scoping.go new file mode 100644 index 0000000000..c9a9b71a9d --- /dev/null +++ b/devshard/cmd/devshardctl/filtercore/scoping.go @@ -0,0 +1,13 @@ +// Package filtercore holds primitives shared between paramvalidators (top-level +// request fields) and messagevalidators (per-message shape rules). Both layers +// dispatch behavior on the routed model id; the dispatch primitive lives here +// so the two packages do not reimplement it. +package filtercore + +import "slices" + +// MatchesModel reports whether routedModel equals any entry in models. Empty +// models slice always returns false. +func MatchesModel(routedModel string, models []string) bool { + return slices.Contains(models, routedModel) +} diff --git a/devshard/cmd/devshardctl/filtercore/scoping_test.go b/devshard/cmd/devshardctl/filtercore/scoping_test.go new file mode 100644 index 0000000000..fc506fe5b0 --- /dev/null +++ b/devshard/cmd/devshardctl/filtercore/scoping_test.go @@ -0,0 +1,25 @@ +package filtercore + +import "testing" + +func TestMatchesModel(t *testing.T) { + cases := []struct { + name string + routed string + models []string + want bool + }{ + {name: "empty list", routed: "model-a", models: nil, want: false}, + {name: "single match", routed: "model-a", models: []string{"model-a"}, want: true}, + {name: "no match", routed: "model-a", models: []string{"model-b"}, want: false}, + {name: "match in multi", routed: "model-b", models: []string{"model-a", "model-b", "model-c"}, want: true}, + {name: "case sensitive", routed: "Model-A", models: []string{"model-a"}, want: false}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + if got := MatchesModel(c.routed, c.models); got != c.want { + t.Fatalf("MatchesModel(%q, %v) = %v, want %v", c.routed, c.models, got, c.want) + } + }) + } +} diff --git a/devshard/cmd/devshardctl/messagevalidators/content.go b/devshard/cmd/devshardctl/messagevalidators/content.go new file mode 100644 index 0000000000..d458eee72d --- /dev/null +++ b/devshard/cmd/devshardctl/messagevalidators/content.go @@ -0,0 +1,124 @@ +package messagevalidators + +import ( + "fmt" + "strings" +) + +// ValidateNonEmptyContent rejects empty strings and empty/malformed content-part +// arrays. Each non-string content must be a []any of typed text parts. +func ValidateNonEmptyContent(content any) error { + switch value := content.(type) { + case string: + if strings.TrimSpace(value) == "" { + return fmt.Errorf("must not be empty") + } + return nil + case []any: + if len(value) == 0 { + return fmt.Errorf("must not be empty") + } + for i, rawPart := range value { + part, ok := rawPart.(map[string]any) + if !ok { + return fmt.Errorf("[%d] must be an object", i) + } + text, err := RequiredTextContentPart(part, i) + if err != nil { + return err + } + if strings.TrimSpace(text) == "" { + return fmt.Errorf("[%d].text must not be empty", i) + } + } + return nil + default: + return fmt.Errorf("must be a string or an array of typed content parts") + } +} + +func ValidateRequiredContentField(message map[string]any) error { + content, exists := message["content"] + if !exists || content == nil { + return fmt.Errorf("is required") + } + return ValidateNonEmptyContent(content) +} + +func ValidateAssistantContentField(message map[string]any, canBeEmpty bool) error { + content, exists := message["content"] + if !exists || content == nil { + if canBeEmpty { + return nil + } + return fmt.Errorf("is required unless tool_calls or function_call is provided") + } + return ValidateNonEmptyContent(content) +} + +// RequiredTextContentPart validates that part has type:"text" and a non-empty text field. +// Returned errors carry no `messages[%d].content` prefix — the caller adds it. +func RequiredTextContentPart(part map[string]any, partIndex int) (string, error) { + partType, err := RequiredNonEmptyString(part, "type") + if err != nil { + return "", fmt.Errorf("[%d].type: %w", partIndex, err) + } + if partType != "text" { + return "", fmt.Errorf("[%d].type has unsupported value %q", partIndex, partType) + } + text, err := RequiredNonEmptyString(part, "text") + if err != nil { + return "", fmt.Errorf("[%d].text: %w", partIndex, err) + } + return text, nil +} + +// CombineTextContentParts joins typed text parts into a single newline-separated string. +// Returned errors carry no `messages[%d].content` prefix — the caller adds it. +func CombineTextContentParts(parts []any) (string, error) { + texts := make([]string, 0, len(parts)) + for partIndex, rawPart := range parts { + part, ok := rawPart.(map[string]any) + if !ok { + return "", fmt.Errorf("[%d] must be an object", partIndex) + } + text, err := RequiredTextContentPart(part, partIndex) + if err != nil { + return "", err + } + texts = append(texts, text) + } + if len(texts) == 0 { + return "", nil + } + return strings.Join(texts, "\n"), nil +} + +func IsEmptyContent(content any) bool { + switch v := content.(type) { + case string: + return strings.TrimSpace(v) == "" + case []any: + return len(v) == 0 + default: + return false + } +} + +func IsAssistantTurnEmpty(msg map[string]any) bool { + if raw, exists := msg["tool_calls"]; exists && raw != nil { + if calls, ok := raw.([]any); ok && len(calls) > 0 { + return false + } + } + if raw, exists := msg["function_call"]; exists && raw != nil { + if fc, ok := raw.(map[string]any); ok && len(fc) > 0 { + return false + } + } + content, exists := msg["content"] + if !exists || content == nil { + return true + } + return IsEmptyContent(content) +} diff --git a/devshard/cmd/devshardctl/messagevalidators/content_test.go b/devshard/cmd/devshardctl/messagevalidators/content_test.go new file mode 100644 index 0000000000..5592013f9a --- /dev/null +++ b/devshard/cmd/devshardctl/messagevalidators/content_test.go @@ -0,0 +1,261 @@ +package messagevalidators + +import ( + "strings" + "testing" +) + +func TestValidateNonEmptyContent(t *testing.T) { + cases := []struct { + name string + content any + errSubstr string + }{ + {"string-ok", "hello", ""}, + {"string-with-think-block", "step answer", ""}, + {"string-empty", "", "must not be empty"}, + {"string-whitespace", " \t\n ", "must not be empty"}, + {"array-of-text-parts-ok", []any{map[string]any{"type": "text", "text": "hello"}}, ""}, + {"array-empty", []any{}, "must not be empty"}, + {"array-element-not-object", []any{"not-an-object"}, "[0] must be an object"}, + {"array-part-missing-type", []any{map[string]any{"text": "no type"}}, "[0].type"}, + {"array-part-wrong-type", []any{map[string]any{"type": "image_url", "text": "x"}}, "unsupported value"}, + {"array-part-empty-text", []any{map[string]any{"type": "text", "text": " "}}, "[0].text"}, + {"number-not-supported", 42, "must be a string or an array"}, + {"object-not-supported", map[string]any{}, "must be a string or an array"}, + {"nil-not-supported", nil, "must be a string or an array"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + err := ValidateNonEmptyContent(tc.content) + if tc.errSubstr == "" { + if err != nil { + t.Fatalf("want no error, got %v", err) + } + return + } + if err == nil { + t.Fatalf("want error containing %q, got nil", tc.errSubstr) + } + if !strings.Contains(err.Error(), tc.errSubstr) { + t.Fatalf("want error containing %q, got %q", tc.errSubstr, err.Error()) + } + }) + } +} + +func TestValidateRequiredContentField(t *testing.T) { + t.Run("missing-rejected", func(t *testing.T) { + err := ValidateRequiredContentField(map[string]any{}) + if err == nil || !strings.Contains(err.Error(), "is required") { + t.Fatalf("want 'is required', got %v", err) + } + }) + t.Run("nil-rejected", func(t *testing.T) { + err := ValidateRequiredContentField(map[string]any{"content": nil}) + if err == nil || !strings.Contains(err.Error(), "is required") { + t.Fatalf("want 'is required', got %v", err) + } + }) + t.Run("string-ok", func(t *testing.T) { + err := ValidateRequiredContentField(map[string]any{"content": "hi"}) + if err != nil { + t.Fatalf("want no error, got %v", err) + } + }) +} + +func TestValidateAssistantContentField(t *testing.T) { + t.Run("missing-with-canBeEmpty-ok", func(t *testing.T) { + // Assistant turn with tool_calls/function_call can omit content. + err := ValidateAssistantContentField(map[string]any{}, true) + if err != nil { + t.Fatalf("want no error, got %v", err) + } + }) + t.Run("missing-without-canBeEmpty-rejected", func(t *testing.T) { + err := ValidateAssistantContentField(map[string]any{}, false) + if err == nil || !strings.Contains(err.Error(), "tool_calls or function_call") { + t.Fatalf("want hint about tool_calls/function_call, got %v", err) + } + }) + t.Run("nil-with-canBeEmpty-ok", func(t *testing.T) { + err := ValidateAssistantContentField(map[string]any{"content": nil}, true) + if err != nil { + t.Fatalf("want no error, got %v", err) + } + }) + t.Run("non-empty-content-still-validated", func(t *testing.T) { + // Even when canBeEmpty=true, present content must not be empty/wrong shape. + err := ValidateAssistantContentField(map[string]any{"content": ""}, true) + if err == nil { + t.Fatal("empty string content must still be rejected") + } + }) +} + +func TestRequiredTextContentPart(t *testing.T) { + t.Run("happy", func(t *testing.T) { + text, err := RequiredTextContentPart(map[string]any{"type": "text", "text": "hello"}, 0) + if err != nil { + t.Fatalf("want no error, got %v", err) + } + if text != "hello" { + t.Fatalf("want 'hello', got %q", text) + } + }) + t.Run("missing-type", func(t *testing.T) { + _, err := RequiredTextContentPart(map[string]any{"text": "hi"}, 2) + if err == nil || !strings.Contains(err.Error(), "[2].type") { + t.Fatalf("want '[2].type', got %v", err) + } + }) + t.Run("wrong-type", func(t *testing.T) { + _, err := RequiredTextContentPart(map[string]any{"type": "image_url", "text": "x"}, 3) + if err == nil || !strings.Contains(err.Error(), "unsupported value") { + t.Fatalf("want 'unsupported value', got %v", err) + } + }) + t.Run("missing-text", func(t *testing.T) { + _, err := RequiredTextContentPart(map[string]any{"type": "text"}, 0) + if err == nil || !strings.Contains(err.Error(), "[0].text") { + t.Fatalf("want '[0].text', got %v", err) + } + }) +} + +func TestCombineTextContentParts(t *testing.T) { + t.Run("single-part", func(t *testing.T) { + got, err := CombineTextContentParts([]any{map[string]any{"type": "text", "text": "hello"}}) + if err != nil { + t.Fatalf("want no error, got %v", err) + } + if got != "hello" { + t.Fatalf("want 'hello', got %q", got) + } + }) + t.Run("multiple-parts-newline-join", func(t *testing.T) { + got, err := CombineTextContentParts([]any{ + map[string]any{"type": "text", "text": "first"}, + map[string]any{"type": "text", "text": "second"}, + }) + if err != nil { + t.Fatalf("want no error, got %v", err) + } + if got != "first\nsecond" { + t.Fatalf("want 'first\\nsecond', got %q", got) + } + }) + t.Run("empty-array-returns-empty-string", func(t *testing.T) { + got, err := CombineTextContentParts([]any{}) + if err != nil { + t.Fatalf("want no error, got %v", err) + } + if got != "" { + t.Fatalf("want empty, got %q", got) + } + }) + t.Run("non-object-element", func(t *testing.T) { + _, err := CombineTextContentParts([]any{"plain-string"}) + if err == nil || !strings.Contains(err.Error(), "[0] must be an object") { + t.Fatalf("want '[0] must be an object', got %v", err) + } + }) + t.Run("propagates-part-error", func(t *testing.T) { + _, err := CombineTextContentParts([]any{map[string]any{"type": "image_url", "text": "x"}}) + if err == nil || !strings.Contains(err.Error(), "unsupported value") { + t.Fatalf("want 'unsupported value', got %v", err) + } + }) + t.Run("preserves-think-block-in-text", func(t *testing.T) { + // MiniMax-M2.7 round-trips ... verbatim in content. + got, err := CombineTextContentParts([]any{ + map[string]any{"type": "text", "text": "reasoning"}, + map[string]any{"type": "text", "text": "answer"}, + }) + if err != nil { + t.Fatalf("want no error, got %v", err) + } + if got != "reasoning\nanswer" { + t.Fatalf("think block must round-trip, got %q", got) + } + }) +} + +func TestIsEmptyContent(t *testing.T) { + cases := []struct { + name string + content any + want bool + }{ + {"empty-string", "", true}, + {"whitespace-string", " ", true}, + {"non-empty-string", "x", false}, + {"empty-array", []any{}, true}, + {"non-empty-array", []any{1}, false}, + {"nil-not-empty", nil, false}, // nil is "missing", not "empty" + {"int-not-empty", 42, false}, + {"object-not-empty", map[string]any{}, false}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + if got := IsEmptyContent(tc.content); got != tc.want { + t.Fatalf("want %v, got %v", tc.want, got) + } + }) + } +} + +func TestIsAssistantTurnEmpty(t *testing.T) { + t.Run("no-fields-empty", func(t *testing.T) { + if !IsAssistantTurnEmpty(map[string]any{}) { + t.Fatal("empty turn must be empty") + } + }) + t.Run("content-only-non-empty", func(t *testing.T) { + if IsAssistantTurnEmpty(map[string]any{"content": "hi"}) { + t.Fatal("turn with content must not be empty") + } + }) + t.Run("tool-calls-non-empty", func(t *testing.T) { + msg := map[string]any{"tool_calls": []any{map[string]any{"id": "x"}}} + if IsAssistantTurnEmpty(msg) { + t.Fatal("turn with tool_calls must not be empty") + } + }) + t.Run("empty-tool-calls-array-still-empty", func(t *testing.T) { + // Empty tool_calls = informationless placeholder. + msg := map[string]any{"tool_calls": []any{}} + if !IsAssistantTurnEmpty(msg) { + t.Fatal("empty tool_calls slot is informationless") + } + }) + t.Run("nil-tool-calls-treated-as-absent", func(t *testing.T) { + msg := map[string]any{"tool_calls": nil} + if !IsAssistantTurnEmpty(msg) { + t.Fatal("nil tool_calls is absent") + } + }) + t.Run("function-call-non-empty", func(t *testing.T) { + msg := map[string]any{"function_call": map[string]any{"name": "fn"}} + if IsAssistantTurnEmpty(msg) { + t.Fatal("turn with function_call must not be empty") + } + }) + t.Run("empty-function-call-object-empty", func(t *testing.T) { + msg := map[string]any{"function_call": map[string]any{}} + if !IsAssistantTurnEmpty(msg) { + t.Fatal("empty function_call object is informationless") + } + }) + t.Run("nil-content-empty", func(t *testing.T) { + if !IsAssistantTurnEmpty(map[string]any{"content": nil}) { + t.Fatal("nil content is empty") + } + }) + t.Run("empty-content-array-empty", func(t *testing.T) { + if !IsAssistantTurnEmpty(map[string]any{"content": []any{}}) { + t.Fatal("empty content array is empty") + } + }) +} diff --git a/devshard/cmd/devshardctl/messagevalidators/fields.go b/devshard/cmd/devshardctl/messagevalidators/fields.go new file mode 100644 index 0000000000..702be36988 --- /dev/null +++ b/devshard/cmd/devshardctl/messagevalidators/fields.go @@ -0,0 +1,44 @@ +package messagevalidators + +import ( + "fmt" + "strings" +) + +// RequiredNonEmptyString returns the trimmed string value at values[field] +// or describes why it's missing/wrong. Returned errors carry no positional +// prefix — the caller (which knows the message/part index) wraps them. +func RequiredNonEmptyString(values map[string]any, field string) (string, error) { + rawValue, exists := values[field] + if !exists || rawValue == nil { + return "", fmt.Errorf("is required") + } + value, ok := rawValue.(string) + if !ok { + return "", fmt.Errorf("must be a string") + } + if strings.TrimSpace(value) == "" { + return "", fmt.Errorf("must not be empty") + } + return value, nil +} + +func OptionalStringField(values map[string]any, field string) error { + rawValue, exists := values[field] + if !exists || rawValue == nil { + return nil + } + if _, ok := rawValue.(string); !ok { + return fmt.Errorf("must be a string") + } + return nil +} + +func EnsureFieldsAbsent(values map[string]any, fields ...string) error { + for _, field := range fields { + if _, exists := values[field]; exists { + return fmt.Errorf("%s is not allowed for this role", field) + } + } + return nil +} diff --git a/devshard/cmd/devshardctl/messagevalidators/fields_test.go b/devshard/cmd/devshardctl/messagevalidators/fields_test.go new file mode 100644 index 0000000000..e855ce028b --- /dev/null +++ b/devshard/cmd/devshardctl/messagevalidators/fields_test.go @@ -0,0 +1,120 @@ +package messagevalidators + +import ( + "strings" + "testing" +) + +func TestRequiredNonEmptyString(t *testing.T) { + cases := []struct { + name string + values map[string]any + field string + want string + errSubstr string + }{ + {"happy", map[string]any{"x": "hello"}, "x", "hello", ""}, + {"trimmed-keeps-original", map[string]any{"x": " hello "}, "x", " hello ", ""}, + {"missing", map[string]any{}, "x", "", "is required"}, + {"explicit-nil", map[string]any{"x": nil}, "x", "", "is required"}, + {"non-string", map[string]any{"x": 42}, "x", "", "must be a string"}, + {"bool-not-string", map[string]any{"x": true}, "x", "", "must be a string"}, + {"empty-string", map[string]any{"x": ""}, "x", "", "must not be empty"}, + {"whitespace-only", map[string]any{"x": " \t\n"}, "x", "", "must not be empty"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + got, err := RequiredNonEmptyString(tc.values, tc.field) + if tc.errSubstr == "" { + if err != nil { + t.Fatalf("want no error, got %v", err) + } + if got != tc.want { + t.Fatalf("want %q, got %q", tc.want, got) + } + return + } + if err == nil { + t.Fatalf("want error containing %q, got nil", tc.errSubstr) + } + if !strings.Contains(err.Error(), tc.errSubstr) { + t.Fatalf("want error containing %q, got %q", tc.errSubstr, err.Error()) + } + }) + } +} + +func TestOptionalStringField(t *testing.T) { + cases := []struct { + name string + values map[string]any + field string + errSubstr string + }{ + {"missing-ok", map[string]any{}, "x", ""}, + {"explicit-nil-ok", map[string]any{"x": nil}, "x", ""}, + {"empty-string-ok", map[string]any{"x": ""}, "x", ""}, + {"valid-string-ok", map[string]any{"x": "hello"}, "x", ""}, + {"non-string-rejected", map[string]any{"x": 42}, "x", "must be a string"}, + {"object-rejected", map[string]any{"x": map[string]any{}}, "x", "must be a string"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + err := OptionalStringField(tc.values, tc.field) + if tc.errSubstr == "" { + if err != nil { + t.Fatalf("want no error, got %v", err) + } + return + } + if err == nil { + t.Fatalf("want error containing %q, got nil", tc.errSubstr) + } + if !strings.Contains(err.Error(), tc.errSubstr) { + t.Fatalf("want error containing %q, got %q", tc.errSubstr, err.Error()) + } + }) + } +} + +func TestEnsureFieldsAbsent(t *testing.T) { + t.Run("all-absent-ok", func(t *testing.T) { + err := EnsureFieldsAbsent(map[string]any{"role": "user"}, "tool_calls", "tool_call_id") + if err != nil { + t.Fatalf("want no error, got %v", err) + } + }) + t.Run("present-rejected", func(t *testing.T) { + err := EnsureFieldsAbsent(map[string]any{"role": "user", "tool_calls": []any{}}, "tool_calls") + if err == nil { + t.Fatal("want error, got nil") + } + if !strings.Contains(err.Error(), "tool_calls is not allowed") { + t.Fatalf("unexpected error: %v", err) + } + }) + t.Run("first-violation-wins", func(t *testing.T) { + // Multiple violations: caller sees the first listed disallowed field. + err := EnsureFieldsAbsent(map[string]any{"a": 1, "b": 2}, "a", "b") + if err == nil { + t.Fatal("want error, got nil") + } + if !strings.Contains(err.Error(), "a is not allowed") { + t.Fatalf("want first-violation 'a', got %v", err) + } + }) + t.Run("nil-value-still-present", func(t *testing.T) { + // An explicit null in the request still counts as "present" — clients that + // emit nil placeholders must not bypass the role policy. + err := EnsureFieldsAbsent(map[string]any{"tool_calls": nil}, "tool_calls") + if err == nil { + t.Fatal("want error, got nil") + } + }) + t.Run("variadic-empty-ok", func(t *testing.T) { + err := EnsureFieldsAbsent(map[string]any{"a": 1}, []string{}...) + if err != nil { + t.Fatalf("want no error, got %v", err) + } + }) +} diff --git a/devshard/cmd/devshardctl/messagevalidators/minimax_tool_message.go b/devshard/cmd/devshardctl/messagevalidators/minimax_tool_message.go new file mode 100644 index 0000000000..a630e7cf03 --- /dev/null +++ b/devshard/cmd/devshardctl/messagevalidators/minimax_tool_message.go @@ -0,0 +1,76 @@ +package messagevalidators + +import ( + "errors" + "fmt" + "strings" +) + +// MiniMax-M2.7 tool result messages carry per-call results as an array of +// {name, type:"text", text} objects instead of OpenAI's single content string +// keyed by tool_call_id. +var ( + ErrMinimaxToolContentShape = errors.New("content: must be a non-empty array of {name,type,text} objects") + ErrMinimaxToolEntryShape = errors.New("content entry: invalid shape") +) + +type MinimaxToolMessage struct { + MaxEntries int + NameMaxLen int + TextMaxSize int +} + +// Validate enforces the MiniMax tool message content shape and per-entry caps. +func (v MinimaxToolMessage) Validate(content any) error { + entries, ok := content.([]any) + if !ok { + return fmt.Errorf("%w: not an array", ErrMinimaxToolContentShape) + } + if len(entries) == 0 { + return fmt.Errorf("%w: array is empty", ErrMinimaxToolContentShape) + } + if v.MaxEntries > 0 && len(entries) > v.MaxEntries { + return fmt.Errorf("%w: %d entries exceeds limit %d", ErrMinimaxToolContentShape, len(entries), v.MaxEntries) + } + for i, rawEntry := range entries { + if err := v.validateEntry(i, rawEntry); err != nil { + return err + } + } + return nil +} + +func (v MinimaxToolMessage) validateEntry(index int, raw any) error { + entry, ok := raw.(map[string]any) + if !ok { + return fmt.Errorf("%w: [%d] must be an object", ErrMinimaxToolEntryShape, index) + } + // Closed allow-list. Stray keys would otherwise reach the upstream parser + // whose union-with-null typing has been a crash vector (SGLang #16057). + for key := range entry { + switch key { + case "name", "type", "text": + default: + return fmt.Errorf("%w: [%d] has unsupported key %q", ErrMinimaxToolEntryShape, index, key) + } + } + name, ok := entry["name"].(string) + if !ok || strings.TrimSpace(name) == "" { + return fmt.Errorf("%w: [%d].name must be a non-empty string", ErrMinimaxToolEntryShape, index) + } + if v.NameMaxLen > 0 && len(name) > v.NameMaxLen { + return fmt.Errorf("%w: [%d].name length %d exceeds limit %d", ErrMinimaxToolEntryShape, index, len(name), v.NameMaxLen) + } + partType, ok := entry["type"].(string) + if !ok || partType != "text" { + return fmt.Errorf("%w: [%d].type must be \"text\"", ErrMinimaxToolEntryShape, index) + } + text, ok := entry["text"].(string) + if !ok { + return fmt.Errorf("%w: [%d].text must be a string", ErrMinimaxToolEntryShape, index) + } + if v.TextMaxSize > 0 && len(text) > v.TextMaxSize { + return fmt.Errorf("%w: [%d].text size %d exceeds limit %d", ErrMinimaxToolEntryShape, index, len(text), v.TextMaxSize) + } + return nil +} diff --git a/devshard/cmd/devshardctl/messagevalidators/minimax_tool_message_test.go b/devshard/cmd/devshardctl/messagevalidators/minimax_tool_message_test.go new file mode 100644 index 0000000000..b165e82b76 --- /dev/null +++ b/devshard/cmd/devshardctl/messagevalidators/minimax_tool_message_test.go @@ -0,0 +1,112 @@ +package messagevalidators + +import ( + "errors" + "strings" + "testing" +) + +func validatorForTest() MinimaxToolMessage { + return MinimaxToolMessage{MaxEntries: 16, NameMaxLen: 64, TextMaxSize: 1024} +} + +func TestMinimaxToolMessage_AcceptsCanonicalShape(t *testing.T) { + v := validatorForTest() + content := []any{ + map[string]any{"name": "get_weather", "type": "text", "text": `{"temperature":"25"}`}, + map[string]any{"name": "search_web", "type": "text", "text": "results"}, + } + if err := v.Validate(content); err != nil { + t.Fatalf("unexpected error: %v", err) + } +} + +func TestMinimaxToolMessage_RejectsNonArray(t *testing.T) { + if err := validatorForTest().Validate("plain string"); !errors.Is(err, ErrMinimaxToolContentShape) { + t.Fatalf("expected ErrMinimaxToolContentShape, got %v", err) + } +} + +func TestMinimaxToolMessage_RejectsEmptyArray(t *testing.T) { + if err := validatorForTest().Validate([]any{}); !errors.Is(err, ErrMinimaxToolContentShape) { + t.Fatalf("expected ErrMinimaxToolContentShape, got %v", err) + } +} + +func TestMinimaxToolMessage_RejectsTooManyEntries(t *testing.T) { + v := MinimaxToolMessage{MaxEntries: 2, NameMaxLen: 64, TextMaxSize: 1024} + content := []any{ + map[string]any{"name": "a", "type": "text", "text": "1"}, + map[string]any{"name": "b", "type": "text", "text": "2"}, + map[string]any{"name": "c", "type": "text", "text": "3"}, + } + if err := v.Validate(content); !errors.Is(err, ErrMinimaxToolContentShape) { + t.Fatalf("expected ErrMinimaxToolContentShape, got %v", err) + } +} + +func TestMinimaxToolMessage_RejectsNonObjectEntry(t *testing.T) { + if err := validatorForTest().Validate([]any{"not an object"}); !errors.Is(err, ErrMinimaxToolEntryShape) { + t.Fatalf("expected ErrMinimaxToolEntryShape, got %v", err) + } +} + +func TestMinimaxToolMessage_RejectsMissingName(t *testing.T) { + content := []any{map[string]any{"type": "text", "text": "result"}} + if err := validatorForTest().Validate(content); !errors.Is(err, ErrMinimaxToolEntryShape) { + t.Fatalf("expected ErrMinimaxToolEntryShape, got %v", err) + } +} + +func TestMinimaxToolMessage_RejectsBlankName(t *testing.T) { + content := []any{map[string]any{"name": "", "type": "text", "text": "result"}} + if err := validatorForTest().Validate(content); !errors.Is(err, ErrMinimaxToolEntryShape) { + t.Fatalf("expected ErrMinimaxToolEntryShape, got %v", err) + } +} + +// A whitespace-only name must be rejected too: it otherwise passes the gateway, +// reaches the MiniMax tool parser, and hangs the request until the deadline. +func TestMinimaxToolMessage_RejectsWhitespaceName(t *testing.T) { + content := []any{map[string]any{"name": " ", "type": "text", "text": "result"}} + if err := validatorForTest().Validate(content); !errors.Is(err, ErrMinimaxToolEntryShape) { + t.Fatalf("expected ErrMinimaxToolEntryShape, got %v", err) + } +} + +func TestMinimaxToolMessage_RejectsOverlongName(t *testing.T) { + v := MinimaxToolMessage{MaxEntries: 16, NameMaxLen: 8, TextMaxSize: 1024} + content := []any{map[string]any{"name": "way_too_long_function_name", "type": "text", "text": "ok"}} + if err := v.Validate(content); !errors.Is(err, ErrMinimaxToolEntryShape) { + t.Fatalf("expected ErrMinimaxToolEntryShape, got %v", err) + } +} + +func TestMinimaxToolMessage_RejectsWrongType(t *testing.T) { + content := []any{map[string]any{"name": "fn", "type": "image_url", "text": "x"}} + if err := validatorForTest().Validate(content); !errors.Is(err, ErrMinimaxToolEntryShape) { + t.Fatalf("expected ErrMinimaxToolEntryShape, got %v", err) + } +} + +func TestMinimaxToolMessage_RejectsNonStringText(t *testing.T) { + content := []any{map[string]any{"name": "fn", "type": "text", "text": 42}} + if err := validatorForTest().Validate(content); !errors.Is(err, ErrMinimaxToolEntryShape) { + t.Fatalf("expected ErrMinimaxToolEntryShape, got %v", err) + } +} + +func TestMinimaxToolMessage_RejectsOverlongText(t *testing.T) { + v := MinimaxToolMessage{MaxEntries: 16, NameMaxLen: 64, TextMaxSize: 4} + content := []any{map[string]any{"name": "fn", "type": "text", "text": strings.Repeat("a", 5)}} + if err := v.Validate(content); !errors.Is(err, ErrMinimaxToolEntryShape) { + t.Fatalf("expected ErrMinimaxToolEntryShape, got %v", err) + } +} + +func TestMinimaxToolMessage_RejectsExtraKeys(t *testing.T) { + content := []any{map[string]any{"name": "fn", "type": "text", "text": "ok", "extra": true}} + if err := validatorForTest().Validate(content); !errors.Is(err, ErrMinimaxToolEntryShape) { + t.Fatalf("expected ErrMinimaxToolEntryShape, got %v", err) + } +} diff --git a/devshard/cmd/devshardctl/messagevalidators/normalizers.go b/devshard/cmd/devshardctl/messagevalidators/normalizers.go new file mode 100644 index 0000000000..95f51c5f4f --- /dev/null +++ b/devshard/cmd/devshardctl/messagevalidators/normalizers.go @@ -0,0 +1,249 @@ +package messagevalidators + +import ( + "fmt" + "slices" +) + +// Wire-format role values; kept as local literals so the package stays free of +// main-package imports. These match what is on the JSON wire and never change. +const ( + roleAssistant = "assistant" + roleTool = "tool" +) + +// OrphanToolMessageDropper drops role:"tool" entries whose tool_call_id has no +// matching prior assistant.tool_call. Mirrors ValidateDocument's pending-set +// accounting so survivors pass the strict check downstream. +type OrphanToolMessageDropper struct{} + +func (OrphanToolMessageDropper) Apply(messages []any) ([]any, bool, error) { + pending := map[string]struct{}{} + filtered := make([]any, 0, len(messages)) + dropped := false + for _, raw := range messages { + msg, ok := raw.(map[string]any) + if !ok { + filtered = append(filtered, raw) + continue + } + role, _ := msg["role"].(string) + switch role { + case roleAssistant: + if calls, ok := msg["tool_calls"].([]any); ok { + for _, rawCall := range calls { + call, ok := rawCall.(map[string]any) + if !ok { + continue + } + if id, ok := call["id"].(string); ok && id != "" { + pending[id] = struct{}{} + } + } + } + case roleTool: + if id, ok := msg["tool_call_id"].(string); ok && id != "" { + if _, matched := pending[id]; !matched { + dropped = true + continue + } + delete(pending, id) + } + } + filtered = append(filtered, raw) + } + return filtered, dropped, nil +} + +// MinimaxOrphanToolMessageDropper drops role:"tool" messages on routes whose +// tool messages carry no tool_call_id (e.g. MiniMax-M2.7). Orphan = tool +// message appearing before any assistant.tool_calls[] block. The block stays +// open across consecutive tool turns and resets on the next non-tool message. +type MinimaxOrphanToolMessageDropper struct{} + +func (MinimaxOrphanToolMessageDropper) Apply(messages []any) ([]any, bool, error) { + inToolBlock := false + filtered := make([]any, 0, len(messages)) + dropped := false + for _, raw := range messages { + msg, ok := raw.(map[string]any) + if !ok { + filtered = append(filtered, raw) + inToolBlock = false + continue + } + role, _ := msg["role"].(string) + switch role { + case roleAssistant: + inToolBlock = false + if calls, ok := msg["tool_calls"].([]any); ok && len(calls) > 0 { + inToolBlock = true + } + case roleTool: + if !inToolBlock { + dropped = true + continue + } + default: + inToolBlock = false + } + filtered = append(filtered, raw) + } + return filtered, dropped, nil +} + +// EmptyAssistantTurnDropper drops role:"assistant" messages with no content +// and no tool_calls / function_call — informationless placeholders left by +// session-resume serializers. +type EmptyAssistantTurnDropper struct{} + +func (EmptyAssistantTurnDropper) Apply(messages []any) ([]any, bool, error) { + filtered := make([]any, 0, len(messages)) + dropped := false + for _, raw := range messages { + msg, ok := raw.(map[string]any) + if !ok { + filtered = append(filtered, raw) + continue + } + if role, _ := msg["role"].(string); role == roleAssistant && IsAssistantTurnEmpty(msg) { + dropped = true + continue + } + filtered = append(filtered, raw) + } + return filtered, dropped, nil +} + +// EmptyContentNormalizer fills/normalizes content for special-case roles: +// - role:"tool" with missing, null, or empty content → ToolSentinel +// - role:"assistant" with empty content AND a tool_calls/function_call payload → null +// +// SkipRoles bypasses messages whose role is listed (e.g. MiniMax tool messages +// carry array content with a non-OpenAI shape and must not be touched here). +type EmptyContentNormalizer struct { + ToolSentinel string + SkipRoles []string +} + +func (n EmptyContentNormalizer) Apply(messages []any) ([]any, bool, error) { + changed := false + for _, raw := range messages { + msg, ok := raw.(map[string]any) + if !ok { + continue + } + role, _ := msg["role"].(string) + if slices.Contains(n.SkipRoles, role) { + continue + } + content, exists := msg["content"] + switch { + case !exists, content == nil: + if role == roleTool { + msg["content"] = n.ToolSentinel + changed = true + } + case IsEmptyContent(content): + switch role { + case roleAssistant: + _, hasToolCalls := msg["tool_calls"] + _, hasFunctionCall := msg["function_call"] + if hasToolCalls || hasFunctionCall { + msg["content"] = nil + changed = true + } + case roleTool: + msg["content"] = n.ToolSentinel + changed = true + } + } + } + return messages, changed, nil +} + +// LegacyToolNameStripper strips the legacy `name` field from role:"tool" +// messages — an artifact of the old role:"function" API. Universal. +type LegacyToolNameStripper struct{} + +func (LegacyToolNameStripper) Apply(messages []any) ([]any, bool, error) { + changed := false + for _, raw := range messages { + msg, ok := raw.(map[string]any) + if !ok { + continue + } + role, _ := msg["role"].(string) + if role != roleTool { + continue + } + if _, exists := msg["name"]; !exists { + continue + } + delete(msg, "name") + changed = true + } + return messages, changed, nil +} + +// MinimaxToolCallIDStripper silently strips tool_call_id from role:"tool" +// messages. Clients dual-emitting tool_call_id for cross-route portability +// keep working — MiniMax itself correlates tool results positionally. +type MinimaxToolCallIDStripper struct{} + +func (MinimaxToolCallIDStripper) Apply(messages []any) ([]any, bool, error) { + changed := false + for _, raw := range messages { + msg, ok := raw.(map[string]any) + if !ok { + continue + } + role, _ := msg["role"].(string) + if role != roleTool { + continue + } + if _, exists := msg["tool_call_id"]; exists { + delete(msg, "tool_call_id") + changed = true + } + } + return messages, changed, nil +} + +// TextPartsFlattener combines a content array of {type:"text",text} parts into +// a single newline-joined string. SkipRoles bypasses messages whose role is +// listed (e.g. MiniMax tool messages keep their {name,type,text}[] shape). +type TextPartsFlattener struct { + SkipRoles []string +} + +func (n TextPartsFlattener) Apply(messages []any) ([]any, bool, error) { + changed := false + for index, raw := range messages { + msg, ok := raw.(map[string]any) + if !ok { + continue + } + role, _ := msg["role"].(string) + if slices.Contains(n.SkipRoles, role) { + continue + } + content, exists := msg["content"] + if !exists || content == nil { + continue + } + parts, ok := content.([]any) + if !ok { + continue + } + combined, err := CombineTextContentParts(parts) + if err != nil { + return nil, false, fmt.Errorf("messages[%d].content%w", index, err) + } + if combined != "" { + msg["content"] = combined + changed = true + } + } + return messages, changed, nil +} diff --git a/devshard/cmd/devshardctl/messagevalidators/normalizers_test.go b/devshard/cmd/devshardctl/messagevalidators/normalizers_test.go new file mode 100644 index 0000000000..489a8635ea --- /dev/null +++ b/devshard/cmd/devshardctl/messagevalidators/normalizers_test.go @@ -0,0 +1,379 @@ +package messagevalidators + +import ( + "strings" + "testing" + + "devshard/cmd/devshardctl/testutil" +) + +// ============================================================ +// OrphanToolMessageDropper +// ============================================================ + +func TestOrphanToolMessageDropper_DropsToolWithUnmatchedID(t *testing.T) { + msgs := []any{ + map[string]any{"role": "user", "content": "hi"}, + map[string]any{"role": "tool", "tool_call_id": "nobody", "content": "stray"}, + map[string]any{"role": "assistant", "content": "hello"}, + } + out, changed, err := OrphanToolMessageDropper{}.Apply(msgs) + if err != nil { + t.Fatalf("want no error, got %v", err) + } + if !changed { + t.Fatal("changed must be true when something is dropped") + } + if len(out) != 2 { + t.Fatalf("want 2 surviving messages, got %d", len(out)) + } + if testutil.MapAt(t, out, 1)["role"] != "assistant" { + t.Fatal("orphan tool message must be dropped, assistant survives") + } +} + +func TestOrphanToolMessageDropper_KeepsMatchedToolMessage(t *testing.T) { + msgs := []any{ + map[string]any{"role": "user", "content": "q"}, + map[string]any{"role": "assistant", "content": "", "tool_calls": []any{ + map[string]any{"id": "c1", "type": "function", "function": map[string]any{"name": "fn"}}, + }}, + map[string]any{"role": "tool", "tool_call_id": "c1", "content": "result"}, + } + out, changed, err := OrphanToolMessageDropper{}.Apply(msgs) + if err != nil { + t.Fatalf("want no error, got %v", err) + } + if changed { + t.Fatal("nothing should change for a well-formed sequence") + } + if len(out) != 3 { + t.Fatalf("want 3 messages, got %d", len(out)) + } +} + +func TestOrphanToolMessageDropper_ConsumesPendingOnMatch(t *testing.T) { + // Same id appearing twice — second tool message is orphan because + // the first consumed the pending entry. + msgs := []any{ + map[string]any{"role": "assistant", "content": "", "tool_calls": []any{ + map[string]any{"id": "c1", "type": "function", "function": map[string]any{"name": "fn"}}, + }}, + map[string]any{"role": "tool", "tool_call_id": "c1", "content": "first"}, + map[string]any{"role": "tool", "tool_call_id": "c1", "content": "second"}, + } + out, changed, _ := OrphanToolMessageDropper{}.Apply(msgs) + if !changed { + t.Fatal("must drop the second tool message") + } + if len(out) != 2 { + t.Fatalf("want 2 survivors, got %d", len(out)) + } +} + +// ============================================================ +// MinimaxOrphanToolMessageDropper +// ============================================================ + +func TestMinimaxOrphanToolMessageDropper_DropsToolBeforeAnyAssistant(t *testing.T) { + msgs := []any{ + map[string]any{"role": "user", "content": "hi"}, + map[string]any{"role": "tool", "content": []any{map[string]any{"name": "fn", "type": "text", "text": "x"}}}, + map[string]any{"role": "assistant", "content": "hello"}, + } + out, changed, _ := MinimaxOrphanToolMessageDropper{}.Apply(msgs) + if !changed { + t.Fatal("orphan tool before assistant must be dropped") + } + if len(out) != 2 { + t.Fatalf("want 2 survivors, got %d", len(out)) + } +} + +func TestMinimaxOrphanToolMessageDropper_KeepsConsecutiveToolsInBlock(t *testing.T) { + // After assistant.tool_calls=[...], tool block stays open across all + // consecutive tool messages (parallel results). + msgs := []any{ + map[string]any{"role": "user", "content": "q"}, + map[string]any{"role": "assistant", "tool_calls": []any{ + map[string]any{"id": "c1"}, map[string]any{"id": "c2"}, + }}, + map[string]any{"role": "tool", "content": []any{map[string]any{"name": "fn", "type": "text", "text": "r1"}}}, + map[string]any{"role": "tool", "content": []any{map[string]any{"name": "fn", "type": "text", "text": "r2"}}}, + } + out, changed, _ := MinimaxOrphanToolMessageDropper{}.Apply(msgs) + if changed { + t.Fatal("both tool messages are part of the block, no drops") + } + if len(out) != 4 { + t.Fatalf("want 4 messages preserved, got %d", len(out)) + } +} + +func TestMinimaxOrphanToolMessageDropper_NonToolMessageClosesBlock(t *testing.T) { + msgs := []any{ + map[string]any{"role": "assistant", "tool_calls": []any{map[string]any{"id": "c1"}}}, + map[string]any{"role": "tool", "content": []any{map[string]any{"name": "fn", "type": "text", "text": "r"}}}, + map[string]any{"role": "user", "content": "follow up"}, + map[string]any{"role": "tool", "content": []any{map[string]any{"name": "fn", "type": "text", "text": "stray"}}}, + } + out, changed, _ := MinimaxOrphanToolMessageDropper{}.Apply(msgs) + if !changed { + t.Fatal("tool after user must be dropped") + } + if len(out) != 3 { + t.Fatalf("want 3 survivors (drop final orphan), got %d", len(out)) + } +} + +func TestMinimaxOrphanToolMessageDropper_EmptyToolCallsDoesNotOpenBlock(t *testing.T) { + msgs := []any{ + map[string]any{"role": "assistant", "content": "no tools", "tool_calls": []any{}}, + map[string]any{"role": "tool", "content": []any{map[string]any{"name": "fn", "type": "text", "text": "x"}}}, + } + out, changed, _ := MinimaxOrphanToolMessageDropper{}.Apply(msgs) + if !changed { + t.Fatal("empty tool_calls does not open a block; tool is orphan") + } + if len(out) != 1 { + t.Fatalf("want 1 survivor, got %d", len(out)) + } +} + +// ============================================================ +// EmptyAssistantTurnDropper +// ============================================================ + +func TestEmptyAssistantTurnDropper(t *testing.T) { + msgs := []any{ + map[string]any{"role": "user", "content": "hi"}, + map[string]any{"role": "assistant"}, // truly empty + map[string]any{"role": "assistant", "content": ""}, // empty string content + map[string]any{"role": "assistant", "content": "answer"}, // keep + map[string]any{"role": "user", "content": ""}, // user role NOT dropped here (validator job) + map[string]any{"role": "assistant", "tool_calls": []any{ // tool_calls makes it non-empty + map[string]any{"id": "c1"}, + }}, + } + out, changed, _ := EmptyAssistantTurnDropper{}.Apply(msgs) + if !changed { + t.Fatal("must drop 2 empty assistant turns") + } + if len(out) != 4 { + t.Fatalf("want 4 survivors, got %d", len(out)) + } + // Survivors: user[0], assistant[answer], user[empty], assistant[tool_calls] + roles := []string{} + for _, m := range out { + roles = append(roles, m.(map[string]any)["role"].(string)) + } + got := strings.Join(roles, ",") + if got != "user,assistant,user,assistant" { + t.Fatalf("survivor roles: %q", got) + } +} + +// ============================================================ +// EmptyContentNormalizer +// ============================================================ + +func TestEmptyContentNormalizer_FillsToolWithSentinel(t *testing.T) { + msgs := []any{ + map[string]any{"role": "tool", "tool_call_id": "c1"}, // missing content + map[string]any{"role": "tool", "tool_call_id": "c2", "content": nil}, + map[string]any{"role": "tool", "tool_call_id": "c3", "content": ""}, + } + _, changed, _ := EmptyContentNormalizer{ToolSentinel: "(no result)"}.Apply(msgs) + if !changed { + t.Fatal("must have rewritten all three tool messages") + } + for i, raw := range msgs { + m := raw.(map[string]any) + if m["content"] != "(no result)" { + t.Fatalf("[%d] want sentinel, got %v", i, m["content"]) + } + } +} + +func TestEmptyContentNormalizer_NullifiesAssistantWithCalls(t *testing.T) { + msgs := []any{ + map[string]any{"role": "assistant", "content": "", "tool_calls": []any{map[string]any{"id": "c1"}}}, + map[string]any{"role": "assistant", "content": "", "function_call": map[string]any{"name": "fn"}}, + } + _, changed, _ := EmptyContentNormalizer{ToolSentinel: "x"}.Apply(msgs) + if !changed { + t.Fatal("must nullify content on assistant turns with call payload") + } + for i, raw := range msgs { + m := raw.(map[string]any) + if m["content"] != nil { + t.Fatalf("[%d] want nil content, got %v", i, m["content"]) + } + } +} + +func TestEmptyContentNormalizer_LeavesAssistantWithoutCalls(t *testing.T) { + // Empty content on assistant turn WITHOUT call payload is left untouched — + // the validator will reject it as malformed. The normalizer must not paper + // over the bug by inventing content. + msg := map[string]any{"role": "assistant", "content": ""} + msgs := []any{msg} + _, changed, _ := EmptyContentNormalizer{ToolSentinel: "x"}.Apply(msgs) + if changed { + t.Fatal("must not touch assistant turn without call payload") + } + if msg["content"] != "" { + t.Fatalf("content must remain empty string, got %v", msg["content"]) + } +} + +func TestEmptyContentNormalizer_SkipRolesBypassesTool(t *testing.T) { + // MiniMax chain: tool messages carry array content with non-OpenAI shape. + // The normalizer must NOT replace it with the sentinel. + msg := map[string]any{"role": "tool", "content": []any{}} + msgs := []any{msg} + _, changed, _ := EmptyContentNormalizer{ + ToolSentinel: "(no result)", + SkipRoles: []string{"tool"}, + }.Apply(msgs) + if changed { + t.Fatal("SkipRoles must bypass tool messages") + } + if _, ok := msg["content"].([]any); !ok { + t.Fatalf("tool content must remain array, got %T", msg["content"]) + } +} + +func TestEmptyContentNormalizer_LeavesUserRoleAlone(t *testing.T) { + // User role isn't a special case: empty content is the validator's + // concern, not the normalizer's. + msg := map[string]any{"role": "user", "content": ""} + msgs := []any{msg} + _, changed, _ := EmptyContentNormalizer{ToolSentinel: "x"}.Apply(msgs) + if changed { + t.Fatal("user role is not normalized") + } +} + +// ============================================================ +// LegacyToolNameStripper +// ============================================================ + +func TestLegacyToolNameStripper(t *testing.T) { + msgs := []any{ + map[string]any{"role": "user", "content": "hi", "name": "kept"}, // not tool — name kept + map[string]any{"role": "tool", "tool_call_id": "c1", "content": "r", "name": "stripped"}, // tool — name stripped + map[string]any{"role": "tool", "tool_call_id": "c2", "content": "r"}, // no name — no-op + map[string]any{"role": "assistant", "content": "x", "name": "assistant-name"}, // not tool — kept + } + _, changed, _ := LegacyToolNameStripper{}.Apply(msgs) + if !changed { + t.Fatal("must strip name from one tool message") + } + if _, has := testutil.MapAt(t, msgs, 0)["name"]; !has { + t.Fatal("user.name must be preserved") + } + if _, has := testutil.MapAt(t, msgs, 1)["name"]; has { + t.Fatal("tool.name must be stripped") + } + if _, has := testutil.MapAt(t, msgs, 3)["name"]; !has { + t.Fatal("assistant.name must be preserved") + } +} + +// ============================================================ +// MinimaxToolCallIDStripper +// ============================================================ + +func TestMinimaxToolCallIDStripper(t *testing.T) { + msgs := []any{ + map[string]any{"role": "user", "tool_call_id": "weird-but-kept", "content": "hi"}, // not tool + map[string]any{"role": "tool", "tool_call_id": "c1", "content": []any{}}, // tool — stripped + map[string]any{"role": "tool", "content": []any{}}, // tool no id — no-op + } + _, changed, _ := MinimaxToolCallIDStripper{}.Apply(msgs) + if !changed { + t.Fatal("must strip from one tool message") + } + if _, has := testutil.MapAt(t, msgs, 0)["tool_call_id"]; !has { + t.Fatal("non-tool role: tool_call_id preserved (validator's problem)") + } + if _, has := testutil.MapAt(t, msgs, 1)["tool_call_id"]; has { + t.Fatal("tool message: tool_call_id must be stripped") + } +} + +// ============================================================ +// TextPartsFlattener +// ============================================================ + +func TestTextPartsFlattener_JoinsTextParts(t *testing.T) { + msg := map[string]any{ + "role": "user", + "content": []any{ + map[string]any{"type": "text", "text": "first"}, + map[string]any{"type": "text", "text": "second"}, + }, + } + _, changed, err := TextPartsFlattener{}.Apply([]any{msg}) + if err != nil { + t.Fatalf("want no error, got %v", err) + } + if !changed { + t.Fatal("must have flattened content") + } + if msg["content"] != "first\nsecond" { + t.Fatalf("want 'first\\nsecond', got %v", msg["content"]) + } +} + +func TestTextPartsFlattener_LeavesStringAlone(t *testing.T) { + msg := map[string]any{"role": "user", "content": "already string"} + _, changed, _ := TextPartsFlattener{}.Apply([]any{msg}) + if changed { + t.Fatal("string content must not trigger change") + } +} + +func TestTextPartsFlattener_SkipRolesBypassesTool(t *testing.T) { + // MiniMax tool messages carry {name,type,text}[] arrays — must NOT be flattened. + msg := map[string]any{ + "role": "tool", + "content": []any{ + map[string]any{"name": "fn", "type": "text", "text": "result"}, + }, + } + _, changed, _ := TextPartsFlattener{SkipRoles: []string{"tool"}}.Apply([]any{msg}) + if changed { + t.Fatal("SkipRoles must skip tool") + } + if _, ok := msg["content"].([]any); !ok { + t.Fatalf("tool content must remain array, got %T", msg["content"]) + } +} + +func TestTextPartsFlattener_ErrorIncludesMessageIndex(t *testing.T) { + msgs := []any{ + map[string]any{"role": "user", "content": "ok"}, + map[string]any{"role": "user", "content": []any{ + map[string]any{"type": "image_url", "text": "x"}, + }}, + } + _, _, err := TextPartsFlattener{}.Apply(msgs) + if err == nil { + t.Fatal("want error from bad part") + } + if !strings.Contains(err.Error(), "messages[1].content") { + t.Fatalf("error must include messages[1].content, got %v", err) + } +} + +func TestTextPartsFlattener_EmptyContentArrayLeftAlone(t *testing.T) { + // CombineTextContentParts returns "" for empty arrays; flattener treats + // "" as "no change" so the empty array is preserved (validator's call). + msg := map[string]any{"role": "user", "content": []any{}} + _, changed, _ := TextPartsFlattener{}.Apply([]any{msg}) + if changed { + t.Fatal("empty content array must not be flattened to empty string") + } +} diff --git a/devshard/cmd/devshardctl/messagevalidators/toolcalls.go b/devshard/cmd/devshardctl/messagevalidators/toolcalls.go new file mode 100644 index 0000000000..a2bc66bf14 --- /dev/null +++ b/devshard/cmd/devshardctl/messagevalidators/toolcalls.go @@ -0,0 +1,82 @@ +package messagevalidators + +import "fmt" + +// ValidateToolCallsField validates the message.tool_calls field and returns +// (collected ids, hasField, error). A null tool_calls is treated as absent +// and silently removed (some SDKs serialize empty slots that way). Errors +// carry no `messages[%d]` prefix — the caller wraps them. +func ValidateToolCallsField(message map[string]any) ([]string, bool, error) { + rawToolCalls, exists := message["tool_calls"] + if !exists { + return nil, false, nil + } + if rawToolCalls == nil { + delete(message, "tool_calls") + return nil, false, nil + } + toolCalls, ok := rawToolCalls.([]any) + if !ok { + return nil, true, fmt.Errorf("tool_calls must be an array") + } + if len(toolCalls) == 0 { + return nil, true, fmt.Errorf("tool_calls must not be empty") + } + seen := map[string]struct{}{} + ids := make([]string, 0, len(toolCalls)) + for callIndex, rawCall := range toolCalls { + call, ok := rawCall.(map[string]any) + if !ok { + return nil, true, fmt.Errorf("tool_calls[%d] must be an object", callIndex) + } + id, err := RequiredNonEmptyString(call, "id") + if err != nil { + return nil, true, fmt.Errorf("tool_calls[%d].id: %w", callIndex, err) + } + if _, exists := seen[id]; exists { + return nil, true, fmt.Errorf("tool_calls[%d].id is duplicated", callIndex) + } + seen[id] = struct{}{} + callType, err := RequiredNonEmptyString(call, "type") + if err != nil { + return nil, true, fmt.Errorf("tool_calls[%d].type: %w", callIndex, err) + } + if callType != "function" { + return nil, true, fmt.Errorf("tool_calls[%d].type must be \"function\"", callIndex) + } + function, ok := call["function"].(map[string]any) + if !ok { + return nil, true, fmt.Errorf("tool_calls[%d].function must be an object", callIndex) + } + if _, err := RequiredNonEmptyString(function, "name"); err != nil { + return nil, true, fmt.Errorf("tool_calls[%d].function.name: %w", callIndex, err) + } + if err := OptionalStringField(function, "arguments"); err != nil { + return nil, true, fmt.Errorf("tool_calls[%d].function.arguments: %w", callIndex, err) + } + ids = append(ids, id) + } + return ids, true, nil +} + +func ValidateFunctionCallField(message map[string]any) (bool, error) { + rawFunctionCall, exists := message["function_call"] + if !exists { + return false, nil + } + if rawFunctionCall == nil { + delete(message, "function_call") + return false, nil + } + functionCall, ok := rawFunctionCall.(map[string]any) + if !ok { + return true, fmt.Errorf("function_call must be an object") + } + if _, err := RequiredNonEmptyString(functionCall, "name"); err != nil { + return true, fmt.Errorf("function_call.name: %w", err) + } + if err := OptionalStringField(functionCall, "arguments"); err != nil { + return true, fmt.Errorf("function_call.arguments: %w", err) + } + return true, nil +} diff --git a/devshard/cmd/devshardctl/messagevalidators/toolcalls_test.go b/devshard/cmd/devshardctl/messagevalidators/toolcalls_test.go new file mode 100644 index 0000000000..b63f7d5860 --- /dev/null +++ b/devshard/cmd/devshardctl/messagevalidators/toolcalls_test.go @@ -0,0 +1,202 @@ +package messagevalidators + +import ( + "strings" + "testing" +) + +func TestValidateToolCallsField_Absent(t *testing.T) { + msg := map[string]any{} + ids, has, err := ValidateToolCallsField(msg) + if err != nil { + t.Fatalf("want no error, got %v", err) + } + if has { + t.Fatal("hasField must be false when field absent") + } + if ids != nil { + t.Fatalf("want nil ids, got %v", ids) + } +} + +func TestValidateToolCallsField_NullTreatedAsAbsent(t *testing.T) { + // LangChain serializes empty slots as `null`; gateway silently drops it. + msg := map[string]any{"tool_calls": nil} + ids, has, err := ValidateToolCallsField(msg) + if err != nil { + t.Fatalf("want no error, got %v", err) + } + if has { + t.Fatal("hasField must be false when value is null") + } + if _, stillThere := msg["tool_calls"]; stillThere { + t.Fatal("null tool_calls must be deleted from message") + } + if ids != nil { + t.Fatalf("want nil ids, got %v", ids) + } +} + +func TestValidateToolCallsField_Happy(t *testing.T) { + msg := map[string]any{ + "tool_calls": []any{ + map[string]any{ + "id": "call_1", + "type": "function", + "function": map[string]any{"name": "fn", "arguments": "{}"}, + }, + map[string]any{ + "id": "call_2", + "type": "function", + "function": map[string]any{"name": "fn", "arguments": ""}, + }, + }, + } + ids, has, err := ValidateToolCallsField(msg) + if err != nil { + t.Fatalf("want no error, got %v", err) + } + if !has { + t.Fatal("hasField must be true") + } + if len(ids) != 2 || ids[0] != "call_1" || ids[1] != "call_2" { + t.Fatalf("want [call_1, call_2], got %v", ids) + } +} + +func TestValidateToolCallsField_Errors(t *testing.T) { + cases := []struct { + name string + msg map[string]any + errSubstr string + }{ + { + "not-array", + map[string]any{"tool_calls": "not-array"}, + "tool_calls must be an array", + }, + { + "empty-array", + map[string]any{"tool_calls": []any{}}, + "tool_calls must not be empty", + }, + { + "element-not-object", + map[string]any{"tool_calls": []any{"x"}}, + "tool_calls[0] must be an object", + }, + { + "missing-id", + map[string]any{"tool_calls": []any{ + map[string]any{"type": "function", "function": map[string]any{"name": "fn"}}, + }}, + "tool_calls[0].id", + }, + { + "duplicate-id", + map[string]any{"tool_calls": []any{ + map[string]any{"id": "x", "type": "function", "function": map[string]any{"name": "fn"}}, + map[string]any{"id": "x", "type": "function", "function": map[string]any{"name": "fn"}}, + }}, + "tool_calls[1].id is duplicated", + }, + { + "wrong-type", + map[string]any{"tool_calls": []any{ + map[string]any{"id": "x", "type": "code_interpreter", "function": map[string]any{"name": "fn"}}, + }}, + "tool_calls[0].type", + }, + { + "function-not-object", + map[string]any{"tool_calls": []any{ + map[string]any{"id": "x", "type": "function", "function": "not-object"}, + }}, + "tool_calls[0].function must be an object", + }, + { + "function-missing-name", + map[string]any{"tool_calls": []any{ + map[string]any{"id": "x", "type": "function", "function": map[string]any{}}, + }}, + "tool_calls[0].function.name", + }, + { + "function-arguments-not-string", + map[string]any{"tool_calls": []any{ + map[string]any{"id": "x", "type": "function", "function": map[string]any{"name": "fn", "arguments": 42}}, + }}, + "tool_calls[0].function.arguments", + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + _, _, err := ValidateToolCallsField(tc.msg) + if err == nil { + t.Fatalf("want error containing %q, got nil", tc.errSubstr) + } + if !strings.Contains(err.Error(), tc.errSubstr) { + t.Fatalf("want error containing %q, got %q", tc.errSubstr, err.Error()) + } + }) + } +} + +func TestValidateFunctionCallField_Absent(t *testing.T) { + has, err := ValidateFunctionCallField(map[string]any{}) + if err != nil { + t.Fatalf("want no error, got %v", err) + } + if has { + t.Fatal("hasField must be false") + } +} + +func TestValidateFunctionCallField_NullTreatedAsAbsent(t *testing.T) { + msg := map[string]any{"function_call": nil} + has, err := ValidateFunctionCallField(msg) + if err != nil { + t.Fatalf("want no error, got %v", err) + } + if has { + t.Fatal("hasField must be false") + } + if _, stillThere := msg["function_call"]; stillThere { + t.Fatal("null function_call must be deleted") + } +} + +func TestValidateFunctionCallField_Happy(t *testing.T) { + has, err := ValidateFunctionCallField(map[string]any{ + "function_call": map[string]any{"name": "fn", "arguments": "{}"}, + }) + if err != nil { + t.Fatalf("want no error, got %v", err) + } + if !has { + t.Fatal("hasField must be true") + } +} + +func TestValidateFunctionCallField_Errors(t *testing.T) { + cases := []struct { + name string + msg map[string]any + errSubstr string + }{ + {"not-object", map[string]any{"function_call": "x"}, "function_call must be an object"}, + {"missing-name", map[string]any{"function_call": map[string]any{}}, "function_call.name"}, + {"args-not-string", map[string]any{"function_call": map[string]any{"name": "fn", "arguments": 42}}, "function_call.arguments"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + _, err := ValidateFunctionCallField(tc.msg) + if err == nil { + t.Fatalf("want error containing %q, got nil", tc.errSubstr) + } + if !strings.Contains(err.Error(), tc.errSubstr) { + t.Fatalf("want error containing %q, got %q", tc.errSubstr, err.Error()) + } + }) + } +} diff --git a/devshard/cmd/devshardctl/messagevalidators/types.go b/devshard/cmd/devshardctl/messagevalidators/types.go new file mode 100644 index 0000000000..cbcfc0ad7a --- /dev/null +++ b/devshard/cmd/devshardctl/messagevalidators/types.go @@ -0,0 +1,14 @@ +// Package messagevalidators holds leaf-level shape/bounds validators for +// per-message content. It mirrors paramvalidators (which validates top-level +// request fields). Validators are model-agnostic — per-model dispatch happens +// upstream in ChatMessageProcessor's role-policy catalog, which picks which +// validator (if any) to call for a given (route, role) pair. +package messagevalidators + +// ContentValidator validates the value of a single message's `content` field. +// Implementations are pure: they do not mutate the message and do not have +// access to cross-message state. Cross-message correlation (e.g. tool_call_id +// matching) is handled by ChatMessageProcessor's per-role policy, not here. +type ContentValidator interface { + Validate(content any) error +} diff --git a/devshard/cmd/devshardctl/paramvalidators/context.go b/devshard/cmd/devshardctl/paramvalidators/context.go index a3b54a71e7..d05e985d38 100644 --- a/devshard/cmd/devshardctl/paramvalidators/context.go +++ b/devshard/cmd/devshardctl/paramvalidators/context.go @@ -6,3 +6,18 @@ type ValidatorContext struct { Document map[string]any RoutedModel string } + +// ParameterContext is the input bundle passed to ParameterHandler.HandleParameter. +// Document is shared with the pipeline; handlers may mutate it (delete the field, +// rewrite its value, etc.). Parameter is the field name the handler is wired against. +type ParameterContext struct { + Document map[string]any + Parameter string + RoutedModel string +} + +// ParameterHandler is the leaf type for stage-driven rewrites of a single field. +// The caller wraps the returned error as HTTP 400. +type ParameterHandler interface { + HandleParameter(ParameterContext) error +} diff --git a/devshard/cmd/devshardctl/paramvalidators/handlers.go b/devshard/cmd/devshardctl/paramvalidators/handlers.go new file mode 100644 index 0000000000..f9df7d7522 --- /dev/null +++ b/devshard/cmd/devshardctl/paramvalidators/handlers.go @@ -0,0 +1,237 @@ +package paramvalidators + +import ( + "fmt" + "math" + + "devshard" +) + +// StripParameter deletes the parameter field from the document unconditionally. +type StripParameter struct{} + +func (StripParameter) HandleParameter(ctx ParameterContext) error { + delete(ctx.Document, ctx.Parameter) + return nil +} + +// ConditionalStripParameter deletes the parameter field when Predicate returns true. +// Predicate is called only when the rule fires; nil predicate is a no-op. +type ConditionalStripParameter struct { + Predicate func(ParameterContext) bool +} + +func (h ConditionalStripParameter) HandleParameter(ctx ParameterContext) error { + if h.Predicate != nil && h.Predicate(ctx) { + delete(ctx.Document, ctx.Parameter) + } + return nil +} + +// SanitizeStringListParameter filters a []string-shaped field. Non-string entries pass +// through unchanged so other validators can flag them. DropFieldIfEmpty removes the +// field entirely when the filter leaves it empty. +type SanitizeStringListParameter struct { + Keep func(string) bool + DropFieldIfEmpty bool +} + +func (h SanitizeStringListParameter) HandleParameter(ctx ParameterContext) error { + raw, ok := ctx.Document[ctx.Parameter].([]any) + if !ok { + return nil + } + cleaned := raw[:0] + for _, item := range raw { + value, ok := item.(string) + if !ok { + cleaned = append(cleaned, item) + continue + } + if h.Keep == nil || h.Keep(value) { + cleaned = append(cleaned, value) + } + } + if len(cleaned) == 0 && h.DropFieldIfEmpty { + delete(ctx.Document, ctx.Parameter) + return nil + } + ctx.Document[ctx.Parameter] = cleaned + return nil +} + +// SanitizeFloatParameter normalizes a numeric knob: parses string/json.Number inputs, +// drops non-finite values when StripNonFinite is set, clamps to [Min, Max]. +type SanitizeFloatParameter struct { + StripNonFinite bool + Min *float64 + Max *float64 +} + +func (h SanitizeFloatParameter) HandleParameter(ctx ParameterContext) error { + value, exists := ctx.Document[ctx.Parameter] + if !exists { + return nil + } + number, ok := devshard.JSONNumericFloat64(value) + if !ok { + delete(ctx.Document, ctx.Parameter) + return nil + } + if h.StripNonFinite && (math.IsNaN(number) || math.IsInf(number, 0)) { + delete(ctx.Document, ctx.Parameter) + return nil + } + if h.Min != nil && number < *h.Min { + number = *h.Min + } + if h.Max != nil && number > *h.Max { + number = *h.Max + } + ctx.Document[ctx.Parameter] = number + return nil +} + +// SanitizeFloatMapParameter applies SanitizeFloatParameter semantics to every value in +// a map-shaped field. Entries that fail clamping are dropped (not clamped) — vLLM +// rejects out-of-range logit_bias entries. +type SanitizeFloatMapParameter struct { + StripNonFinite bool + Min *float64 + Max *float64 + DropFieldIfEmpty bool + MaxEntries int +} + +func (h SanitizeFloatMapParameter) HandleParameter(ctx ParameterContext) error { + raw, ok := ctx.Document[ctx.Parameter].(map[string]any) + if !ok { + return nil + } + if h.MaxEntries > 0 && len(raw) > h.MaxEntries { + return fmt.Errorf("%s: map size %d exceeds limit %d", ctx.Parameter, len(raw), h.MaxEntries) + } + for key, value := range raw { + number, ok := devshard.JSONNumericFloat64(value) + if !ok { + continue + } + if h.StripNonFinite && (math.IsNaN(number) || math.IsInf(number, 0)) { + delete(raw, key) + continue + } + if h.Min != nil && number < *h.Min { + delete(raw, key) + continue + } + if h.Max != nil && number > *h.Max { + delete(raw, key) + } + } + if len(raw) == 0 && h.DropFieldIfEmpty { + delete(ctx.Document, ctx.Parameter) + return nil + } + ctx.Document[ctx.Parameter] = raw + return nil +} + +// ForceLiteralParameter writes Value into the document. OverwriteOnly leaves the +// field untouched when it isn't already present. +type ForceLiteralParameter struct { + Value any + OverwriteOnly bool +} + +func (h ForceLiteralParameter) HandleParameter(ctx ParameterContext) error { + if h.OverwriteOnly { + if _, exists := ctx.Document[ctx.Parameter]; !exists { + return nil + } + } + ctx.Document[ctx.Parameter] = h.Value + return nil +} + +// CapUintParameter caps a uint64-shaped field to Max. +type CapUintParameter struct { + Max uint64 +} + +func (h CapUintParameter) HandleParameter(ctx ParameterContext) error { + value, ok := numericJSONValueAsUint64FromMap(ctx.Document, ctx.Parameter) + if !ok { + return nil + } + if value > h.Max { + ctx.Document[ctx.Parameter] = h.Max + } + return nil +} + +// ClampUintToFieldParameter clamps the parameter to the value of another field +// in the same document (e.g. thinking_token_budget ≤ max_tokens). A zero MaxField +// value or missing MaxField is treated as "no clamp". +type ClampUintToFieldParameter struct { + MaxField string +} + +func (h ClampUintToFieldParameter) HandleParameter(ctx ParameterContext) error { + value, ok := numericJSONValueAsUint64FromMap(ctx.Document, ctx.Parameter) + if !ok { + return nil + } + maxValue, ok := numericJSONValueAsUint64FromMap(ctx.Document, h.MaxField) + if !ok || maxValue == 0 { + return nil + } + if value > maxValue { + ctx.Document[ctx.Parameter] = maxValue + } + return nil +} + +// ValidateUintParameter rejects requests whose field is present but cannot be parsed +// as a non-negative integer that fits in uint64. Pass-through when the field is absent. +type ValidateUintParameter struct{} + +func (h ValidateUintParameter) HandleParameter(ctx ParameterContext) error { + raw, exists := ctx.Document[ctx.Parameter] + if !exists || raw == nil { + return nil + } + if _, ok := devshard.JSONNumericUint64(raw); !ok { + return fmt.Errorf("%s: must be a non-negative integer", ctx.Parameter) + } + return nil +} + +// LengthCapListParameter bounds the number of entries in an array field, and +// optionally the byte length of each string entry. MaxEntries=0 disables the +// array cap; MaxEntryLen=0 disables the per-string cap. +type LengthCapListParameter struct { + MaxEntries int + MaxEntryLen int +} + +func (h LengthCapListParameter) HandleParameter(ctx ParameterContext) error { + raw, ok := ctx.Document[ctx.Parameter].([]any) + if !ok { + return nil + } + if h.MaxEntries > 0 && len(raw) > h.MaxEntries { + return fmt.Errorf("%s: array length %d exceeds limit %d", ctx.Parameter, len(raw), h.MaxEntries) + } + if h.MaxEntryLen > 0 { + for i, item := range raw { + s, ok := item.(string) + if !ok { + continue + } + if len(s) > h.MaxEntryLen { + return fmt.Errorf("%s[%d]: string length %d exceeds limit %d", ctx.Parameter, i, len(s), h.MaxEntryLen) + } + } + } + return nil +} diff --git a/devshard/cmd/devshardctl/paramvalidators/handlers_test.go b/devshard/cmd/devshardctl/paramvalidators/handlers_test.go new file mode 100644 index 0000000000..a1d0e8c346 --- /dev/null +++ b/devshard/cmd/devshardctl/paramvalidators/handlers_test.go @@ -0,0 +1,450 @@ +package paramvalidators + +import ( + "math" + "strings" + "testing" + + "devshard/cmd/devshardctl/testutil" +) + +// run is a tiny wrapper that builds a ParameterContext and dispatches the handler. +func run(h ParameterHandler, doc map[string]any, param string) error { + return h.HandleParameter(ParameterContext{ + Document: doc, + Parameter: param, + RoutedModel: "", + }) +} + +// ============================================================ +// StripParameter +// ============================================================ + +func TestStripParameter(t *testing.T) { + doc := map[string]any{"x": "y", "keep": 1} + if err := run(StripParameter{}, doc, "x"); err != nil { + t.Fatal(err) + } + if _, has := doc["x"]; has { + t.Fatal("x must be stripped") + } + if _, has := doc["keep"]; !has { + t.Fatal("other fields must remain") + } +} + +func TestStripParameter_NoopWhenAbsent(t *testing.T) { + doc := map[string]any{"keep": 1} + if err := run(StripParameter{}, doc, "x"); err != nil { + t.Fatal(err) + } +} + +// ============================================================ +// ConditionalStripParameter +// ============================================================ + +func TestConditionalStripParameter_PredicateTrue(t *testing.T) { + doc := map[string]any{"min_tokens": 10, "stop_token_ids": []any{42}} + h := ConditionalStripParameter{ + Predicate: func(ctx ParameterContext) bool { + _, ok := ctx.Document["stop_token_ids"] + return ok + }, + } + if err := run(h, doc, "min_tokens"); err != nil { + t.Fatal(err) + } + if _, has := doc["min_tokens"]; has { + t.Fatal("min_tokens must be stripped when stop_token_ids present") + } +} + +func TestConditionalStripParameter_PredicateFalse(t *testing.T) { + doc := map[string]any{"min_tokens": 10} + h := ConditionalStripParameter{ + Predicate: func(ctx ParameterContext) bool { + _, ok := ctx.Document["stop_token_ids"] + return ok + }, + } + if err := run(h, doc, "min_tokens"); err != nil { + t.Fatal(err) + } + if _, has := doc["min_tokens"]; !has { + t.Fatal("min_tokens must remain when predicate is false") + } +} + +func TestConditionalStripParameter_NilPredicateNoop(t *testing.T) { + doc := map[string]any{"x": 1} + if err := run(ConditionalStripParameter{}, doc, "x"); err != nil { + t.Fatal(err) + } + if _, has := doc["x"]; !has { + t.Fatal("nil predicate is a no-op") + } +} + +// ============================================================ +// SanitizeStringListParameter +// ============================================================ + +func TestSanitizeStringListParameter_Filters(t *testing.T) { + doc := map[string]any{"bad_words": []any{"keep", "", " ", "also-keep"}} + h := SanitizeStringListParameter{ + Keep: func(s string) bool { return strings.TrimSpace(s) != "" }, + DropFieldIfEmpty: true, + } + if err := run(h, doc, "bad_words"); err != nil { + t.Fatal(err) + } + out := doc["bad_words"].([]any) + if len(out) != 2 || out[0] != "keep" || out[1] != "also-keep" { + t.Fatalf("want [keep, also-keep], got %v", out) + } +} + +func TestSanitizeStringListParameter_DropsFieldWhenEmpty(t *testing.T) { + doc := map[string]any{"bad_words": []any{"", " "}} + h := SanitizeStringListParameter{ + Keep: func(s string) bool { return strings.TrimSpace(s) != "" }, + DropFieldIfEmpty: true, + } + if err := run(h, doc, "bad_words"); err != nil { + t.Fatal(err) + } + if _, has := doc["bad_words"]; has { + t.Fatal("empty post-filter array must be removed when DropFieldIfEmpty=true") + } +} + +func TestSanitizeStringListParameter_NonStringElementsPreserved(t *testing.T) { + // Non-strings pass through so later validators can flag them; the filter + // only acts on strings. + doc := map[string]any{"x": []any{"a", 42, true, "b"}} + h := SanitizeStringListParameter{Keep: func(s string) bool { return true }} + if err := run(h, doc, "x"); err != nil { + t.Fatal(err) + } + out := doc["x"].([]any) + if len(out) != 4 { + t.Fatalf("want 4 elements, got %d", len(out)) + } +} + +func TestSanitizeStringListParameter_AbsentFieldNoop(t *testing.T) { + doc := map[string]any{} + if err := run(SanitizeStringListParameter{}, doc, "x"); err != nil { + t.Fatal(err) + } +} + +// ============================================================ +// SanitizeFloatParameter +// ============================================================ + +func TestSanitizeFloatParameter_Clamps(t *testing.T) { + doc := map[string]any{"t": 5.0} + h := SanitizeFloatParameter{Min: testutil.FloatPtr(0), Max: testutil.FloatPtr(2.0)} + if err := run(h, doc, "t"); err != nil { + t.Fatal(err) + } + if doc["t"].(float64) != 2.0 { + t.Fatalf("want clamped to 2.0, got %v", doc["t"]) + } +} + +func TestSanitizeFloatParameter_StripsNonFinite(t *testing.T) { + for _, v := range []float64{math.NaN(), math.Inf(1), math.Inf(-1)} { + doc := map[string]any{"t": v} + h := SanitizeFloatParameter{StripNonFinite: true} + if err := run(h, doc, "t"); err != nil { + t.Fatal(err) + } + if _, has := doc["t"]; has { + t.Fatalf("non-finite %v must be stripped", v) + } + } +} + +func TestSanitizeFloatParameter_NonNumericStripped(t *testing.T) { + doc := map[string]any{"t": "not-a-number"} + if err := run(SanitizeFloatParameter{}, doc, "t"); err != nil { + t.Fatal(err) + } + if _, has := doc["t"]; has { + t.Fatal("non-numeric must be stripped") + } +} + +func TestSanitizeFloatParameter_StringNumericCoerced(t *testing.T) { + // Some SDKs serialize numbers as strings for high-precision fields. + doc := map[string]any{"t": "1.5"} + if err := run(SanitizeFloatParameter{}, doc, "t"); err != nil { + t.Fatal(err) + } + if doc["t"].(float64) != 1.5 { + t.Fatalf("want 1.5, got %v", doc["t"]) + } +} + +// ============================================================ +// SanitizeFloatMapParameter +// ============================================================ + +func TestSanitizeFloatMapParameter_DropsOutOfRange(t *testing.T) { + doc := map[string]any{"logit_bias": map[string]any{ + "a": 50.0, // out of [-100, 100]? No, in range + "b": 150.0, // > 100 → drop + "c": -200.0, // < -100 → drop + "d": 0.0, + }} + h := SanitizeFloatMapParameter{Min: testutil.FloatPtr(-100), Max: testutil.FloatPtr(100)} + if err := run(h, doc, "logit_bias"); err != nil { + t.Fatal(err) + } + out := doc["logit_bias"].(map[string]any) + if _, has := out["b"]; has { + t.Fatal("b must be dropped") + } + if _, has := out["c"]; has { + t.Fatal("c must be dropped") + } + if len(out) != 2 { + t.Fatalf("want 2 surviving entries, got %d", len(out)) + } +} + +func TestSanitizeFloatMapParameter_DropsNonFinite(t *testing.T) { + doc := map[string]any{"m": map[string]any{ + "a": 1.0, + "b": math.NaN(), + "c": math.Inf(1), + }} + h := SanitizeFloatMapParameter{StripNonFinite: true} + if err := run(h, doc, "m"); err != nil { + t.Fatal(err) + } + out := doc["m"].(map[string]any) + if len(out) != 1 { + t.Fatalf("want 1 surviving, got %d", len(out)) + } +} + +func TestSanitizeFloatMapParameter_RejectsOversizeMap(t *testing.T) { + doc := map[string]any{"m": map[string]any{}} + for i := 0; i < 5; i++ { + doc["m"].(map[string]any)[string(rune('a'+i))] = 1.0 + } + h := SanitizeFloatMapParameter{MaxEntries: 3} + err := run(h, doc, "m") + if err == nil || !strings.Contains(err.Error(), "exceeds limit") { + t.Fatalf("want 'exceeds limit', got %v", err) + } +} + +func TestSanitizeFloatMapParameter_DropFieldIfEmpty(t *testing.T) { + doc := map[string]any{"m": map[string]any{"a": math.NaN(), "b": math.Inf(1)}} + h := SanitizeFloatMapParameter{StripNonFinite: true, DropFieldIfEmpty: true} + if err := run(h, doc, "m"); err != nil { + t.Fatal(err) + } + if _, has := doc["m"]; has { + t.Fatal("post-filter empty map must be removed") + } +} + +// ============================================================ +// ForceLiteralParameter +// ============================================================ + +func TestForceLiteralParameter_Overwrites(t *testing.T) { + doc := map[string]any{"logprobs": false} + if err := run(ForceLiteralParameter{Value: true}, doc, "logprobs"); err != nil { + t.Fatal(err) + } + if doc["logprobs"] != true { + t.Fatalf("want true, got %v", doc["logprobs"]) + } +} + +func TestForceLiteralParameter_CreatesWhenAbsent(t *testing.T) { + doc := map[string]any{} + if err := run(ForceLiteralParameter{Value: 5}, doc, "n"); err != nil { + t.Fatal(err) + } + if doc["n"] != 5 { + t.Fatalf("want 5, got %v", doc["n"]) + } +} + +func TestForceLiteralParameter_OverwriteOnlySkipsAbsent(t *testing.T) { + doc := map[string]any{} + h := ForceLiteralParameter{Value: 0.0, OverwriteOnly: true} + if err := run(h, doc, "frequency_penalty"); err != nil { + t.Fatal(err) + } + if _, has := doc["frequency_penalty"]; has { + t.Fatal("OverwriteOnly must not create absent field") + } +} + +func TestForceLiteralParameter_OverwriteOnlyTouchesPresent(t *testing.T) { + doc := map[string]any{"frequency_penalty": 0.5} + h := ForceLiteralParameter{Value: 0.0, OverwriteOnly: true} + if err := run(h, doc, "frequency_penalty"); err != nil { + t.Fatal(err) + } + if doc["frequency_penalty"] != 0.0 { + t.Fatalf("want 0.0, got %v", doc["frequency_penalty"]) + } +} + +// ============================================================ +// CapUintParameter +// ============================================================ + +func TestCapUintParameter(t *testing.T) { + cases := []struct { + name string + in any + max uint64 + want any + }{ + {"caps-when-over", float64(100), 10, uint64(10)}, + {"leaves-when-under", float64(5), 10, float64(5)}, + {"leaves-when-equal", float64(10), 10, float64(10)}, + {"non-numeric-noop", "x", 10, "x"}, + {"negative-noop", float64(-1), 10, float64(-1)}, // not coerced → not capped + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + doc := map[string]any{"n": tc.in} + if err := run(CapUintParameter{Max: tc.max}, doc, "n"); err != nil { + t.Fatal(err) + } + if doc["n"] != tc.want { + t.Fatalf("want %v, got %v", tc.want, doc["n"]) + } + }) + } +} + +// ============================================================ +// ClampUintToFieldParameter +// ============================================================ + +func TestClampUintToFieldParameter(t *testing.T) { + t.Run("clamps-to-other-field", func(t *testing.T) { + doc := map[string]any{"min_tokens": float64(1000), "max_tokens": float64(100)} + if err := run(ClampUintToFieldParameter{MaxField: "max_tokens"}, doc, "min_tokens"); err != nil { + t.Fatal(err) + } + if doc["min_tokens"] != uint64(100) { + t.Fatalf("want clamped to 100, got %v", doc["min_tokens"]) + } + }) + t.Run("max-field-absent-noop", func(t *testing.T) { + doc := map[string]any{"min_tokens": float64(1000)} + if err := run(ClampUintToFieldParameter{MaxField: "max_tokens"}, doc, "min_tokens"); err != nil { + t.Fatal(err) + } + if doc["min_tokens"] != float64(1000) { + t.Fatal("must not clamp when max field is absent") + } + }) + t.Run("max-field-zero-noop", func(t *testing.T) { + // max_tokens=0 means "no max set" — don't clamp to 0. + doc := map[string]any{"min_tokens": float64(1000), "max_tokens": float64(0)} + if err := run(ClampUintToFieldParameter{MaxField: "max_tokens"}, doc, "min_tokens"); err != nil { + t.Fatal(err) + } + if doc["min_tokens"] != float64(1000) { + t.Fatal("max=0 must not clamp") + } + }) +} + +// ============================================================ +// ValidateUintParameter +// ============================================================ + +func TestValidateUintParameter(t *testing.T) { + cases := []struct { + name string + in any + errSubstr string + }{ + {"valid-int", float64(42), ""}, + {"absent", nil, ""}, + {"negative-rejected", float64(-1), "must be a non-negative integer"}, + {"float-rejected", float64(1.5), "must be a non-negative integer"}, + {"string-rejected", "42", "must be a non-negative integer"}, + {"bool-rejected", true, "must be a non-negative integer"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + doc := map[string]any{} + if tc.name != "absent" { + doc["seed"] = tc.in + } + err := run(ValidateUintParameter{}, doc, "seed") + if tc.errSubstr == "" { + if err != nil { + t.Fatalf("want no error, got %v", err) + } + return + } + if err == nil { + t.Fatalf("want error containing %q, got nil", tc.errSubstr) + } + if !strings.Contains(err.Error(), tc.errSubstr) { + t.Fatalf("want %q, got %q", tc.errSubstr, err.Error()) + } + }) + } +} + +// ============================================================ +// LengthCapListParameter +// ============================================================ + +func TestLengthCapListParameter_CapsArray(t *testing.T) { + doc := map[string]any{"stop": []any{"a", "b", "c"}} + err := run(LengthCapListParameter{MaxEntries: 2}, doc, "stop") + if err == nil || !strings.Contains(err.Error(), "array length") { + t.Fatalf("want 'array length' error, got %v", err) + } +} + +func TestLengthCapListParameter_CapsEntryLen(t *testing.T) { + doc := map[string]any{"stop": []any{"short", strings.Repeat("x", 100)}} + err := run(LengthCapListParameter{MaxEntryLen: 10}, doc, "stop") + if err == nil || !strings.Contains(err.Error(), "string length") { + t.Fatalf("want 'string length' error, got %v", err) + } +} + +func TestLengthCapListParameter_NonStringEntriesSkipped(t *testing.T) { + // stop_token_ids is an int array — string cap doesn't apply. + doc := map[string]any{"stop_token_ids": []any{1, 2, 3, 4}} + if err := run(LengthCapListParameter{MaxEntries: 0, MaxEntryLen: 5}, doc, "stop_token_ids"); err != nil { + t.Fatal(err) + } +} + +func TestLengthCapListParameter_WithinCapOK(t *testing.T) { + doc := map[string]any{"stop": []any{"a", "b"}} + if err := run(LengthCapListParameter{MaxEntries: 5, MaxEntryLen: 10}, doc, "stop"); err != nil { + t.Fatal(err) + } +} + +func TestLengthCapListParameter_AbsentFieldNoop(t *testing.T) { + doc := map[string]any{} + if err := run(LengthCapListParameter{MaxEntries: 1}, doc, "stop"); err != nil { + t.Fatal(err) + } +} diff --git a/devshard/cmd/devshardctl/paramvalidators/numeric.go b/devshard/cmd/devshardctl/paramvalidators/numeric.go new file mode 100644 index 0000000000..3fa3017396 --- /dev/null +++ b/devshard/cmd/devshardctl/paramvalidators/numeric.go @@ -0,0 +1,14 @@ +package paramvalidators + +import "devshard" + +// numericJSONValueAsUint64FromMap reads document[field] and coerces it to a uint64 +// using the shared devshard primitive. Convenience helper for handlers that read +// integer fields out of the raw document by name. +func numericJSONValueAsUint64FromMap(document map[string]any, field string) (uint64, bool) { + value, ok := document[field] + if !ok { + return 0, false + } + return devshard.JSONNumericUint64(value) +} diff --git a/devshard/cmd/devshardctl/paramvalidators/response_format.go b/devshard/cmd/devshardctl/paramvalidators/response_format.go index cc61193883..a4982961e3 100644 --- a/devshard/cmd/devshardctl/paramvalidators/response_format.go +++ b/devshard/cmd/devshardctl/paramvalidators/response_format.go @@ -98,7 +98,7 @@ func (v ResponseFormatValidator) validateJSONSchemaWrapper(rf map[string]any) er return fmt.Errorf("%w: must be an object", ErrResponseFormatJSONSchema) } name, ok := wrapper["name"].(string) - if !ok || name == "" { + if !ok || strings.TrimSpace(name) == "" { return fmt.Errorf("%w: must be a non-empty string", ErrResponseFormatName) } if len(name) > v.MaxNameLen { diff --git a/devshard/cmd/devshardctl/paramvalidators/response_format_test.go b/devshard/cmd/devshardctl/paramvalidators/response_format_test.go index af2c9d918e..b47e4af5ff 100644 --- a/devshard/cmd/devshardctl/paramvalidators/response_format_test.go +++ b/devshard/cmd/devshardctl/paramvalidators/response_format_test.go @@ -77,6 +77,7 @@ func TestResponseFormatValidatorRejects(t *testing.T) { {name: "unknown type", body: `{"response_format":{"type":"banana"}}`, wantErr: ErrResponseFormatType}, {name: "json_schema wrapper missing", body: `{"response_format":{"type":"json_schema"}}`, wantErr: ErrResponseFormatJSONSchema}, {name: "json_schema missing name", body: `{"response_format":{"type":"json_schema","json_schema":{"schema":{"type":"object"}}}}`, wantErr: ErrResponseFormatName}, + {name: "json_schema whitespace name", body: `{"response_format":{"type":"json_schema","json_schema":{"name":" ","schema":{"type":"object"}}}}`, wantErr: ErrResponseFormatName}, {name: "json_schema name has bad chars", body: `{"response_format":{"type":"json_schema","json_schema":{"name":"bad name","schema":{"type":"object"}}}}`, wantErr: ErrResponseFormatName}, {name: "json_schema name too long", body: `{"response_format":{"type":"json_schema","json_schema":{"name":"` + strings.Repeat("a", 65) + `","schema":{"type":"object"}}}}`, wantErr: ErrResponseFormatName}, {name: "json_schema missing schema", body: `{"response_format":{"type":"json_schema","json_schema":{"name":"r"}}}`, wantErr: ErrResponseFormatSchemaShape}, diff --git a/devshard/cmd/devshardctl/paramvalidators/structured_outputs.go b/devshard/cmd/devshardctl/paramvalidators/structured_outputs.go index ac1c9ba891..a2906397df 100644 --- a/devshard/cmd/devshardctl/paramvalidators/structured_outputs.go +++ b/devshard/cmd/devshardctl/paramvalidators/structured_outputs.go @@ -1,10 +1,13 @@ package paramvalidators import ( + "encoding/json" "errors" "fmt" "regexp" "strings" + + "devshard/cmd/devshardctl/filtercore" ) var ( @@ -86,10 +89,8 @@ func (v StructuredOutputsValidator) Validate(vctx ValidatorContext) error { if !exists { return nil } - for _, m := range v.RejectedModels { - if m == vctx.RoutedModel { - return fmt.Errorf("%w", ErrStructuredOutputsNotSupportedOnRoute) - } + if filtercore.MatchesModel(vctx.RoutedModel, v.RejectedModels) { + return fmt.Errorf("%w", ErrStructuredOutputsNotSupportedOnRoute) } obj, ok := raw.(map[string]any) if !ok { @@ -279,13 +280,21 @@ func (v StructuredOutputsValidator) validateGrammar(value any) error { return nil } +// validateStructuralTag accepts only the OBJECT form (vLLM's +// StructuralTagResponseFormat: {type, structures[], triggers[]}). A JSON-encoded +// STRING form crashes the engine with an HTTP 500 on the request handler, so it +// is rejected here regardless of route. Size is bounded on the serialized object. func (v StructuredOutputsValidator) validateStructuralTag(value any) error { - s, ok := value.(string) + obj, ok := value.(map[string]any) if !ok { - return fmt.Errorf("%w: must be a string", ErrStructuredOutputsStructuralTagShape) + return fmt.Errorf("%w: must be an object (a JSON-encoded string crashes the engine)", ErrStructuredOutputsStructuralTagShape) + } + encoded, err := json.Marshal(obj) + if err != nil { + return fmt.Errorf("%w: %v", ErrStructuredOutputsStructuralTagShape, err) } - if len(s) > v.MaxStructuralTagLen { - return fmt.Errorf("%w: %d > %d", ErrStructuredOutputsStructuralTagLength, len(s), v.MaxStructuralTagLen) + if len(encoded) > v.MaxStructuralTagLen { + return fmt.Errorf("%w: %d > %d", ErrStructuredOutputsStructuralTagLength, len(encoded), v.MaxStructuralTagLen) } return nil } diff --git a/devshard/cmd/devshardctl/paramvalidators/structured_outputs_test.go b/devshard/cmd/devshardctl/paramvalidators/structured_outputs_test.go index ea0d41fbca..56053ccfb6 100644 --- a/devshard/cmd/devshardctl/paramvalidators/structured_outputs_test.go +++ b/devshard/cmd/devshardctl/paramvalidators/structured_outputs_test.go @@ -268,20 +268,27 @@ func TestStructuredOutputsValidatorJSONObject(t *testing.T) { func TestStructuredOutputsValidatorStructuralTag(t *testing.T) { v := defaultStructuredOutputsValidator() - t.Run("accepts a normal tag", func(t *testing.T) { - body := `{"structured_outputs":{"structural_tag":"..."}}` + t.Run("accepts the object form", func(t *testing.T) { + body := `{"structured_outputs":{"structural_tag":{"type":"structural_tag","structures":[],"triggers":[]}}}` require.NoError(t, v.Validate(ValidatorContext{Document: parseDocument(t, body)})) }) - t.Run("rejects non-string", func(t *testing.T) { + t.Run("rejects the engine-crashing string form", func(t *testing.T) { + // A JSON-encoded string structural_tag returns HTTP 500 from vLLM; reject it here. + body := `{"structured_outputs":{"structural_tag":"..."}}` + err := v.Validate(ValidatorContext{Document: parseDocument(t, body)}) + require.ErrorIs(t, err, ErrStructuredOutputsStructuralTagShape) + }) + + t.Run("rejects a non-object scalar", func(t *testing.T) { body := `{"structured_outputs":{"structural_tag":42}}` err := v.Validate(ValidatorContext{Document: parseDocument(t, body)}) require.ErrorIs(t, err, ErrStructuredOutputsStructuralTagShape) }) - t.Run("rejects oversize", func(t *testing.T) { + t.Run("rejects oversize object", func(t *testing.T) { long := strings.Repeat("a", 4*1024+1) - body := `{"structured_outputs":{"structural_tag":"` + long + `"}}` + body := `{"structured_outputs":{"structural_tag":{"type":"structural_tag","blob":"` + long + `"}}}` err := v.Validate(ValidatorContext{Document: parseDocument(t, body)}) require.ErrorIs(t, err, ErrStructuredOutputsStructuralTagLength) }) diff --git a/devshard/cmd/devshardctl/paramvalidators/thinking.go b/devshard/cmd/devshardctl/paramvalidators/thinking.go index 61f4a07b13..6dc172acf9 100644 --- a/devshard/cmd/devshardctl/paramvalidators/thinking.go +++ b/devshard/cmd/devshardctl/paramvalidators/thinking.go @@ -3,6 +3,8 @@ package paramvalidators import ( "errors" "fmt" + + "devshard/cmd/devshardctl/filtercore" ) // ErrThinkingShape covers the wrapper-level rejection. ErrThinkingType covers the inner @@ -70,12 +72,7 @@ func resolveThinkingType(typeStr string) (bool, bool) { } func (v ThinkingValidator) shouldMirror(routedModel string) bool { - for _, m := range v.MirrorToTemplateKwargsForModels { - if m == routedModel { - return true - } - } - return false + return filtercore.MatchesModel(routedModel, v.MirrorToTemplateKwargsForModels) } func mirrorThinkingToTemplateKwargs(document map[string]any, enabled bool) error { diff --git a/devshard/cmd/devshardctl/paramvalidators/tool_choice.go b/devshard/cmd/devshardctl/paramvalidators/tool_choice.go index 5fa634871b..8cc7e53b69 100644 --- a/devshard/cmd/devshardctl/paramvalidators/tool_choice.go +++ b/devshard/cmd/devshardctl/paramvalidators/tool_choice.go @@ -3,6 +3,7 @@ package paramvalidators import ( "errors" "fmt" + "strings" ) var ( @@ -37,7 +38,7 @@ func (v ToolChoiceValidator) Validate(vctx ValidatorContext) error { return fmt.Errorf("%w: function must be an object", ErrToolChoiceFunctionShape) } name, ok := fn["name"].(string) - if !ok || name == "" { + if !ok || strings.TrimSpace(name) == "" { return fmt.Errorf("%w: function.name must be a non-empty string", ErrToolChoiceFunctionShape) } if v.MaxNameLen > 0 && len(name) > v.MaxNameLen { diff --git a/devshard/cmd/devshardctl/paramvalidators/tool_choice_test.go b/devshard/cmd/devshardctl/paramvalidators/tool_choice_test.go index b9829d3135..9937f9da56 100644 --- a/devshard/cmd/devshardctl/paramvalidators/tool_choice_test.go +++ b/devshard/cmd/devshardctl/paramvalidators/tool_choice_test.go @@ -46,6 +46,7 @@ func TestToolChoiceValidatorRejects(t *testing.T) { {name: "object function not object", body: `{"tool_choice":{"type":"function","function":"x"}}`, wantErr: ErrToolChoiceFunctionShape}, {name: "object missing name", body: `{"tool_choice":{"type":"function","function":{}}}`, wantErr: ErrToolChoiceFunctionShape}, {name: "object empty name", body: `{"tool_choice":{"type":"function","function":{"name":""}}}`, wantErr: ErrToolChoiceFunctionShape}, + {name: "object whitespace name", body: `{"tool_choice":{"type":"function","function":{"name":" "}}}`, wantErr: ErrToolChoiceFunctionShape}, {name: "object non-string name", body: `{"tool_choice":{"type":"function","function":{"name":42}}}`, wantErr: ErrToolChoiceFunctionShape}, {name: "name exceeds length cap", body: `{"tool_choice":{"type":"function","function":{"name":"` + longString(65) + `"}}}`, wantErr: ErrToolChoiceFunctionShape}, } diff --git a/devshard/cmd/devshardctl/paramvalidators/tools.go b/devshard/cmd/devshardctl/paramvalidators/tools.go index 5856bb760c..4311639d6e 100644 --- a/devshard/cmd/devshardctl/paramvalidators/tools.go +++ b/devshard/cmd/devshardctl/paramvalidators/tools.go @@ -3,6 +3,7 @@ package paramvalidators import ( "errors" "fmt" + "strings" ) // Tool-specific sentinels. Schema-walk rejections (depth/nodes/ref/enum/branch/size) flow @@ -80,7 +81,7 @@ func (v ToolsValidator) Validate(vctx ValidatorContext) error { return fmt.Errorf("%w: tools[%d].function must be an object", ErrToolFunctionShape, i) } name, ok := fn["name"].(string) - if !ok || name == "" { + if !ok || strings.TrimSpace(name) == "" { return fmt.Errorf("%w (tools[%d])", ErrToolFunctionName, i) } // OpenAI Structured Outputs flag. vLLM accepts but does not honor it diff --git a/devshard/cmd/devshardctl/paramvalidators/tools_test.go b/devshard/cmd/devshardctl/paramvalidators/tools_test.go index 7ab4291650..74750a8450 100644 --- a/devshard/cmd/devshardctl/paramvalidators/tools_test.go +++ b/devshard/cmd/devshardctl/paramvalidators/tools_test.go @@ -65,6 +65,7 @@ func TestToolsValidatorRejects(t *testing.T) { // function.name required. {name: "function name missing", body: `{"tools":[{"type":"function","function":{}}]}`, wantErr: ErrToolFunctionName}, {name: "function name empty", body: `{"tools":[{"type":"function","function":{"name":""}}]}`, wantErr: ErrToolFunctionName}, + {name: "function name whitespace", body: `{"tools":[{"type":"function","function":{"name":" "}}]}`, wantErr: ErrToolFunctionName}, {name: "function name not a string", body: `{"tools":[{"type":"function","function":{"name":42}}]}`, wantErr: ErrToolFunctionName}, {name: "depth exceeds limit", body: `{"tools":[` + toolWithParams(nestedPropertiesSchema(17)) + `]}`, wantErr: ErrSchemaDepth}, {name: "deep recursion attack hidden in tool", body: `{"tools":[` + toolWithParams(nestedPropertiesSchema(200)) + `]}`, wantErr: ErrSchemaDepth}, diff --git a/devshard/cmd/devshardctl/request_filters.go b/devshard/cmd/devshardctl/request_filters.go index cfac4a0dcc..142aee53ba 100644 --- a/devshard/cmd/devshardctl/request_filters.go +++ b/devshard/cmd/devshardctl/request_filters.go @@ -90,10 +90,10 @@ func (p ChatRequestPipeline) Normalize(body []byte, adminAuthenticated bool, lim if err := p.parameters.Apply(RequestFilterStagePreValidation, ctx); err != nil { return nil, chatRequest{}, err } - if err := p.messages.NormalizeDocument(&ctx.Document); err != nil { + if err := p.messages.NormalizeDocument(&ctx.Document, ctx.RoutedModel); err != nil { return nil, chatRequest{}, err } - if err := p.messages.ValidateDocument(&ctx.Document); err != nil { + if err := p.messages.ValidateDocument(&ctx.Document, ctx.RoutedModel); err != nil { return nil, chatRequest{}, err } if err := ctx.DecodeRequest(); err != nil { diff --git a/devshard/cmd/devshardctl/request_filters_config.go b/devshard/cmd/devshardctl/request_filters_config.go index f0d5b03048..a24aae7b63 100644 --- a/devshard/cmd/devshardctl/request_filters_config.go +++ b/devshard/cmd/devshardctl/request_filters_config.go @@ -20,7 +20,7 @@ const ( // wrappers (~9-10 levels) plus a small allowance for client-side structuring. const MaxRequestNestingDepth = 32 -// Per-parameter bounds wired into the catalog. Values match supported-params.md. +// Per-parameter bounds wired into the catalog. Values match docs/chat-api/README.md. const ( MessagesMaxEntries = 2048 @@ -83,10 +83,20 @@ const ( // Below this floor Kimi-K2.6 emits only (special token vLLM drops from content). kimiMaxTokensMin uint64 = 16 + + MinimaxToolMessageMaxEntries = 16 + MinimaxToolMessageNameMaxLen = 64 + // 64 KiB per-entry defensive cap; no vendor recommendation (MiniMax-4 fixes shape, + // not size). Sized to fit common agent tool-results (file reads, build logs). + MinimaxToolMessageTextMaxSize = 64 * 1024 ) -// Routed model identifiers. The catalog wires per-model behavior keyed on these strings. -const kimiK26ModelID = "moonshotai/Kimi-K2.6" +// Routed model identifiers. The parameter catalog and the message processor +// both dispatch on these strings. +const ( + kimiK26ModelID = "moonshotai/Kimi-K2.6" + miniMaxM27ModelID = "MiniMaxAI/MiniMax-M2.7" +) // Sentinel content used by message normalization when an upstream tool result is empty. const emptyToolResultContent = "" diff --git a/devshard/cmd/devshardctl/request_filters_messages.go b/devshard/cmd/devshardctl/request_filters_messages.go index b8749b4326..ff4eba8787 100644 --- a/devshard/cmd/devshardctl/request_filters_messages.go +++ b/devshard/cmd/devshardctl/request_filters_messages.go @@ -1,8 +1,7 @@ package main import ( - "fmt" - "strings" + "devshard/cmd/devshardctl/messagevalidators" ) type MessageRole string @@ -23,21 +22,88 @@ const ( MessageContentOptionalWithCalls ) +// MessageRolePolicy bundles validate-side rules for a (route, role) pair. +// Per-model overrides plug into ChatMessageProcessor.modelRoles and replace +// the default entry verbatim for that role on that route. type MessageRolePolicy struct { Role MessageRole DisallowedFields []string RequireName bool RequireToolCallID bool ContentRule MessageContentRule + // ContentValidator, when non-nil, replaces the default role-specific + // content shape check. Used by MiniMax-M2.7 tool messages whose content + // is a {name,type,text}[] array instead of a string. + ContentValidator messagevalidators.ContentValidator } +// MessageNormalizer rewrites the messages array before ValidateDocument runs. +// Returns the (possibly filtered/rewritten) messages, whether anything +// changed, and an unrecoverable error (which the caller wraps as HTTP 400). +type MessageNormalizer interface { + Apply(messages []any) ([]any, bool, error) +} + +// ChatMessageProcessor coordinates per-model normalization and validation. +// Mirrors defaultVLLMParameterCatalog's per-model dispatch pattern for the +// message side of the pipeline. type ChatMessageProcessor struct { - roles map[string]MessageRolePolicy + defaultRoles map[string]MessageRolePolicy + modelRoles map[string]map[string]MessageRolePolicy + defaultNormalizers []MessageNormalizer + modelNormalizers map[string][]MessageNormalizer } var defaultMessageProcessor = defaultChatMessageProcessor() func defaultChatMessageProcessor() ChatMessageProcessor { + return ChatMessageProcessor{ + defaultRoles: openAIRolePolicies(), + // Per-model role-policy overrides. Adding a model = add one row. + modelRoles: map[string]map[string]MessageRolePolicy{ + // MiniMax-M2.7 tool messages omit tool_call_id and carry results as a + // {name,type,text}[] array — see docs/chat-api/minimax-m2.7.md. + miniMaxM27ModelID: { + string(MessageRoleTool): { + Role: MessageRoleTool, + DisallowedFields: []string{"tool_calls", "function_call"}, + RequireToolCallID: false, + ContentRule: MessageContentRequired, + ContentValidator: messagevalidators.MinimaxToolMessage{ + MaxEntries: MinimaxToolMessageMaxEntries, + NameMaxLen: MinimaxToolMessageNameMaxLen, + TextMaxSize: MinimaxToolMessageTextMaxSize, + }, + }, + }, + }, + defaultNormalizers: []MessageNormalizer{ + messagevalidators.OrphanToolMessageDropper{}, + messagevalidators.EmptyAssistantTurnDropper{}, + messagevalidators.EmptyContentNormalizer{ToolSentinel: emptyToolResultContent}, + messagevalidators.LegacyToolNameStripper{}, + messagevalidators.TextPartsFlattener{}, + }, + // Per-model normalizer chain overrides. Adding a model = add one row. + modelNormalizers: map[string][]MessageNormalizer{ + miniMaxM27ModelID: { + messagevalidators.MinimaxOrphanToolMessageDropper{}, + messagevalidators.EmptyAssistantTurnDropper{}, + messagevalidators.EmptyContentNormalizer{ + ToolSentinel: emptyToolResultContent, + SkipRoles: []string{string(MessageRoleTool)}, + }, + messagevalidators.LegacyToolNameStripper{}, + messagevalidators.MinimaxToolCallIDStripper{}, + messagevalidators.TextPartsFlattener{ + SkipRoles: []string{string(MessageRoleTool)}, + }, + }, + }, + } +} + +func openAIRolePolicies() map[string]MessageRolePolicy { policies := []MessageRolePolicy{ {Role: MessageRoleDeveloper, DisallowedFields: []string{"tool_calls", "tool_call_id", "function_call"}, ContentRule: MessageContentRequired}, {Role: MessageRoleSystem, DisallowedFields: []string{"tool_calls", "tool_call_id", "function_call"}, ContentRule: MessageContentRequired}, @@ -50,53 +116,47 @@ func defaultChatMessageProcessor() ChatMessageProcessor { for _, policy := range policies { byRole[string(policy.Role)] = policy } - return ChatMessageProcessor{roles: byRole} + return byRole } -// NormalizeDocument only fixes shapes we intentionally accept, for example empty tool content or text-part arrays. -func (p ChatMessageProcessor) NormalizeDocument(document *ChatRequestDocument) error { +func (p ChatMessageProcessor) resolveRoles(routedModel string) map[string]MessageRolePolicy { + overrides := p.modelRoles[routedModel] + if len(overrides) == 0 { + return p.defaultRoles + } + merged := make(map[string]MessageRolePolicy, len(p.defaultRoles)) + for k, v := range p.defaultRoles { + merged[k] = v + } + for k, v := range overrides { + merged[k] = v + } + return merged +} + +func (p ChatMessageProcessor) resolveNormalizers(routedModel string) []MessageNormalizer { + if list, ok := p.modelNormalizers[routedModel]; ok { + return list + } + return p.defaultNormalizers +} + +// NormalizeDocument fixes message shapes the gateway intentionally accepts +// (orphan tool drops, empty tool content sentinels, content-array flattening). +// The per-route normalizer chain decides what runs. +func (p ChatMessageProcessor) NormalizeDocument(document *ChatRequestDocument, routedModel string) error { messages, ok := document.Array("messages") if !ok { return nil } changed := false - if filtered, dropped := p.dropOrphanToolMessages(messages); dropped { - messages = filtered - changed = true - } - if filtered, dropped := p.dropEmptyAssistantTurns(messages); dropped { - messages = filtered - changed = true - } - for index, rawMessage := range messages { - message, ok := rawMessage.(map[string]any) - if !ok { - continue - } - role, _ := message["role"].(string) - if p.normalizeMissingContent(message, role) { - changed = true - } - if p.normalizeEmptyContent(message, role) { - changed = true - } - if p.normalizeStripLegacyToolName(message, role) { - changed = true - } - content, exists := message["content"] - if !exists || content == nil { - continue - } - parts, ok := content.([]any) - if !ok { - continue - } - combined, err := combineTextContentParts(parts, index) + for _, n := range p.resolveNormalizers(routedModel) { + rewritten, ch, err := n.Apply(messages) if err != nil { - return err + return wrapBadChatRequest(err) } - if combined != "" { - message["content"] = combined + if ch { + messages = rewritten changed = true } } @@ -106,154 +166,10 @@ func (p ChatMessageProcessor) NormalizeDocument(document *ChatRequestDocument) e return nil } -// Drops role:"tool" entries whose tool_call_id has no matching prior assistant.tool_call. -// Mirrors ValidateDocument's pending-set accounting so survivors pass the strict check. -func (p ChatMessageProcessor) dropOrphanToolMessages(messages []any) ([]any, bool) { - pending := map[string]struct{}{} - filtered := make([]any, 0, len(messages)) - dropped := false - for _, raw := range messages { - msg, ok := raw.(map[string]any) - if !ok { - filtered = append(filtered, raw) - continue - } - role, _ := msg["role"].(string) - switch role { - case string(MessageRoleAssistant): - if calls, ok := msg["tool_calls"].([]any); ok { - for _, rawCall := range calls { - call, ok := rawCall.(map[string]any) - if !ok { - continue - } - if id, ok := call["id"].(string); ok && id != "" { - pending[id] = struct{}{} - } - } - } - case string(MessageRoleTool): - if id, ok := msg["tool_call_id"].(string); ok && id != "" { - if _, matched := pending[id]; !matched { - dropped = true - continue - } - delete(pending, id) - } - } - filtered = append(filtered, raw) - } - return filtered, dropped -} - -// Drops role:"assistant" messages with no content and no tool_calls/function_call — -// informationless placeholders left by session-resume serializers. -func (p ChatMessageProcessor) dropEmptyAssistantTurns(messages []any) ([]any, bool) { - filtered := make([]any, 0, len(messages)) - dropped := false - for _, raw := range messages { - msg, ok := raw.(map[string]any) - if !ok { - filtered = append(filtered, raw) - continue - } - if role, _ := msg["role"].(string); role == string(MessageRoleAssistant) && isAssistantTurnEmpty(msg) { - dropped = true - continue - } - filtered = append(filtered, raw) - } - return filtered, dropped -} - -func isAssistantTurnEmpty(msg map[string]any) bool { - if raw, exists := msg["tool_calls"]; exists && raw != nil { - if calls, ok := raw.([]any); ok && len(calls) > 0 { - return false - } - } - if raw, exists := msg["function_call"]; exists && raw != nil { - if fc, ok := raw.(map[string]any); ok && len(fc) > 0 { - return false - } - } - content, exists := msg["content"] - if !exists || content == nil { - return true - } - return isEmptyContent(content) -} - -// Strips legacy `name` from role:"tool" messages (artifact of the old role:"function" API). -func (p ChatMessageProcessor) normalizeStripLegacyToolName(message map[string]any, role string) bool { - if role != string(MessageRoleTool) { - return false - } - if _, exists := message["name"]; !exists { - return false - } - delete(message, "name") - return true -} - -func (p ChatMessageProcessor) normalizeMissingContent(message map[string]any, role string) bool { - if _, exists := message["content"]; exists { - return false - } - if role == string(MessageRoleTool) { - message["content"] = emptyToolResultContent - return true - } - return false -} - -// Empty assistant content (string/array/null) is allowed only when a call payload is present; -// empty tool content is normalized to a sentinel string. -func (p ChatMessageProcessor) normalizeEmptyContent(message map[string]any, role string) bool { - content, exists := message["content"] - if !exists { - return false - } - if content == nil { - if role == string(MessageRoleTool) { - message["content"] = emptyToolResultContent - return true - } - return false - } - if !isEmptyContent(content) { - return false - } - switch role { - case string(MessageRoleAssistant): - if _, hasToolCalls := message["tool_calls"]; hasToolCalls { - message["content"] = nil - return true - } - if _, hasFunctionCall := message["function_call"]; hasFunctionCall { - message["content"] = nil - return true - } - case string(MessageRoleTool): - message["content"] = emptyToolResultContent - return true - } - return false -} - -func isEmptyContent(content any) bool { - switch v := content.(type) { - case string: - return strings.TrimSpace(v) == "" - case []any: - return len(v) == 0 - default: - return false - } -} - -// ValidateDocument enforces the OpenAI-compatible message contract and makes sure tool responses match earlier assistant tool_calls. -func (p ChatMessageProcessor) ValidateDocument(document *ChatRequestDocument) error { +// ValidateDocument enforces the per-role policies resolved against the routed +// model. RequireToolCallID and ContentValidator are honored as policy fields +// so per-model overrides reuse the same validation skeleton. +func (p ChatMessageProcessor) ValidateDocument(document *ChatRequestDocument, routedModel string) error { rawMessages, exists := document.Array("messages") if !exists { return badChatRequest("messages is required") @@ -261,6 +177,7 @@ func (p ChatMessageProcessor) ValidateDocument(document *ChatRequestDocument) er if len(rawMessages) == 0 { return badChatRequest("messages must not be empty") } + roles := p.resolveRoles(routedModel) pendingToolCalls := map[string]struct{}{} for i, rawMessage := range rawMessages { @@ -268,257 +185,82 @@ func (p ChatMessageProcessor) ValidateDocument(document *ChatRequestDocument) er if !ok { return badChatRequest("messages[%d] must be an object", i) } - role, err := requiredNonEmptyString(message, "role") + role, err := messagevalidators.RequiredNonEmptyString(message, "role") if err != nil { return badChatRequest("messages[%d].role: %v", i, err) } - policy, ok := p.roles[role] + policy, ok := roles[role] if !ok { return badChatRequest("messages[%d].role has unsupported value %q", i, role) } - if err := ensureFieldsAbsent(message, policy.DisallowedFields...); err != nil { + if err := messagevalidators.EnsureFieldsAbsent(message, policy.DisallowedFields...); err != nil { return badChatRequest("messages[%d]: %v", i, err) } - - switch policy.Role { - case MessageRoleDeveloper, MessageRoleSystem, MessageRoleUser: - if err := validateRequiredContent(message); err != nil { - return badChatRequest("messages[%d].content: %v", i, err) - } - case MessageRoleAssistant: - toolCallIDs, hasToolCalls, err := validateToolCallsField(message, i) - if err != nil { - return err - } - hasFunctionCall, err := validateFunctionCallField(message, i) - if err != nil { - return err - } - if err := validateAssistantContent(message, hasToolCalls || hasFunctionCall); err != nil { - return badChatRequest("messages[%d].content: %v", i, err) - } - for _, id := range toolCallIDs { - pendingToolCalls[id] = struct{}{} - } - case MessageRoleTool: - toolCallID, err := requiredNonEmptyString(message, "tool_call_id") - if err != nil { - return badChatRequest("messages[%d].tool_call_id: %v", i, err) - } - if _, ok := pendingToolCalls[toolCallID]; !ok { - return badChatRequest("messages[%d].tool_call_id does not match any previous assistant tool_calls", i) - } - delete(pendingToolCalls, toolCallID) - if err := validateRequiredContent(message); err != nil { - return badChatRequest("messages[%d].content: %v", i, err) - } - case MessageRoleFunction: - if _, err := requiredNonEmptyString(message, "name"); err != nil { - return badChatRequest("messages[%d].name: %v", i, err) - } - if err := validateRequiredContent(message); err != nil { - return badChatRequest("messages[%d].content: %v", i, err) - } + if err := p.validateRoleSpecific(message, i, policy, pendingToolCalls); err != nil { + return err } } return nil } -func validateOpenAICompatChatMessages(request map[string]any) error { - return defaultMessageProcessor.ValidateDocument(&ChatRequestDocument{raw: request}) -} - -func validateToolCallsField(message map[string]any, messageIndex int) ([]string, bool, error) { - rawToolCalls, exists := message["tool_calls"] - if !exists { - return nil, false, nil - } - // Treat explicit null as absent (some SDKs serialize empty slots that way). - if rawToolCalls == nil { - delete(message, "tool_calls") - return nil, false, nil - } - toolCalls, ok := rawToolCalls.([]any) - if !ok { - return nil, true, badChatRequest("messages[%d].tool_calls must be an array", messageIndex) - } - if len(toolCalls) == 0 { - return nil, true, badChatRequest("messages[%d].tool_calls must not be empty", messageIndex) - } - seen := map[string]struct{}{} - ids := make([]string, 0, len(toolCalls)) - for callIndex, rawCall := range toolCalls { - call, ok := rawCall.(map[string]any) - if !ok { - return nil, true, badChatRequest("messages[%d].tool_calls[%d] must be an object", messageIndex, callIndex) - } - id, err := requiredNonEmptyString(call, "id") +// validateRoleSpecific keeps ValidateDocument's iteration loop legible: each +// case reads as a single responsibility. +func (p ChatMessageProcessor) validateRoleSpecific(message map[string]any, i int, policy MessageRolePolicy, pendingToolCalls map[string]struct{}) error { + switch policy.Role { + case MessageRoleDeveloper, MessageRoleSystem, MessageRoleUser: + if err := messagevalidators.ValidateRequiredContentField(message); err != nil { + return badChatRequest("messages[%d].content: %v", i, err) + } + case MessageRoleAssistant: + toolCallIDs, hasToolCalls, err := messagevalidators.ValidateToolCallsField(message) if err != nil { - return nil, true, badChatRequest("messages[%d].tool_calls[%d].id: %v", messageIndex, callIndex, err) - } - if _, exists := seen[id]; exists { - return nil, true, badChatRequest("messages[%d].tool_calls[%d].id is duplicated", messageIndex, callIndex) + return badChatRequest("messages[%d].%v", i, err) } - seen[id] = struct{}{} - callType, err := requiredNonEmptyString(call, "type") + hasFunctionCall, err := messagevalidators.ValidateFunctionCallField(message) if err != nil { - return nil, true, badChatRequest("messages[%d].tool_calls[%d].type: %v", messageIndex, callIndex, err) + return badChatRequest("messages[%d].%v", i, err) } - if callType != "function" { - return nil, true, badChatRequest("messages[%d].tool_calls[%d].type must be \"function\"", messageIndex, callIndex) + if err := messagevalidators.ValidateAssistantContentField(message, hasToolCalls || hasFunctionCall); err != nil { + return badChatRequest("messages[%d].content: %v", i, err) } - function, ok := call["function"].(map[string]any) - if !ok { - return nil, true, badChatRequest("messages[%d].tool_calls[%d].function must be an object", messageIndex, callIndex) - } - if _, err := requiredNonEmptyString(function, "name"); err != nil { - return nil, true, badChatRequest("messages[%d].tool_calls[%d].function.name: %v", messageIndex, callIndex, err) - } - if err := optionalStringField(function, "arguments"); err != nil { - return nil, true, badChatRequest("messages[%d].tool_calls[%d].function.arguments: %v", messageIndex, callIndex, err) - } - ids = append(ids, id) - } - return ids, true, nil -} - -func validateFunctionCallField(message map[string]any, messageIndex int) (bool, error) { - rawFunctionCall, exists := message["function_call"] - if !exists { - return false, nil - } - if rawFunctionCall == nil { - delete(message, "function_call") - return false, nil - } - functionCall, ok := rawFunctionCall.(map[string]any) - if !ok { - return true, badChatRequest("messages[%d].function_call must be an object", messageIndex) - } - if _, err := requiredNonEmptyString(functionCall, "name"); err != nil { - return true, badChatRequest("messages[%d].function_call.name: %v", messageIndex, err) - } - if err := optionalStringField(functionCall, "arguments"); err != nil { - return true, badChatRequest("messages[%d].function_call.arguments: %v", messageIndex, err) - } - return true, nil -} - -func validateAssistantContent(message map[string]any, canBeEmpty bool) error { - content, exists := message["content"] - if !exists || content == nil { - if canBeEmpty { - return nil + for _, id := range toolCallIDs { + pendingToolCalls[id] = struct{}{} } - return fmt.Errorf("is required unless tool_calls or function_call is provided") - } - return validateNonEmptyContent(content) -} - -func validateRequiredContent(message map[string]any) error { - content, exists := message["content"] - if !exists || content == nil { - return fmt.Errorf("is required") - } - return validateNonEmptyContent(content) -} - -func validateNonEmptyContent(content any) error { - switch value := content.(type) { - case string: - if strings.TrimSpace(value) == "" { - return fmt.Errorf("must not be empty") - } - return nil - case []any: - if len(value) == 0 { - return fmt.Errorf("must not be empty") - } - for i, rawPart := range value { - part, ok := rawPart.(map[string]any) - if !ok { - return fmt.Errorf("[%d] must be an object", i) - } - text, err := requiredTextContentPart(part, i) + case MessageRoleTool: + if policy.RequireToolCallID { + toolCallID, err := messagevalidators.RequiredNonEmptyString(message, "tool_call_id") if err != nil { - return err + return badChatRequest("messages[%d].tool_call_id: %v", i, err) } - if strings.TrimSpace(text) == "" { - return fmt.Errorf("[%d].text must not be empty", i) + if _, ok := pendingToolCalls[toolCallID]; !ok { + return badChatRequest("messages[%d].tool_call_id does not match any previous assistant tool_calls", i) } + delete(pendingToolCalls, toolCallID) } - return nil - default: - return fmt.Errorf("must be a string or an array of typed content parts") - } -} - -// We only accept text parts here because the gateway normalizes typed text content into a plain string before forwarding. -func requiredTextContentPart(part map[string]any, partIndex int) (string, error) { - partType, err := requiredNonEmptyString(part, "type") - if err != nil { - return "", fmt.Errorf("[%d].type: %w", partIndex, err) - } - if partType != "text" { - return "", fmt.Errorf("[%d].type has unsupported value %q", partIndex, partType) - } - text, err := requiredNonEmptyString(part, "text") - if err != nil { - return "", fmt.Errorf("[%d].text: %w", partIndex, err) - } - return text, nil -} - -func combineTextContentParts(parts []any, messageIndex int) (string, error) { - texts := make([]string, 0, len(parts)) - for partIndex, rawPart := range parts { - part, ok := rawPart.(map[string]any) - if !ok { - return "", badChatRequest("messages[%d].content[%d] must be an object", messageIndex, partIndex) + if policy.ContentValidator != nil { + content, exists := message["content"] + if !exists { + return badChatRequest("messages[%d].content: is required", i) + } + if err := policy.ContentValidator.Validate(content); err != nil { + return badChatRequest("messages[%d].%v", i, err) + } + } else if err := messagevalidators.ValidateRequiredContentField(message); err != nil { + return badChatRequest("messages[%d].content: %v", i, err) } - text, err := requiredTextContentPart(part, partIndex) - if err != nil { - return "", badChatRequest("messages[%d].content%v", messageIndex, err) + case MessageRoleFunction: + if policy.RequireName { + if _, err := messagevalidators.RequiredNonEmptyString(message, "name"); err != nil { + return badChatRequest("messages[%d].name: %v", i, err) + } } - texts = append(texts, text) - } - if len(texts) == 0 { - return "", nil - } - return strings.Join(texts, "\n"), nil -} - -func ensureFieldsAbsent(values map[string]any, fields ...string) error { - for _, field := range fields { - if _, exists := values[field]; exists { - return fmt.Errorf("%s is not allowed for this role", field) + if err := messagevalidators.ValidateRequiredContentField(message); err != nil { + return badChatRequest("messages[%d].content: %v", i, err) } } return nil } -func requiredNonEmptyString(values map[string]any, field string) (string, error) { - rawValue, exists := values[field] - if !exists || rawValue == nil { - return "", fmt.Errorf("is required") - } - value, ok := rawValue.(string) - if !ok { - return "", fmt.Errorf("must be a string") - } - if strings.TrimSpace(value) == "" { - return "", fmt.Errorf("must not be empty") - } - return value, nil -} - -func optionalStringField(values map[string]any, field string) error { - rawValue, exists := values[field] - if !exists || rawValue == nil { - return nil - } - if _, ok := rawValue.(string); !ok { - return fmt.Errorf("must be a string") - } - return nil +func validateOpenAICompatChatMessages(request map[string]any) error { + return defaultMessageProcessor.ValidateDocument(&ChatRequestDocument{raw: request}, "") } diff --git a/devshard/cmd/devshardctl/request_filters_parameters.go b/devshard/cmd/devshardctl/request_filters_parameters.go index 88df1f26d1..2e1b4c99f4 100644 --- a/devshard/cmd/devshardctl/request_filters_parameters.go +++ b/devshard/cmd/devshardctl/request_filters_parameters.go @@ -3,13 +3,12 @@ package main import ( "bytes" "encoding/json" - "math" "slices" - "strconv" "strings" "sync" "devshard" + "devshard/cmd/devshardctl/filtercore" "devshard/cmd/devshardctl/paramvalidators" ) @@ -41,137 +40,28 @@ type ParameterHandler interface { Apply(*RequestFilterContext, VLLMParameter) error } -type StripParameterHandler struct{} - -func (StripParameterHandler) Apply(ctx *RequestFilterContext, parameter VLLMParameter) error { - ctx.Document.Delete(parameter.Name) - return nil -} - -type ConditionalStripParameterHandler struct { - Predicate func(*RequestFilterContext) bool -} - -func (h ConditionalStripParameterHandler) Apply(ctx *RequestFilterContext, parameter VLLMParameter) error { - if h.Predicate != nil && h.Predicate(ctx) { - ctx.Document.Delete(parameter.Name) - } - return nil +// ParameterHandlerAdapter wraps a paramvalidators.ParameterHandler (which operates on a +// raw map[string]any) so the catalog can drive it through the standard ParameterHandler +// contract. Mirrors DocumentValidatorHandler's role for DocumentValidator. +type ParameterHandlerAdapter struct { + Handler paramvalidators.ParameterHandler } -type SanitizeStringListParameterHandler struct { - Keep func(string) bool - DropFieldIfEmpty bool -} - -func (h SanitizeStringListParameterHandler) Apply(ctx *RequestFilterContext, parameter VLLMParameter) error { - raw, ok := ctx.Document.Array(parameter.Name) - if !ok { - return nil - } - cleaned := raw[:0] - for _, item := range raw { - value, ok := item.(string) - if !ok { - cleaned = append(cleaned, item) - continue - } - if h.Keep == nil || h.Keep(value) { - cleaned = append(cleaned, value) - } - } - if len(cleaned) == 0 && h.DropFieldIfEmpty { - ctx.Document.Delete(parameter.Name) - return nil - } - ctx.Document.Set(parameter.Name, cleaned) - return nil -} - -// SanitizeFloatParameterHandler normalizes numeric knobs from either JSON numbers or string-encoded numbers. -type SanitizeFloatParameterHandler struct { - StripNonFinite bool - Min *float64 - Max *float64 -} - -func (h SanitizeFloatParameterHandler) Apply(ctx *RequestFilterContext, parameter VLLMParameter) error { - value, ok := ctx.Document.Get(parameter.Name) - if !ok { - return nil - } - number, ok := numericJSONValueAsFloat64(value) - if !ok { - ctx.Document.Delete(parameter.Name) - return nil - } - if h.StripNonFinite && (math.IsNaN(number) || math.IsInf(number, 0)) { - ctx.Document.Delete(parameter.Name) +func (h ParameterHandlerAdapter) Apply(ctx *RequestFilterContext, parameter VLLMParameter) error { + if h.Handler == nil { return nil } - if h.Min != nil && number < *h.Min { - number = *h.Min - } - if h.Max != nil && number > *h.Max { - number = *h.Max - } - ctx.Document.Set(parameter.Name, number) - return nil -} - -type SanitizeFloatMapParameterHandler struct { - StripNonFinite bool - Min *float64 - Max *float64 - DropFieldIfEmpty bool - MaxEntries int -} - -func (h SanitizeFloatMapParameterHandler) Apply(ctx *RequestFilterContext, parameter VLLMParameter) error { - raw, ok := ctx.Document.Object(parameter.Name) - if !ok { - return nil - } - if h.MaxEntries > 0 && len(raw) > h.MaxEntries { - return badChatRequest("%s: map size %d exceeds limit %d", parameter.Name, len(raw), h.MaxEntries) - } - for key, value := range raw { - number, ok := numericJSONValueAsFloat64(value) - if !ok { - continue - } - if h.StripNonFinite && (math.IsNaN(number) || math.IsInf(number, 0)) { - delete(raw, key) - continue - } - if h.Min != nil && number < *h.Min { - delete(raw, key) - continue - } - if h.Max != nil && number > *h.Max { - delete(raw, key) - } - } - if len(raw) == 0 && h.DropFieldIfEmpty { - ctx.Document.Delete(parameter.Name) - return nil - } - ctx.Document.Set(parameter.Name, raw) - return nil -} - -type ForceLiteralParameterHandler struct { - Value any - OverwriteOnly bool -} - -func (h ForceLiteralParameterHandler) Apply(ctx *RequestFilterContext, parameter VLLMParameter) error { - if h.OverwriteOnly { - if _, exists := ctx.Document.Get(parameter.Name); !exists { - return nil - } + var handlerErr error + ctx.Document.LockedScope(func(raw map[string]any) { + handlerErr = h.Handler.HandleParameter(paramvalidators.ParameterContext{ + Document: raw, + Parameter: parameter.Name, + RoutedModel: ctx.RoutedModel, + }) + }) + if handlerErr != nil { + return wrapBadChatRequest(handlerErr) } - ctx.Document.Set(parameter.Name, h.Value) return nil } @@ -185,13 +75,11 @@ type ModelScopedParameterHandler struct { } func (h ModelScopedParameterHandler) Apply(ctx *RequestFilterContext, parameter VLLMParameter) error { - for _, m := range h.Models { - if m == ctx.RoutedModel { - if h.Handler == nil { - return nil - } - return h.Handler.Apply(ctx, parameter) + if filtercore.MatchesModel(ctx.RoutedModel, h.Models) { + if h.Handler == nil { + return nil } + return h.Handler.Apply(ctx, parameter) } if h.UnmatchedHandler == nil { return nil @@ -199,107 +87,30 @@ func (h ModelScopedParameterHandler) Apply(ctx *RequestFilterContext, parameter return h.UnmatchedHandler.Apply(ctx, parameter) } -type CapUintParameterHandler struct { - Max uint64 -} - -func (h CapUintParameterHandler) Apply(ctx *RequestFilterContext, parameter VLLMParameter) error { - value, ok := numericJSONValueAsUint64FromDocument(&ctx.Document, parameter.Name) - if !ok { - return nil - } - if value > h.Max { - ctx.Document.Set(parameter.Name, h.Max) - } - return nil -} - -type ClampUintToFieldParameterHandler struct { - MaxField string -} - -func (h ClampUintToFieldParameterHandler) Apply(ctx *RequestFilterContext, parameter VLLMParameter) error { - value, ok := numericJSONValueAsUint64FromDocument(&ctx.Document, parameter.Name) - if !ok { - return nil - } - maxValue, ok := numericJSONValueAsUint64FromDocument(&ctx.Document, h.MaxField) - if !ok || maxValue == 0 { - return nil - } - if value > maxValue { - ctx.Document.Set(parameter.Name, maxValue) - } - return nil -} - // MinUintParameterHandler clamps a uint parameter UP to Min when the value is present // and below the floor. Pass-through when the field is absent or already >= Min. +// Used by the Kimi-K2.6 max_tokens floor; lives here (rather than in paramvalidators) +// because it has only one call-site and was added after the bulk-handler migration. type MinUintParameterHandler struct { Min uint64 } func (h MinUintParameterHandler) Apply(ctx *RequestFilterContext, parameter VLLMParameter) error { - value, ok := numericJSONValueAsUint64FromDocument(&ctx.Document, parameter.Name) + raw, ok := ctx.Document.Get(parameter.Name) if !ok { return nil } - if value < h.Min { - ctx.Document.Set(parameter.Name, h.Min) - } - return nil -} - -// ValidateUintParameterHandler rejects the request if the field is present but its value -// cannot be parsed as a non-negative integer that fits in uint64. Pass-through when the -// field is absent. Used for fields like `seed` where vLLM expects a uint64 and we want to -// catch garbage types at the gateway boundary rather than relying on the upstream's error -// path. -type ValidateUintParameterHandler struct{} - -func (h ValidateUintParameterHandler) Apply(ctx *RequestFilterContext, parameter VLLMParameter) error { - raw, exists := ctx.Document.Get(parameter.Name) - if !exists || raw == nil { - return nil - } - if _, ok := devshard.JSONNumericUint64(raw); !ok { - return badChatRequest("%s: must be a non-negative integer", parameter.Name) - } - return nil -} - -// LengthCapListParameterHandler bounds the number of entries in a JSON array, and -// optionally the byte length of each string entry. Used for fields like `stop`, -// `stop_token_ids`, and `bad_words` -- vLLM scans every entry against every generated -// token, so unbounded arrays linearly slow inference. MaxEntries=0 disables the array cap, -// MaxEntryLen=0 disables the per-string cap (use 0 for int-only arrays). -type LengthCapListParameterHandler struct { - MaxEntries int - MaxEntryLen int -} - -func (h LengthCapListParameterHandler) Apply(ctx *RequestFilterContext, parameter VLLMParameter) error { - raw, ok := ctx.Document.Array(parameter.Name) + value, ok := devshard.JSONNumericUint64(raw) if !ok { return nil } - if h.MaxEntries > 0 && len(raw) > h.MaxEntries { - return badChatRequest("%s: array length %d exceeds limit %d", parameter.Name, len(raw), h.MaxEntries) - } - if h.MaxEntryLen > 0 { - for i, item := range raw { - s, ok := item.(string) - if !ok { - continue - } - if len(s) > h.MaxEntryLen { - return badChatRequest("%s[%d]: string length %d exceeds limit %d", parameter.Name, i, len(s), h.MaxEntryLen) - } - } + if value < h.Min { + ctx.Document.Set(parameter.Name, h.Min) } return nil } + // DocumentValidator: validators in paramvalidators expose this contract. May mutate // vctx.Document for per-model rewrites alongside shape checks. type DocumentValidator interface { @@ -546,7 +357,7 @@ func (ctx *RequestFilterContext) DecodeRequest() error { // TestNormalizeChatRequestDefaultsAndCapsOutputTokens. // // Other fields are re-read so caps applied by PostLimits rules (for example `n` via -// CapUintParameterHandler) propagate into the projection. +// paramvalidators.CapUintParameter through the adapter) propagate into the projection. func (ctx *RequestFilterContext) SyncRequestView() error { var req chatRequest if err := readChatRequestFields(&ctx.Document, &req); err != nil { @@ -610,24 +421,24 @@ func defaultVLLMParameterCatalog() VLLMParameterCatalog { parameters := slices.Concat( []VLLMParameter{ newParameter("messages"). - withRule(RequestFilterStagePreValidation, LengthCapListParameterHandler{MaxEntries: MessagesMaxEntries}), + withRule(RequestFilterStagePreValidation, ParameterHandlerAdapter{Handler: paramvalidators.LengthCapListParameter{MaxEntries: MessagesMaxEntries}}), newParameter("seed"). - withRule(RequestFilterStagePreValidation, ValidateUintParameterHandler{}), + withRule(RequestFilterStagePreValidation, ParameterHandlerAdapter{Handler: paramvalidators.ValidateUintParameter{}}), newParameter("n"). - withRule(RequestFilterStagePostLimits, CapUintParameterHandler{Max: MaxChatRequestChoices}). + withRule(RequestFilterStagePostLimits, ParameterHandlerAdapter{Handler: paramvalidators.CapUintParameter{Max: MaxChatRequestChoices}}). withRule(RequestFilterStagePostLimits, DocumentValidatorHandler{ Validator: paramvalidators.GreedySamplingValidator{}, }), newParameter("temperature"). - withRule(RequestFilterStagePostLimits, SanitizeFloatParameterHandler{StripNonFinite: true, Max: floatPointer(MaxTemperature)}), + withRule(RequestFilterStagePostLimits, ParameterHandlerAdapter{Handler: paramvalidators.SanitizeFloatParameter{StripNonFinite: true, Max: floatPointer(MaxTemperature)}}), newParameter("repetition_penalty"). - withRule(RequestFilterStagePostLimits, SanitizeFloatParameterHandler{StripNonFinite: true, Max: floatPointer(MaxRepetitionPenalty)}), + withRule(RequestFilterStagePostLimits, ParameterHandlerAdapter{Handler: paramvalidators.SanitizeFloatParameter{StripNonFinite: true, Max: floatPointer(MaxRepetitionPenalty)}}), newParameter("logit_bias"). - withRule(RequestFilterStagePostLimits, SanitizeFloatMapParameterHandler{StripNonFinite: true, Min: floatPointer(LogitBiasMinValue), Max: floatPointer(LogitBiasMaxValue), DropFieldIfEmpty: true, MaxEntries: LogitBiasMaxEntries}), + withRule(RequestFilterStagePostLimits, ParameterHandlerAdapter{Handler: paramvalidators.SanitizeFloatMapParameter{StripNonFinite: true, Min: floatPointer(LogitBiasMinValue), Max: floatPointer(LogitBiasMaxValue), DropFieldIfEmpty: true, MaxEntries: LogitBiasMaxEntries}}), newParameter("stop"). - withRule(RequestFilterStagePreValidation, LengthCapListParameterHandler{MaxEntries: StopMaxEntries, MaxEntryLen: StopMaxEntryLen}), + withRule(RequestFilterStagePreValidation, ParameterHandlerAdapter{Handler: paramvalidators.LengthCapListParameter{MaxEntries: StopMaxEntries, MaxEntryLen: StopMaxEntryLen}}), newParameter("stop_token_ids"). - withRule(RequestFilterStagePreValidation, LengthCapListParameterHandler{MaxEntries: StopTokenIdsMaxEntries}), + withRule(RequestFilterStagePreValidation, ParameterHandlerAdapter{Handler: paramvalidators.LengthCapListParameter{MaxEntries: StopTokenIdsMaxEntries}}), newParameter("reasoning"). withRule(RequestFilterStagePreValidation, DocumentValidatorHandler{ Validator: paramvalidators.ReasoningValidator{}, @@ -641,13 +452,26 @@ func defaultVLLMParameterCatalog() VLLMParameterCatalog { }). withRule(RequestFilterStagePreValidation, ModelScopedParameterHandler{ Models: nil, - UnmatchedHandler: StripParameterHandler{}, + UnmatchedHandler: ParameterHandlerAdapter{Handler: paramvalidators.StripParameter{}}, }), + // MiniMax-M2.7 has no chat_template knob for enable_thinking (vLLM #36778); + // strip on this route before EnableThinkingValidator runs. newParameter("enable_thinking"). + withRule(RequestFilterStagePreValidation, ModelScopedParameterHandler{ + Models: []string{miniMaxM27ModelID}, + Handler: ParameterHandlerAdapter{Handler: paramvalidators.StripParameter{}}, + }). withRule(RequestFilterStagePreValidation, DocumentValidatorHandler{ Validator: paramvalidators.EnableThinkingValidator{}, }), + // thinking: Kimi mirrors to chat_template_kwargs.thinking; MiniMax-M2.7 has no + // equivalent knob (interleaved thinking is structural to the chat template) so + // strip the field before ThinkingValidator runs. Other routes normalize+keep. newParameter("thinking"). + withRule(RequestFilterStagePreValidation, ModelScopedParameterHandler{ + Models: []string{miniMaxM27ModelID}, + Handler: ParameterHandlerAdapter{Handler: paramvalidators.StripParameter{}}, + }). withRule(RequestFilterStagePreValidation, DocumentValidatorHandler{ Validator: paramvalidators.ThinkingValidator{ MirrorToTemplateKwargsForModels: []string{kimiK26ModelID}, @@ -664,7 +488,7 @@ func defaultVLLMParameterCatalog() VLLMParameterCatalog { newParameter("thinking_token_budget"). withRule(RequestFilterStagePreValidation, ModelScopedParameterHandler{ Models: []string{kimiK26ModelID}, - UnmatchedHandler: StripParameterHandler{}, + UnmatchedHandler: ParameterHandlerAdapter{Handler: paramvalidators.StripParameter{}}, }). withRule(RequestFilterStagePostLimits, ModelScopedParameterHandler{ Models: []string{kimiK26ModelID}, @@ -674,8 +498,8 @@ func defaultVLLMParameterCatalog() VLLMParameterCatalog { }, }, }). - withRule(RequestFilterStagePostLimits, CapUintParameterHandler{Max: kimiThinkingTokenBudgetMax}). - withRule(RequestFilterStagePostLimits, ClampUintToFieldParameterHandler{MaxField: "max_tokens"}), + withRule(RequestFilterStagePostLimits, ParameterHandlerAdapter{Handler: paramvalidators.CapUintParameter{Max: kimiThinkingTokenBudgetMax}}). + withRule(RequestFilterStagePostLimits, ParameterHandlerAdapter{Handler: paramvalidators.ClampUintToFieldParameter{MaxField: "max_tokens"}}), newParameter("tools"). withRule(RequestFilterStagePreValidation, DocumentValidatorHandler{ Validator: paramvalidators.ToolsValidator{ @@ -693,20 +517,21 @@ func defaultVLLMParameterCatalog() VLLMParameterCatalog { Validator: paramvalidators.ToolChoiceValidator{MaxNameLen: ToolChoiceMaxNameLen}, }), newParameter("min_tokens"). - withRule(RequestFilterStagePreValidation, ConditionalStripParameterHandler{ - Predicate: func(ctx *RequestFilterContext) bool { - return ctx.Document.Has("stop_token_ids") + withRule(RequestFilterStagePreValidation, ParameterHandlerAdapter{Handler: paramvalidators.ConditionalStripParameter{ + Predicate: func(ctx paramvalidators.ParameterContext) bool { + _, ok := ctx.Document["stop_token_ids"] + return ok }, - }). - withRule(RequestFilterStagePostLimits, ClampUintToFieldParameterHandler{MaxField: "max_tokens"}), + }}). + withRule(RequestFilterStagePostLimits, ParameterHandlerAdapter{Handler: paramvalidators.ClampUintToFieldParameter{MaxField: "max_tokens"}}), newParameter("bad_words"). - withRule(RequestFilterStagePreValidation, SanitizeStringListParameterHandler{ + withRule(RequestFilterStagePreValidation, ParameterHandlerAdapter{Handler: paramvalidators.SanitizeStringListParameter{ Keep: func(value string) bool { return strings.TrimSpace(value) != "" }, DropFieldIfEmpty: true, - }). - withRule(RequestFilterStagePreValidation, LengthCapListParameterHandler{MaxEntries: BadWordsMaxEntries, MaxEntryLen: BadWordsMaxEntryLen}), + }}). + withRule(RequestFilterStagePreValidation, ParameterHandlerAdapter{Handler: paramvalidators.LengthCapListParameter{MaxEntries: BadWordsMaxEntries, MaxEntryLen: BadWordsMaxEntryLen}}), // OpenAI Chat Completions standard observability fields. No inference-side // semantics on the vLLM upstream; clients send them for end-user tracking, // distributed tracing, agent control, and streaming token accounting. @@ -736,11 +561,11 @@ func defaultVLLMParameterCatalog() VLLMParameterCatalog { Validator: paramvalidators.StreamOptionsValidator{}, }), newParameter("return_token_ids"). - withRule(RequestFilterStagePostLimits, ForceLiteralParameterHandler{Value: true}), + withRule(RequestFilterStagePostLimits, ParameterHandlerAdapter{Handler: paramvalidators.ForceLiteralParameter{Value: true}}), newParameter("logprobs"). - withRule(RequestFilterStagePostLimits, ForceLiteralParameterHandler{Value: true}), + withRule(RequestFilterStagePostLimits, ParameterHandlerAdapter{Handler: paramvalidators.ForceLiteralParameter{Value: true}}), newParameter("top_logprobs"). - withRule(RequestFilterStagePostLimits, ForceLiteralParameterHandler{Value: TopLogprobsForcedValue}), + withRule(RequestFilterStagePostLimits, ParameterHandlerAdapter{Handler: paramvalidators.ForceLiteralParameter{Value: TopLogprobsForcedValue}}), newParameter("response_format"). withRule(RequestFilterStagePreValidation, DocumentValidatorHandler{ Validator: paramvalidators.ResponseFormatValidator{ @@ -779,7 +604,17 @@ func defaultVLLMParameterCatalog() VLLMParameterCatalog { DefaultMaxLen: SafetyIdentifierMaxLen, }, }, - UnmatchedHandler: StripParameterHandler{}, + UnmatchedHandler: ParameterHandlerAdapter{Handler: paramvalidators.StripParameter{}}, + }), + // MiniMax-M2.7 native extension lifted from extra_body. On the MiniMax + // route the field passes through to vLLM verbatim (controls whether + // reasoning is emitted as inline ... in content or as a + // separate reasoning_details[] array). Stripped on other routes since + // Kimi/Qwen vLLM serves don't know the field. See docs/chat-api/minimax-m2.7.md. + newParameter("reasoning_split"). + withRule(RequestFilterStagePreValidation, ModelScopedParameterHandler{ + Models: []string{miniMaxM27ModelID}, + UnmatchedHandler: ParameterHandlerAdapter{Handler: paramvalidators.StripParameter{}}, }), }, []VLLMParameter{ @@ -806,11 +641,11 @@ func defaultVLLMParameterCatalog() VLLMParameterCatalog { newParameters([]string{"top_p", "top_k", "min_p"}, ParameterRule{ Stage: RequestFilterStagePostLimits, - Handler: SanitizeFloatParameterHandler{StripNonFinite: true}, + Handler: ParameterHandlerAdapter{Handler: paramvalidators.SanitizeFloatParameter{StripNonFinite: true}}, }, ), newParameters([]string{"service_tier", "store", "provider", "plugins", "prompt_cache_key", "cache_key", "extra_headers", "thinking_config", "think"}, - ParameterRule{Stage: RequestFilterStagePreValidation, Handler: StripParameterHandler{}}, + ParameterRule{Stage: RequestFilterStagePreValidation, Handler: ParameterHandlerAdapter{Handler: paramvalidators.StripParameter{}}}, ), // frequency_penalty / presence_penalty share identical rules: catalog clamp // [-2, 2] for all models + per-Kimi force-rewrite to 0.0 (Moonshot's K2.6 wire @@ -818,13 +653,13 @@ func defaultVLLMParameterCatalog() VLLMParameterCatalog { newParameters([]string{"frequency_penalty", "presence_penalty"}, ParameterRule{ Stage: RequestFilterStagePostLimits, - Handler: SanitizeFloatParameterHandler{StripNonFinite: true, Min: floatPointer(PenaltyMin), Max: floatPointer(PenaltyMax)}, + Handler: ParameterHandlerAdapter{Handler: paramvalidators.SanitizeFloatParameter{StripNonFinite: true, Min: floatPointer(PenaltyMin), Max: floatPointer(PenaltyMax)}}, }, ParameterRule{ Stage: RequestFilterStagePostLimits, Handler: ModelScopedParameterHandler{ Models: []string{kimiK26ModelID}, - Handler: ForceLiteralParameterHandler{Value: KimiK2PenaltyForcedValue, OverwriteOnly: true}, + Handler: ParameterHandlerAdapter{Handler: paramvalidators.ForceLiteralParameter{Value: KimiK2PenaltyForcedValue, OverwriteOnly: true}}, }, }, ), @@ -930,32 +765,3 @@ func newParameters(names []string, rules ...ParameterRule) []VLLMParameter { func floatPointer(value float64) *float64 { return &value } - -func numericJSONValueAsFloat64(value any) (float64, bool) { - switch v := value.(type) { - case float64: - return v, true - case int: - return float64(v), true - case int64: - return float64(v), true - case uint64: - return float64(v), true - case json.Number: - number, err := v.Float64() - return number, err == nil - case string: - number, err := strconv.ParseFloat(strings.TrimSpace(v), 64) - return number, err == nil - default: - return 0, false - } -} - -func numericJSONValueAsUint64FromDocument(document *ChatRequestDocument, field string) (uint64, bool) { - value, ok := document.Get(field) - if !ok { - return 0, false - } - return devshard.JSONNumericUint64(value) -} diff --git a/devshard/cmd/devshardctl/request_filters_test.go b/devshard/cmd/devshardctl/request_filters_test.go index 6c0cbb1d46..2f4387a255 100644 --- a/devshard/cmd/devshardctl/request_filters_test.go +++ b/devshard/cmd/devshardctl/request_filters_test.go @@ -2583,3 +2583,332 @@ func TestStructuredOutputsCatalogEntryRunsInPreValidationStage(t *testing.T) { } require.True(t, found, "structured_outputs catalog entry missing") } + +// ==================================================================== +// MiniMax-M2.7 route — see docs/chat-api/minimax-m2.7.md +// ==================================================================== + +func TestNormalizeForMinimaxStripsThinking(t *testing.T) { + body := `{"model":"MiniMaxAI/MiniMax-M2.7","messages":[{"role":"user","content":"hi"}],"thinking":{"type":"enabled"}}` + out, _, err := normalizeChatRequestForModel([]byte(body), miniMaxM27ModelID) + require.NoError(t, err) + var raw map[string]any + require.NoError(t, json.Unmarshal(out, &raw)) + require.NotContains(t, raw, "thinking") + require.NotContains(t, raw, "chat_template_kwargs") +} + +func TestNormalizeForMinimaxStripsEnableThinking(t *testing.T) { + body := `{"model":"MiniMaxAI/MiniMax-M2.7","messages":[{"role":"user","content":"hi"}],"enable_thinking":true}` + out, _, err := normalizeChatRequestForModel([]byte(body), miniMaxM27ModelID) + require.NoError(t, err) + var raw map[string]any + require.NoError(t, json.Unmarshal(out, &raw)) + require.NotContains(t, raw, "enable_thinking") + require.NotContains(t, raw, "chat_template_kwargs") +} + +func TestNormalizeForMinimaxStripsThinkingTokenBudget(t *testing.T) { + body := `{"model":"MiniMaxAI/MiniMax-M2.7","messages":[{"role":"user","content":"hi"}],"max_tokens":4096,"thinking_token_budget":1024}` + out, _, err := normalizeChatRequestForModel([]byte(body), miniMaxM27ModelID) + require.NoError(t, err) + var raw map[string]any + require.NoError(t, json.Unmarshal(out, &raw)) + require.NotContains(t, raw, "thinking_token_budget") +} + +func TestNormalizeForMinimaxDoesNotForceZeroPenalties(t *testing.T) { + body := `{"model":"MiniMaxAI/MiniMax-M2.7","messages":[{"role":"user","content":"hi"}],"frequency_penalty":0.5,"presence_penalty":-0.5}` + out, _, err := normalizeChatRequestForModel([]byte(body), miniMaxM27ModelID) + require.NoError(t, err) + var raw map[string]any + require.NoError(t, json.Unmarshal(out, &raw)) + require.EqualValues(t, 0.5, raw["frequency_penalty"]) + require.EqualValues(t, -0.5, raw["presence_penalty"]) +} + +func TestNormalizeForMinimaxStripsSafetyIdentifier(t *testing.T) { + body := `{"model":"MiniMaxAI/MiniMax-M2.7","messages":[{"role":"user","content":"hi"}],"safety_identifier":"abuse-track-hash"}` + out, _, err := normalizeChatRequestForModel([]byte(body), miniMaxM27ModelID) + require.NoError(t, err) + var raw map[string]any + require.NoError(t, json.Unmarshal(out, &raw)) + require.NotContains(t, raw, "safety_identifier") +} + +func TestNormalizeForMinimaxAcceptsToolMessageArrayShape(t *testing.T) { + body := `{ + "model":"MiniMaxAI/MiniMax-M2.7", + "messages":[ + {"role":"user","content":"weather in Paris?"}, + {"role":"assistant","content":"","tool_calls":[{"id":"call_1","type":"function","function":{"name":"get_weather","arguments":"{}"}}]}, + {"role":"tool","content":[{"name":"get_weather","type":"text","text":"{\"temp\":\"18\"}"}]} + ] + }` + out, _, err := normalizeChatRequestForModel([]byte(body), miniMaxM27ModelID) + require.NoError(t, err) + var raw map[string]any + require.NoError(t, json.Unmarshal(out, &raw)) + msgs, ok := raw["messages"].([]any) + require.True(t, ok) + require.Len(t, msgs, 3) + toolMsg := msgs[2].(map[string]any) + content, ok := toolMsg["content"].([]any) + require.True(t, ok, "M2.7 tool content must remain an array (no flatten)") + require.Len(t, content, 1) + entry := content[0].(map[string]any) + require.Equal(t, "get_weather", entry["name"]) + require.Equal(t, "text", entry["type"]) +} + +func TestNormalizeForMinimaxStripsToolCallIDFromToolMessage(t *testing.T) { + body := `{ + "model":"MiniMaxAI/MiniMax-M2.7", + "messages":[ + {"role":"user","content":"hi"}, + {"role":"assistant","content":"","tool_calls":[{"id":"call_1","type":"function","function":{"name":"fn","arguments":"{}"}}]}, + {"role":"tool","tool_call_id":"call_1","content":[{"name":"fn","type":"text","text":"ok"}]} + ] + }` + out, _, err := normalizeChatRequestForModel([]byte(body), miniMaxM27ModelID) + require.NoError(t, err) + var raw map[string]any + require.NoError(t, json.Unmarshal(out, &raw)) + msgs := raw["messages"].([]any) + toolMsg := msgs[2].(map[string]any) + require.NotContains(t, toolMsg, "tool_call_id", "M2.7 strips tool_call_id silently for dual-emit compat") +} + +func TestNormalizeForMinimaxDropsOrphanToolMessage(t *testing.T) { + // Orphan = no preceding assistant.tool_calls[] block. + body := `{ + "model":"MiniMaxAI/MiniMax-M2.7", + "messages":[ + {"role":"user","content":"hi"}, + {"role":"tool","content":[{"name":"fn","type":"text","text":"strayed"}]}, + {"role":"assistant","content":"hello"} + ] + }` + out, _, err := normalizeChatRequestForModel([]byte(body), miniMaxM27ModelID) + require.NoError(t, err) + var raw map[string]any + require.NoError(t, json.Unmarshal(out, &raw)) + msgs := raw["messages"].([]any) + require.Len(t, msgs, 2, "orphan tool message must be dropped") + require.Equal(t, "user", msgs[0].(map[string]any)["role"]) + require.Equal(t, "assistant", msgs[1].(map[string]any)["role"]) +} + +func TestNormalizeForMinimaxPreservesThinkBlocksInAssistantContent(t *testing.T) { + // MiniMax-M2.7 interleaved-thinking constraint: ... in + // assistant content history must round-trip byte-identical. + body := `{ + "model":"MiniMaxAI/MiniMax-M2.7", + "messages":[ + {"role":"user","content":"first question"}, + {"role":"assistant","content":"let me reason about this here is the answer"}, + {"role":"user","content":"follow up"} + ] + }` + out, _, err := normalizeChatRequestForModel([]byte(body), miniMaxM27ModelID) + require.NoError(t, err) + var raw map[string]any + require.NoError(t, json.Unmarshal(out, &raw)) + msgs := raw["messages"].([]any) + asst := msgs[1].(map[string]any) + require.Equal(t, "let me reason about this here is the answer", asst["content"]) +} + +func TestValidateForMinimaxRejectsBareStringToolContent(t *testing.T) { + body := `{ + "model":"MiniMaxAI/MiniMax-M2.7", + "messages":[ + {"role":"user","content":"hi"}, + {"role":"assistant","content":"","tool_calls":[{"id":"call_1","type":"function","function":{"name":"fn","arguments":"{}"}}]}, + {"role":"tool","content":"bare string instead of array"} + ] + }` + _, _, err := normalizeChatRequestForModel([]byte(body), miniMaxM27ModelID) + require.Error(t, err) + require.Contains(t, err.Error(), "content") + require.Equal(t, http.StatusBadRequest, chatRequestErrorStatus(err, http.StatusInternalServerError)) +} + +func TestValidateForMinimaxRejectsToolEntryMissingName(t *testing.T) { + body := `{ + "model":"MiniMaxAI/MiniMax-M2.7", + "messages":[ + {"role":"user","content":"hi"}, + {"role":"assistant","content":"","tool_calls":[{"id":"call_1","type":"function","function":{"name":"fn","arguments":"{}"}}]}, + {"role":"tool","content":[{"type":"text","text":"missing name"}]} + ] + }` + _, _, err := normalizeChatRequestForModel([]byte(body), miniMaxM27ModelID) + require.Error(t, err) + require.Contains(t, err.Error(), "name") +} + +func TestValidateForMinimaxRejectsToolEntryWrongType(t *testing.T) { + body := `{ + "model":"MiniMaxAI/MiniMax-M2.7", + "messages":[ + {"role":"user","content":"hi"}, + {"role":"assistant","content":"","tool_calls":[{"id":"call_1","type":"function","function":{"name":"fn","arguments":"{}"}}]}, + {"role":"tool","content":[{"name":"fn","type":"image_url","text":"x"}]} + ] + }` + _, _, err := normalizeChatRequestForModel([]byte(body), miniMaxM27ModelID) + require.Error(t, err) + require.Contains(t, err.Error(), "type") +} + +func TestValidateForMinimaxRejectsExtraKeysInToolEntry(t *testing.T) { + body := `{ + "model":"MiniMaxAI/MiniMax-M2.7", + "messages":[ + {"role":"user","content":"hi"}, + {"role":"assistant","content":"","tool_calls":[{"id":"call_1","type":"function","function":{"name":"fn","arguments":"{}"}}]}, + {"role":"tool","content":[{"name":"fn","type":"text","text":"ok","unexpected":true}]} + ] + }` + _, _, err := normalizeChatRequestForModel([]byte(body), miniMaxM27ModelID) + require.Error(t, err) + require.Contains(t, err.Error(), "unsupported key") +} + +func TestNormalizeForOpenAIRouteStillRequiresToolCallID(t *testing.T) { + // Sanity: the catalog default policy keeps OpenAI-compat behavior on non-M2.7 routes. + body := `{ + "messages":[ + {"role":"user","content":"hi"}, + {"role":"assistant","content":"","tool_calls":[{"id":"call_1","type":"function","function":{"name":"fn","arguments":"{}"}}]}, + {"role":"tool","content":[{"name":"fn","type":"text","text":"ok"}]} + ] + }` + _, _, err := normalizeChatRequest([]byte(body)) + require.Error(t, err, "OpenAI route requires tool_call_id; array-content tool message should fail validation") +} + +func TestNormalizeForMinimaxPassesReasoningSplitFromExtraBody(t *testing.T) { + // extra_body.reasoning_split is a MiniMax native extension; gateway lifts it + // to top-level (universal extra_body unwrap) and forwards to vLLM verbatim. + body := `{"model":"MiniMaxAI/MiniMax-M2.7","messages":[{"role":"user","content":"hi"}],"extra_body":{"reasoning_split":true}}` + out, _, err := normalizeChatRequestForModel([]byte(body), miniMaxM27ModelID) + require.NoError(t, err) + var raw map[string]any + require.NoError(t, json.Unmarshal(out, &raw)) + require.NotContains(t, raw, "extra_body") + require.Equal(t, true, raw["reasoning_split"], "reasoning_split must survive the closed-allowlist check on MiniMax") +} + +func TestNormalizeForMinimaxPassesReasoningSplitTopLevel(t *testing.T) { + body := `{"model":"MiniMaxAI/MiniMax-M2.7","messages":[{"role":"user","content":"hi"}],"reasoning_split":false}` + out, _, err := normalizeChatRequestForModel([]byte(body), miniMaxM27ModelID) + require.NoError(t, err) + var raw map[string]any + require.NoError(t, json.Unmarshal(out, &raw)) + require.Equal(t, false, raw["reasoning_split"]) +} + +func TestNormalizeStripsReasoningSplitOnDefaultRoute(t *testing.T) { + // Kimi/Qwen vLLM servers do not know reasoning_split; the gateway strips it so + // stray clients don't trip an upstream parser error. Assert it is actually gone + // on both the empty default route and an explicit non-MiniMax (Kimi) route. + for _, routedModel := range []string{"", kimiK26ModelID} { + body := `{"messages":[{"role":"user","content":"hi"}],"reasoning_split":true}` + out, _, err := normalizeChatRequestForModel([]byte(body), routedModel) + require.NoError(t, err) + var raw map[string]any + require.NoError(t, json.Unmarshal(out, &raw)) + require.NotContains(t, raw, "reasoning_split", "route=%q", routedModel) + } +} + +func TestNormalizeForMinimaxPreservesReasoningDetailsOnAssistantTurn(t *testing.T) { + // MiniMax-M2.7 emits reasoning_details[] on the response when reasoning_split=true; + // clients round-trip it back into history. Gateway MUST NOT strip — stripping + // breaks interleaved-thinking continuity. + body := `{ + "model":"MiniMaxAI/MiniMax-M2.7", + "messages":[ + {"role":"user","content":"hi"}, + {"role":"assistant","content":"final answer","reasoning_details":[{"type":"text","text":"step 1"},{"type":"text","text":"step 2"}]} + ] + }` + out, _, err := normalizeChatRequestForModel([]byte(body), miniMaxM27ModelID) + require.NoError(t, err) + var raw map[string]any + require.NoError(t, json.Unmarshal(out, &raw)) + msgs := raw["messages"].([]any) + asst := msgs[1].(map[string]any) + details, ok := asst["reasoning_details"].([]any) + require.True(t, ok, "reasoning_details must round-trip on assistant turn") + require.Len(t, details, 2) +} + +func TestValidateForMinimaxRejectsToolMessageExceedingMaxEntries(t *testing.T) { + entries := make([]string, 0, MinimaxToolMessageMaxEntries+1) + for i := 0; i < MinimaxToolMessageMaxEntries+1; i++ { + entries = append(entries, `{"name":"fn","type":"text","text":"r"}`) + } + body := `{ + "model":"MiniMaxAI/MiniMax-M2.7", + "messages":[ + {"role":"user","content":"hi"}, + {"role":"assistant","content":"","tool_calls":[{"id":"call_1","type":"function","function":{"name":"fn","arguments":"{}"}}]}, + {"role":"tool","content":[` + strings.Join(entries, ",") + `]} + ] + }` + _, _, err := normalizeChatRequestForModel([]byte(body), miniMaxM27ModelID) + require.Error(t, err) + require.Contains(t, err.Error(), "exceeds limit") +} + +func TestValidateForMinimaxRejectsToolMessageNameTooLong(t *testing.T) { + tooLongName := strings.Repeat("x", MinimaxToolMessageNameMaxLen+1) + body := `{ + "model":"MiniMaxAI/MiniMax-M2.7", + "messages":[ + {"role":"user","content":"hi"}, + {"role":"assistant","content":"","tool_calls":[{"id":"call_1","type":"function","function":{"name":"fn","arguments":"{}"}}]}, + {"role":"tool","content":[{"name":"` + tooLongName + `","type":"text","text":"ok"}]} + ] + }` + _, _, err := normalizeChatRequestForModel([]byte(body), miniMaxM27ModelID) + require.Error(t, err) + require.Contains(t, err.Error(), "name length") +} + +func TestValidateForMinimaxRejectsToolMessageTextTooLarge(t *testing.T) { + tooLongText := strings.Repeat("a", MinimaxToolMessageTextMaxSize+1) + body := `{ + "model":"MiniMaxAI/MiniMax-M2.7", + "messages":[ + {"role":"user","content":"hi"}, + {"role":"assistant","content":"","tool_calls":[{"id":"call_1","type":"function","function":{"name":"fn","arguments":"{}"}}]}, + {"role":"tool","content":[{"name":"fn","type":"text","text":"` + tooLongText + `"}]} + ] + }` + _, _, err := normalizeChatRequestForModel([]byte(body), miniMaxM27ModelID) + require.Error(t, err) + require.Contains(t, err.Error(), "text size") +} + +func TestNormalizeForMinimaxStripsToolsFunctionStrict(t *testing.T) { + // ToolsValidator silently strips function.strict on every route; confirm it + // holds on the MiniMax route (vLLM minimax_m2 parser ignores the field). + body := `{ + "model":"MiniMaxAI/MiniMax-M2.7", + "messages":[{"role":"user","content":"hi"}], + "tools":[{"type":"function","function":{"name":"fn","strict":true,"parameters":{"type":"object","properties":{}}}}] + }` + out, _, err := normalizeChatRequestForModel([]byte(body), miniMaxM27ModelID) + require.NoError(t, err) + var raw map[string]any + require.NoError(t, json.Unmarshal(out, &raw)) + tools, ok := raw["tools"].([]any) + require.True(t, ok) + require.Len(t, tools, 1) + fn := tools[0].(map[string]any)["function"].(map[string]any) + require.NotContains(t, fn, "strict") +} diff --git a/devshard/cmd/devshardctl/testutil/testutil.go b/devshard/cmd/devshardctl/testutil/testutil.go new file mode 100644 index 0000000000..beb9ce5706 --- /dev/null +++ b/devshard/cmd/devshardctl/testutil/testutil.go @@ -0,0 +1,20 @@ +// Package testutil holds small assertion helpers shared across devshardctl test +// packages. It imports "testing" on purpose: it is test-support code (like +// net/http/httptest) and is only ever linked into _test binaries that import it. +package testutil + +import "testing" + +// FloatPtr returns a pointer to v, for building optional float parameters in tests. +func FloatPtr(v float64) *float64 { return &v } + +// MapAt returns items[index] as a map[string]any, failing the test if the element +// is missing or not a map. +func MapAt(t *testing.T, items []any, index int) map[string]any { + t.Helper() + value, ok := items[index].(map[string]any) + if !ok { + t.Fatalf("items[%d] is not a map: %T", index, items[index]) + } + return value +} diff --git a/devshard/json.go b/devshard/json.go index 90b969a206..87400eb670 100644 --- a/devshard/json.go +++ b/devshard/json.go @@ -6,6 +6,8 @@ import ( "encoding/json" "fmt" "sort" + "strconv" + "strings" ) // CanonicalizeJSON returns a deterministic JSON encoding with sorted keys and @@ -41,6 +43,31 @@ func CanonicalPromptHash(prompt []byte) ([]byte, error) { return h[:], nil } +// JSONNumericFloat64 converts a JSON-decoded numeric value to float64. Accepts +// float64, int / int64 / uint64, json.Number, and string (trimmed and parsed) +// because some clients serialize numerics as strings for fields like +// temperature / top_p. +func JSONNumericFloat64(value any) (float64, bool) { + switch v := value.(type) { + case float64: + return v, true + case int: + return float64(v), true + case int64: + return float64(v), true + case uint64: + return float64(v), true + case json.Number: + number, err := v.Float64() + return number, err == nil + case string: + number, err := strconv.ParseFloat(strings.TrimSpace(v), 64) + return number, err == nil + default: + return 0, false + } +} + // JSONNumericUint64 converts a JSON-decoded numeric value to uint64. Accepts // float64 (only when integer-valued and non-negative), int / int64 / uint64, // and json.Number. Returns (0, false) for any other type or out-of-range value. diff --git a/devshard/json_test.go b/devshard/json_test.go index f5ca5f6cb8..e52736534d 100644 --- a/devshard/json_test.go +++ b/devshard/json_test.go @@ -135,6 +135,44 @@ func TestCanonicalPromptHash_StoredPayloadMatchesDirectHash(t *testing.T) { "validator sha256(stored_canonical) must equal user CanonicalPromptHash(original)") } +func TestJSONNumericFloat64(t *testing.T) { + cases := []struct { + name string + in any + want float64 + ok bool + }{ + {"float64_positive", 1.5, 1.5, true}, + {"float64_zero", float64(0), 0, true}, + {"float64_negative", -2.5, -2.5, true}, + {"int_positive", 7, 7, true}, + {"int_negative", -3, -3, true}, + {"int64_positive", int64(42), 42, true}, + {"uint64", uint64(99), 99, true}, + {"json_Number_float", json.Number("3.14"), 3.14, true}, + {"json_Number_integer", json.Number("42"), 42, true}, + {"json_Number_negative", json.Number("-1.5"), -1.5, true}, + {"json_Number_garbage", json.Number("abc"), 0, false}, + {"string_float", "1.5", 1.5, true}, + {"string_integer", "100", 100, true}, + {"string_trimmed", " 2.5 ", 2.5, true}, + {"string_negative", "-0.25", -0.25, true}, + {"string_invalid", "abc", 0, false}, + {"string_empty", "", 0, false}, + {"bool", true, 0, false}, + {"nil", nil, 0, false}, + {"slice", []int{1, 2}, 0, false}, + {"map", map[string]int{"x": 1}, 0, false}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + got, ok := JSONNumericFloat64(c.in) + require.Equal(t, c.ok, ok, "ok mismatch") + require.Equal(t, c.want, got, "value mismatch") + }) + } +} + func TestJSONNumericUint64(t *testing.T) { cases := []struct { name string diff --git a/docs/chat-api/README.md b/docs/chat-api/README.md index 0bbeaffc28..1a2058b6ae 100644 --- a/docs/chat-api/README.md +++ b/docs/chat-api/README.md @@ -1,10 +1,11 @@ # Gonka Chat Completions API -OpenAI-compatible chat completions, routed to Kimi-K2.6 / Qwen3-235B via vLLM. This doc covers universal parameter behavior. For per-model overrides see [Kimi-K2.6](kimi-k2.6.md) / [Qwen3-235B](qwen3-235b-a22b-instruct-2507.md). +OpenAI-compatible chat completions, routed to Kimi-K2.6 / Qwen3-235B / MiniMax-M2.7 via vLLM. This doc covers universal parameter behavior. For per-model overrides see [Kimi-K2.6](kimi-k2.6.md) / [Qwen3-235B](qwen3-235b-a22b-instruct-2507.md) / [MiniMax-M2.7](minimax-m2.7.md). ## Quick navigation - [Per-model overrides: Kimi-K2.6](kimi-k2.6.md) - [Per-model overrides: Qwen3-235B-A22B-Instruct-2507](qwen3-235b-a22b-instruct-2507.md) +- [Per-model overrides: MiniMax-M2.7](minimax-m2.7.md) - [Why was my param stripped/rejected?](troubleshooting.md) - [Client agents compatibility](agents.md) - [Source citations](references.md) @@ -46,7 +47,7 @@ OpenAI-compatible chat completions, routed to Kimi-K2.6 / Qwen3-235B via vLLM. T | `top_logprobs` | int | — | force `5`; observability pipeline | — | | `return_token_ids` | bool | — | force `true`; required for stream-derived `enforced_tokens` reconstruction on Kimi-K2.6 reasoning routes (without it, ``/`` are silently dropped from SSE while still counted in `usage.completion_tokens`). Resulting `prompt_token_ids` / `choices[].token_ids` are stripped from the client-facing response | [[vLLM-19]](references.md#vllm), [[vLLM-20]](references.md#vllm) | | `response_format` | object | — | shape-bounded (depth ≤16, nodes ≤128, branch arms ≤16, enum ≤256, size ≤16 KiB); `$ref`/`$defs`/`definitions` forbidden; `pattern` ≤512 B + must compile as regex; `json_schema.name` non-empty ≤64 chars matching `^[A-Za-z0-9_.-]+$`; schema must be an object | [[OpenAI-1]](references.md#openai), [[CVE-2]](references.md#security-advisories) | -| `structured_outputs` | object | — | validated against vLLM envelope (`json`/`regex`/`choice`/`grammar`/`json_object`/`structural_tag`); CVE-driven caps per sub-field — see [Qwen native extensions](qwen3-235b-a22b-instruct-2507.md#native-extensions); **rejected on Kimi-K2.6 route** ([why](troubleshooting.md#reject-structured_outputs-kimi)); **rejected if combined with `response_format`** ([why](troubleshooting.md#reject-structured_outputs-with-response_format)) | [[vLLM-16]](references.md#vllm), [[vLLM-18]](references.md#vllm), [[CVE-3]](references.md#security-advisories), [[CVE-4]](references.md#security-advisories), [[CVE-8]](references.md#security-advisories) | +| `structured_outputs` | object | — | validated against vLLM envelope (`json`/`regex`/`choice`/`grammar`/`json_object`/`structural_tag`); CVE-driven caps per sub-field — see [Qwen native extensions](qwen3-235b-a22b-instruct-2507.md#native-extensions); **rejected on Kimi-K2.6 route** ([why](troubleshooting.md#reject-structured_outputs-kimi)); **accepted on MiniMax-M2.7 route** — vLLM enforces it ([details](troubleshooting.md#accept-structured_outputs-minimax)); `structural_tag` must be the object form (string form is rejected — crashes the engine); **rejected if combined with `response_format`** ([why](troubleshooting.md#reject-structured_outputs-with-response_format)) | [[vLLM-16]](references.md#vllm), [[vLLM-18]](references.md#vllm), [[CVE-3]](references.md#security-advisories), [[CVE-4]](references.md#security-advisories), [[CVE-8]](references.md#security-advisories) | | `tools` | array | — | shape-bounded: function schema depth ≤16, nodes ≤256, branch arms ≤16, enum ≤256, size ≤16 KiB; `$ref`/`$defs`/`definitions` forbidden; `pattern` ≤512 B + regex compile; `function.name` ≤64 B; `tools[].function.strict` silent-stripped (vLLM parsers ignore) | [[OpenAI-1]](references.md#openai), [[CVE-2]](references.md#security-advisories) | | `tool_choice` | string\|object | "auto" if tools | shape-strict; `function.name` ≤64 B; `"required"` coerced ([why](troubleshooting.md#coerce-tool-choice-required)) | [[OpenAI-1]](references.md#openai) | | `parallel_tool_calls` | bool | — | pass-through | [[OpenAI-1]](references.md#openai) | @@ -103,6 +104,7 @@ Enforced by the gateway's message validator: - **Lenient SDK compat:** orphan `role: "tool"` messages — those whose `tool_call_id` was never emitted by a prior `assistant.tool_calls[].id` — are silently dropped before validation. Long agent conversations sometimes lose part of a multi-tool fan-out during client-side history compaction. - **Lenient SDK compat:** empty `role: "assistant"` turns — no `content` AND no `tool_calls` AND no `function_call` — are silently dropped. The model can't observe an informationless turn; the drop is a semantic no-op. - **Strict (no lenient compat):** duplicate `tool_calls[].id` within a single assistant message is rejected per OpenAI spec — see [troubleshooting](troubleshooting.md#reject-duplicate-tool-call-id). +- **Route-specific shape:** the `MiniMaxAI/MiniMax-M2.7` route uses a different `role:"tool"` contract — `content` is a `{name,type,text}[]` array, `tool_call_id` is absent (silently stripped if present), and the orphan-drop policy is "no preceding assistant.tool_calls[] block" rather than "no matching tool_call_id". See [accept-tool-message-minimax-shape](troubleshooting.md#accept-tool-message-minimax-shape) and the [MiniMax-M2.7 per-model doc](minimax-m2.7.md). ## Errors diff --git a/docs/chat-api/agents.md b/docs/chat-api/agents.md index e082dda91f..c98d99706c 100644 --- a/docs/chat-api/agents.md +++ b/docs/chat-api/agents.md @@ -28,6 +28,14 @@ Cause: model (typically Kimi-K2.6) emitted duplicate symbolic ids in one assista Cause: Kimi-K2.6 reads from `chat_template_kwargs.thinking` only — the gateway mirrors automatically from top-level `thinking.type`. The top-level field is dropped after mirroring. To override the mirrored value, pre-set `chat_template_kwargs.thinking` directly in your request body. +### "My tool message is rejected on MiniMax-M2.7" + +Cause: MiniMax-M2.7 uses a different `role:"tool"` contract from OpenAI — `content` must be an array of `{name, type:"text", text}` entries, with no `tool_call_id`. Bare-string `content` or OpenAI-style `tool_call_id`-correlated messages are rejected with HTTP 400. See [accept-tool-message-minimax-shape](troubleshooting.md#accept-tool-message-minimax-shape) and the [MiniMax-M2.7 per-model doc](minimax-m2.7.md). Clients dual-emitting `tool_call_id` for portability across routes are fine — the gateway silently strips it. + +### "My `` blocks disappeared on MiniMax-M2.7" + +They shouldn't. MiniMax-M2.7 emits reasoning inline as `...` in `content`, and the gateway preserves these byte-identical across multi-turn history. If you're seeing them stripped, you're probably running them through a client-side reasoning filter — keep them in conversation history for chain-of-thought continuity ([[MiniMax-5]](references.md#minimax)). + ### "My cache key disappeared" Cause: `cache_key` / `prompt_cache_key` silently stripped. vLLM uses a different field (`cache_salt`) for cache isolation, and the aliasing PR is unmerged — see [troubleshooting](troubleshooting.md#strip-cache_key) for the full chain of upstream gaps. diff --git a/docs/chat-api/minimax-m2.7.md b/docs/chat-api/minimax-m2.7.md new file mode 100644 index 0000000000..25b524cec2 --- /dev/null +++ b/docs/chat-api/minimax-m2.7.md @@ -0,0 +1,88 @@ +# MiniMax-M2.7 (`MiniMaxAI/MiniMax-M2.7`) — overrides & extensions + +Provider: MiniMax AI. This doc documents how MiniMax-M2.7 deviates from the [universal contract](README.md). For params that behave the same as universal, see the universal contract directly. + +Mirrors the structure of [Kimi-K2.6](kimi-k2.6.md) and [Qwen3-235B](qwen3-235b-a22b-instruct-2507.md). Chain-side wiring (HfCommit pin, ModelArgs, PoC params) lives in `inference-chain/app/upgrades/v0_2_13/upgrades.go`. + +## Model facts + +| Property | Value | Source | +|----------|-------|--------| +| Provider | MiniMax AI | [[MiniMax-1]](references.md#minimax) | +| vLLM route id | `MiniMaxAI/MiniMax-M2.7` | — | +| Total params | 229 B (sparse MoE; ~10 B activated, inherited from M2 base) | [[MiniMax-2]](references.md#minimax) | +| Tensor types | F32 / BF16 / F8_E4M3 (FP8) | [[MiniMax-2]](references.md#minimax) | +| Max context per sequence | 196 K tokens (180 K served) | [[MiniMax-3]](references.md#minimax) | +| Tool-call parser (vLLM) | `minimax_m2` | [[MiniMax-3]](references.md#minimax), [[vLLM-21]](references.md#vllm) | +| Reasoning parser (vLLM) | `minimax_m2_append_think` (per MiniMax guide); see [parser caveat](#deployment-requirements) | [[MiniMax-3]](references.md#minimax) | +| Native thinking | yes — **interleaved** `...` tags embedded in `content` | [[MiniMax-2]](references.md#minimax), [[MiniMax-4]](references.md#minimax) | +| Recommended sampling | `temperature=1.0, top_p=0.95, top_k=40` | [[MiniMax-2]](references.md#minimax) | + +## Deployment requirements + +Infrastructure-level constraints that must hold BEFORE this route is served — these are enforced by vLLM engine configuration / flags, NOT by the gateway: + +- **HfCommit pinned** — `--trust-remote-code` is required by the model card ([[MiniMax-3]](references.md#minimax)), which puts the deployment inside the blast radius of [[CVE-12]](references.md#security-advisories) (vLLM hardcoded trust_remote_code bypass via malicious model repositories). Mitigation: pin the HuggingFace `revision=` to a verified weights checkpoint. Never serve `revision=main`. +- **Tool-call parser MUST be `minimax_m2`** ([[vLLM-21]](references.md#vllm)). Drives the wire-format the model emits (`` XML, converted to OpenAI `tool_calls[]` on the response by vLLM). +- **Reasoning parser caveat** — official MiniMax recommendation is `--reasoning-parser minimax_m2_append_think` ([[MiniMax-3]](references.md#minimax)). vLLM upstream has known bugs on M2.5+ with this parser ([[vLLM-23]](references.md#vllm), [[vLLM-24]](references.md#vllm), [[vLLM-27]](references.md#vllm)): `extract_reasoning_streaming` assumes no opening `` tag (M2.5+ emits one), and reasoning can be missing from SSE deltas. The visible-content stream still includes the reasoning, just not separated into `reasoning_content`. For the gateway this means `...` blocks land **inline in `content`** on responses — the pass-through design already handles this. +- **Pure TP8 is NOT supported** — vLLM recipe forbids it; H100 TP4+EP4 is the supported topology ([[vLLM-21]](references.md#vllm)). +- **AMD path (MI300X/MI325X/MI350X/MI355X)** requires `VLLM_ROCM_USE_AITER=1 VLLM_ROCM_SHUFFLE_KV_CACHE_LAYOUT=1`; first AITER launch JITs for several minutes ([[vLLM-21]](references.md#vllm)). +- **Avoid Ampere (A100) FP8** — M2.7 FP8 loads on vLLM v0.19.0 but crashes on nightly ([[vLLM-22]](references.md#vllm)). Either stay on a pinned tagged release or use BF16 on Ampere. +- **Disable `pythonic` and `qwen3_coder` tool-call parsers** — same vendor-agnostic CVE history that applies to Kimi/Qwen routes ([[CVE-1]](references.md#security-advisories), [[CVE-7]](references.md#security-advisories)). + +## Parameter overrides + +*Delta from [universal contract](README.md#supported-parameters-universal-behavior). Listed params behave differently on this route; everything else matches universal.* + +| Param | Universal | On MiniMax-M2.7 | Why | +|-------|-----------|-----------------|-----| +| `tools[].function.strict` | (silent-strip in universal via `ToolsValidator`) | silent-strip — vLLM `minimax_m2` parser ignores | [[vLLM-21]](references.md#vllm) | +| `thinking` (top-level Anthropic-style object) | mirrored to `chat_template_kwargs.thinking` on Kimi route | **silent-strip on this route** — MiniMax has no `chat_template_kwargs` toggle for thinking; thinking is always on and emitted inline as `...` in `content`. Clients wanting to suppress thinking display must parse + filter client-side. | [[MiniMax-2]](references.md#minimax), [why](troubleshooting.md#strip-thinking-minimax) | +| `thinking_token_budget` | injected/clamped on Kimi; silent-strip on Qwen | **silent-strip** — no equivalent knob in MiniMax chat template | [[MiniMax-2]](references.md#minimax) | +| `enable_thinking` (top-level) | translated to `chat_template_kwargs.enable_thinking` on Qwen | **silent-strip on this route** — MiniMax does not honor; thinking is structural to the chat template, not configurable per request ([[vLLM-25]](references.md#vllm)). | [why](troubleshooting.md#strip-enable_thinking-minimax) | +| `safety_identifier` | strip (universal); pass-through on Kimi | strip (no MiniMax abuse-tracking API contract) | [[MiniMax-1]](references.md#minimax) | +| Content of `role:"tool"` messages | universal: string or text-part array, flattened to string; `tool_call_id` required | **MiniMax shape**: `content: [{name, type:"text", text}]` array; no `tool_call_id` (silently stripped if dual-emitted). Per-entry caps: ≤16 entries × `name` ≤64 B × `text` ≤64 KiB; closed allow-list of keys. | [[MiniMax-4]](references.md#minimax), [why](troubleshooting.md#accept-tool-message-minimax-shape) | + +## Native extensions + +*Params unique to this route — no equivalent in the universal contract.* + +| Param | Type | Behavior | Source | +|-------|------|----------|--------| +| `extra_body.reasoning_split` | bool | If `true`, vLLM/MiniMax emit reasoning as a separate `reasoning_details[]` array on the response instead of inline `...` blocks in `content`. Gateway unwraps `extra_body` per universal contract; the lifted key passes through to vLLM. Document client responsibility: round-trip `reasoning_details` (or `` blocks) verbatim into history. | [[MiniMax-5]](references.md#minimax) | +| `messages[].reasoning_details` | array | Assistant-turn reasoning history when client uses `reasoning_split=true`. **Pass-through** — assistant-side validator does not strip. **Critical:** stripping these breaks M2.7 interleaved-thinking continuity. | [[MiniMax-5]](references.md#minimax) | +| `...` blocks in assistant `content` | inline tags | **Pass-through verbatim** when round-tripped in history. The gateway message validator MUST NOT strip these from prior assistant turns or rewrite their internals. | [[MiniMax-2]](references.md#minimax), [[MiniMax-5]](references.md#minimax) | +| Tool-call output XML (``) | tag stream | Emitted by the model verbatim; the vLLM `minimax_m2` tool-call parser converts to OpenAI `tool_calls[]` format on the response. Gateway pass-through — no special handling required on the response path. | [[MiniMax-4]](references.md#minimax), [[vLLM-21]](references.md#vllm) | +| MiniMax-only response fields (`base_resp`, `output_sensitive`, `input_sensitive`) | object / bool | **Pass-through** on the response. Document existence; clients may ignore. | [[MiniMax-5]](references.md#minimax) | + +## Rejected MiniMax-platform-only keys + +*Fields the MiniMax direct API (platform.minimax.io) accepts that the gateway rejects with HTTP 400 via the closed top-level allowlist. Listed so clients porting from the MiniMax platform see why their request fails.* + +| Param | Why rejected | +|-------|--------------| +| `partial` | Assistant-prefill / continuation flag on the MiniMax platform. Rejected because it bypasses the gateway's per-role message validation (which treats the trailing turn as a new prompt, not a forced model continuation) and would expose vLLM to a chat-template integrity hazard that the OpenAI Chat Completions contract does not anticipate. Clients wanting equivalent behavior must shape the request through the normal `messages[]` array. [[MiniMax-5]](references.md#minimax) | +| `web_search` / `enable_search` / `search_kwargs` | MiniMax platform-side network-search features; vLLM does not implement them. Silent-stripping would mislead clients into thinking search ran. Fail-loud is the safer default. [[MiniMax-1]](references.md#minimax) | +| `mask_sensitive_info` | MiniMax platform-side content masking; not implemented in vLLM. Same fail-loud rationale. [[MiniMax-1]](references.md#minimax) | + +## Structured outputs + +| Field | Status | Note | +|-------|--------|------| +| `response_format` | ✅ supported (see universal) | xgrammar via vLLM; full schema bounds enforced. Compatible with thinking models since xgrammar runs on the `content`-emission phase, after thinking. | +| `structured_outputs` | ✅ **accepted on this route** | vLLM enforces the constraint on M2.7 (verified with discriminating/control requests across `json`/`regex`/`choice`/`grammar`/`json_object`). `structural_tag` must be the object form — the JSON-encoded string form is rejected (crashes the engine). See [accept-structured_outputs-minimax](troubleshooting.md#accept-structured_outputs-minimax). | + +## Known model-side bugs we work around + +- **Streaming tool_calls malformation** ([[vLLM-26]](references.md#vllm), [[SGLang-1]](references.md#sglang)): under SSE streaming a single tool call is sometimes emitted as two `tool_calls[]` entries — one with `name=null` and duplicated `arguments`. PR #35895 fixed the `stream_interval > 1` case; the `name=null` malformation may still occur with concurrent calls. Documented as a known client-responsibility issue. +- **Tool-call parser fails on `str | null` param types** ([[SGLang-2]](references.md#sglang)): `minimax_m2_tool_parser._convert_param_value` errors on union-with-null param values; clients should avoid `["string","null"]` in tool parameter schemas. +- **Reasoning missing in stream mode** ([[vLLM-27]](references.md#vllm)): when serving with `--reasoning-parser minimax_m2_append_think` and `stream=true`, reasoning_content is sometimes absent from delta chunks. Upstream bug; no gateway mitigation. +- **`thinking` cannot be disabled** ([[vLLM-25]](references.md#vllm)): even with `chat_template_kwargs.enable_thinking=false`, M2.5+ still emits `` blocks. Confirms the strip-`enable_thinking` policy. +- **A100 / Ampere FP8 regression** ([[vLLM-22]](references.md#vllm)): M2.7 FP8 fails on Ampere with vLLM nightly. Mitigated by deployment-side version pin; no gateway responsibility. + +## See also +- [Troubleshooting](troubleshooting.md) +- [References](references.md) +- [Universal contract](README.md) +- [Kimi-K2.6 overrides](kimi-k2.6.md) +- [Qwen3-235B overrides](qwen3-235b-a22b-instruct-2507.md) diff --git a/docs/chat-api/references.md b/docs/chat-api/references.md index 760fe75914..30616e1f55 100644 --- a/docs/chat-api/references.md +++ b/docs/chat-api/references.md @@ -8,10 +8,12 @@ Namespaces: - `[vLLM-N]` vLLM - `[Moonshot-N]` Moonshot (includes Kimi model line) - `[Qwen-N]` Qwen +- `[MiniMax-N]` MiniMax AI (MiniMax-M2 line) +- `[SGLang-N]` SGLang (cross-engine parser bug references) - `[OpenRouter-N]` OpenRouter - `[CVE-N]` security advisories -Industry/community sources (Ollama blog, OpenAI community thread, arxiv papers) are inline links in `troubleshooting.md` and `agents.md`, not here. Captured-requests evidence is referenced inline by request-id. +Industry/community sources (Ollama blog, OpenAI community thread, arxiv papers) are inline links in `troubleshooting.md` and `agents.md`, not here. Captured-requests evidence is referenced inline by request-id. Chain governance changes (model registration, ModelArgs pins) are cited inline by PR number under the appropriate per-model doc, not in this file. ## OpenAI @@ -48,6 +50,14 @@ Industry/community sources (Ollama blog, OpenAI community thread, arxiv papers) - **[vLLM-18]** [Structured outputs feature docs](https://docs.vllm.ai/en/latest/features/structured_outputs.html) — `structured_outputs` supersedes `guided_json`/`guided_regex`/`guided_grammar`/`guided_choice`. - **[vLLM-19]** [PR #29074 — kimi_k2 reasoning parser: emit DeltaMessage when return_token_ids=true](https://github.com/vllm-project/vllm/pull/29074) — changes `extract_reasoning_streaming` to emit an empty `DeltaMessage()` (with the token id attached) instead of `None` for single-token deltas carrying ``/``. Without `return_token_ids=true`, those tokens are silently dropped from the SSE stream while still counted in `usage.completion_tokens`, producing a hidden-token gap that breaks stream-derived `enforced_tokens` reconstruction. - **[vLLM-20]** [kimi_k2_reasoning_parser.py source](https://github.com/vllm-project/vllm/blob/main/vllm/reasoning/kimi_k2_reasoning_parser.py) — the parser whose `extract_reasoning_streaming` returns `None` (= suppresses event) when a delta is exactly `` or ``. Tokens still counted in usage; vLLM-19 added the `return_token_ids` escape valve. +- **[vLLM-21]** [vLLM Recipes — MiniMax-M2 Series Usage Guide](https://docs.vllm.ai/projects/recipes/en/latest/MiniMax/MiniMax-M2.html) — official vLLM recipe covering M2/M2.1/M2.5/M2.7 deployment (parsers, topology, compilation-config, AMD AITER env vars). +- **[vLLM-22]** [Issue #39610 — MiniMax-M2.7/Qwen3.5 FP8 fail on A100/Ampere with nightly](https://github.com/vllm-project/vllm/issues/39610) — regression confirmed vs v0.19.0. +- **[vLLM-23]** [Issue #38212 — MiniMaxM2ReasoningParser broken for M2.5](https://github.com/vllm-project/vllm/issues/38212) — `extract_reasoning_streaming` assumes no opening `` tag; M2.5/.7 do emit it. +- **[vLLM-24]** [Issue #34625 — Reasoning Parser not working with MiniMax M2.5](https://github.com/vllm-project/vllm/issues/34625) — same parser class affects M2.5+ reasoning extraction. +- **[vLLM-25]** [Issue #36778 — Thinking cannot be disabled on M2.5 via chat_template_kwargs](https://github.com/vllm-project/vllm/issues/36778) — drives strip-`enable_thinking` policy on the M2.7 route. +- **[vLLM-26]** [PR #35895 — minimax_m2 tool parser fix for stream_interval > 1](https://github.com/vllm-project/vllm/pull/35895) — covers one streaming malformation class. +- **[vLLM-27]** [Issue #36632 — MiniMax-M2.5 reasoning missing in chat completions stream](https://github.com/vllm-project/vllm/issues/36632) — `--reasoning-parser minimax_m2_append_think` skips reasoning content in SSE deltas. +- **[vLLM-28]** [Issue #28963 — MiniMax tool parsing errors](https://github.com/vllm-project/vllm/issues/28963) — error in `_convert_param_value`. ## Moonshot @@ -62,6 +72,19 @@ Industry/community sources (Ollama blog, OpenAI community thread, arxiv papers) - **[Qwen-2]** [Qwen3-235B-A22B-Instruct-2507-FP8 model card](https://huggingface.co/Qwen/Qwen3-235B-A22B-Instruct-2507-FP8) — FP8 quantised variant model card; same non-thinking confirmation. - **[Qwen-3]** [Qwen vLLM deployment docs](https://qwen.readthedocs.io/en/latest/deployment/vllm.html) — canonical placement for `enable_thinking` inside `chat_template_kwargs`; notes it is not OpenAI API compatible at the top level. +## MiniMax + +- **[MiniMax-1]** [MiniMax platform docs index](https://platform.minimax.io/docs) — overview of MiniMax-hosted API surface. +- **[MiniMax-2]** [MiniMax-M2.7 model card](https://huggingface.co/MiniMaxAI/MiniMax-M2.7) — architecture (229B params, F32/BF16/F8_E4M3 tensor types), sampling recommendations, interleaved `...` constraint. +- **[MiniMax-3]** [MiniMax-M2.7 vLLM deployment guide](https://huggingface.co/MiniMaxAI/MiniMax-M2.7/blob/main/docs/vllm_deploy_guide.md) — exact CLI invocations, GPU sizing (220 GB weights + 240 GB/1M tokens), 196K max context per sequence, parser flags. +- **[MiniMax-4]** [MiniMax-M2.7 tool calling guide](https://huggingface.co/MiniMaxAI/MiniMax-M2.7/blob/main/docs/tool_calling_guide.md) — `` output format; `role:"tool"` with `content:[{name,type,text}]` array (no `tool_call_id`). +- **[MiniMax-5]** [MiniMax M2 Tool Use & Interleaved Thinking docs](https://platform.minimax.io/docs/guides/text-m2-function-call) — `extra_body.reasoning_split`, `reasoning_details[]` response field, multi-turn history rules. + +## SGLang + +- **[SGLang-1]** [Issue #23071 — MiniMax-M2 streaming tool_calls malformed](https://github.com/sgl-project/sglang/issues/23071) — under SSE streaming a single tool call sometimes splits into two entries (`name=null` + duplicated arguments). +- **[SGLang-2]** [Issue #16057 — MiniMax M2 tool parser failure on `str | null` union param types](https://github.com/sgl-project/sglang/issues/16057) — parser crashes on union-with-null in tool parameter schema; potential DoS vector via crafted tool schema. + ## OpenRouter - **[OpenRouter-1]** [Chat Completion API reference](https://openrouter.ai/docs/api/api-reference/chat/send-chat-completion-request) — OpenRouter wire schema for `/api/v1/chat/completions`. @@ -84,4 +107,7 @@ Industry/community sources (Ollama blog, OpenAI community thread, arxiv papers) - **[CVE-9]** [CVE-2026-34756 / GHSA-3mwp-wvh9-7528 (vLLM)](https://github.com/vllm-project/vllm/security/advisories/GHSA-3mwp-wvh9-7528) — unbounded `n` causes OOM; `CapUintParameterHandler` clamps `n ≤ 5`. - **[CVE-10]** [CVE-2026-44222 / GHSA-hpv8-x276-m59f (vLLM)](https://github.com/vllm-project/vllm/security/advisories/GHSA-hpv8-x276-m59f) — special-token literals crash VL models; requires content sanitizer for Kimi-K2.6 multimodal path. - **[CVE-11]** [CVE-2026-44223 / GHSA-83vm-p52w-f9pw (vLLM)](https://github.com/vllm-project/vllm/security/advisories/GHSA-83vm-p52w-f9pw) — penalty fields crash EngineCore with `extract_hidden_states` spec decode; pin vLLM ≥ 0.20.0. +- **[CVE-12]** [CVE-2026-27893 / RAXE-2026-044 (vLLM)](https://raxe.ai/labs/advisories/RAXE-2026-044) — vLLM hardcoded trust_remote_code bypass enables RCE via malicious model repositories; direct mitigation for the MiniMax-M2.7 route is the chain-pinned `HfCommit` SHA in the governance model config. +- **[CVE-13]** [CVE-2026-22778 (vLLM)](https://www.ox.security/blog/cve-2026-22778-vllm-rce-vulnerability/) — RCE via crafted video link in multimodal content; gateway mitigates by rejecting non-text content parts on all text-only routes (M2.7 included). +- **[CVE-14]** [CVE-2025-62164 / GHSA-mrw7-hf4f-83pf (vLLM)](https://github.com/advisories/GHSA-mrw7-hf4f-83pf) — tensor deserialization → DoS / potential RCE; pin vLLM ≥ patched release. diff --git a/docs/chat-api/troubleshooting.md b/docs/chat-api/troubleshooting.md index d0b5c22698..59fef10a92 100644 --- a/docs/chat-api/troubleshooting.md +++ b/docs/chat-api/troubleshooting.md @@ -7,6 +7,7 @@ Every parameter that is stripped / rejected / normalized at the gateway is docum | HTTP / behavior | Common cause | Anchor | |-----------------|--------------|--------| | 400 `"" is currently rejected by the Gonka network` | non-allowlist parameter | [#reject-unknown-param](#reject-unknown-param) | +| 400 `...name must be a non-empty string` on a whitespace-only name | names are trimmed before the non-empty check | [#reject-whitespace-names](#reject-whitespace-names) | | 400 `messages[N].tool_calls[M].id is duplicated` | Kimi-K2.6 model-side bug | [#reject-duplicate-tool-call-id](#reject-duplicate-tool-call-id) | | 400 on `tags` | undocumented field | [#reject-tags](#reject-tags) | | 400 on `guided_json` / `guided_regex` / etc. | vLLM-native structured decoding | [#reject-guided-decoding](#reject-guided-decoding) | @@ -200,6 +201,48 @@ Every parameter that is stripped / rejected / normalized at the gateway is docum --- +### #strip-thinking-minimax + +**What**: top-level `thinking: {...}` (Anthropic-style wrapper) silent-stripped on the MiniMax-M2.7 route before `ThinkingValidator` runs. + +**Why**: MiniMax-M2.7 has no `chat_template_kwargs` switch for thinking — interleaved reasoning is structural to the chat template and always-on. The model emits `...` blocks inline in `content` by design ([[MiniMax-2]](references.md#minimax)). Mirroring `thinking` into template kwargs (as on Kimi-K2.6) or normalizing it (as on Qwen-style routes) is a no-op on this route. Silent-strip is the closest behavior to the model's actual contract. + +**When to restore**: when MiniMax adds a documented per-request thinking toggle. + +**Fix (client-side)**: stop sending `thinking`; for clients that need to suppress the visible thinking display, parse + filter `...` blocks client-side from the response content. + +**Captured-requests**: n/a — pre-deployment design decision. + +--- + +### #strip-enable_thinking-minimax + +**What**: top-level `enable_thinking: bool` silent-stripped on the MiniMax-M2.7 route before `EnableThinkingValidator` runs (which would otherwise translate it into `chat_template_kwargs.enable_thinking`). + +**Why**: vLLM Issue [vLLM-25](references.md#vllm) ([#36778](https://github.com/vllm-project/vllm/issues/36778)) confirms `chat_template_kwargs.enable_thinking=false` does NOT disable thinking on M2.5+. Forwarding the translated kwarg would silently mislead clients into believing they'd disabled reasoning when in fact the model still emits `` blocks. Strip avoids the misleading appearance of effect. + +**When to restore**: when the upstream Issue is fixed and the kwarg is honored on the M2 line. + +**Fix (client-side)**: stop sending `enable_thinking` on this route — see [strip-thinking-minimax](#strip-thinking-minimax) for the broader story. + +**Captured-requests**: n/a — pre-deployment design decision. + +--- + +### #strip-tool_call_id-minimax + +**What**: `tool_call_id` on `role:"tool"` messages silent-stripped on the MiniMax-M2.7 route during message normalization. + +**Why**: MiniMax-M2.7 tool messages correlate by `name` + positional order within a tool-result block, not by `tool_call_id` ([[MiniMax-4]](references.md#minimax)). Clients porting OpenAI code may dual-emit the field. Forwarding it is harmless (vLLM ignores) but rejecting it would force every OpenAI-compat client to fork their tool-message serializer per route. Silent-strip preserves compat without implying semantics we don't honor. + +**When to restore**: never — the field has no consumer on this route. + +**Fix (client-side)**: omit `tool_call_id` for cleanest payloads. If you must dual-emit (e.g. shared serializer across routes), it's a free pass-through to silent-strip. + +**Captured-requests**: n/a — anticipated dual-emission from porting clients. + +--- + ## Validates-then-strips ### #strip-reasoning_effort @@ -411,9 +454,57 @@ Every parameter that is stripped / rejected / normalized at the gateway is docum **Captured-requests**: n/a — no captures observed. +--- + +### #accept-structured_outputs-minimax + +**What**: `structured_outputs` is accepted on the `MiniMaxAI/MiniMax-M2.7` route (not rejected). Only Kimi-K2.6 rejects the field. + +**Why**: The vLLM structured-output manager actively enforces the constraint on this route — verified end-to-end with paired discriminating/control requests: the `json`, `regex`, `choice`, `grammar`, and `json_object` kinds each steered output away from the natural answer the control produced. The engine holds the constraint, so there is no need to gate the field at the gateway. + +**Caveat — `structural_tag` must be the OBJECT form**: a JSON-encoded *string* `structural_tag` crashes vLLM with an HTTP 500 on the request handler, so the gateway rejects the string form with a 400 on every route. Send the object shape (`{"type":"structural_tag","structures":[...],"triggers":[...]}`). + +**Caveat — `response_format` conflict**: `structured_outputs` still cannot be combined with `response_format` (see [#reject-structured_outputs-with-response_format](#reject-structured_outputs-with-response_format)). + +**Captured-requests**: n/a — pre-deployment design decision. + +--- + +### #reject-whitespace-names + +**What**: HTTP 400 on a name field that is present but contains only whitespace — e.g. `tools[].function.name`, `tool_choice.function.name`, `response_format.json_schema.name`, or a MiniMax tool-result entry `name`. The error reads `...name must be a non-empty string`. + +**Why**: Every name validator trims with `strings.TrimSpace` *before* its non-empty check. Without the trim a whitespace-only name passes the gateway, reaches the engine as an effectively empty / nonexistent tool or schema name, and the request produces no usable output — it hangs until the deadline (0 bytes) and ties up a request slot/escrow. This is the same hang class as `max_tokens: 0` and `n: 0`. The engine is not crashed; the request just never completes. + +**Maintainer note**: when adding any new name/identifier field to a validator, check it with `strings.TrimSpace(value) == ""` (or `messagevalidators.RequiredNonEmptyString`), never a bare `== ""` — a bare empty-string check is a whitespace-bypass that re-opens this hang. + +**Fix (client-side)**: send a real (non-blank) name. + +--- + +### #accept-tool-message-minimax-shape + +**What**: On the MiniMaxAI/MiniMax-M2.7 route, `role:"tool"` messages MUST carry `content` as an array of `{name, type:"text", text}` objects (the MiniMax-native shape per [[MiniMax-4]](references.md#minimax)) — not the OpenAI string + `tool_call_id` shape. Bare-string content is rejected with HTTP 400. + +**Why**: MiniMax's tool-calling contract correlates tool results by per-entry `name` + positional order inside a tool-result block; there is no `tool_call_id`. The MinimaxToolMessage content validator (registered in the per-model role policy override) enforces the entry shape and caps (≤16 entries, name ≤64 B, text ≤64 KiB, closed allow-list of keys to defend against [[SGLang-2]](references.md#sglang) union-with-null crash class). `tool_call_id`, if dual-emitted by a client porting from OpenAI, is silently stripped — see [strip-tool_call_id-minimax](#strip-tool_call_id-minimax). + +**When to restore**: when MiniMax adds OpenAI-compat tool-message handling to their parser. + +**Fix (client-side)**: emit the M2.7 shape: +```json +{"role": "tool", + "content": [ + {"name": "", "type": "text", "text": ""} + ]} +``` +Multiple parallel tool results in one block: one array entry per call. + +**Captured-requests**: n/a — pre-deployment design decision. + ## Per-model gotchas Brief pointers to deeper notes in per-model docs: - **Kimi-K2.6**: [Known model-side bugs we work around](kimi-k2.6.md#known-model-side-bugs-we-work-around) - **Qwen3-235B-A22B-Instruct-2507**: [Known model-side bugs we work around](qwen3-235b-a22b-instruct-2507.md#known-model-side-bugs-we-work-around) +- **MiniMaxAI/MiniMax-M2.7**: [Known model-side bugs we work around](minimax-m2.7.md#known-model-side-bugs-we-work-around) From d55dd16dc0debd38465d6cfe24bd13d73d38b6cc Mon Sep 17 00:00:00 2001 From: Daniil Yankouski Date: Tue, 30 Jun 2026 12:08:45 +0200 Subject: [PATCH 04/13] fix(devshard): skip escrow rotation for models absent from the network (#1375) --- devshard/cmd/devshardctl/escrow_rotator.go | 23 ++++ .../cmd/devshardctl/gateway_store_test.go | 130 ++++++++++++++++++ 2 files changed, 153 insertions(+) diff --git a/devshard/cmd/devshardctl/escrow_rotator.go b/devshard/cmd/devshardctl/escrow_rotator.go index 1300a419ab..794dad694d 100644 --- a/devshard/cmd/devshardctl/escrow_rotator.go +++ b/devshard/cmd/devshardctl/escrow_rotator.go @@ -5,6 +5,7 @@ import ( "errors" "fmt" "log" + "slices" "strconv" "strings" "time" @@ -263,6 +264,12 @@ func (g *Gateway) ensureRotationEscrows(ctx context.Context, settings GatewaySet } } result.ExistingCount = count + if count < target { + if served, known := g.rotationModelServedByNetwork(model.ModelID); known && !served { + log.Printf("escrow_rotation_skip_model_absent role=%s epoch=%d model=%q reason=model_not_in_network", role, epoch, model.ModelID) + return result, nil + } + } if count < target && g.rotationCreateFailed(model.ModelID, role, epoch) { return result, errEscrowRotationCreateSuppressed } @@ -277,6 +284,22 @@ func (g *Gateway) ensureRotationEscrows(ctx context.Context, settings GatewaySet return result, nil } +// rotationModelServedByNetwork reports whether the model is served; known is false on cold start (empty model set) so callers don't skip a genuinely-served model. +func (g *Gateway) rotationModelServedByNetwork(modelID string) (served bool, known bool) { + if g == nil || g.capacity == nil { + return false, false + } + networkModels := g.capacity.Models() + if len(networkModels) == 0 { + return false, false + } + modelID = strings.TrimSpace(modelID) + if slices.Contains(networkModels, modelID) { + return true, true + } + return false, true +} + func (g *Gateway) createRotationEscrow(ctx context.Context, settings GatewaySettings, model EscrowRotationModelSettings, role string, epoch uint64) (*CreateDevshardEscrowResult, error) { signer, _, err := signerFromRequestKey("", model.PrivateKeyEnv) if err != nil { diff --git a/devshard/cmd/devshardctl/gateway_store_test.go b/devshard/cmd/devshardctl/gateway_store_test.go index 9b150ba8a3..b2702d6aa2 100644 --- a/devshard/cmd/devshardctl/gateway_store_test.go +++ b/devshard/cmd/devshardctl/gateway_store_test.go @@ -447,6 +447,136 @@ func TestEscrowRotationFinishDoesNotSettleTempWhenRegularCreateFails(t *testing. require.Equal(t, 0, settleAttempts) } +func TestEscrowRotationSkipsCreateWhenModelAbsentFromNetwork(t *testing.T) { + store, err := NewGatewayStore(filepath.Join(t.TempDir(), "gateway.db")) + require.NoError(t, err) + t.Cleanup(func() { + require.NoError(t, store.Close()) + }) + + settings := GatewaySettings{ + ChainREST: "http://node:1317", + PublicAPI: "http://api:9000", + DefaultModel: "Qwen/Test", + DefaultRequestMaxTokens: 1000, + MaxConcurrentRequests: 2, + EscrowRotation: EscrowRotationSettings{ + Enabled: true, + SettlementEnabled: true, + Models: []EscrowRotationModelSettings{{ + ModelID: "Qwen/Removed", + TempCount: 1, + TargetCount: 16, + Amount: 1000, + PrivateKeyEnv: "DEVSHARD_PRIVATE_KEY", + }}, + }, + }.WithTuningDefaults() + // A stale temp bridge escrow remains for a model the network no longer + // serves; finishing rotation must NOT broadcast a regular create (which + // the chain rejects with ErrEpochGroupDataNotFound and still burns the + // gas fee) but MUST still settle the stranded temp escrow to recover funds. + require.NoError(t, store.Initialize(settings, []GatewayDevshardState{{ + RuntimeConfig: RuntimeConfig{ID: "12", PrivateKeyHex: "secret", Model: "Qwen/Removed"}, + Active: true, + RotationRole: rotationRoleTemp, + RotationEpoch: 10, + }})) + + oldCreate := gatewayCreateRotationEscrow + oldSettle := gatewaySettleDevshardOnChain + createAttempts := 0 + var settled []string + gatewayCreateRotationEscrow = func(*Gateway, context.Context, GatewaySettings, EscrowRotationModelSettings, string, uint64) (*CreateDevshardEscrowResult, error) { + createAttempts++ + return &CreateDevshardEscrowResult{EscrowID: uint64(90 + createAttempts), TxHash: "OK"}, nil + } + gatewaySettleDevshardOnChain = func(_ *Gateway, _ context.Context, id string, _ adminSettleEscrowRequest) (*SettleDevshardEscrowResult, error) { + settled = append(settled, id) + return nil, nil + } + t.Cleanup(func() { + gatewayCreateRotationEscrow = oldCreate + gatewaySettleDevshardOnChain = oldSettle + }) + + // The network serves a different model; "Qwen/Removed" is positively absent. + capacity := NewCapacityState() + capacity.SetHostWeightViews( + map[string]float64{"host": 1}, + map[string]float64{"host": 1}, + map[string]map[string]float64{"Qwen/Present": {"host": 1}}, + map[string]map[string]float64{"Qwen/Present": {"host": 1}}, + ) + + g := &Gateway{store: store, capacity: capacity, rotationFailures: make(map[string]struct{})} + g.finishBridgeEscrows(ChainPhaseSnapshot{EpochIndex: 11}, settings) + + require.Equal(t, 0, createAttempts, "must not create escrow for a model absent from the network") + require.Equal(t, []string{"12"}, settled, "stranded temp escrow must still be settled") +} + +func TestEscrowRotationCreatesWhenModelPresentInNetwork(t *testing.T) { + store, err := NewGatewayStore(filepath.Join(t.TempDir(), "gateway.db")) + require.NoError(t, err) + t.Cleanup(func() { + require.NoError(t, store.Close()) + }) + + settings := GatewaySettings{ + ChainREST: "http://node:1317", + PublicAPI: "http://api:9000", + DefaultModel: "Qwen/Test", + DefaultRequestMaxTokens: 1000, + MaxConcurrentRequests: 2, + EscrowRotation: EscrowRotationSettings{ + Enabled: true, + SettlementEnabled: true, + Models: []EscrowRotationModelSettings{{ + ModelID: "Qwen/Test", + TempCount: 1, + TargetCount: 2, + Amount: 1000, + PrivateKeyEnv: "DEVSHARD_PRIVATE_KEY", + }}, + }, + }.WithTuningDefaults() + require.NoError(t, store.Initialize(settings, []GatewayDevshardState{{ + RuntimeConfig: RuntimeConfig{ID: "12", PrivateKeyHex: "secret", Model: "Qwen/Test"}, + Active: true, + RotationRole: rotationRoleTemp, + RotationEpoch: 10, + }})) + + oldCreate := gatewayCreateRotationEscrow + oldSettle := gatewaySettleDevshardOnChain + createAttempts := 0 + gatewayCreateRotationEscrow = func(*Gateway, context.Context, GatewaySettings, EscrowRotationModelSettings, string, uint64) (*CreateDevshardEscrowResult, error) { + createAttempts++ + return &CreateDevshardEscrowResult{EscrowID: uint64(90 + createAttempts), TxHash: "OK"}, nil + } + gatewaySettleDevshardOnChain = func(*Gateway, context.Context, string, adminSettleEscrowRequest) (*SettleDevshardEscrowResult, error) { + return nil, nil + } + t.Cleanup(func() { + gatewayCreateRotationEscrow = oldCreate + gatewaySettleDevshardOnChain = oldSettle + }) + + capacity := NewCapacityState() + capacity.SetHostWeightViews( + map[string]float64{"host": 1}, + map[string]float64{"host": 1}, + map[string]map[string]float64{"Qwen/Test": {"host": 1}}, + map[string]map[string]float64{"Qwen/Test": {"host": 1}}, + ) + + g := &Gateway{store: store, capacity: capacity, rotationFailures: make(map[string]struct{})} + g.finishBridgeEscrows(ChainPhaseSnapshot{EpochIndex: 11}, settings) + + require.Equal(t, 2, createAttempts, "must create escrows for a model the network serves") +} + func TestEscrowRotationFinishSettlesTempFromCurrentLatestEpoch(t *testing.T) { store, err := NewGatewayStore(filepath.Join(t.TempDir(), "gateway.db")) require.NoError(t, err) From b173572ce4defa6e319f3ebad357929f485ba60e Mon Sep 17 00:00:00 2001 From: Daniil Yankouski Date: Thu, 2 Jul 2026 11:43:03 +0200 Subject: [PATCH 05/13] fix(devshard): clamp chat n into [1,5] so n=0 no longer hangs the request (#1316) --- devshard/cmd/devshardctl/paramvalidators/handlers.go | 4 ++++ devshard/cmd/devshardctl/request_filters_parameters.go | 2 +- devshard/cmd/devshardctl/request_filters_test.go | 7 +++++++ 3 files changed, 12 insertions(+), 1 deletion(-) diff --git a/devshard/cmd/devshardctl/paramvalidators/handlers.go b/devshard/cmd/devshardctl/paramvalidators/handlers.go index f9df7d7522..810fc1459c 100644 --- a/devshard/cmd/devshardctl/paramvalidators/handlers.go +++ b/devshard/cmd/devshardctl/paramvalidators/handlers.go @@ -155,6 +155,7 @@ func (h ForceLiteralParameter) HandleParameter(ctx ParameterContext) error { // CapUintParameter caps a uint64-shaped field to Max. type CapUintParameter struct { + Min uint64 Max uint64 } @@ -163,6 +164,9 @@ func (h CapUintParameter) HandleParameter(ctx ParameterContext) error { if !ok { return nil } + if value < h.Min { + ctx.Document[ctx.Parameter] = h.Min + } if value > h.Max { ctx.Document[ctx.Parameter] = h.Max } diff --git a/devshard/cmd/devshardctl/request_filters_parameters.go b/devshard/cmd/devshardctl/request_filters_parameters.go index 2e1b4c99f4..7a3237a637 100644 --- a/devshard/cmd/devshardctl/request_filters_parameters.go +++ b/devshard/cmd/devshardctl/request_filters_parameters.go @@ -425,7 +425,7 @@ func defaultVLLMParameterCatalog() VLLMParameterCatalog { newParameter("seed"). withRule(RequestFilterStagePreValidation, ParameterHandlerAdapter{Handler: paramvalidators.ValidateUintParameter{}}), newParameter("n"). - withRule(RequestFilterStagePostLimits, ParameterHandlerAdapter{Handler: paramvalidators.CapUintParameter{Max: MaxChatRequestChoices}}). + withRule(RequestFilterStagePostLimits, ParameterHandlerAdapter{Handler: paramvalidators.CapUintParameter{Min: 1, Max: MaxChatRequestChoices}}). withRule(RequestFilterStagePostLimits, DocumentValidatorHandler{ Validator: paramvalidators.GreedySamplingValidator{}, }), diff --git a/devshard/cmd/devshardctl/request_filters_test.go b/devshard/cmd/devshardctl/request_filters_test.go index 2f4387a255..db8982954e 100644 --- a/devshard/cmd/devshardctl/request_filters_test.go +++ b/devshard/cmd/devshardctl/request_filters_test.go @@ -203,6 +203,13 @@ func TestNormalizeChatRequestCapsChoices(t *testing.T) { require.NotContains(t, string(body), `"n"`) } +func TestNormalizeChatRequestClampsZeroChoicesToOne(t *testing.T) { + body, req, err := normalizeChatRequest([]byte(`{"n":0,"messages":[{"role":"user","content":"hi"}]}`)) + require.NoError(t, err) + require.EqualValues(t, 1, req.N) + require.Contains(t, string(body), `"n":1`) +} + func TestNormalizeChatRequestClampsMinTokensAboveEffectiveMax(t *testing.T) { oldDefault := DefaultRequestMaxTokens oldCap := RequestMaxTokensCap From 13a533c93fb089b8ad51431b1c4217541f28b71f Mon Sep 17 00:00:00 2001 From: Daniil Yankouski Date: Thu, 2 Jul 2026 11:51:24 +0200 Subject: [PATCH 06/13] fix(devshard): validate & clamp chat-completion params at the gateway instead of vLLM (#1318) * feat(devshard): validate and clamp chat-completion params at the gateway, unify and extract filter handlers * fix(devshard): mirror max_completion_tokens into max_tokens so the Kimi thinking-budget bounds reasoning (no empty output) --- .../devshardctl/paramvalidators/handlers.go | 68 +++++++++ .../paramvalidators/validate_test.go | 102 +++++++++++++ devshard/cmd/devshardctl/request_filters.go | 1 + .../cmd/devshardctl/request_filters_config.go | 4 + .../devshardctl/request_filters_parameters.go | 113 +++++++++----- .../cmd/devshardctl/request_filters_test.go | 139 +++++++++++++++++- docs/chat-api/README.md | 32 ++-- docs/chat-api/troubleshooting.md | 39 +++++ 8 files changed, 440 insertions(+), 58 deletions(-) create mode 100644 devshard/cmd/devshardctl/paramvalidators/validate_test.go diff --git a/devshard/cmd/devshardctl/paramvalidators/handlers.go b/devshard/cmd/devshardctl/paramvalidators/handlers.go index 810fc1459c..e9fc6a679a 100644 --- a/devshard/cmd/devshardctl/paramvalidators/handlers.go +++ b/devshard/cmd/devshardctl/paramvalidators/handlers.go @@ -210,6 +210,74 @@ func (h ValidateUintParameter) HandleParameter(ctx ParameterContext) error { return nil } +// RejectNumberParameter rejects the request when the parameter is present and parses as a +// number but Allow returns false for it. Non-numeric or absent values pass through so the +// dedicated type/shape validators own those cases. Used for range gates whose lower bound is +// exclusive (e.g. top_p > 0), where clamping to the bound would itself be an illegal value. +type RejectNumberParameter struct { + Allow func(float64) bool + Message string +} + +func (h RejectNumberParameter) HandleParameter(ctx ParameterContext) error { + value, exists := ctx.Document[ctx.Parameter] + if !exists { + return nil + } + number, ok := devshard.JSONNumericFloat64(value) + if !ok { + return nil + } + if h.Allow == nil || h.Allow(number) { + return nil + } + return fmt.Errorf("%s: %s", ctx.Parameter, h.Message) +} + +// ValidateScalarParameter rejects the request when the parameter is present (non-null) but +// Valid returns false for its raw JSON value. Catches wrong-typed scalars at the boundary +// instead of forwarding them for an opaque upstream 400. +type ValidateScalarParameter struct { + Valid func(any) bool + Message string +} + +func (h ValidateScalarParameter) HandleParameter(ctx ParameterContext) error { + value, exists := ctx.Document[ctx.Parameter] + if !exists || value == nil { + return nil + } + if h.Valid == nil || h.Valid(value) { + return nil + } + return fmt.Errorf("%s: %s", ctx.Parameter, h.Message) +} + +// ValidateListElementsParameter rejects the request when any array element fails Valid. +// Pass-through when the field is absent or not an array. +type ValidateListElementsParameter struct { + Valid func(any) bool + Message string +} + +func (h ValidateListElementsParameter) HandleParameter(ctx ParameterContext) error { + raw, ok := ctx.Document[ctx.Parameter].([]any) + if !ok { + return nil + } + for index, item := range raw { + if h.Valid != nil && !h.Valid(item) { + return fmt.Errorf("%s[%d]: %s", ctx.Parameter, index, h.Message) + } + } + return nil +} + +// JSON type predicates for the Validate*Parameter handlers above. +func IsJSONBool(value any) bool { _, ok := value.(bool); return ok } +func IsJSONString(value any) bool { _, ok := value.(string); return ok } +func IsJSONUint(value any) bool { _, ok := devshard.JSONNumericUint64(value); return ok } + // LengthCapListParameter bounds the number of entries in an array field, and // optionally the byte length of each string entry. MaxEntries=0 disables the // array cap; MaxEntryLen=0 disables the per-string cap. diff --git a/devshard/cmd/devshardctl/paramvalidators/validate_test.go b/devshard/cmd/devshardctl/paramvalidators/validate_test.go new file mode 100644 index 0000000000..0445bbbe63 --- /dev/null +++ b/devshard/cmd/devshardctl/paramvalidators/validate_test.go @@ -0,0 +1,102 @@ +package paramvalidators + +import ( + "strings" + "testing" +) + +// ============================================================ +// RejectNumberParameter +// ============================================================ + +func TestRejectNumberParameter(t *testing.T) { + positive := RejectNumberParameter{Allow: func(v float64) bool { return v > 0 }, Message: "must be greater than 0"} + + // Absent and non-numeric values pass through — dedicated shape validators own those. + if err := run(positive, map[string]any{}, "top_p"); err != nil { + t.Fatalf("absent field must pass through, got %v", err) + } + if err := run(positive, map[string]any{"top_p": "not-a-number"}, "top_p"); err != nil { + t.Fatalf("non-numeric must pass through, got %v", err) + } + if err := run(positive, map[string]any{"top_p": 0.5}, "top_p"); err != nil { + t.Fatalf("allowed value must pass, got %v", err) + } + + err := run(positive, map[string]any{"top_p": 0}, "top_p") + if err == nil || !strings.Contains(err.Error(), "top_p") || !strings.Contains(err.Error(), "greater than 0") { + t.Fatalf("non-positive must be rejected with a field-named error, got %v", err) + } +} + +func TestRejectNumberParameter_TopKPredicate(t *testing.T) { + topK := RejectNumberParameter{Allow: func(v float64) bool { return v == -1 || v >= 1 }, Message: "must be -1 or a positive integer"} + for _, good := range []any{-1, 1, 50} { + if err := run(topK, map[string]any{"top_k": good}, "top_k"); err != nil { + t.Fatalf("top_k=%v must pass, got %v", good, err) + } + } + for _, bad := range []any{0, -2} { + if err := run(topK, map[string]any{"top_k": bad}, "top_k"); err == nil { + t.Fatalf("top_k=%v must be rejected", bad) + } + } +} + +// ============================================================ +// ValidateScalarParameter +// ============================================================ + +func TestValidateScalarParameter(t *testing.T) { + mustBeBool := ValidateScalarParameter{Valid: IsJSONBool, Message: "must be a boolean"} + + // Absent or explicit null passes through. + if err := run(mustBeBool, map[string]any{}, "stream"); err != nil { + t.Fatalf("absent must pass, got %v", err) + } + if err := run(mustBeBool, map[string]any{"stream": nil}, "stream"); err != nil { + t.Fatalf("null must pass, got %v", err) + } + if err := run(mustBeBool, map[string]any{"stream": true}, "stream"); err != nil { + t.Fatalf("bool must pass, got %v", err) + } + if err := run(mustBeBool, map[string]any{"stream": "yes"}, "stream"); err == nil { + t.Fatal("non-bool must be rejected") + } +} + +// ============================================================ +// ValidateListElementsParameter +// ============================================================ + +func TestValidateListElementsParameter(t *testing.T) { + elementsMustBeString := ValidateListElementsParameter{Valid: IsJSONString, Message: "must be a string"} + + // Absent or non-array passes through. + if err := run(elementsMustBeString, map[string]any{}, "stop"); err != nil { + t.Fatalf("absent must pass, got %v", err) + } + if err := run(elementsMustBeString, map[string]any{"stop": "single"}, "stop"); err != nil { + t.Fatalf("non-array must pass, got %v", err) + } + if err := run(elementsMustBeString, map[string]any{"stop": []any{"a", "b"}}, "stop"); err != nil { + t.Fatalf("all-string array must pass, got %v", err) + } + + err := run(elementsMustBeString, map[string]any{"stop": []any{"a", 5, "c"}}, "stop") + if err == nil || !strings.Contains(err.Error(), "stop[1]") { + t.Fatalf("bad element must be rejected with its index, got %v", err) + } +} + +func TestJSONTypePredicates(t *testing.T) { + if !IsJSONBool(true) || IsJSONBool("x") { + t.Fatal("IsJSONBool") + } + if !IsJSONString("x") || IsJSONString(1) { + t.Fatal("IsJSONString") + } + if !IsJSONUint(5) || IsJSONUint(-1) || IsJSONUint(3.5) { + t.Fatal("IsJSONUint") + } +} diff --git a/devshard/cmd/devshardctl/request_filters.go b/devshard/cmd/devshardctl/request_filters.go index 142aee53ba..98084e2677 100644 --- a/devshard/cmd/devshardctl/request_filters.go +++ b/devshard/cmd/devshardctl/request_filters.go @@ -139,6 +139,7 @@ func (p ChatRequestPipeline) applyOutputTokenLimits(ctx *RequestFilterContext) { case hasMaxCompletionTokens: maxCompletionTokens := capOutputTokens(ctx.Request.MaxCompletionTokens, true, ctx.AdminAuthenticated, limits) ctx.Document.Set("max_completion_tokens", maxCompletionTokens) + ctx.Document.Set("max_tokens", maxCompletionTokens) ctx.Request.MaxCompletionTokens = maxCompletionTokens ctx.Request.MaxTokens = maxCompletionTokens default: diff --git a/devshard/cmd/devshardctl/request_filters_config.go b/devshard/cmd/devshardctl/request_filters_config.go index a24aae7b63..d5462222cd 100644 --- a/devshard/cmd/devshardctl/request_filters_config.go +++ b/devshard/cmd/devshardctl/request_filters_config.go @@ -5,7 +5,11 @@ const ( MaxChatRequestBodySize = 10 * 1024 * 1024 MaxLoggedResponseFormatBytes = 2048 * 1024 MaxChatRequestChoices = 5 + MinTemperature = 0.0 MaxTemperature = 2.0 + MinPMin = 0.0 + MinPMax = 1.0 + TopPMax = 1.0 MaxRepetitionPenalty = 2.0 ) diff --git a/devshard/cmd/devshardctl/request_filters_parameters.go b/devshard/cmd/devshardctl/request_filters_parameters.go index 7a3237a637..1b5daa9a07 100644 --- a/devshard/cmd/devshardctl/request_filters_parameters.go +++ b/devshard/cmd/devshardctl/request_filters_parameters.go @@ -110,7 +110,6 @@ func (h MinUintParameterHandler) Apply(ctx *RequestFilterContext, parameter VLLM return nil } - // DocumentValidator: validators in paramvalidators expose this contract. May mutate // vctx.Document for per-model rewrites alongside shape checks. type DocumentValidator interface { @@ -343,18 +342,11 @@ func (ctx *RequestFilterContext) DecodeRequest() error { return nil } -// SyncRequestView refreshes ctx.Request after PostLimits rules ran. Why we explicitly -// preserve the token fields instead of re-reading them from the document: -// -// - When the client sends only `max_completion_tokens` (no `max_tokens`), -// `applyOutputTokenLimits` sets `ctx.Request.MaxTokens` from the resolved -// `max_completion_tokens` (see request_filters.go:139) but does NOT write a -// corresponding `max_tokens` key into the document. Re-reading the document would -// therefore reset `req.MaxTokens` to 0. -// - In the other three branches of `applyOutputTokenLimits`, the document DOES carry -// the same value, so preserving from `ctx.Request` is a no-op. Net effect: this -// branch only matters for the max-completion-only path, locked in by -// TestNormalizeChatRequestDefaultsAndCapsOutputTokens. +// SyncRequestView refreshes ctx.Request after PostLimits rules ran. The token fields +// are preserved from ctx.Request rather than re-read because applyOutputTokenLimits is +// their source of truth; all four of its branches now write the same max_tokens / +// max_completion_tokens into the document (the max-completion-only branch mirrors into +// max_tokens too), so this preservation is a harmless no-op safety net. // // Other fields are re-read so caps applied by PostLimits rules (for example `n` via // paramvalidators.CapUintParameter through the adapter) propagate into the projection. @@ -416,6 +408,26 @@ type VLLMParameterCatalog struct { var defaultParameterCatalog = defaultVLLMParameterCatalog() +// Shared stateless parameter handlers reused across catalog entries. The rejectNumber gates +// enforce exclusive-lower-bound ranges (clamping to the bound would itself be an illegal +// value, so reject instead); the mustBe*/elementsMustBe* validators reject wrong-typed +// scalars and array elements at the gateway boundary rather than forwarding them for an +// opaque upstream 400. +var ( + rejectNonPositiveNumber = ParameterHandlerAdapter{Handler: paramvalidators.RejectNumberParameter{ + Allow: func(value float64) bool { return value > 0 }, + Message: "must be greater than 0", + }} + rejectInvalidTopK = ParameterHandlerAdapter{Handler: paramvalidators.RejectNumberParameter{ + Allow: func(value float64) bool { return value == -1 || value >= 1 }, + Message: "must be -1 or a positive integer", + }} + mustBeBool = ParameterHandlerAdapter{Handler: paramvalidators.ValidateScalarParameter{Valid: paramvalidators.IsJSONBool, Message: "must be a boolean"}} + mustBeUint = ParameterHandlerAdapter{Handler: paramvalidators.ValidateUintParameter{}} + elementsMustBeUint = ParameterHandlerAdapter{Handler: paramvalidators.ValidateListElementsParameter{Valid: paramvalidators.IsJSONUint, Message: "must be an integer token id"}} + elementsMustBeString = ParameterHandlerAdapter{Handler: paramvalidators.ValidateListElementsParameter{Valid: paramvalidators.IsJSONString, Message: "must be a string"}} +) + // The catalog is the single source of truth for how each supported OpenAI/vLLM field is treated. func defaultVLLMParameterCatalog() VLLMParameterCatalog { parameters := slices.Concat( @@ -423,22 +435,25 @@ func defaultVLLMParameterCatalog() VLLMParameterCatalog { newParameter("messages"). withRule(RequestFilterStagePreValidation, ParameterHandlerAdapter{Handler: paramvalidators.LengthCapListParameter{MaxEntries: MessagesMaxEntries}}), newParameter("seed"). - withRule(RequestFilterStagePreValidation, ParameterHandlerAdapter{Handler: paramvalidators.ValidateUintParameter{}}), + withRule(RequestFilterStagePreValidation, mustBeUint), newParameter("n"). withRule(RequestFilterStagePostLimits, ParameterHandlerAdapter{Handler: paramvalidators.CapUintParameter{Min: 1, Max: MaxChatRequestChoices}}). withRule(RequestFilterStagePostLimits, DocumentValidatorHandler{ Validator: paramvalidators.GreedySamplingValidator{}, }), newParameter("temperature"). - withRule(RequestFilterStagePostLimits, ParameterHandlerAdapter{Handler: paramvalidators.SanitizeFloatParameter{StripNonFinite: true, Max: floatPointer(MaxTemperature)}}), + withRule(RequestFilterStagePostLimits, ParameterHandlerAdapter{Handler: paramvalidators.SanitizeFloatParameter{StripNonFinite: true, Min: floatPointer(MinTemperature), Max: floatPointer(MaxTemperature)}}), newParameter("repetition_penalty"). - withRule(RequestFilterStagePostLimits, ParameterHandlerAdapter{Handler: paramvalidators.SanitizeFloatParameter{StripNonFinite: true, Max: floatPointer(MaxRepetitionPenalty)}}), + withRule(RequestFilterStagePostLimits, ParameterHandlerAdapter{Handler: paramvalidators.SanitizeFloatParameter{StripNonFinite: true, Max: floatPointer(MaxRepetitionPenalty)}}). + withRule(RequestFilterStagePostLimits, rejectNonPositiveNumber), newParameter("logit_bias"). withRule(RequestFilterStagePostLimits, ParameterHandlerAdapter{Handler: paramvalidators.SanitizeFloatMapParameter{StripNonFinite: true, Min: floatPointer(LogitBiasMinValue), Max: floatPointer(LogitBiasMaxValue), DropFieldIfEmpty: true, MaxEntries: LogitBiasMaxEntries}}), newParameter("stop"). + withRule(RequestFilterStagePreValidation, elementsMustBeString). withRule(RequestFilterStagePreValidation, ParameterHandlerAdapter{Handler: paramvalidators.LengthCapListParameter{MaxEntries: StopMaxEntries, MaxEntryLen: StopMaxEntryLen}}), newParameter("stop_token_ids"). - withRule(RequestFilterStagePreValidation, ParameterHandlerAdapter{Handler: paramvalidators.LengthCapListParameter{MaxEntries: StopTokenIdsMaxEntries}}), + withRule(RequestFilterStagePreValidation, ParameterHandlerAdapter{Handler: paramvalidators.LengthCapListParameter{MaxEntries: StopTokenIdsMaxEntries}}). + withRule(RequestFilterStagePreValidation, elementsMustBeUint), newParameter("reasoning"). withRule(RequestFilterStagePreValidation, DocumentValidatorHandler{ Validator: paramvalidators.ReasoningValidator{}, @@ -517,6 +532,7 @@ func defaultVLLMParameterCatalog() VLLMParameterCatalog { Validator: paramvalidators.ToolChoiceValidator{MaxNameLen: ToolChoiceMaxNameLen}, }), newParameter("min_tokens"). + withRule(RequestFilterStagePreValidation, mustBeUint). withRule(RequestFilterStagePreValidation, ParameterHandlerAdapter{Handler: paramvalidators.ConditionalStripParameter{ Predicate: func(ctx paramvalidators.ParameterContext) bool { _, ok := ctx.Document["stop_token_ids"] @@ -525,6 +541,7 @@ func defaultVLLMParameterCatalog() VLLMParameterCatalog { }}). withRule(RequestFilterStagePostLimits, ParameterHandlerAdapter{Handler: paramvalidators.ClampUintToFieldParameter{MaxField: "max_tokens"}}), newParameter("bad_words"). + withRule(RequestFilterStagePreValidation, elementsMustBeString). withRule(RequestFilterStagePreValidation, ParameterHandlerAdapter{Handler: paramvalidators.SanitizeStringListParameter{ Keep: func(value string) bool { return strings.TrimSpace(value) != "" @@ -581,18 +598,18 @@ func defaultVLLMParameterCatalog() VLLMParameterCatalog { newParameter("structured_outputs"). withRule(RequestFilterStagePreValidation, DocumentValidatorHandler{ Validator: paramvalidators.StructuredOutputsValidator{ - RejectedModels: []string{kimiK26ModelID}, - MaxDepth: StructuredOutputsMaxDepth, - MaxSize: StructuredOutputsMaxSize, - MaxNodes: StructuredOutputsMaxNodes, - MaxBranch: StructuredOutputsMaxBranch, - MaxEnum: StructuredOutputsMaxEnum, - MaxPatternLen: StructuredOutputsMaxPatternLen, - MaxChoiceEntries: StructuredOutputsMaxChoiceEntries, - MaxChoiceEntryLen: StructuredOutputsMaxChoiceEntryLen, - MaxGrammarLen: StructuredOutputsMaxGrammarLen, - MaxGrammarNesting: StructuredOutputsMaxGrammarNesting, - MaxStructuralTagLen: StructuredOutputsMaxStructuralTagLen, + RejectedModels: []string{kimiK26ModelID}, + MaxDepth: StructuredOutputsMaxDepth, + MaxSize: StructuredOutputsMaxSize, + MaxNodes: StructuredOutputsMaxNodes, + MaxBranch: StructuredOutputsMaxBranch, + MaxEnum: StructuredOutputsMaxEnum, + MaxPatternLen: StructuredOutputsMaxPatternLen, + MaxChoiceEntries: StructuredOutputsMaxChoiceEntries, + MaxChoiceEntryLen: StructuredOutputsMaxChoiceEntryLen, + MaxGrammarLen: StructuredOutputsMaxGrammarLen, + MaxGrammarNesting: StructuredOutputsMaxGrammarNesting, + MaxStructuralTagLen: StructuredOutputsMaxStructuralTagLen, }, }), newParameter("safety_identifier"). @@ -619,30 +636,46 @@ func defaultVLLMParameterCatalog() VLLMParameterCatalog { }, []VLLMParameter{ // PreValidation so the floor lands before applyOutputTokenLimits (caps down) and the - // thinking_token_budget defaulter (derives ttb from max_tokens). + // thinking_token_budget defaulter (derives ttb from max_tokens). Kimi clamps a + // too-small budget up to its floor (think-burn mitigation); other models reject a + // non-positive budget outright since they have no such floor. newParameter("max_tokens"). withRule(RequestFilterStagePreValidation, ModelScopedParameterHandler{ - Models: []string{kimiK26ModelID}, - Handler: MinUintParameterHandler{Min: kimiMaxTokensMin}, + Models: []string{kimiK26ModelID}, + Handler: MinUintParameterHandler{Min: kimiMaxTokensMin}, + UnmatchedHandler: rejectNonPositiveNumber, }), newParameter("max_completion_tokens"). withRule(RequestFilterStagePreValidation, ModelScopedParameterHandler{ - Models: []string{kimiK26ModelID}, - Handler: MinUintParameterHandler{Min: kimiMaxTokensMin}, + Models: []string{kimiK26ModelID}, + Handler: MinUintParameterHandler{Min: kimiMaxTokensMin}, + UnmatchedHandler: rejectNonPositiveNumber, }), + // Sampling knobs with per-field ranges: min_p clamps into [0, 1]; top_p clamps + // down to 1 but rejects a non-positive value (exclusive lower bound); top_k + // accepts -1 (disabled) or any integer >= 1. + newParameter("min_p"). + withRule(RequestFilterStagePostLimits, ParameterHandlerAdapter{Handler: paramvalidators.SanitizeFloatParameter{StripNonFinite: true, Min: floatPointer(MinPMin), Max: floatPointer(MinPMax)}}), + newParameter("top_p"). + withRule(RequestFilterStagePostLimits, ParameterHandlerAdapter{Handler: paramvalidators.SanitizeFloatParameter{StripNonFinite: true, Max: floatPointer(TopPMax)}}). + withRule(RequestFilterStagePostLimits, rejectNonPositiveNumber), + newParameter("top_k"). + withRule(RequestFilterStagePostLimits, ParameterHandlerAdapter{Handler: paramvalidators.SanitizeFloatParameter{StripNonFinite: true}}). + withRule(RequestFilterStagePostLimits, rejectInvalidTopK), }, + // model and stream are type-checked during the typed parse (string / bool); register + // them as known so the whitelist keeps them. newParameters([]string{ "model", "stream", + }), + // The remaining boolean flags are pass-through fields, so validate their type here. + newParameters([]string{ "skip_special_tokens", "detokenize", "parallel_tool_calls", - }), - newParameters([]string{"top_p", "top_k", "min_p"}, - ParameterRule{ - Stage: RequestFilterStagePostLimits, - Handler: ParameterHandlerAdapter{Handler: paramvalidators.SanitizeFloatParameter{StripNonFinite: true}}, - }, + }, + ParameterRule{Stage: RequestFilterStagePreValidation, Handler: mustBeBool}, ), newParameters([]string{"service_tier", "store", "provider", "plugins", "prompt_cache_key", "cache_key", "extra_headers", "thinking_config", "think"}, ParameterRule{Stage: RequestFilterStagePreValidation, Handler: ParameterHandlerAdapter{Handler: paramvalidators.StripParameter{}}}, diff --git a/devshard/cmd/devshardctl/request_filters_test.go b/devshard/cmd/devshardctl/request_filters_test.go index db8982954e..0c5509a295 100644 --- a/devshard/cmd/devshardctl/request_filters_test.go +++ b/devshard/cmd/devshardctl/request_filters_test.go @@ -46,7 +46,7 @@ func TestNormalizeChatRequestDefaultsAndCapsOutputTokens(t *testing.T) { require.NoError(t, err) require.EqualValues(t, 64, req.MaxTokens) require.EqualValues(t, 64, req.MaxCompletionTokens) - require.NotContains(t, string(body), `"max_tokens"`) + require.Contains(t, string(body), `"max_tokens":64`) require.Contains(t, string(body), `"max_completion_tokens":64`) body, req, err = normalizeChatRequest([]byte(`{"max_tokens":10001,"max_completion_tokens":20000,"messages":[{"role":"user","content":"hello"}]}`)) @@ -126,7 +126,7 @@ func TestPrepareChatRequestBodyAdminAuthKeepsMaxCompletionTokensAboveDefault(t * require.NoError(t, err) require.EqualValues(t, 30_000, chatReq.MaxTokens) require.EqualValues(t, 30_000, chatReq.MaxCompletionTokens) - require.NotContains(t, string(body), `"max_tokens"`) + require.Contains(t, string(body), `"max_tokens":30000`) require.Contains(t, string(body), `"max_completion_tokens":30000`) } @@ -282,6 +282,141 @@ func TestNormalizeChatRequestKeepsTemperatureWithinMax(t *testing.T) { require.EqualValues(t, 1.5, raw["temperature"]) } +func TestNormalizeChatRequestClampsTemperatureBelowMin(t *testing.T) { + body, _, err := normalizeChatRequest([]byte(`{ + "messages": [{"role": "user", "content": "hi"}], + "temperature": -1 + }`)) + require.NoError(t, err) + + var raw map[string]any + require.NoError(t, json.Unmarshal(body, &raw)) + require.EqualValues(t, 0.0, raw["temperature"]) +} + +func TestNormalizeChatRequestClampsMinPIntoRange(t *testing.T) { + body, _, err := normalizeChatRequest([]byte(`{"messages":[{"role":"user","content":"hi"}],"min_p":-0.5}`)) + require.NoError(t, err) + var raw map[string]any + require.NoError(t, json.Unmarshal(body, &raw)) + require.EqualValues(t, 0.0, raw["min_p"]) + + body, _, err = normalizeChatRequest([]byte(`{"messages":[{"role":"user","content":"hi"}],"min_p":2}`)) + require.NoError(t, err) + require.NoError(t, json.Unmarshal(body, &raw)) + require.EqualValues(t, 1.0, raw["min_p"]) +} + +func TestNormalizeChatRequestTopPClampsHighRejectsNonPositive(t *testing.T) { + body, _, err := normalizeChatRequest([]byte(`{"messages":[{"role":"user","content":"hi"}],"top_p":1.5}`)) + require.NoError(t, err) + var raw map[string]any + require.NoError(t, json.Unmarshal(body, &raw)) + require.EqualValues(t, 1.0, raw["top_p"]) + + for _, bad := range []string{"0", "-0.2"} { + _, _, err := normalizeChatRequest([]byte(`{"messages":[{"role":"user","content":"hi"}],"top_p":` + bad + `}`)) + require.Error(t, err, "top_p=%s", bad) + require.Contains(t, err.Error(), "top_p") + } +} + +func TestNormalizeChatRequestRejectsNonPositiveRepetitionPenalty(t *testing.T) { + _, _, err := normalizeChatRequest([]byte(`{"messages":[{"role":"user","content":"hi"}],"repetition_penalty":0}`)) + require.Error(t, err) + require.Contains(t, err.Error(), "repetition_penalty") + + body, _, err := normalizeChatRequest([]byte(`{"messages":[{"role":"user","content":"hi"}],"repetition_penalty":5}`)) + require.NoError(t, err) + var raw map[string]any + require.NoError(t, json.Unmarshal(body, &raw)) + require.EqualValues(t, 2.0, raw["repetition_penalty"]) +} + +func TestNormalizeChatRequestRejectsInvalidTopK(t *testing.T) { + for _, bad := range []string{"0", "-2"} { + _, _, err := normalizeChatRequest([]byte(`{"messages":[{"role":"user","content":"hi"}],"top_k":` + bad + `}`)) + require.Error(t, err, "top_k=%s", bad) + require.Contains(t, err.Error(), "top_k") + } + for _, good := range []string{"-1", "1", "50"} { + _, _, err := normalizeChatRequest([]byte(`{"messages":[{"role":"user","content":"hi"}],"top_k":` + good + `}`)) + require.NoError(t, err, "top_k=%s", good) + } +} + +func TestNormalizeChatRequestRejectsZeroMaxTokens(t *testing.T) { + _, _, err := normalizeChatRequest([]byte(`{"messages":[{"role":"user","content":"hi"}],"max_tokens":0}`)) + require.Error(t, err) + require.Contains(t, err.Error(), "max_tokens") + + _, _, err = normalizeChatRequest([]byte(`{"messages":[{"role":"user","content":"hi"}],"max_completion_tokens":0}`)) + require.Error(t, err) + require.Contains(t, err.Error(), "max_completion_tokens") +} + +func TestNormalizeChatRequestKimiClampsZeroMaxTokensInsteadOfRejecting(t *testing.T) { + body, _, err := normalizeChatRequestForModel([]byte(`{"messages":[{"role":"user","content":"hi"}],"max_tokens":0}`), kimiK26ModelID) + require.NoError(t, err) + var raw map[string]any + require.NoError(t, json.Unmarshal(body, &raw)) + require.EqualValues(t, kimiMaxTokensMin, raw["max_tokens"]) +} + +// Regression (found via e2e against the live Kimi route): a Kimi request that +// sends only max_completion_tokens must floor it AND mirror into max_tokens, so +// the thinking_token_budget defaulter (which derives from max_tokens) bounds +// reasoning. Without the mirror the whole budget is spent thinking → empty content. +func TestNormalizeChatRequestKimiMaxCompletionTokensZeroMirrorsToMaxTokens(t *testing.T) { + body, req, err := normalizeChatRequestForModel( + []byte(`{"messages":[{"role":"user","content":"hi"}],"max_completion_tokens":0}`), kimiK26ModelID) + require.NoError(t, err) + var raw map[string]any + require.NoError(t, json.Unmarshal(body, &raw)) + require.EqualValues(t, kimiMaxTokensMin, raw["max_tokens"], "max_tokens mirrored + floored") + require.EqualValues(t, kimiMaxTokensMin, raw["max_completion_tokens"], "max_completion_tokens floored") + require.EqualValues(t, kimiMaxTokensMin, req.MaxTokens) + require.Contains(t, raw, "thinking_token_budget", "thinking budget derives from the mirrored max_tokens") +} + +func TestNormalizeChatRequestRejectsNonBoolFlags(t *testing.T) { + for _, field := range []string{"skip_special_tokens", "detokenize", "parallel_tool_calls"} { + _, _, err := normalizeChatRequest([]byte(`{"messages":[{"role":"user","content":"hi"}],"` + field + `":"yes"}`)) + require.Error(t, err, field) + require.Contains(t, err.Error(), field) + } + _, _, err := normalizeChatRequest([]byte(`{"messages":[{"role":"user","content":"hi"}],"skip_special_tokens":true,"parallel_tool_calls":false}`)) + require.NoError(t, err) +} + +func TestNormalizeChatRequestRejectsNonIntStopTokenIds(t *testing.T) { + _, _, err := normalizeChatRequest([]byte(`{"messages":[{"role":"user","content":"hi"}],"stop_token_ids":[1,"two",3]}`)) + require.Error(t, err) + require.Contains(t, err.Error(), "stop_token_ids") + + _, _, err = normalizeChatRequest([]byte(`{"messages":[{"role":"user","content":"hi"}],"stop_token_ids":[1,2,3]}`)) + require.NoError(t, err) +} + +func TestNormalizeChatRequestRejectsNonStringStopAndBadWords(t *testing.T) { + _, _, err := normalizeChatRequest([]byte(`{"messages":[{"role":"user","content":"hi"}],"stop":["ok",5]}`)) + require.Error(t, err) + require.Contains(t, err.Error(), "stop") + + _, _, err = normalizeChatRequest([]byte(`{"messages":[{"role":"user","content":"hi"}],"bad_words":["ok",5]}`)) + require.Error(t, err) + require.Contains(t, err.Error(), "bad_words") +} + +func TestNormalizeChatRequestRejectsNonIntMinTokens(t *testing.T) { + _, _, err := normalizeChatRequest([]byte(`{"messages":[{"role":"user","content":"hi"}],"min_tokens":3.5}`)) + require.Error(t, err) + require.Contains(t, err.Error(), "min_tokens") + + _, _, err = normalizeChatRequest([]byte(`{"messages":[{"role":"user","content":"hi"}],"min_tokens":5,"max_tokens":100}`)) + require.NoError(t, err) +} + func TestNormalizeChatRequestClampsFrequencyAndPresencePenalty(t *testing.T) { // OpenAI/vLLM accept [-2.0, 2.0] for both. Catalog clamps; out-of-range is rewritten, // not rejected. Per-Kimi force-zero is exercised separately under ApplyKimiRequestOverrides. diff --git a/docs/chat-api/README.md b/docs/chat-api/README.md index 1a2058b6ae..7387b417cd 100644 --- a/docs/chat-api/README.md +++ b/docs/chat-api/README.md @@ -28,21 +28,21 @@ OpenAI-compatible chat completions, routed to Kimi-K2.6 / Qwen3-235B / MiniMax-M |-------|------|---------|----------|--------| | `model` | string | — | Required; route key | [[OpenAI-1]](references.md#openai) | | `messages` | array | — | Required; OpenAI shape (see [Messages contract](#messages-contract)); ≤2048 entries | [[OpenAI-1]](references.md#openai) | -| `temperature` | float | 1.0 | cap ≤ 2.0; sanitize (NaN/Inf strip) | [[OpenAI-1]](references.md#openai) | -| `top_p` | float | 1.0 | sanitize (NaN/Inf strip) | [[OpenAI-1]](references.md#openai) | -| `top_k` | int | — | sanitize float; vLLM extension | [[vLLM-1]](references.md#vllm) | -| `min_p` | float | — | sanitize; vLLM extension | [[vLLM-1]](references.md#vllm) | +| `temperature` | float | 1.0 | clamp to `[0, 2]`; sanitize (NaN/Inf strip) | [[OpenAI-1]](references.md#openai) | +| `top_p` | float | 1.0 | clamp `>1` → `1`; reject `≤0` (must be in `(0, 1]` — [why](troubleshooting.md#reject-out-of-range-sampling)); sanitize (NaN/Inf strip) | [[OpenAI-1]](references.md#openai) | +| `top_k` | int | — | must be `-1` (disabled) or `≥1`, else reject ([why](troubleshooting.md#reject-out-of-range-sampling)); sanitize float; vLLM extension | [[vLLM-1]](references.md#vllm) | +| `min_p` | float | — | clamp to `[0, 1]`; sanitize; vLLM extension | [[vLLM-1]](references.md#vllm) | | `frequency_penalty` | float | 0.0 | clamp `[-2, 2]`; see [Kimi override](kimi-k2.6.md#parameter-overrides) for force-rewrite | [[OpenAI-1]](references.md#openai) | | `presence_penalty` | float | 0.0 | clamp `[-2, 2]`; see [Kimi override](kimi-k2.6.md#parameter-overrides) for force-rewrite | [[OpenAI-1]](references.md#openai) | -| `repetition_penalty` | float | 1.0 | cap ≤ 2.0; vLLM extension | [[vLLM-1]](references.md#vllm) | +| `repetition_penalty` | float | 1.0 | clamp `>2` → `2`; reject `≤0` (must be `>0` — [why](troubleshooting.md#reject-out-of-range-sampling)); vLLM extension | [[vLLM-1]](references.md#vllm) | | `logit_bias` | object | — | ≤1024 entries; value range `[-100, 100]` | [[OpenAI-1]](references.md#openai) | -| `max_tokens` | int | — | clamp | [[OpenAI-1]](references.md#openai) | -| `max_completion_tokens` | int | — | alias for max_tokens | [[OpenAI-1]](references.md#openai) | -| `stream` | bool | false | pass-through | [[OpenAI-1]](references.md#openai) | +| `max_tokens` | int | — | must be `≥1` (reject `0` — [why](troubleshooting.md#reject-nonpositive-max-tokens)); capped to model max; Kimi-K2.6 floors small values to 16 ([why](troubleshooting.md#kimi-empty-content-think-burn)) | [[OpenAI-1]](references.md#openai) | +| `max_completion_tokens` | int | — | alias for max_tokens (same `≥1` reject / cap / Kimi-floor handling) | [[OpenAI-1]](references.md#openai) | +| `stream` | bool | false | must be a boolean (else reject — [why](troubleshooting.md#reject-malformed-param-types)); pass-through | [[OpenAI-1]](references.md#openai) | | `stream_options` | object | — | strip when stream≠true; whitelist `include_usage`; strip `continuous_usage_stats` | [[OpenAI-1]](references.md#openai) | -| `stop` | str\|array | — | pass-through; ≤16 entries × 256 B each | [[OpenAI-1]](references.md#openai) | +| `stop` | str\|array | — | array entries must be strings (else reject — [why](troubleshooting.md#reject-malformed-param-types)); ≤16 entries × 256 B each | [[OpenAI-1]](references.md#openai) | | `n` | int | 1 | hard cap ≤5; coerce to 1 when temperature==0 ([why](troubleshooting.md#coerce-n-when-temperature-zero)) | [[OpenAI-1]](references.md#openai), [[CVE-9]](references.md#security-advisories) | -| `seed` | uint64 | — | pass-through | [[OpenAI-1]](references.md#openai) | +| `seed` | uint64 | — | must be a non-negative integer (else reject — [why](troubleshooting.md#reject-malformed-param-types)); pass-through | [[OpenAI-1]](references.md#openai) | | `logprobs` | bool | — | force `true`; observability pipeline | — | | `top_logprobs` | int | — | force `5`; observability pipeline | — | | `return_token_ids` | bool | — | force `true`; required for stream-derived `enforced_tokens` reconstruction on Kimi-K2.6 reasoning routes (without it, ``/`` are silently dropped from SSE while still counted in `usage.completion_tokens`). Resulting `prompt_token_ids` / `choices[].token_ids` are stripped from the client-facing response | [[vLLM-19]](references.md#vllm), [[vLLM-20]](references.md#vllm) | @@ -50,7 +50,7 @@ OpenAI-compatible chat completions, routed to Kimi-K2.6 / Qwen3-235B / MiniMax-M | `structured_outputs` | object | — | validated against vLLM envelope (`json`/`regex`/`choice`/`grammar`/`json_object`/`structural_tag`); CVE-driven caps per sub-field — see [Qwen native extensions](qwen3-235b-a22b-instruct-2507.md#native-extensions); **rejected on Kimi-K2.6 route** ([why](troubleshooting.md#reject-structured_outputs-kimi)); **accepted on MiniMax-M2.7 route** — vLLM enforces it ([details](troubleshooting.md#accept-structured_outputs-minimax)); `structural_tag` must be the object form (string form is rejected — crashes the engine); **rejected if combined with `response_format`** ([why](troubleshooting.md#reject-structured_outputs-with-response_format)) | [[vLLM-16]](references.md#vllm), [[vLLM-18]](references.md#vllm), [[CVE-3]](references.md#security-advisories), [[CVE-4]](references.md#security-advisories), [[CVE-8]](references.md#security-advisories) | | `tools` | array | — | shape-bounded: function schema depth ≤16, nodes ≤256, branch arms ≤16, enum ≤256, size ≤16 KiB; `$ref`/`$defs`/`definitions` forbidden; `pattern` ≤512 B + regex compile; `function.name` ≤64 B; `tools[].function.strict` silent-stripped (vLLM parsers ignore) | [[OpenAI-1]](references.md#openai), [[CVE-2]](references.md#security-advisories) | | `tool_choice` | string\|object | "auto" if tools | shape-strict; `function.name` ≤64 B; `"required"` coerced ([why](troubleshooting.md#coerce-tool-choice-required)) | [[OpenAI-1]](references.md#openai) | -| `parallel_tool_calls` | bool | — | pass-through | [[OpenAI-1]](references.md#openai) | +| `parallel_tool_calls` | bool | — | must be a boolean (else reject — [why](troubleshooting.md#reject-malformed-param-types)); pass-through | [[OpenAI-1]](references.md#openai) | | `user` | string | — | byte-length ≤512 | [[OpenAI-1]](references.md#openai) | | `safety_identifier` | string | — | silent-strip; see [Kimi override](kimi-k2.6.md#parameter-overrides) for pass-through ([why](troubleshooting.md#strip-safety_identifier)) | [[OpenAI-6]](references.md#openai) | | `metadata` | object | — | OpenAI bounds: ≤16 keys × 64-char × 512-char vals | [[OpenAI-1]](references.md#openai) | @@ -67,11 +67,11 @@ OpenAI-compatible chat completions, routed to Kimi-K2.6 / Qwen3-235B / MiniMax-M | `enable_thinking` | bool | — | translate to chat_template_kwargs ([why](troubleshooting.md#translate-enable_thinking)) | [[Qwen-3]](references.md#qwen) | | `thinking_config` | object | — | silent-strip ([why](troubleshooting.md#strip-thinking_config)) | — | | `think` | bool | — | silent-strip ([why](troubleshooting.md#strip-think)) | — | -| `min_tokens` | int | — | vLLM extension; clamp to ≤max_tokens; conditional strip when stop_token_ids set | [[vLLM-1]](references.md#vllm) | -| `bad_words` | string array | — | vLLM extension; ≤64 entries × 128 B per entry | [[vLLM-1]](references.md#vllm) | -| `stop_token_ids` | int array | — | vLLM extension; ≤64 | [[vLLM-1]](references.md#vllm) | -| `skip_special_tokens` | bool | — | vLLM extension; pass-through | [[vLLM-1]](references.md#vllm) | -| `detokenize` | bool | — | vLLM extension; pass-through | [[vLLM-1]](references.md#vllm) | +| `min_tokens` | int | — | vLLM extension; must be a non-negative integer (else reject — [why](troubleshooting.md#reject-malformed-param-types)); clamp to ≤max_tokens; conditional strip when stop_token_ids set | [[vLLM-1]](references.md#vllm) | +| `bad_words` | string array | — | vLLM extension; entries must be strings (else reject — [why](troubleshooting.md#reject-malformed-param-types)); ≤64 entries × 128 B per entry | [[vLLM-1]](references.md#vllm) | +| `stop_token_ids` | int array | — | vLLM extension; entries must be non-negative integers (else reject — [why](troubleshooting.md#reject-malformed-param-types)); ≤64 | [[vLLM-1]](references.md#vllm) | +| `skip_special_tokens` | bool | — | vLLM extension; must be a boolean (else reject — [why](troubleshooting.md#reject-malformed-param-types)) | [[vLLM-1]](references.md#vllm) | +| `detokenize` | bool | — | vLLM extension; must be a boolean (else reject — [why](troubleshooting.md#reject-malformed-param-types)) | [[vLLM-1]](references.md#vllm) | | `chat_template_kwargs` | object | — | depth ≤16, nodes ≤128, size ≤16 KiB; key denylist (`chat_template`, `tokenize`, `tools`, `documents`, `conversation`, `continue_final_message`, `padding`, `truncation`, `max_length`, `return_tensors`, `return_dict`) — CVE-2025-61620 / CVE-2025-62426 mitigation | [[vLLM-1]](references.md#vllm), [[CVE-5]](references.md#security-advisories), [[CVE-6]](references.md#security-advisories) | *Parameters with truly model-exclusive behavior (`thinking`, `thinking_token_budget`, `messages[].reasoning_content`) are documented in the per-model docs — see [Kimi-K2.6](kimi-k2.6.md) and [Qwen3-235B-A22B-Instruct-2507](qwen3-235b-a22b-instruct-2507.md). For params with universal baseline behavior plus per-model adjustments (`frequency_penalty`, `presence_penalty`, `safety_identifier`), the baseline appears above and the per-model override is linked from the row.* diff --git a/docs/chat-api/troubleshooting.md b/docs/chat-api/troubleshooting.md index 59fef10a92..30464f495b 100644 --- a/docs/chat-api/troubleshooting.md +++ b/docs/chat-api/troubleshooting.md @@ -21,6 +21,9 @@ Every parameter that is stripped / rejected / normalized at the gateway is docum | `extra_body` keys appear at top level | OpenAI Python SDK passthrough | [#unwrap-extra_body](#unwrap-extra_body) | | `enable_thinking` lifts into `chat_template_kwargs` | Qwen3 canonical placement | [#translate-enable_thinking](#translate-enable_thinking) | | `reasoning` object decomposed to top-level `reasoning_effort` | OpenRouter unified-reasoning convention | [#translate-reasoning](#translate-reasoning) | +| 400 on out-of-range `top_p` / `repetition_penalty` / `top_k` | value outside backend-accepted range | [#reject-out-of-range-sampling](#reject-out-of-range-sampling) | +| 400 on `max_tokens: 0` (non-Kimi route) | zero output budget | [#reject-nonpositive-max-tokens](#reject-nonpositive-max-tokens) | +| 400 on wrong-typed param (bool / int / array element) | type mismatch caught at the gateway | [#reject-malformed-param-types](#reject-malformed-param-types) | ## Silent strips @@ -501,6 +504,42 @@ Multiple parallel tool results in one block: one array entry per call. **Captured-requests**: n/a — pre-deployment design decision. +--- + +### #reject-out-of-range-sampling + +**What**: HTTP 400 on `top_p ≤ 0`, `repetition_penalty ≤ 0`, or a `top_k` that is neither `-1` nor `≥ 1`. + +**Why**: These sampling knobs have ranges the backend enforces: `top_p` must be in `(0, 1]` and `repetition_penalty` must be `> 0` per the OpenAI/vLLM wire schema [[OpenAI-1]](references.md#openai), [[vLLM-1]](references.md#vllm); `top_k` accepts `-1` (disabled) or any integer `≥ 1`. The low end is an *exclusive* bound, so clamping to it would itself be invalid (e.g. `top_p = 0` is not a legal value) — the gateway returns a clear, field-named 400 at the boundary instead of forwarding a value the engine rejects with an opaque error. Values above the *upper* bound are clamped instead (`top_p > 1 → 1`, `repetition_penalty > 2 → 2`), and `temperature` / `min_p` are clamped into `[0, 2]` / `[0, 1]`. + +**Fix (client-side)**: send `top_p` in `(0, 1]`, `repetition_penalty > 0`, and `top_k` as `-1` or a positive integer. + +**Captured-requests**: n/a — no captures observed. + +--- + +### #reject-nonpositive-max-tokens + +**What**: HTTP 400 on `max_tokens: 0` (or `max_completion_tokens: 0`) for non-Kimi routes. + +**Why**: A zero output budget is not a meaningful request — vLLM emits no content, and the gateway's redundancy layer then waits for a winner that can never arrive, so the request hangs to the deadline instead of failing fast. The gateway requires `max_tokens ≥ 1` [[OpenAI-1]](references.md#openai). Kimi-K2.6 instead floors small budgets to 16 as part of its think-burn mitigation (see [#kimi-empty-content-think-burn](#kimi-empty-content-think-burn)), so `0` becomes `16` on that route rather than a 400. + +**Fix (client-side)**: send `max_tokens ≥ 1`, or omit it to take the route default. + +**Captured-requests**: n/a — no captures observed. + +--- + +### #reject-malformed-param-types + +**What**: HTTP 400 when a parameter carries the wrong JSON type: non-boolean `stream` / `skip_special_tokens` / `detokenize` / `parallel_tool_calls`; non-integer `seed` / `min_tokens`; non-string elements in `stop` / `bad_words`; non-integer elements in `stop_token_ids`. + +**Why**: vLLM rejects these with type errors at the engine boundary, producing opaque upstream 400s. The gateway type-checks them up front against the OpenAI/vLLM wire schema [[OpenAI-1]](references.md#openai), [[vLLM-1]](references.md#vllm) so the client gets an immediate, field-named error instead of a backend round-trip. + +**Fix (client-side)**: send each field with its declared type — booleans for the flags, non-negative integers for `seed` / `min_tokens` and `stop_token_ids` elements, strings for `stop` / `bad_words` elements. + +**Captured-requests**: n/a — no captures observed. + ## Per-model gotchas Brief pointers to deeper notes in per-model docs: From e73121a694b767243f981358b43da7592c0a6e0c Mon Sep 17 00:00:00 2001 From: Daniil Yankouski Date: Mon, 6 Jul 2026 11:07:08 +0200 Subject: [PATCH 07/13] fix(devshard): reassemble fragmented SSE events for raceWriter content classification (#1240) * fix(devshard): reassemble fragmented SSE events for raceWriter content classification * fix(devshard): resolve comments * fix(devshard): bound SSE classify buffer with per-attempt 1 MiB + global 100 MiB atomic cap * fix(devshard): log reassembly-buffer drops when classify caps are exceeded * fix(devshard): sample reassembled SSE buffer, parallel subtests, stdlib strconv/rand * fix(devshard): flush newline-less final SSE event on stream completion * fix(devshard): flush newline-less final SSE event and release dropped classify buffers * fix(devshard): isolate SSE classify buffers per participant, make caps env-tunable, degrade gracefully on cap drop --- devshard/cmd/devshardctl/main.go | 1 + devshard/cmd/devshardctl/redundancy.go | 272 +++++++++-- devshard/cmd/devshardctl/sse_fixtures_test.go | 439 ++++++++++++++++++ 3 files changed, 679 insertions(+), 33 deletions(-) create mode 100644 devshard/cmd/devshardctl/sse_fixtures_test.go diff --git a/devshard/cmd/devshardctl/main.go b/devshard/cmd/devshardctl/main.go index 003672eabb..bcc7eca441 100644 --- a/devshard/cmd/devshardctl/main.go +++ b/devshard/cmd/devshardctl/main.go @@ -104,6 +104,7 @@ var gatewayRuntimeBuilder = buildRuntime func main() { ConfigurePoCRequestMode(os.Getenv("DEVSHARD_POC_REQUEST_MODE")) ConfigureCapacityAwareLimits(os.Getenv("DEVSHARD_CAPACITY_AWARE_LIMITS")) + configureClassifyCapsFromEnv() flags := parseCLIFlags() runtimeOpts := mustLoadRuntimeOptions(flags) gatewayStore := mustOpenGatewayStore(runtimeOpts.baseStorageDir) diff --git a/devshard/cmd/devshardctl/redundancy.go b/devshard/cmd/devshardctl/redundancy.go index 7e7c7430e2..129d11ae98 100644 --- a/devshard/cmd/devshardctl/redundancy.go +++ b/devshard/cmd/devshardctl/redundancy.go @@ -872,6 +872,14 @@ type inflight struct { // different attempt wins or the attempt ends with no content. pendingBuf []byte + // classifyPartial keeps the tail after the last '\n' from prior Writes; + // used to reassemble fragmented SSE events for the classifier. + classifyPartial []byte + classifyCapLog sync.Once + // participantClassifyBytes is the shared per-participant byte counter this + // attempt contributes to; resolved at creation. Nil disables the cap. + participantClassifyBytes *atomic.Int64 + // contentSource labels the field that produced the first content event // ("delta.content", "delta.reasoning_content", "delta.tool_calls", or the // streaming-only convertible shape "message.content"). Set exactly once @@ -1241,6 +1249,219 @@ type raceWriter struct { inf *inflight } +const ( + defaultMaxClassifyPartial = 1 << 20 // 1 MiB per attempt + defaultMaxClassifyPartialParticipant = 10 << 20 // 10 MiB per participant + defaultMaxClassifyPartialGlobal = 100 << 20 // 100 MiB process-wide +) + +// Reassembly-buffer caps, tunable at startup via configureClassifyCapsFromEnv. +// These are machine-scale memory knobs, not governance policy. +var ( + maxClassifyPartial = defaultMaxClassifyPartial + maxClassifyPartialParticipant int64 = defaultMaxClassifyPartialParticipant + maxClassifyPartialGlobal int64 = defaultMaxClassifyPartialGlobal +) + +// configureClassifyCapsFromEnv overrides the reassembly caps from the +// environment; unset variables keep the conservative defaults. +func configureClassifyCapsFromEnv() { + maxClassifyPartial = int(readInt64Env("GATEWAY_CLASSIFY_MAX_ATTEMPT_BYTES", int64(defaultMaxClassifyPartial))) + maxClassifyPartialParticipant = readInt64Env("GATEWAY_CLASSIFY_MAX_PARTICIPANT_BYTES", defaultMaxClassifyPartialParticipant) + maxClassifyPartialGlobal = readInt64Env("GATEWAY_CLASSIFY_MAX_GLOBAL_BYTES", defaultMaxClassifyPartialGlobal) +} + +// classifyPartialBytes is the live total of every inflight's classifyPartial. +var classifyPartialBytes atomic.Int64 + +// participantClassify bounds live reassembly bytes per participant so one +// hostile participant can't exhaust the shared global pool and starve others. +var participantClassify = &participantClassifyTracker{counters: map[string]*atomic.Int64{}} + +type participantClassifyTracker struct { + mu sync.Mutex + counters map[string]*atomic.Int64 +} + +// counterFor returns the participant's byte counter, creating it on first use. +// Entries are never removed; the participant set is the bounded validator set. +func (t *participantClassifyTracker) counterFor(participantKey string) *atomic.Int64 { + t.mu.Lock() + defer t.mu.Unlock() + counter := t.counters[participantKey] + if counter == nil { + counter = &atomic.Int64{} + t.counters[participantKey] = counter + } + return counter +} + +// takeParseable returns the prefix of (classifyPartial + p) ending at the last +// '\n'; the trailing fragment is stashed for the next call. +func (rw *raceWriter) takeParseable(p []byte) []byte { + lastNL := bytes.LastIndexByte(p, '\n') + if lastNL == -1 { + if rw.growClassify(p) { + return nil + } + // Cap hit: classify the raw chunk best-effort (incomplete fragments won't parse). + rw.dropClassify() + return p + } + + var parseable []byte + if len(rw.inf.classifyPartial) == 0 { + parseable = p[:lastNL+1] + } else { + buf := make([]byte, 0, len(rw.inf.classifyPartial)+lastNL+1) + buf = append(buf, rw.inf.classifyPartial...) + buf = append(buf, p[:lastNL+1]...) + parseable = buf + } + if !rw.replaceClassify(p[lastNL+1:]) { + rw.dropClassify() + } + return parseable +} + +// classifyParseable records the first content/non-retriable-error signal for +// this attempt, returning whether the buffer carried either. +func (rw *raceWriter) classifyParseable(parseable []byte) (hasContent, hasError bool) { + if len(parseable) == 0 { + return false, false + } + if src, ok := sseChunkContentSource(parseable); ok { + hasContent = true + if rw.inf.contentSource == "" { + rw.inf.contentSource = src + } + } else if details, ok := sseChunkErrorDetails(parseable); ok { + src := "error" + if details.Type != "" { + src = "error." + details.Type + } + hasError = !isRetriableCapabilityErrorMessage(details.Message) + if rw.inf.errorSource == "" { + rw.inf.errorSource = src + rw.inf.errorCode = details.Code + rw.inf.errorType = details.Type + rw.inf.errorMessage = details.Message + rw.inf.errorBodySample = append(rw.inf.errorBodySample, parseable...) + } + } + return hasContent, hasError +} + +// flushClassify classifies a newline-less final SSE event left in classifyPartial +// once the stream ends, so a truncated tail isn't misread as empty_stream. +func (rw *raceWriter) flushClassify() { + if rw.inf.probe || len(rw.inf.classifyPartial) == 0 { + return + } + hasContent, hasError := rw.classifyParseable(rw.inf.classifyPartial) + rw.dropClassify() + if hasContent || hasError { + rw.inf.contentChunks.Add(1) + } +} + +// flushClassifyAndCheckEmpty flushes a buffered newline-less final event before deciding empty, so a truncated final content/error event isn't misread as empty_stream. Call on the send goroutine after SendOnly returns. +func (rw *raceWriter) flushClassifyAndCheckEmpty() bool { + rw.flushClassify() + return isEmptyStreamAttempt(rw.inf) +} + +// logClassifyDrop reports the first reassembly-buffer drop for this attempt. +// Capped at once per attempt so a persistently newline-less or oversized +// transport can't spam the log. +func (rw *raceWriter) logClassifyDrop(reason string) { + rw.inf.classifyCapLog.Do(func() { + ctx := context.Background() + if rw.group != nil { + ctx = rw.group.logCtx + } + logInferenceStage(ctx, rw.inf.escrowID, rw.nonce, "classify_buffer_dropped", + "host", rw.inf.hostID, + "reason", reason, + "buffered", len(rw.inf.classifyPartial), + "global_bytes", classifyPartialBytes.Load(), + ) + }) +} + +// growClassify appends tail to classifyPartial if all caps still fit. +func (rw *raceWriter) growClassify(tail []byte) bool { + if len(rw.inf.classifyPartial)+len(tail) > maxClassifyPartial { + rw.logClassifyDrop("per_attempt_cap") + return false + } + if reason := rw.inf.reserveClassifyBytes(int64(len(tail))); reason != "" { + rw.logClassifyDrop(reason) + return false + } + rw.inf.classifyPartial = append(rw.inf.classifyPartial, tail...) + return true +} + +// replaceClassify swaps classifyPartial for buf, adjusting byte accounting. +func (rw *raceWriter) replaceClassify(buf []byte) bool { + if len(buf) > maxClassifyPartial { + rw.logClassifyDrop("per_attempt_cap") + return false + } + delta := int64(len(buf) - len(rw.inf.classifyPartial)) + if delta > 0 { + if reason := rw.inf.reserveClassifyBytes(delta); reason != "" { + rw.logClassifyDrop(reason) + return false + } + } else if delta < 0 { + rw.inf.adjustClassifyBytes(delta) + } + rw.inf.classifyPartial = append(rw.inf.classifyPartial[:0], buf...) + return true +} + +// dropClassify releases the per-attempt buffer and its accounted bytes. +func (rw *raceWriter) dropClassify() { + rw.inf.releaseClassifyPartial() +} + +// reserveClassifyBytes charges n bytes against the participant then global cap, +// rolling back on the first cap hit. Returns the drop reason, "" on success. +func (inf *inflight) reserveClassifyBytes(n int64) string { + participant := inf.participantClassifyBytes + if participant != nil && participant.Add(n) > maxClassifyPartialParticipant { + participant.Add(-n) + return "participant_cap" + } + if classifyPartialBytes.Add(n) > maxClassifyPartialGlobal { + classifyPartialBytes.Add(-n) + if participant != nil { + participant.Add(-n) + } + return "global_cap" + } + return "" +} + +// adjustClassifyBytes adds delta to the global and per-participant counters. +func (inf *inflight) adjustClassifyBytes(delta int64) { + classifyPartialBytes.Add(delta) + if inf.participantClassifyBytes != nil { + inf.participantClassifyBytes.Add(delta) + } +} + +// releaseClassifyPartial frees the reassembly buffer's backing array and +// decrements the counters. Nils the slice so cap doesn't outlive the gauge. +func (inf *inflight) releaseClassifyPartial() { + if n := len(inf.classifyPartial); n > 0 { + inf.adjustClassifyBytes(-int64(n)) + } + inf.classifyPartial = nil +} + func (rw *raceWriter) ctxErr() error { if rw.group == nil || rw.group.writeCtx == nil { return nil @@ -1269,25 +1490,7 @@ func (rw *raceWriter) Write(p []byte) (int, error) { var chunkHasContent bool var chunkHasError bool if !rw.inf.probe { - if src, ok := sseChunkContentSource(p); ok { - chunkHasContent = true - if rw.inf.contentSource == "" { - rw.inf.contentSource = src - } - } else if details, ok := sseChunkErrorDetails(p); ok { - src := "error" - if details.Type != "" { - src = "error." + details.Type - } - chunkHasError = !isRetriableCapabilityErrorMessage(details.Message) - if rw.inf.errorSource == "" { - rw.inf.errorSource = src - rw.inf.errorCode = details.Code - rw.inf.errorType = details.Type - rw.inf.errorMessage = details.Message - rw.inf.errorBodySample = append(rw.inf.errorBodySample, p...) - } - } + chunkHasContent, chunkHasError = rw.classifyParseable(rw.takeParseable(p)) } if chunkHasContent || chunkHasError { rw.inf.contentChunks.Add(1) @@ -1572,19 +1775,20 @@ func (e *Redundancy) prepareInflight(ctx context.Context, params user.InferenceP participantKey := e.session.HostParticipantKey(res.prepared.HostIdx()) noWinner, noWinnerOK := e.noWinnerStatusForParticipant(participantKey) inf := &inflight{ - prepared: res.prepared, - hostIdx: res.prepared.HostIdx(), - hostID: e.session.HostLabel(res.prepared.HostIdx()), - nonce: res.prepared.Nonce(), - escrowID: e.devshardID, - probe: res.isProbe, - suspicious: noWinnerOK, - noWinnerReason: noWinner.reason, - noWinnerQuarantineMode: noWinner.quarantineMode, - noWinnerFailureStrikes: noWinner.failureStrikes, - done: make(chan struct{}), - receiptCh: make(chan struct{}), - firstTokenCh: make(chan struct{}), + prepared: res.prepared, + hostIdx: res.prepared.HostIdx(), + hostID: e.session.HostLabel(res.prepared.HostIdx()), + nonce: res.prepared.Nonce(), + escrowID: e.devshardID, + probe: res.isProbe, + suspicious: noWinnerOK, + noWinnerReason: noWinner.reason, + noWinnerQuarantineMode: noWinner.quarantineMode, + noWinnerFailureStrikes: noWinner.failureStrikes, + done: make(chan struct{}), + receiptCh: make(chan struct{}), + firstTokenCh: make(chan struct{}), + participantClassifyBytes: participantClassify.counterFor(participantKey), } e.recordAccountingAttempt(ctx, inf) return inf, nil @@ -1647,6 +1851,8 @@ func (e *Redundancy) startInflight(ctx context.Context, inf *inflight, race *rac go func() { defer close(inf.done) defer cancel() + // Sole owner of classifyPartial: release on every exit path (incl. the early error return); content is classified synchronously via flushClassifyAndCheckEmpty below. + defer inf.releaseClassifyPartial() logInferenceStage(ctx, inf.escrowID, inf.nonce, "started", "host", inf.hostID) inf.resp, inf.err = e.session.SendOnly(attemptCtx, inf.prepared, rw, receiptHandler) streamBytes := int64(0) @@ -1678,7 +1884,7 @@ func (e *Redundancy) startInflight(ctx context.Context, inf *inflight, race *rac // stream_bytes_read > 0 but output_chunks == 0 because only devshard // receipt/meta events were parsed and no inference data was forwarded to // the race writer. - if isEmptyStreamAttempt(inf) { + if rw.flushClassifyAndCheckEmpty() { responseBodySample, responseSampleTruncated := bodySampleForLog(inf.pendingBuf, emptyStreamBodySampleLimit) inf.emptyResponseBodySample = responseBodySample inf.emptyResponseBodySampleTruncated = responseSampleTruncated diff --git a/devshard/cmd/devshardctl/sse_fixtures_test.go b/devshard/cmd/devshardctl/sse_fixtures_test.go new file mode 100644 index 0000000000..9601980f38 --- /dev/null +++ b/devshard/cmd/devshardctl/sse_fixtures_test.go @@ -0,0 +1,439 @@ +package main + +import ( + "bytes" + "context" + "math/rand/v2" + "strconv" + "sync/atomic" + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +// Minimal vLLM-shaped SSE fixtures for raceWriter classification tests. +const sseContentStream = "" + + `data: {"id":"chatcmpl-1","object":"chat.completion.chunk","created":1,"model":"Qwen/Qwen3","choices":[{"index":0,"delta":{"role":"assistant","content":""},"finish_reason":null}]}` + "\n\n" + + `data: {"id":"chatcmpl-1","object":"chat.completion.chunk","created":1,"model":"Qwen/Qwen3","choices":[{"index":0,"delta":{"content":"ok"},"finish_reason":null}]}` + "\n\n" + + `data: {"id":"chatcmpl-1","object":"chat.completion.chunk","created":1,"model":"Qwen/Qwen3","choices":[{"index":0,"delta":{"content":""},"finish_reason":"stop"}]}` + "\n\n" + + `data: {"id":"chatcmpl-1","object":"chat.completion.chunk","created":1,"model":"Qwen/Qwen3","choices":[],"usage":{"prompt_tokens":3,"total_tokens":5,"completion_tokens":2}}` + "\n\n" + + `data: [DONE]` + "\n\n" + +const sseToolCallsStream = "" + + `data: {"id":"chatcmpl-2","object":"chat.completion.chunk","created":2,"model":"Qwen/Qwen3","choices":[{"index":0,"delta":{"role":"assistant","content":""},"finish_reason":null}]}` + "\n\n" + + `data: {"id":"chatcmpl-2","object":"chat.completion.chunk","created":2,"model":"Qwen/Qwen3","choices":[{"index":0,"delta":{"tool_calls":[{"id":"call-1","type":"function","index":0,"function":{"name":"get_weather"}}]},"finish_reason":null}]}` + "\n\n" + + `data: {"id":"chatcmpl-2","object":"chat.completion.chunk","created":2,"model":"Qwen/Qwen3","choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"function":{"arguments":"{\"q\":\"sf\"}"}}]},"finish_reason":null}]}` + "\n\n" + + `data: {"id":"chatcmpl-2","object":"chat.completion.chunk","created":2,"model":"Qwen/Qwen3","choices":[{"index":0,"delta":{"content":""},"finish_reason":"tool_calls"}]}` + "\n\n" + + `data: {"id":"chatcmpl-2","object":"chat.completion.chunk","created":2,"model":"Qwen/Qwen3","choices":[],"usage":{"prompt_tokens":20,"total_tokens":40,"completion_tokens":20}}` + "\n\n" + + `data: [DONE]` + "\n\n" + +var sseEmbeddedFixtures = []struct { + name string + body string + wantSource string +}{ + {"delta.content", sseContentStream, "delta.content"}, + {"delta.tool_calls", sseToolCallsStream, "delta.tool_calls"}, +} + +// Stream whose final content event carries no trailing newline (truncated +// upstream, mid-event proxy close). Write only classifies up to the last '\n', +// so the "ok" event lands in classifyPartial and is recovered by flushClassify. +const sseNewlineLessFinalContent = "" + + `data: {"id":"chatcmpl-3","object":"chat.completion.chunk","created":3,"model":"Qwen/Qwen3","choices":[{"index":0,"delta":{"role":"assistant","content":""},"finish_reason":null}]}` + "\n\n" + + `data: {"id":"chatcmpl-3","object":"chat.completion.chunk","created":3,"model":"Qwen/Qwen3","choices":[{"index":0,"delta":{"content":"ok"},"finish_reason":null}]}` + +// Same shape, but the newline-less final event is a non-retriable error. +const sseNewlineLessFinalError = "" + + `data: {"error":{"message":"boom","type":"server_error","code":"internal"}}` + +func TestSseBlobClassifierNotEmpty(t *testing.T) { + for _, fx := range sseEmbeddedFixtures { + t.Run(fx.name, func(t *testing.T) { + src, ok := sseChunkContentSource([]byte(fx.body)) + require.True(t, ok, "blob classifier returned EMPTY") + require.Equal(t, fx.wantSource, src) + }) + } +} + +// Regression: classifier must work for any chunk size, including sub-event. +func TestSseRaceWriterAllChunkSizes(t *testing.T) { + chunkSizes := []int{1, 64, 256, 1024, 4096, 8192} + for _, fx := range sseEmbeddedFixtures { + for _, sz := range chunkSizes { + t.Run(fx.name+"/chunk="+strconv.Itoa(sz), func(t *testing.T) { + t.Parallel() + inf := mkRaceWriterInflight(t) + rw := mkRaceWriter(t, inf) + body := []byte(fx.body) + for i := 0; i < len(body); i += sz { + end := i + sz + if end > len(body) { + end = len(body) + } + _, err := rw.Write(body[i:end]) + require.NoError(t, err) + } + require.False(t, isEmptyStreamAttempt(inf), + "classified empty (cc=%d oc=%d)", + inf.contentChunks.Load(), inf.outputChunks.Load()) + require.Equal(t, fx.wantSource, inf.contentSource) + }) + } + } +} + +// Per-attempt cap: oversized newline-less stream is dropped, global counter balanced. +func TestSseRaceWriterClassifyCapDrops(t *testing.T) { + before := classifyPartialBytes.Load() + inf := mkRaceWriterInflight(t) + rw := mkRaceWriter(t, inf) + _, err := rw.Write(bytes.Repeat([]byte("x"), maxClassifyPartial+1)) + require.NoError(t, err) + require.Equal(t, 0, len(inf.classifyPartial)) + require.Equal(t, before, classifyPartialBytes.Load()) +} + +// replaceClassify drops the trailing fragment when it exceeds the per-attempt cap. +func TestSseRaceWriterReplaceClassifyPerAttemptCapDrops(t *testing.T) { + before := classifyPartialBytes.Load() + inf := mkRaceWriterInflight(t) + rw := mkRaceWriter(t, inf) + chunk := append([]byte("\n"), bytes.Repeat([]byte("x"), maxClassifyPartial+1)...) + _, err := rw.Write(chunk) + require.NoError(t, err) + require.Equal(t, 0, len(inf.classifyPartial)) + require.Equal(t, before, classifyPartialBytes.Load()) +} + +// growClassify drops a newline-less chunk when the global cap is already reached. +func TestSseRaceWriterGrowClassifyGlobalCapDrops(t *testing.T) { + before := classifyPartialBytes.Load() + defer classifyPartialBytes.Store(before) + classifyPartialBytes.Store(maxClassifyPartialGlobal) + inf := mkRaceWriterInflight(t) + rw := mkRaceWriter(t, inf) + _, err := rw.Write([]byte("data: partial-no-newline")) + require.NoError(t, err) + require.Equal(t, 0, len(inf.classifyPartial)) + require.Equal(t, maxClassifyPartialGlobal, classifyPartialBytes.Load()) +} + +// replaceClassify drops the trailing fragment when the global cap is reached. +func TestSseRaceWriterReplaceClassifyGlobalCapDrops(t *testing.T) { + before := classifyPartialBytes.Load() + defer classifyPartialBytes.Store(before) + classifyPartialBytes.Store(maxClassifyPartialGlobal) + inf := mkRaceWriterInflight(t) + rw := mkRaceWriter(t, inf) + _, err := rw.Write([]byte("\ntail")) + require.NoError(t, err) + require.Equal(t, 0, len(inf.classifyPartial)) + require.Equal(t, maxClassifyPartialGlobal, classifyPartialBytes.Load()) +} + +// Dropping the buffer must free its backing array, not just shrink len — else +// real memory outlives the gauge and can exceed the global cap. +func TestSseRaceWriterDropReleasesBackingArray(t *testing.T) { + before := classifyPartialBytes.Load() + inf := mkRaceWriterInflight(t) + rw := mkRaceWriter(t, inf) + _, err := rw.Write(bytes.Repeat([]byte("x"), maxClassifyPartial-10)) + require.NoError(t, err) + require.Greater(t, cap(inf.classifyPartial), 0) + _, err = rw.Write(bytes.Repeat([]byte("x"), 20)) + require.NoError(t, err) + require.Equal(t, 0, cap(inf.classifyPartial), "drop must release the backing array") + require.Equal(t, before, classifyPartialBytes.Load()) +} + +// releaseClassifyPartial frees a len-0-but-large-cap buffer (the send-goroutine defer after a drop) and leaves the gauge unchanged when len is already 0. +func TestReleaseClassifyPartialFreesLenZeroBuffer(t *testing.T) { + before := classifyPartialBytes.Load() + inf := mkRaceWriterInflight(t) + inf.classifyPartial = make([]byte, 0, maxClassifyPartial) + inf.releaseClassifyPartial() + require.Nil(t, inf.classifyPartial) + require.Equal(t, before, classifyPartialBytes.Load()) +} + +// Realistic transport shape: arbitrary chunk boundaries from TLS/proxy flushes. +func TestSseRaceWriterRandomChunking(t *testing.T) { + for fxIndex, fx := range sseEmbeddedFixtures { + t.Run(fx.name, func(t *testing.T) { + t.Parallel() + // Own rng per subtest: *rand.Rand is not safe for concurrent use. + rng := rand.New(rand.NewPCG(42, uint64(fxIndex))) + inf := mkRaceWriterInflight(t) + rw := mkRaceWriter(t, inf) + body := []byte(fx.body) + for i := 0; i < len(body); { + sz := 1 + rng.IntN(64) + end := i + sz + if end > len(body) { + end = len(body) + } + _, err := rw.Write(body[i:end]) + require.NoError(t, err) + i = end + } + require.False(t, isEmptyStreamAttempt(inf), + "classified empty (cc=%d oc=%d)", + inf.contentChunks.Load(), inf.outputChunks.Load()) + require.Equal(t, fx.wantSource, inf.contentSource) + }) + } +} + +// Regression: a content event delivered as the final, newline-less write is +// stashed in classifyPartial during Write and would leave the attempt looking +// empty. flushClassify (run once the upstream stream ends) must recover it. +func TestSseRaceWriterFlushRecoversNewlineLessContent(t *testing.T) { + inf := mkRaceWriterInflight(t) + rw := mkRaceWriter(t, inf) + + _, err := rw.Write([]byte(sseNewlineLessFinalContent)) + require.NoError(t, err) + // Before the flush the trailing content event is unclassified. + require.True(t, isEmptyStreamAttempt(inf), + "precondition: newline-less final event should be unclassified until flush") + + rw.flushClassify() + + require.False(t, isEmptyStreamAttempt(inf), + "flush must classify the newline-less final content event") + require.Equal(t, "delta.content", inf.contentSource) + require.Equal(t, 0, len(inf.classifyPartial), "flush must release the buffer") +} + +// Regression: flushClassifyAndCheckEmpty must flush the buffered final event before reading contentChunks (a prior deferred flush ran after the decision, misclassifying content as empty). +func TestFlushClassifyAndCheckEmptyFlushesBeforeDeciding(t *testing.T) { + inf := mkRaceWriterInflight(t) + rw := mkRaceWriter(t, inf) + + _, err := rw.Write([]byte(sseNewlineLessFinalContent)) + require.NoError(t, err) + require.True(t, isEmptyStreamAttempt(inf), + "precondition: unflushed newline-less final event looks empty") + + require.False(t, rw.flushClassifyAndCheckEmpty(), + "must flush the buffered final content before the empty-stream decision") + require.Equal(t, "delta.content", inf.contentSource) +} + +// Regression: a newline-less final error event must resolve to an error stream, not empty, through the same flush-before-decide path. +func TestFlushClassifyAndCheckEmptyFinalErrorIsNotEmpty(t *testing.T) { + inf := mkRaceWriterInflight(t) + rw := mkRaceWriter(t, inf) + + _, err := rw.Write([]byte(sseNewlineLessFinalError)) + require.NoError(t, err) + + require.False(t, rw.flushClassifyAndCheckEmpty(), + "final error event must not be classified as empty_stream") + require.True(t, isErrorStreamAttempt(inf)) +} + +// Regression: the same recovery must apply to a newline-less final error event +// so the attempt is classified as an error stream, not an empty one. +func TestSseRaceWriterFlushRecoversNewlineLessError(t *testing.T) { + inf := mkRaceWriterInflight(t) + rw := mkRaceWriter(t, inf) + + _, err := rw.Write([]byte(sseNewlineLessFinalError)) + require.NoError(t, err) + + rw.flushClassify() + + require.True(t, isErrorStreamAttempt(inf), "flush must classify the final error event") + require.False(t, isEmptyStreamAttempt(inf)) + require.Equal(t, "error.server_error", inf.errorSource) +} + +// flushClassify is a no-op when nothing was buffered (normal newline-terminated +// stream) and stays balanced against the global counter. +func TestSseRaceWriterFlushNoBufferedTail(t *testing.T) { + before := classifyPartialBytes.Load() + inf := mkRaceWriterInflight(t) + rw := mkRaceWriter(t, inf) + + _, err := rw.Write([]byte(sseContentStream)) + require.NoError(t, err) + require.False(t, isEmptyStreamAttempt(inf)) + source := inf.contentSource + contentChunks := inf.contentChunks.Load() + + rw.flushClassify() + + require.Equal(t, source, inf.contentSource, "flush must not change an already-classified attempt") + require.Equal(t, contentChunks, inf.contentChunks.Load(), "flush must not double-count content") + require.Equal(t, before, classifyPartialBytes.Load()) +} + +// A probe attempt never classifies content, so flushClassify must leave it untouched. +func TestSseRaceWriterFlushProbeNoop(t *testing.T) { + inf := mkRaceWriterInflight(t) + inf.probe = true + rw := mkRaceWriter(t, inf) + + _, err := rw.Write([]byte(sseNewlineLessFinalContent)) + require.NoError(t, err) + + rw.flushClassify() + + require.Equal(t, int64(0), inf.contentChunks.Load(), "probe flush must not count content") + require.Equal(t, "", inf.contentSource) +} + +// On a cap drop, the raw write chunk is still classified best-effort rather than +// silently skipped — a complete content line survives even when it can't buffer. +func TestSseRaceWriterGracefulDegradationOnCapDrop(t *testing.T) { + inf := mkRaceWriterInflight(t) + rw := mkRaceWriter(t, inf) + _, err := rw.Write(bytes.Repeat([]byte("x"), maxClassifyPartial-10)) + require.NoError(t, err) + _, err = rw.Write([]byte(`data: {"choices":[{"delta":{"content":"ok"}}]}`)) + require.NoError(t, err) + require.False(t, isEmptyStreamAttempt(inf), "capped content chunk must still classify") + require.Equal(t, "delta.content", inf.contentSource) +} + +// A hostile participant that fills its own budget must not starve another +// attempt from a different participant sharing the global pool. +func TestSseRaceWriterParticipantCapIsolatesParticipants(t *testing.T) { + restore := saveClassifyCaps() + defer restore() + maxClassifyPartial = 100 + maxClassifyPartialParticipant = 150 + maxClassifyPartialGlobal = 1 << 30 + + hostile := &atomic.Int64{} + honest := &atomic.Int64{} + infHostile := mkRaceWriterInflight(t) + infHostile.participantClassifyBytes = hostile + infHonest := mkRaceWriterInflight(t) + infHonest.participantClassifyBytes = honest + + _, err := mkRaceWriter(t, infHostile).Write(bytes.Repeat([]byte("x"), 100)) + require.NoError(t, err) + require.Equal(t, int64(100), hostile.Load()) + + // The honest participant still has its full budget available. + _, err = mkRaceWriter(t, infHonest).Write(bytes.Repeat([]byte("y"), 100)) + require.NoError(t, err) + require.Equal(t, 100, len(infHonest.classifyPartial), "honest attempt must not be starved") + require.Equal(t, int64(100), honest.Load()) +} + +// The same participant is capped across concurrent attempts sharing its counter. +func TestSseRaceWriterParticipantCapDropsOverBudget(t *testing.T) { + restore := saveClassifyCaps() + defer restore() + maxClassifyPartial = 100 + maxClassifyPartialParticipant = 150 + maxClassifyPartialGlobal = 1 << 30 + + shared := &atomic.Int64{} + first := mkRaceWriterInflight(t) + first.participantClassifyBytes = shared + second := mkRaceWriterInflight(t) + second.participantClassifyBytes = shared + + _, err := mkRaceWriter(t, first).Write(bytes.Repeat([]byte("x"), 100)) + require.NoError(t, err) + _, err = mkRaceWriter(t, second).Write(bytes.Repeat([]byte("y"), 100)) + require.NoError(t, err) + + require.Equal(t, 0, len(second.classifyPartial), "participant cap must drop the over-budget attempt") + require.Equal(t, int64(100), shared.Load(), "over-budget bytes rolled back") +} + +// A global-cap drop must roll back the participant counter it already charged. +func TestSseRaceWriterGlobalCapRollsBackParticipant(t *testing.T) { + restore := saveClassifyCaps() + defer restore() + maxClassifyPartial = 1000 + maxClassifyPartialParticipant = 1000 + maxClassifyPartialGlobal = 50 + + participant := &atomic.Int64{} + inf := mkRaceWriterInflight(t) + inf.participantClassifyBytes = participant + before := classifyPartialBytes.Load() + + _, err := mkRaceWriter(t, inf).Write(bytes.Repeat([]byte("x"), 100)) + require.NoError(t, err) + + require.Equal(t, 0, len(inf.classifyPartial), "global cap drops the attempt") + require.Equal(t, int64(0), participant.Load(), "participant counter rolled back") + require.Equal(t, before, classifyPartialBytes.Load()) +} + +// Releasing the buffer decrements both the global and participant counters. +func TestReleaseClassifyPartialDecrementsParticipant(t *testing.T) { + restore := saveClassifyCaps() + defer restore() + maxClassifyPartial = 1000 + maxClassifyPartialParticipant = 1000 + maxClassifyPartialGlobal = 1 << 30 + + participant := &atomic.Int64{} + inf := mkRaceWriterInflight(t) + inf.participantClassifyBytes = participant + + _, err := mkRaceWriter(t, inf).Write(bytes.Repeat([]byte("x"), 100)) + require.NoError(t, err) + require.Equal(t, int64(100), participant.Load()) + + inf.releaseClassifyPartial() + + require.Equal(t, int64(0), participant.Load(), "release decrements participant counter") + require.Nil(t, inf.classifyPartial) +} + +func TestConfigureClassifyCapsFromEnv(t *testing.T) { + restore := saveClassifyCaps() + defer restore() + t.Setenv("GATEWAY_CLASSIFY_MAX_ATTEMPT_BYTES", "2048") + t.Setenv("GATEWAY_CLASSIFY_MAX_PARTICIPANT_BYTES", "4096") + t.Setenv("GATEWAY_CLASSIFY_MAX_GLOBAL_BYTES", "8192") + + configureClassifyCapsFromEnv() + + require.Equal(t, 2048, maxClassifyPartial) + require.Equal(t, int64(4096), maxClassifyPartialParticipant) + require.Equal(t, int64(8192), maxClassifyPartialGlobal) +} + +// saveClassifyCaps snapshots the tunable caps and returns a restore func. +func saveClassifyCaps() func() { + attempt, participant, global := maxClassifyPartial, maxClassifyPartialParticipant, maxClassifyPartialGlobal + return func() { + maxClassifyPartial = attempt + maxClassifyPartialParticipant = participant + maxClassifyPartialGlobal = global + } +} + +func mkRaceWriterInflight(t testing.TB) *inflight { + t.Helper() + return &inflight{ + hostID: "fixture-host", + escrowID: "fixture-escrow", + nonce: 1, + done: make(chan struct{}), + receiptCh: make(chan struct{}), + firstTokenCh: make(chan struct{}), + receiptTime: time.Now(), + } +} + +func mkRaceWriter(t testing.TB, inf *inflight) *raceWriter { + t.Helper() + ctx := context.Background() + var sink bytes.Buffer + rg := newRaceGroup(ctx, ctx, inf.escrowID, &sink) + return &raceWriter{group: rg, nonce: inf.nonce, inf: inf} +} From 6eab14a9d62458881e622f1437156602317c92c8 Mon Sep 17 00:00:00 2001 From: Daniil Yankouski Date: Mon, 6 Jul 2026 20:29:56 +0200 Subject: [PATCH 08/13] count background race finalizers in the drain barrier (#1403) * fix(devshard): count background race finalizers in the drain barrier * fix(devshard): rename vars to make more clear naming --- .../cmd/devshardctl/capacity_state_test.go | 2 +- devshard/cmd/devshardctl/escrow_rotator.go | 4 +- devshard/cmd/devshardctl/gateway.go | 85 +++++++++----- .../gateway_race_cleanup_barrier_test.go | 110 ++++++++++++++++++ .../gateway_runtime_retire_test.go | 8 +- devshard/cmd/devshardctl/gateway_test.go | 26 ++--- devshard/cmd/devshardctl/metrics.go | 2 +- devshard/cmd/devshardctl/redundancy.go | 39 +++++-- 8 files changed, 220 insertions(+), 56 deletions(-) create mode 100644 devshard/cmd/devshardctl/gateway_race_cleanup_barrier_test.go diff --git a/devshard/cmd/devshardctl/capacity_state_test.go b/devshard/cmd/devshardctl/capacity_state_test.go index 7686734627..df88317b69 100644 --- a/devshard/cmd/devshardctl/capacity_state_test.go +++ b/devshard/cmd/devshardctl/capacity_state_test.go @@ -367,7 +367,7 @@ func TestGatewayLimiterUnlimitedBaselinePreservedUnderScale(t *testing.T) { func TestDevshardRuntimeLoadIsActivePerWeight(t *testing.T) { rt := &devshardRuntime{} - rt.activeRequests.Store(2) + rt.activeUserRequests.Store(2) rt.reservedTokens.Store(3) // reserved tokens no longer factor in. require.InDelta(t, 0.5, rt.load(4.0), 1e-9) diff --git a/devshard/cmd/devshardctl/escrow_rotator.go b/devshard/cmd/devshardctl/escrow_rotator.go index 794dad694d..2d0e1ab1cf 100644 --- a/devshard/cmd/devshardctl/escrow_rotator.go +++ b/devshard/cmd/devshardctl/escrow_rotator.go @@ -405,9 +405,9 @@ func (g *Gateway) settleDevshardOnChain(ctx context.Context, id string, req admi log.Printf("devshard_settle_start escrow=%s", id) g.mu.Lock() rt, ok := g.runtimes[id] - if ok && rt.activeRequests.Load() > 0 { + if ok && rt.escrowHasBackgroundWork() { g.mu.Unlock() - log.Printf("devshard_settle_blocked escrow=%s reason=active_requests count=%d", id, rt.activeRequests.Load()) + log.Printf("devshard_settle_blocked escrow=%s reason=background_work active_requests=%d pending_race_cleanup=%d", id, rt.activeUserRequests.Load(), rt.pendingRaceCleanup.Load()) return nil, errDevshardBusy } if ok { diff --git a/devshard/cmd/devshardctl/gateway.go b/devshard/cmd/devshardctl/gateway.go index 7a6b93db45..04b8dd4bbe 100644 --- a/devshard/cmd/devshardctl/gateway.go +++ b/devshard/cmd/devshardctl/gateway.go @@ -80,9 +80,12 @@ type devshardRuntime struct { // same escrow. participantSlotCounts map[string]int - active atomic.Bool - activeRequests atomic.Int64 - reservedTokens atomic.Int64 + active atomic.Bool + activeUserRequests atomic.Int64 + reservedTokens atomic.Int64 + + // pendingRaceCleanup counts background race cleanups (refund + loser-signature persistence) still in flight + pendingRaceCleanup atomic.Int64 // settlementPending marks an escrow that has been deactivated and must // be settled once its in-flight requests drain. settlementReason is @@ -99,6 +102,11 @@ type devshardRuntime struct { activeConfigured bool } +// escrowHasBackgroundWork reports whether foreground requests or background race cleanups are in flight; settle and store-close must wait until it is false. +func (rt *devshardRuntime) escrowHasBackgroundWork() bool { + return rt.activeUserRequests.Load() > 0 || rt.pendingRaceCleanup.Load() > 0 +} + type runtimeStatus struct { ID string `json:"id"` Model string `json:"model"` @@ -422,7 +430,7 @@ func (rt *devshardRuntime) snapshot() runtimeStatus { ID: rt.id, Model: rt.model, Active: rt.active.Load(), - ActiveRequests: rt.activeRequests.Load(), + ActiveRequests: rt.activeUserRequests.Load(), ReservedTokens: rt.reservedTokens.Load(), SettlementPending: rt.settlementPending.Load(), } @@ -444,8 +452,8 @@ func (rt *devshardRuntime) snapshot() runtimeStatus { return status } -// TODO: the (reservedTokens*1000 + activeRequests) formula is missleading, -// let's just leave activeRequests here, and leave a todo comment, that +// TODO: the (reservedTokens*1000 + activeUserRequests) formula is missleading, +// let's just leave activeUserRequests here, and leave a todo comment, that // we might need to change it, so that if limits for tokens or cuncurrent // requests are set, we need to measure if the escrow is further from // the limists @@ -453,8 +461,8 @@ func (rt *devshardRuntime) snapshot() runtimeStatus { // load returns the capacity-aware load score for this runtime. Lower // is better; the picker selects the runtime with the smallest load. // -// Score is simply activeRequests / W(e): -// - activeRequests is the live count of in-flight inferences this +// Score is simply activeUserRequests / W(e): +// - activeUserRequests is the live count of in-flight inferences this // runtime owns (incremented on dispatch, decremented on // completion). It's the most direct, low-latency signal of "is // this runtime busy right now". @@ -477,7 +485,7 @@ func (rt *devshardRuntime) load(weight float64) float64 { if weight <= 0 { return math.Inf(+1) } - return float64(rt.activeRequests.Load()) / weight + return float64(rt.activeUserRequests.Load()) / weight } func NewGateway(runtimes []*devshardRuntime, limiter *GatewayLimiter, defaultModel string) *Gateway { @@ -573,6 +581,7 @@ func (g *Gateway) attachRuntimeSharedState(rt *devshardRuntime) { } g.attachMetrics(rt) g.attachEscrowChecker(rt) + g.attachRaceCleanupBarrier(rt) if g.capacity != nil { g.capacity.SetEscrowMembership(rt.id, rt.participantSlotCounts) } @@ -1342,7 +1351,7 @@ func (g *Gateway) handleDevshard(w http.ResponseWriter, r *http.Request) { return } if innerPath == "/v1/finalize" && r.Method == http.MethodPost { - if rt.activeRequests.Load() > 0 { + if rt.escrowHasBackgroundWork() { http.Error(w, fmt.Sprintf(`{"error":{"message":"devshard %s has active requests"}}`, devshardID), http.StatusConflict) return } @@ -1586,7 +1595,7 @@ func (g *Gateway) formatCandidateWeightsLocked(candidates []*devshardRuntime, re // Fallback rules: // - No capacity state attached, or escrow not registered with the // state (no slot/membership info): use neutral weight 1.0 so the -// picker degrades to a pure activeRequests comparison. +// picker degrades to a pure activeUserRequests comparison. // - Escrow registered but W(e) == 0 (every host is PoC-excluded or // fully throttled): honor the 0 so the runtime drops to +Inf load // and stops receiving traffic until at least one host recovers. @@ -1604,32 +1613,47 @@ func (g *Gateway) reserveRuntime(rt *devshardRuntime, inputTokens int64) { } func (g *Gateway) reserveRuntimeLocked(rt *devshardRuntime, inputTokens int64) { - rt.activeRequests.Add(1) + rt.activeUserRequests.Add(1) rt.reservedTokens.Add(inputTokens) } func (g *Gateway) releaseRuntime(rt *devshardRuntime, inputTokens int64) { - remaining := rt.activeRequests.Add(-1) + remaining := rt.activeUserRequests.Add(-1) rt.reservedTokens.Add(-inputTokens) - // active=false (set during enqueue) blocks new reservations, so the count - // only drains downward — remaining==0 is the exact "last request finished" - // edge. scheduleAutoSettlement dedups, so a double-fire is harmless. - if remaining != 0 { + // Stay non-quiet while a background race cleanup is still refunding/persisting; its own releaseRaceCleanup re-checks the drain (whichever hits zero last fires; dedup makes a double-fire harmless). + if remaining != 0 || rt.pendingRaceCleanup.Load() != 0 { return } + g.runtimeDrained(rt) +} +// runtimeDrained fires the deferred settle/retire once a runtime is quiet; scheduleAutoSettlement/retireRuntime dedup, so a double-fire from the two drain paths is harmless. +func (g *Gateway) runtimeDrained(rt *devshardRuntime) { if rt.settlementPending.Load() { log.Printf("settlement_drain_complete escrow=%s reason=%s", rt.id, rt.settlementReason) g.scheduleAutoSettlement(rt.id, rt.settlementReason) return } - if rt.retirePending.Load() { log.Printf("runtime_retire_drain_complete escrow=%s reason=%s", rt.id, rt.retireReason) g.retireRuntime(rt.id, rt.retireReason) } } +// startRaceCleanup registers a background race cleanup against the drain barrier; it must run synchronously before the cleanup goroutine spawns (and before the winning handler returns). +func (g *Gateway) startRaceCleanup(rt *devshardRuntime) { + rt.pendingRaceCleanup.Add(1) +} + +// releaseRaceCleanup clears a race cleanup from the barrier and fires the deferred settle/retire if the runtime is now quiet. +func (g *Gateway) releaseRaceCleanup(rt *devshardRuntime) { + remaining := rt.pendingRaceCleanup.Add(-1) + if remaining != 0 || rt.activeUserRequests.Load() != 0 { + return + } + g.runtimeDrained(rt) +} + func (rt *devshardRuntime) validateRequestedModel(requestModel string) error { if rt == nil || requestModel == "" || requestModel == rt.model { return nil @@ -2865,7 +2889,7 @@ func (g *Gateway) handleAdminDevshardParticipants(w http.ResponseWriter, r *http } model := rt.model active := rt.active.Load() - activeRequests := rt.activeRequests.Load() + activeUserRequests := rt.activeUserRequests.Load() participantKeys := runtimeParticipantKeys(rt) slotCounts := make(map[string]int, len(rt.participantSlotCounts)) for key, count := range rt.participantSlotCounts { @@ -2910,7 +2934,7 @@ func (g *Gateway) handleAdminDevshardParticipants(w http.ResponseWriter, r *http "id": id, "model": model, "active": active, - "active_requests": activeRequests, + "active_requests": activeUserRequests, "participant_count": len(participants), "available_count": availableCount, "blocked_count": blockedCount, @@ -3280,7 +3304,7 @@ func (g *Gateway) handleAdminCleanDevshard(w http.ResponseWriter, r *http.Reques return } if rt, ok := g.runtimes[id]; ok { - if rt.activeRequests.Load() > 0 { + if rt.escrowHasBackgroundWork() { http.Error(w, fmt.Sprintf(`{"error":{"message":"devshard %s has active requests"}}`, id), http.StatusConflict) return } @@ -3381,10 +3405,10 @@ func (g *Gateway) retireRuntimeLocked(id, reason string) *devshardRuntime { log.Printf("runtime_retire_skipped escrow=%s reason=%q cause=not_registered", id, reason) return nil } - if inFlight := rt.activeRequests.Load(); inFlight > 0 { + if rt.escrowHasBackgroundWork() { rt.retireReason = reason rt.retirePending.Store(true) - log.Printf("runtime_retire_deferred escrow=%s reason=%q active_requests=%d", id, reason, inFlight) + log.Printf("runtime_retire_deferred escrow=%s reason=%q active_requests=%d pending_race_cleanup=%d", id, reason, rt.activeUserRequests.Load(), rt.pendingRaceCleanup.Load()) return nil } delete(g.runtimes, id) @@ -3409,6 +3433,15 @@ func (g *Gateway) attachMetrics(rt *devshardRuntime) { rt.proxy.redundancy.devshardID = rt.id } +// attachRaceCleanupBarrier wires the runtime's background race cleanups into the drain barrier so settle/store-close wait for them, not just for foreground requests. +func (g *Gateway) attachRaceCleanupBarrier(rt *devshardRuntime) { + if g == nil || rt == nil || rt.proxy == nil || rt.proxy.redundancy == nil { + return + } + rt.proxy.redundancy.onRaceCleanupStart = func() { g.startRaceCleanup(rt) } + rt.proxy.redundancy.onRaceCleanupDone = func() { g.releaseRaceCleanup(rt) } +} + func (g *Gateway) attachEscrowChecker(rt *devshardRuntime) { if g == nil || rt == nil || rt.proxy == nil || rt.proxy.redundancy == nil { return @@ -3481,9 +3514,9 @@ func (g *Gateway) deactivateAndSettleDevshardByID(id, reason string) { g.mu.Lock() rt, ok := g.runtimes[id] g.mu.Unlock() - if ok && rt.activeRequests.Load() > 0 { - log.Printf("settlement_queued_waiting_for_drain escrow=%s reason=%s active_requests=%d", - id, reason, rt.activeRequests.Load()) + if ok && rt.escrowHasBackgroundWork() { + log.Printf("settlement_queued_waiting_for_drain escrow=%s reason=%s active_requests=%d pending_race_cleanup=%d", + id, reason, rt.activeUserRequests.Load(), rt.pendingRaceCleanup.Load()) return } g.scheduleAutoSettlement(id, reason) diff --git a/devshard/cmd/devshardctl/gateway_race_cleanup_barrier_test.go b/devshard/cmd/devshardctl/gateway_race_cleanup_barrier_test.go new file mode 100644 index 0000000000..cf69ff65d5 --- /dev/null +++ b/devshard/cmd/devshardctl/gateway_race_cleanup_barrier_test.go @@ -0,0 +1,110 @@ +package main + +import ( + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +// A speculative race returns to the client on the winner and hands its still +// pending losers to a background race cleanup (finishRaceWhenPendingDone) that +// keeps refunding reserved cost and persisting loser signatures on +// context.Background() for up to SecondaryWaitAfterWinner. That cleanup holds no +// activeUserRequests slot, so the drain barrier must track it separately: +// settling (which reads the balance the cleanup is still mutating) and retiring +// (which closes the SQLite store the cleanup is still writing to) must both wait +// until the cleanup drains too. + +// TestReleaseRuntimeDefersSettleWhilePendingRaceCleanup pins the money branch: a +// runtime with a pending race cleanup must NOT settle when its last foreground +// request drains — only once the cleanup completes as well. +func TestReleaseRuntimeDefersSettleWhilePendingRaceCleanup(t *testing.T) { + rt := gatewayTestRuntimeForLimits(t, "12", balanceMinimumThreshold-1, nonceDeactivationLimit-1) + g, _, settled := gatewayTestDepletionGateway(t, rt) + + g.reserveRuntime(rt, 1) // the one in-flight foreground request + g.startRaceCleanup(rt) // its background race cleanup, spawned before it returns + rt.settlementReason = "low_balance" + rt.settlementPending.Store(true) + + g.releaseRuntime(rt, 1) // foreground request returns; cleanup still in flight + require.Never(t, func() bool { return settled.Load() > 0 }, 200*time.Millisecond, 20*time.Millisecond, + "settlement must not fire while a background race cleanup is still refunding/persisting") + + g.releaseRaceCleanup(rt) // cleanup drains + require.Eventually(t, func() bool { return settled.Load() == 1 }, time.Second, 10*time.Millisecond, + "settlement must fire exactly once, after the cleanup drains") +} + +// TestReleaseRuntimeDefersRetireWhilePendingRaceCleanup pins the durability +// branch: retirement (which closes the per-runtime SQLite store) must defer +// while a race cleanup is still writing loser signatures, and fire once it drains. +func TestReleaseRuntimeDefersRetireWhilePendingRaceCleanup(t *testing.T) { + g, rt := newRetireTestGateway("12") + g.reserveRuntime(rt, 0) // the one in-flight foreground request + g.startRaceCleanup(rt) // its background race cleanup + rt.retireReason = "balance exhausted" + rt.retirePending.Store(true) + + g.releaseRuntime(rt, 0) // foreground request returns; cleanup still in flight + _, stillRegistered := g.runtimes["12"] + require.True(t, stillRegistered, "retire must defer while a race cleanup is in flight") + + g.releaseRaceCleanup(rt) // cleanup drains + _, stillRegistered = g.runtimes["12"] + require.False(t, stillRegistered, "retire fires once the cleanup drains") + require.Empty(t, g.runtimeOrder) +} + +// TestGoTrackedRaceCleanupStartsBarrierSynchronously pins the ordering contract +// the fix depends on: the start hook must fire synchronously, before the cleanup +// goroutine is spawned, so a winning foreground handler that returns and drops +// activeUserRequests immediately afterwards can never observe the runtime as +// quiet while this cleanup is still in flight. +func TestGoTrackedRaceCleanupStartsBarrierSynchronously(t *testing.T) { + var started, done atomic.Int64 + release := make(chan struct{}) + redundancy := &Redundancy{ + onRaceCleanupStart: func() { started.Add(1) }, + onRaceCleanupDone: func() { done.Add(1) }, + } + + redundancy.goTrackedRaceCleanup(func() { <-release }) + + require.Equal(t, int64(1), started.Load(), "start hook must fire before goTrackedRaceCleanup returns") + require.Equal(t, int64(0), done.Load(), "done hook must not fire while the cleanup runs") + + close(release) + require.Eventually(t, func() bool { return done.Load() == 1 }, time.Second, 10*time.Millisecond, + "done hook must fire once the cleanup completes") +} + +// TestConcurrentDrainSettlesExactlyOnce guards the two-counter drain: when the +// last foreground request and the last race cleanup reach zero concurrently, +// exactly one of the racing drain paths must settle — never zero (a lost +// wakeup) and, thanks to dedup, never twice. +func TestConcurrentDrainSettlesExactlyOnce(t *testing.T) { + rt := gatewayTestRuntimeForLimits(t, "12", balanceMinimumThreshold-1, nonceDeactivationLimit-1) + g, _, settled := gatewayTestDepletionGateway(t, rt) + + g.reserveRuntime(rt, 1) + g.startRaceCleanup(rt) + rt.settlementReason = "low_balance" + rt.settlementPending.Store(true) + + var ready sync.WaitGroup + ready.Add(2) + start := make(chan struct{}) + go func() { ready.Done(); <-start; g.releaseRuntime(rt, 1) }() + go func() { ready.Done(); <-start; g.releaseRaceCleanup(rt) }() + ready.Wait() + close(start) // release both drains as simultaneously as the scheduler allows + + require.Eventually(t, func() bool { return settled.Load() == 1 }, time.Second, 10*time.Millisecond, + "a concurrent drain must settle exactly once") + require.Never(t, func() bool { return settled.Load() > 1 }, 200*time.Millisecond, 20*time.Millisecond, + "dedup must keep a double drain from settling twice") +} diff --git a/devshard/cmd/devshardctl/gateway_runtime_retire_test.go b/devshard/cmd/devshardctl/gateway_runtime_retire_test.go index 680706b11c..80c3f20818 100644 --- a/devshard/cmd/devshardctl/gateway_runtime_retire_test.go +++ b/devshard/cmd/devshardctl/gateway_runtime_retire_test.go @@ -43,7 +43,7 @@ func TestRetireRuntimeRemovesRuntimeFromRegistry(t *testing.T) { // the runtime registered) until the request count drains to zero. func TestRetireRuntimeDefersWhileRequestsInFlight(t *testing.T) { g, rt := newRetireTestGateway("12") - rt.activeRequests.Store(1) + rt.activeUserRequests.Store(1) require.False(t, g.retireRuntime("12", "busy")) _, stillRegistered := g.runtimes["12"] @@ -51,7 +51,7 @@ func TestRetireRuntimeDefersWhileRequestsInFlight(t *testing.T) { require.True(t, rt.retirePending.Load(), "deferred retire must record its intent") require.Equal(t, "busy", rt.retireReason) - rt.activeRequests.Store(0) + rt.activeUserRequests.Store(0) require.True(t, g.retireRuntime("12", "drained")) _, stillRegistered = g.runtimes["12"] require.False(t, stillRegistered) @@ -61,7 +61,7 @@ func TestRetireRuntimeDefersWhileRequestsInFlight(t *testing.T) { // the last request drains through releaseRuntime. func TestReleaseRuntimeRetiresAfterDrain(t *testing.T) { g, rt := newRetireTestGateway("12") - rt.activeRequests.Store(1) + rt.activeUserRequests.Store(1) require.False(t, g.retireRuntime("12", "balance exhausted")) _, stillRegistered := g.runtimes["12"] @@ -78,7 +78,7 @@ func TestReleaseRuntimeRetiresAfterDrain(t *testing.T) { // isolation: only retirePending set (no settlement), drain → retire. func TestReleaseRuntimeRetiresWithOnlyRetirePending(t *testing.T) { g, rt := newRetireTestGateway("12") - rt.activeRequests.Store(1) + rt.activeUserRequests.Store(1) rt.retireReason = "balance exhausted" rt.retirePending.Store(true) diff --git a/devshard/cmd/devshardctl/gateway_test.go b/devshard/cmd/devshardctl/gateway_test.go index d5bc7c71f3..001b693cea 100644 --- a/devshard/cmd/devshardctl/gateway_test.go +++ b/devshard/cmd/devshardctl/gateway_test.go @@ -336,19 +336,19 @@ func TestParseDevshardPath(t *testing.T) { } func TestGatewayChooseRuntimeUsesLowestLoad(t *testing.T) { - // Load score is activeRequests / W(e). Both runtimes share W(e)=1 + // Load score is activeUserRequests / W(e). Both runtimes share W(e)=1 // (no capacity model wired). The picker should prefer the runtime // with fewer in-flight requests. a := &devshardRuntime{id: "6", model: "m"} b := &devshardRuntime{id: "12", model: "m"} - a.activeRequests.Store(5) - b.activeRequests.Store(1) + a.activeUserRequests.Store(5) + b.activeUserRequests.Store(1) g := NewGateway([]*devshardRuntime{a, b}, NewGatewayLimiter(0, 0), "m") chosen, err := g.reserveRuntimeForModel("m", 5) require.NoError(t, err) require.Equal(t, "12", chosen.id) - require.EqualValues(t, 2, chosen.activeRequests.Load()) + require.EqualValues(t, 2, chosen.activeUserRequests.Load()) require.EqualValues(t, 5, chosen.reservedTokens.Load()) } @@ -807,7 +807,7 @@ func TestAdminDeactivateDevshardAllowsActiveRequestsAndStopsNewChat(t *testing.T w.WriteHeader(http.StatusNoContent) }), } - rt.activeRequests.Store(1) + rt.activeUserRequests.Store(1) g := NewGateway([]*devshardRuntime{rt}, NewGatewayLimiter(0, 0), "Qwen/Test") g.store = store @@ -817,7 +817,7 @@ func TestAdminDeactivateDevshardAllowsActiveRequestsAndStopsNewChat(t *testing.T require.Equal(t, http.StatusOK, rec.Code) require.False(t, rt.active.Load()) - require.EqualValues(t, 1, rt.activeRequests.Load()) + require.EqualValues(t, 1, rt.activeUserRequests.Load()) state, ok, err := store.LoadState() require.NoError(t, err) @@ -1144,7 +1144,7 @@ func TestGatewayHandleDevshardFinalizeRequiresNoActiveRequests(t *testing.T) { w.WriteHeader(http.StatusNoContent) }), } - rt.activeRequests.Store(1) + rt.activeUserRequests.Store(1) g := NewGateway([]*devshardRuntime{rt}, NewGatewayLimiter(0, 0), "Qwen/Test") g.store = store @@ -1156,7 +1156,7 @@ func TestGatewayHandleDevshardFinalizeRequiresNoActiveRequests(t *testing.T) { require.False(t, forwarded) require.True(t, rt.active.Load()) - rt.activeRequests.Store(0) + rt.activeUserRequests.Store(0) rec = httptest.NewRecorder() g.handleDevshard(rec, req) @@ -1228,8 +1228,8 @@ func TestGatewayHandlePooledChatSetsChosenDevshardHeader(t *testing.T) { w.WriteHeader(http.StatusCreated) }), } - slow.activeRequests.Store(10) - fast.activeRequests.Store(0) + slow.activeUserRequests.Store(10) + fast.activeUserRequests.Store(0) g := NewGateway([]*devshardRuntime{slow, fast}, NewGatewayLimiter(0, 0), "Qwen/Test") g.settings.ModelLimits = []GatewayModelLimitSettings{{ModelID: "Qwen/Test", AccessMode: string(gatewayAccessModeOpen)}} @@ -1478,7 +1478,7 @@ func TestGatewayHandlePooledChatRejectsUnsupportedModel(t *testing.T) { require.Contains(t, rec.Body.String(), `unsupported model \"Nope/Unsupported\"`) require.Contains(t, rec.Body.String(), "Qwen/Test") require.False(t, forwarded) - require.EqualValues(t, 0, rt.activeRequests.Load()) + require.EqualValues(t, 0, rt.activeUserRequests.Load()) } func TestGatewayHandlePooledChatRejectsUnsupportedModelBeforeDefaultAdminOnly(t *testing.T) { @@ -1503,7 +1503,7 @@ func TestGatewayHandlePooledChatRejectsUnsupportedModelBeforeDefaultAdminOnly(t require.Contains(t, rec.Body.String(), `unsupported model \"Nope/Unsupported\"`) require.NotContains(t, rec.Body.String(), "admin API key") require.False(t, forwarded) - require.EqualValues(t, 0, rt.activeRequests.Load()) + require.EqualValues(t, 0, rt.activeUserRequests.Load()) } func TestGatewayHandleDevshardChatRejectsUnsupportedModel(t *testing.T) { @@ -1528,7 +1528,7 @@ func TestGatewayHandleDevshardChatRejectsUnsupportedModel(t *testing.T) { require.Contains(t, rec.Body.String(), `unsupported model \"Nope/Unsupported\"`) require.Contains(t, rec.Body.String(), "Qwen/Test") require.False(t, forwarded) - require.EqualValues(t, 0, rt.activeRequests.Load()) + require.EqualValues(t, 0, rt.activeUserRequests.Load()) } func TestGatewayPooledChatRefreshesCapacityScaleBeforeAcquire(t *testing.T) { diff --git a/devshard/cmd/devshardctl/metrics.go b/devshard/cmd/devshardctl/metrics.go index 5e7b6511c1..e972540502 100644 --- a/devshard/cmd/devshardctl/metrics.go +++ b/devshard/cmd/devshardctl/metrics.go @@ -840,7 +840,7 @@ func (c *gatewayMetricsCollector) Collect(ch chan<- prometheus.Metric) { } labels := []string{rt.id, rt.model} ch <- prometheus.MustNewConstMetric(c.runtimeActiveDesc, prometheus.GaugeValue, active, labels...) - ch <- prometheus.MustNewConstMetric(c.runtimeRequestsDesc, prometheus.GaugeValue, float64(rt.activeRequests.Load()), labels...) + ch <- prometheus.MustNewConstMetric(c.runtimeRequestsDesc, prometheus.GaugeValue, float64(rt.activeUserRequests.Load()), labels...) ch <- prometheus.MustNewConstMetric(c.runtimeReservedDesc, prometheus.GaugeValue, float64(rt.reservedTokens.Load()), labels...) blocked := 0 if c.gateway.participantLimiter != nil { diff --git a/devshard/cmd/devshardctl/redundancy.go b/devshard/cmd/devshardctl/redundancy.go index 129d11ae98..649f763f26 100644 --- a/devshard/cmd/devshardctl/redundancy.go +++ b/devshard/cmd/devshardctl/redundancy.go @@ -537,6 +537,8 @@ type Redundancy struct { metrics *DevshardMetrics onEscrowMissing func() // called (at most once per request) when a host reports escrow not found onBalanceExhausted func() // called (once) when local state hits insufficient balance + onRaceCleanupStart func() // fires synchronously before a race cleanup goroutine spawns + onRaceCleanupDone func() // fires when a race cleanup goroutine finishes balanceExhaustedOnce sync.Once picker *sessionPicker participantLimiter *ParticipantRequestLimiter @@ -2251,7 +2253,9 @@ func (e *Redundancy) awaitRace(streamCtx, settleCtx context.Context, attempts [] "max_wait_ms", SecondaryWaitAfterWinner.Milliseconds(), "decision", decision.Reason, ) - go e.finishRaceWhenPendingDone(settleCtx, attempts, params, decision, winner, raceFinishOptions{recordFailureSamples: true}) + e.goTrackedRaceCleanup(func() { + e.finishRaceWhenPendingDone(settleCtx, attempts, params, decision, winner, raceFinishOptions{recordFailureSamples: true}) + }) return nil } } @@ -2376,9 +2380,11 @@ func (e *Redundancy) awaitRace(streamCtx, settleCtx context.Context, attempts [] } e.markPhaseTransitionAbort(inf) e.recordWinnerTerminalFailureOnce(inf, params, w) - go e.finishRaceWhenPendingDone(settleCtx, attempts, params, decision, w, raceFinishOptions{ - forceTreatAsFailure: true, - recordFailureSamples: true, + e.goTrackedRaceCleanup(func() { + e.finishRaceWhenPendingDone(settleCtx, attempts, params, decision, w, raceFinishOptions{ + forceTreatAsFailure: true, + recordFailureSamples: true, + }) }) logRequestStage(settleCtx, "winner_failed_after_content", "escrow", e.devshardID, "winner_nonce", w, "error", err) return err @@ -2442,7 +2448,7 @@ func (e *Redundancy) awaitRace(streamCtx, settleCtx context.Context, attempts [] recordFailureSamples: true, nonStreamingReducedTokenTimeout: true, } - go func() { + e.goTrackedRaceCleanup(func() { if err := e.finishRaceOutcome(settleCtx, attempts, params, decision, 0, opts); err != nil { var timeoutErr *nonStreamingReducedMaxTokensTimeoutError if errors.As(err, &timeoutErr) { @@ -2450,7 +2456,7 @@ func (e *Redundancy) awaitRace(streamCtx, settleCtx context.Context, attempts [] } logRequestStage(settleCtx, "background_finish_failed", "escrow", e.devshardID, "error", err) } - }() + }) return &nonStreamingReducedMaxTokensTimeoutError{} case <-stallC: now := time.Now() @@ -2519,7 +2525,9 @@ func (e *Redundancy) awaitRace(streamCtx, settleCtx context.Context, attempts [] } pending := pendingInflights(attempts) logRequestStage(settleCtx, "request_stream_canceled", "escrow", e.devshardID, "winner_nonce", winner, "pending", len(pending), "decision", decision.Reason, "error", streamCtx.Err()) - go e.finishRaceWhenPendingDone(settleCtx, attempts, params, decision, winner, raceFinishOptions{}) + e.goTrackedRaceCleanup(func() { + e.finishRaceWhenPendingDone(settleCtx, attempts, params, decision, winner, raceFinishOptions{}) + }) return streamCtx.Err() } @@ -2690,6 +2698,19 @@ type raceFinishOptions struct { nonStreamingReducedTokenTimeout bool } +// goTrackedRaceCleanup runs a background race cleanup detached while keeping the drain barrier aware of it; onRaceCleanupStart fires synchronously so the winning handler can never see the runtime as quiet mid-cleanup. +func (e *Redundancy) goTrackedRaceCleanup(fn func()) { + if e.onRaceCleanupStart != nil { + e.onRaceCleanupStart() + } + go func() { + if e.onRaceCleanupDone != nil { + defer e.onRaceCleanupDone() + } + fn() + }() +} + func (e *Redundancy) finishRaceWhenPendingDone(ctx context.Context, attempts []*inflight, params user.InferenceParams, decision Decision, winnerNonce uint64, opts raceFinishOptions) { bgCtx, _ := ensureRequestLogContext(context.Background()) bgCtx = logging.PropagateRequestID(bgCtx, ctx) @@ -3567,7 +3588,7 @@ func (e *Redundancy) finishRaceOutcome(ctx context.Context, attempts []*inflight StartedAt: params.StartedAt, } if anySucceeded { - go func() { + e.goTrackedRaceCleanup(func() { bgCtx, _ := ensureRequestLogContext(context.Background()) bgCtx = logging.PropagateRequestID(bgCtx, ctx) for _, inf := range failed { @@ -3617,7 +3638,7 @@ func (e *Redundancy) finishRaceOutcome(ctx context.Context, attempts []*inflight } } e.logRequestSettled(bgCtx, winnerNonce, decision, "success") - }() + }) } } From 0cb8b4ea1b6e1ac4f29905c63202dc57d1b20ad4 Mon Sep 17 00:00:00 2001 From: Daniil Yankouski Date: Mon, 6 Jul 2026 21:01:40 +0200 Subject: [PATCH 09/13] Recover orphaned rotation escrows on DB persist failure instead of re-minting on chain (#1343) * fix(devshard): recover orphaned rotation escrows on DB persist failure instead of re-minting on chain * fix(devshard): don't clear rotation commitment on transient 404; only after unordered-tx TTL elapses to avoid orphaning escrows * fix(devshard): GetTxEscrowID not found error --- devshard/cmd/devshardctl/chain_tx_rest.go | 68 ++++- .../cmd/devshardctl/chain_tx_rest_test.go | 115 +++++++- .../cmd/devshardctl/escrow_recovery_test.go | 251 ++++++++++++++++++ devshard/cmd/devshardctl/escrow_rotator.go | 216 +++++++++++++-- devshard/cmd/devshardctl/gateway.go | 9 +- .../gateway_runtime_retire_test.go | 2 +- devshard/cmd/devshardctl/gateway_store.go | 96 +++++++ .../cmd/devshardctl/gateway_store_test.go | 23 +- 8 files changed, 734 insertions(+), 46 deletions(-) create mode 100644 devshard/cmd/devshardctl/escrow_recovery_test.go diff --git a/devshard/cmd/devshardctl/chain_tx_rest.go b/devshard/cmd/devshardctl/chain_tx_rest.go index c4fd182ddc..66d03dc0f6 100644 --- a/devshard/cmd/devshardctl/chain_tx_rest.go +++ b/devshard/cmd/devshardctl/chain_tx_rest.go @@ -3,7 +3,9 @@ package main import ( "bytes" "context" + "crypto/sha256" "encoding/base64" + "encoding/hex" "encoding/json" "errors" "fmt" @@ -113,7 +115,49 @@ func NewRESTChainTxClient(cfg RESTChainTxConfig) (*RESTChainTxClient, error) { }, nil } -func (c *RESTChainTxClient) CreateDevshardEscrow(ctx context.Context, signer *signing.Secp256k1Signer, amount uint64, modelID string) (*CreateDevshardEscrowResult, error) { +// txHashFromBytes returns the cosmos tx hash (uppercase-hex SHA-256 of the tx +// bytes), computable before broadcast and equal to the hash the node returns. +func txHashFromBytes(txBytes []byte) string { + sum := sha256.Sum256(txBytes) + return strings.ToUpper(hex.EncodeToString(sum[:])) +} + +// errTxNotFound marks a tx that is not on chain (404) — distinct from one that +// committed but failed. The tx may still land until its unordered TTL elapses. +var errTxNotFound = errors.New("tx not found on chain") + +// GetTxEscrowID resolves a create tx hash to its escrow_id in one lookup; +// found=false when the tx is absent or failed (no escrow), error when unreachable. +func (c *RESTChainTxClient) GetTxEscrowID(ctx context.Context, txHash string) (uint64, bool, error) { + var lastErr error + hasNotFoundError := false + for _, baseURL := range c.txQueryBaseURLs() { + var payload txResponseEnvelope + err := c.getJSONFromBaseURL(ctx, baseURL, "/cosmos/tx/v1beta1/txs/"+url.PathEscape(txHash), &payload) + if err != nil { + if isNotFoundError(err) { + hasNotFoundError = true // this endpoint lacks it; try the fallback before deciding + continue + } + lastErr = fmt.Errorf("%s: %w", baseURL, err) + continue + } + if payload.TxResponse.Code != 0 { + return 0, false, nil // tx committed but failed → no escrow created + } + if escrowID, ok := payload.TxResponse.createdEscrowID(); ok { + return escrowID, true, nil + } + lastErr = fmt.Errorf("tx %s committed via %s but escrow_id event was not found", txHash, baseURL) + } + // Only conclude "not on chain" when every reachable endpoint agreed on 404. + if lastErr == nil && hasNotFoundError { + return 0, false, errTxNotFound + } + return 0, false, lastErr +} + +func (c *RESTChainTxClient) CreateDevshardEscrow(ctx context.Context, signer *signing.Secp256k1Signer, amount uint64, modelID string, onPrepared func(txHash string) error) (*CreateDevshardEscrowResult, error) { if c == nil { return nil, fmt.Errorf("chain tx client is nil") } @@ -145,10 +189,20 @@ func (c *RESTChainTxClient) CreateDevshardEscrow(ctx context.Context, signer *si if err != nil { return nil, err } - txHash, err := c.broadcastTx(ctx, txBytes) + // Record the intent (precomputed hash) before the irreversible broadcast; abort if it fails. + txHash := txHashFromBytes(txBytes) + if onPrepared != nil { + if err := onPrepared(txHash); err != nil { + return nil, fmt.Errorf("record escrow create intent before broadcast: %w", err) + } + } + broadcastHash, err := c.broadcastTx(ctx, txBytes) if err != nil { return nil, err } + if !strings.EqualFold(broadcastHash, txHash) { + return nil, fmt.Errorf("tx hash mismatch: precomputed %s, node returned %s", txHash, broadcastHash) + } escrowID, err := c.waitForCreatedEscrowID(ctx, txHash) if err != nil { return nil, err @@ -332,6 +386,16 @@ func (c *RESTChainTxClient) getJSONFromBaseURL(ctx context.Context, baseURL, pat return json.NewDecoder(resp.Body).Decode(out) } +// isNotFoundError reports whether a chain GET failed with 404 / not-found rather +// than a transient error. +func isNotFoundError(err error) bool { + if err == nil { + return false + } + s := strings.ToLower(err.Error()) + return strings.Contains(s, "status 404") || strings.Contains(s, "not found") +} + func (c *RESTChainTxClient) postJSON(ctx context.Context, path string, in any, out any) error { data, err := json.Marshal(in) if err != nil { diff --git a/devshard/cmd/devshardctl/chain_tx_rest_test.go b/devshard/cmd/devshardctl/chain_tx_rest_test.go index 2798720727..f0ec68d2ec 100644 --- a/devshard/cmd/devshardctl/chain_tx_rest_test.go +++ b/devshard/cmd/devshardctl/chain_tx_rest_test.go @@ -20,6 +20,7 @@ func TestRESTChainTxClient_CreateDevshardEscrow(t *testing.T) { require.NoError(t, err) var broadcastSeen bool + var expectedHash string server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { switch r.URL.Path { case "/cosmos/auth/v1beta1/accounts/" + signer.Address(): @@ -44,18 +45,20 @@ func TestRESTChainTxClient_CreateDevshardEscrow(t *testing.T) { require.NotEmpty(t, txBytes) assertUnorderedTx(t, txBytes) broadcastSeen = true + // A real node returns SHA-256(txBytes); the client precomputes the same. + expectedHash = txHashFromBytes(txBytes) writeTestJSON(t, w, map[string]any{ "tx_response": map[string]any{ "code": 0, - "txhash": "ABC123", + "txhash": expectedHash, }, }) - case "/cosmos/tx/v1beta1/txs/ABC123": + case "/cosmos/tx/v1beta1/txs/" + expectedHash: require.True(t, broadcastSeen) writeTestJSON(t, w, map[string]any{ "tx_response": map[string]any{ "code": 0, - "txhash": "ABC123", + "txhash": expectedHash, "events": []map[string]any{{ "type": "devshard_escrow_created", "attributes": []map[string]string{{ @@ -81,10 +84,11 @@ func TestRESTChainTxClient_CreateDevshardEscrow(t *testing.T) { }) require.NoError(t, err) - result, err := client.CreateDevshardEscrow(t.Context(), signer, 1_000_000, "Qwen/Test") + result, err := client.CreateDevshardEscrow(t.Context(), signer, 1_000_000, "Qwen/Test", nil) require.NoError(t, err) require.Equal(t, uint64(42), result.EscrowID) - require.Equal(t, "ABC123", result.TxHash) + require.NotEmpty(t, expectedHash) + require.Equal(t, expectedHash, result.TxHash) require.Equal(t, signer.Address(), result.Creator) } @@ -93,6 +97,7 @@ func TestRESTChainTxClient_CreateDevshardEscrowUsesTxQueryFallback(t *testing.T) require.NoError(t, err) var broadcastSeen bool + var expectedHash string broadcastServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { switch r.URL.Path { case "/cosmos/auth/v1beta1/accounts/" + signer.Address(): @@ -116,13 +121,14 @@ func TestRESTChainTxClient_CreateDevshardEscrowUsesTxQueryFallback(t *testing.T) require.NoError(t, err) assertUnorderedTx(t, txBytes) broadcastSeen = true + expectedHash = txHashFromBytes(txBytes) writeTestJSON(t, w, map[string]any{ "tx_response": map[string]any{ "code": 0, - "txhash": "FALLBACK123", + "txhash": expectedHash, }, }) - case "/cosmos/tx/v1beta1/txs/FALLBACK123": + case "/cosmos/tx/v1beta1/txs/" + expectedHash: http.Error(w, `{"code":2,"message":"transaction indexing is disabled"}`, http.StatusInternalServerError) default: http.NotFound(w, r) @@ -131,12 +137,12 @@ func TestRESTChainTxClient_CreateDevshardEscrowUsesTxQueryFallback(t *testing.T) defer broadcastServer.Close() queryServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - require.Equal(t, "/cosmos/tx/v1beta1/txs/FALLBACK123", r.URL.Path) + require.Equal(t, "/cosmos/tx/v1beta1/txs/"+expectedHash, r.URL.Path) require.True(t, broadcastSeen) writeTestJSON(t, w, map[string]any{ "tx_response": map[string]any{ "code": 0, - "txhash": "FALLBACK123", + "txhash": expectedHash, "events": []map[string]any{{ "type": "devshard_escrow_created", "attributes": []map[string]string{{ @@ -160,13 +166,100 @@ func TestRESTChainTxClient_CreateDevshardEscrowUsesTxQueryFallback(t *testing.T) }) require.NoError(t, err) - result, err := client.CreateDevshardEscrow(t.Context(), signer, 1_000_000, "Qwen/Test") + result, err := client.CreateDevshardEscrow(t.Context(), signer, 1_000_000, "Qwen/Test", nil) require.NoError(t, err) require.Equal(t, uint64(43), result.EscrowID) - require.Equal(t, "FALLBACK123", result.TxHash) + require.NotEmpty(t, expectedHash) + require.Equal(t, expectedHash, result.TxHash) require.Equal(t, signer.Address(), result.Creator) } +// escrowIDQueryServer returns a query endpoint that replies with a committed +// create tx carrying an escrow_id event for txHash, and 404 for anything else. +func escrowIDQueryServer(t *testing.T, txHash string, escrowID uint64) *httptest.Server { + t.Helper() + return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/cosmos/tx/v1beta1/txs/"+txHash { + http.NotFound(w, r) + return + } + writeTestJSON(t, w, map[string]any{ + "tx_response": map[string]any{ + "code": 0, + "txhash": txHash, + "events": []map[string]any{{ + "type": "devshard_escrow_created", + "attributes": []map[string]string{{"key": "escrow_id", "value": fmt.Sprintf("%d", escrowID)}}, + }}, + }, + }) + })) +} + +// TestRESTChainTxClient_GetTxEscrowIDTriesFallbackOn404 pins the recovery path: +// when the primary node has not indexed the create tx (404), the escrow_id must +// still be resolved from the fallback query URL rather than reported missing. +func TestRESTChainTxClient_GetTxEscrowIDTriesFallbackOn404(t *testing.T) { + const txHash = "ABC123" + primary := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + http.NotFound(w, r) // primary hasn't indexed the tx yet + })) + defer primary.Close() + fallback := escrowIDQueryServer(t, txHash, 77) + defer fallback.Close() + + client, err := NewRESTChainTxClient(RESTChainTxConfig{BaseURL: primary.URL, TxQueryURL: fallback.URL, ChainID: "gonka-test"}) + require.NoError(t, err) + + escrowID, found, err := client.GetTxEscrowID(t.Context(), txHash) + require.NoError(t, err) + require.True(t, found) + require.Equal(t, uint64(77), escrowID) +} + +// TestRESTChainTxClient_GetTxEscrowIDNotFoundWhenAllEndpoints404 confirms the +// only-after-all-endpoints semantics: errTxNotFound is returned solely when +// every reachable endpoint agrees the tx is absent. +func TestRESTChainTxClient_GetTxEscrowIDNotFoundWhenAllEndpoints404(t *testing.T) { + notFound := func() *httptest.Server { + return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { http.NotFound(w, r) })) + } + primary := notFound() + defer primary.Close() + fallback := notFound() + defer fallback.Close() + + client, err := NewRESTChainTxClient(RESTChainTxConfig{BaseURL: primary.URL, TxQueryURL: fallback.URL, ChainID: "gonka-test"}) + require.NoError(t, err) + + _, found, err := client.GetTxEscrowID(t.Context(), "ABC123") + require.False(t, found) + require.ErrorIs(t, err, errTxNotFound) +} + +// TestRESTChainTxClient_GetTxEscrowIDInconclusiveOnFallbackError guards against +// orphaning: a 404 on the primary plus a real error on the fallback is +// inconclusive, so a non-errTxNotFound error is returned and the caller keeps +// the commitment for a later retry instead of clearing it. +func TestRESTChainTxClient_GetTxEscrowIDInconclusiveOnFallbackError(t *testing.T) { + primary := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + http.NotFound(w, r) + })) + defer primary.Close() + fallback := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + http.Error(w, `{"message":"transaction indexing is disabled"}`, http.StatusInternalServerError) + })) + defer fallback.Close() + + client, err := NewRESTChainTxClient(RESTChainTxConfig{BaseURL: primary.URL, TxQueryURL: fallback.URL, ChainID: "gonka-test"}) + require.NoError(t, err) + + _, found, err := client.GetTxEscrowID(t.Context(), "ABC123") + require.False(t, found) + require.Error(t, err) + require.NotErrorIs(t, err, errTxNotFound) +} + func TestRESTChainTxClient_SettleDevshardEscrow(t *testing.T) { signer, err := signing.GenerateKey() require.NoError(t, err) diff --git a/devshard/cmd/devshardctl/escrow_recovery_test.go b/devshard/cmd/devshardctl/escrow_recovery_test.go new file mode 100644 index 0000000000..fbe4a50f80 --- /dev/null +++ b/devshard/cmd/devshardctl/escrow_recovery_test.go @@ -0,0 +1,251 @@ +package main + +import ( + "context" + "errors" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// The root-cause layer: the gateway store must open in WAL with a busy timeout +// so concurrent writes wait instead of failing with "database is locked". +func TestGatewayStoreUsesWALAndBusyTimeout(t *testing.T) { + store, err := NewGatewayStore(filepath.Join(t.TempDir(), "gateway.db")) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, store.Close()) }) + + var journalMode string + require.NoError(t, store.db.QueryRow("PRAGMA journal_mode").Scan(&journalMode)) + assert.Equal(t, "wal", strings.ToLower(journalMode)) + + var busyTimeout int + require.NoError(t, store.db.QueryRow("PRAGMA busy_timeout").Scan(&busyTimeout)) + assert.Equal(t, 5000, busyTimeout) +} + +func recoveryTestSettings() GatewaySettings { + return GatewaySettings{ + ChainREST: "http://node:1317", + PublicAPI: "http://api:9000", + DefaultModel: "Qwen/Test", + DefaultRequestMaxTokens: 1000, + MaxConcurrentRequests: 2, + EscrowRotation: EscrowRotationSettings{ + Enabled: true, + SettlementEnabled: true, + Models: []EscrowRotationModelSettings{{ + ModelID: "Qwen/Test", + TempCount: 1, + TargetCount: 1, + Amount: 1000, + PrivateKeyEnv: "DEVSHARD_PRIVATE_KEY", + }}, + }, + }.WithTuningDefaults() +} + +func stubRuntimeBuilder(t *testing.T) { + t.Helper() + saved := gatewayRuntimeBuilder + gatewayRuntimeBuilder = func(cfg RuntimeConfig, _, _ string, _ *PerfTracker) (*devshardRuntime, error) { + return &devshardRuntime{id: cfg.ID, model: cfg.Model}, nil + } + t.Cleanup(func() { gatewayRuntimeBuilder = saved }) +} + +// stubCreateOnChain replaces the on-chain create with a fake that mimics the real +// flow: it invokes onPrepared(txHash) (so the commitment is written) and aborts +// without a result if that fails — exactly as CreateDevshardEscrow does. +func stubCreateOnChain(t *testing.T, txHash string, escrowID uint64) { + t.Helper() + saved := gatewayCreateEscrowOnChain + gatewayCreateEscrowOnChain = func(_ *Gateway, _ context.Context, _ GatewaySettings, _ EscrowRotationModelSettings, onPrepared func(string) error) (*CreateDevshardEscrowResult, error) { + if onPrepared != nil { + if err := onPrepared(txHash); err != nil { + return nil, err + } + } + return &CreateDevshardEscrowResult{EscrowID: escrowID, TxHash: txHash}, nil + } + t.Cleanup(func() { gatewayCreateEscrowOnChain = saved }) +} + +func stubQueryTxEscrowID(t *testing.T, fn func(string) (uint64, bool, error)) { + t.Helper() + saved := gatewayQueryTxEscrowID + gatewayQueryTxEscrowID = func(_ context.Context, _ GatewaySettings, txHash string) (uint64, bool, error) { + return fn(txHash) + } + t.Cleanup(func() { gatewayQueryTxEscrowID = saved }) +} + +func newRecoveryGateway(t *testing.T) (*Gateway, *GatewayStore, GatewaySettings) { + t.Helper() + store, err := NewGatewayStore(filepath.Join(t.TempDir(), "gateway.db")) + require.NoError(t, err) + t.Cleanup(func() { _ = store.Close() }) + settings := recoveryTestSettings() + require.NoError(t, store.Initialize(settings, nil)) + stubRuntimeBuilder(t) + g := &Gateway{store: store, runtimes: map[string]*devshardRuntime{}} + return g, store, settings +} + +func devshardIDs(t *testing.T, store *GatewayStore) map[string]GatewayDevshardState { + t.Helper() + state, ok, err := store.LoadState() + require.NoError(t, err) + require.True(t, ok) + return gatewayDevshardsByID(state.Devshards) +} + +// Happy path: intent commitment is written before the chain tx, the escrow is +// persisted, and the commitment is cleared. +func TestCreateRotationEscrowIntentFirstThenPersistAndClear(t *testing.T) { + g, store, settings := newRecoveryGateway(t) + stubCreateOnChain(t, "TXAAA", 777) + model := normalizedEscrowRotationModels(settings)[0] + + _, err := g.createRotationEscrow(context.Background(), settings, model, rotationRoleTemp, 10) + require.NoError(t, err) + + require.Contains(t, devshardIDs(t, store), "777", "escrow persisted") + commitments, err := store.LoadCommitments() + require.NoError(t, err) + assert.Empty(t, commitments, "commitment cleared after persist") +} + +// If the intent commitment cannot be written, no chain tx is broadcast and no +// escrow is created — the safe failure. +func TestCreateRotationEscrowAbortsWhenCommitmentWriteFails(t *testing.T) { + g, store, settings := newRecoveryGateway(t) + stubCreateOnChain(t, "TXBBB", 778) + model := normalizedEscrowRotationModels(settings)[0] + + // Make every write fail, as a locked/read-only DB would. + _, err := store.db.Exec("PRAGMA query_only=ON") + require.NoError(t, err) + + _, err = g.createRotationEscrow(context.Background(), settings, model, rotationRoleTemp, 10) + require.Error(t, err) + + _, _ = store.db.Exec("PRAGMA query_only=OFF") + assert.NotContains(t, devshardIDs(t, store), "778", "no escrow created when intent write fails") + commitments, _ := store.LoadCommitments() + assert.Empty(t, commitments) +} + +// If the post-create persist fails, the commitment survives and reconcile later +// recovers the escrow from chain by its tx hash — escrow_id is never lost. +func TestCreateRotationEscrowPersistFailureRecoversViaCommitment(t *testing.T) { + g, store, settings := newRecoveryGateway(t) + stubCreateOnChain(t, "TXCCC", 888) + model := normalizedEscrowRotationModels(settings)[0] + + // Force the persist to fail (runtime build error) on the create path. + savedBuilder := gatewayRuntimeBuilder + gatewayRuntimeBuilder = func(RuntimeConfig, string, string, *PerfTracker) (*devshardRuntime, error) { + return nil, errors.New("persist boom") + } + _, err := g.createRotationEscrow(context.Background(), settings, model, rotationRoleTemp, 10) + require.Error(t, err) + gatewayRuntimeBuilder = savedBuilder // restore (stubRuntimeBuilder's success) + + require.NotContains(t, devshardIDs(t, store), "888", "not persisted yet") + commitments, err := store.LoadCommitments() + require.NoError(t, err) + require.Len(t, commitments, 1, "commitment kept for recovery") + assert.Equal(t, "TXCCC", commitments[0].TxHash) + + // Reconcile: chain resolves the tx hash to the escrow_id. + stubQueryTxEscrowID(t, func(txHash string) (uint64, bool, error) { + assert.Equal(t, "TXCCC", txHash) + return 888, true, nil + }) + g.reconcileCommitments(context.Background(), settings) + + assert.Contains(t, devshardIDs(t, store), "888", "recovered via commitment") + commitments, _ = store.LoadCommitments() + assert.Empty(t, commitments, "commitment cleared after recovery") +} + +// Reconcile persists an escrow for a pending commitment and clears it. +func TestReconcileCommitmentsRecoversFromChain(t *testing.T) { + g, store, settings := newRecoveryGateway(t) + require.NoError(t, store.SaveCommitment(GatewayEscrowCommitment{ + TxHash: "TXDDD", Model: "Qwen/Test", Role: rotationRoleTemp, Epoch: 11, PrivateKeyEnv: "DEVSHARD_PRIVATE_KEY", + })) + stubQueryTxEscrowID(t, func(string) (uint64, bool, error) { return 999, true, nil }) + + g.reconcileCommitments(context.Background(), settings) + + assert.Contains(t, devshardIDs(t, store), "999") + commitments, _ := store.LoadCommitments() + assert.Empty(t, commitments) +} + +// A commitment whose tx committed but failed (no escrow created) is cleared +// immediately — the failure is final, the tx can never produce an escrow. +func TestReconcileCommitmentsClearsWhenTxCommittedFailed(t *testing.T) { + g, store, settings := newRecoveryGateway(t) + require.NoError(t, store.SaveCommitment(GatewayEscrowCommitment{TxHash: "TXEEE", Model: "Qwen/Test", Role: rotationRoleTemp})) + stubQueryTxEscrowID(t, func(string) (uint64, bool, error) { return 0, false, nil }) + + g.reconcileCommitments(context.Background(), settings) + + commitments, _ := store.LoadCommitments() + assert.Empty(t, commitments, "commitment cleared when tx committed but created no escrow") +} + +// A fresh commitment whose tx is not (yet) on chain must be KEPT: an unordered +// tx can still land until its TTL elapses, and a fresh 404 is usually mempool / +// index lag, not a failed broadcast. Clearing now would orphan a real escrow. +func TestReconcileCommitmentsKeepsFreshTxNotFound(t *testing.T) { + g, store, settings := newRecoveryGateway(t) + require.NoError(t, store.SaveCommitment(GatewayEscrowCommitment{TxHash: "TXFRESH", Model: "Qwen/Test", Role: rotationRoleTemp})) + stubQueryTxEscrowID(t, func(string) (uint64, bool, error) { return 0, false, errTxNotFound }) + + g.reconcileCommitments(context.Background(), settings) + + commitments, err := store.LoadCommitments() + require.NoError(t, err) + require.Len(t, commitments, 1, "fresh not-found commitment retained until its tx can no longer land") + assert.Equal(t, "TXFRESH", commitments[0].TxHash) +} + +// Once the commitment is older than the unordered-tx window, a not-found tx can +// never land — only then is it safe to clear. +func TestReconcileCommitmentsClearsExpiredTxNotFound(t *testing.T) { + g, store, settings := newRecoveryGateway(t) + old := time.Now().UTC().Add(-1 * time.Hour).Format(time.RFC3339Nano) + _, err := store.db.Exec(` + INSERT INTO escrow_rotation_commitments (tx_hash, model, role, epoch, private_key_env, block_height, created_at) + VALUES (?, ?, ?, ?, ?, ?, ?)`, + "TXOLD", "Qwen/Test", rotationRoleTemp, 0, "", 0, old) + require.NoError(t, err) + stubQueryTxEscrowID(t, func(string) (uint64, bool, error) { return 0, false, errTxNotFound }) + + g.reconcileCommitments(context.Background(), settings) + + commitments, _ := store.LoadCommitments() + assert.Empty(t, commitments, "commitment cleared once the unordered tx can no longer land") +} + +// A transient chain error leaves the commitment for the next pass. +func TestReconcileCommitmentsKeepsCommitmentOnChainError(t *testing.T) { + g, store, settings := newRecoveryGateway(t) + require.NoError(t, store.SaveCommitment(GatewayEscrowCommitment{TxHash: "TXFFF", Model: "Qwen/Test", Role: rotationRoleTemp})) + stubQueryTxEscrowID(t, func(string) (uint64, bool, error) { return 0, false, errors.New("chain unreachable") }) + + g.reconcileCommitments(context.Background(), settings) + + commitments, err := store.LoadCommitments() + require.NoError(t, err) + require.Len(t, commitments, 1, "commitment retained when chain cannot be queried") + assert.Equal(t, "TXFFF", commitments[0].TxHash) +} diff --git a/devshard/cmd/devshardctl/escrow_rotator.go b/devshard/cmd/devshardctl/escrow_rotator.go index 2d0e1ab1cf..46bec94574 100644 --- a/devshard/cmd/devshardctl/escrow_rotator.go +++ b/devshard/cmd/devshardctl/escrow_rotator.go @@ -18,16 +18,29 @@ const ( rotationRoleTemp = "temp" defaultEscrowRotationInterval = 15 * time.Second + + rotationBreakerMaxCooldownTicks = 4 + + escrowWriteRetries = 10 + escrowWriteRetryBackoff = 200 * time.Millisecond ) var ( errDevshardBusy = errors.New("devshard has active requests") - errEscrowRotationCreateSuppressed = errors.New("escrow rotation create already failed for this epoch") + errDevshardAlreadyExists = errors.New("devshard already exists") + errEscrowRotationCreateSuppressed = errors.New("escrow rotation create suppressed (backoff)") gatewayCreateRotationEscrow = (*Gateway).createRotationEscrow + gatewayCreateEscrowOnChain = (*Gateway).createEscrowOnChain gatewayCreateDepletionEscrow func(*Gateway, context.Context, GatewaySettings, EscrowRotationModelSettings, string, uint64) (*CreateDevshardEscrowResult, error) gatewaySettleDevshardOnChain = (*Gateway).settleDevshardOnChain + gatewayQueryTxEscrowID = defaultQueryTxEscrowID ) +type rotationBreaker struct { + consecutiveFailures int + cooldownTicks int +} + func (g *Gateway) startEscrowRotatorIfEnabled() { g.mu.Lock() defer g.mu.Unlock() @@ -88,6 +101,7 @@ func (g *Gateway) rotateEscrowsOnce() { g.mu.Lock() settings := g.settings g.mu.Unlock() + g.reconcileCommitments(context.Background(), settings) rotation := settings.EscrowRotation if !rotation.Enabled { return @@ -270,17 +284,18 @@ func (g *Gateway) ensureRotationEscrows(ctx context.Context, settings GatewaySet return result, nil } } - if count < target && g.rotationCreateFailed(model.ModelID, role, epoch) { + if count < target && g.rotationCreateGated(model.ModelID, role) { return result, errEscrowRotationCreateSuppressed } for count < target { if _, err := gatewayCreateRotationEscrow(g, ctx, settings, model, role, epoch); err != nil { - g.recordRotationCreateFailure(model.ModelID, role, epoch) + g.recordRotationCreateFailure(model.ModelID, role) return result, err } count++ result.CreatedCount++ } + g.resetRotationBreaker(model.ModelID, role) return result, nil } @@ -300,7 +315,7 @@ func (g *Gateway) rotationModelServedByNetwork(modelID string) (served bool, kno return false, true } -func (g *Gateway) createRotationEscrow(ctx context.Context, settings GatewaySettings, model EscrowRotationModelSettings, role string, epoch uint64) (*CreateDevshardEscrowResult, error) { +func (g *Gateway) createEscrowOnChain(ctx context.Context, settings GatewaySettings, model EscrowRotationModelSettings, onPrepared func(txHash string) error) (*CreateDevshardEscrowResult, error) { signer, _, err := signerFromRequestKey("", model.PrivateKeyEnv) if err != nil { return nil, err @@ -309,14 +324,34 @@ func (g *Gateway) createRotationEscrow(ctx context.Context, settings GatewaySett if err != nil { return nil, err } - result, err := txClient.CreateDevshardEscrow(ctx, signer, model.Amount, model.ModelID) + return txClient.CreateDevshardEscrow(ctx, signer, model.Amount, model.ModelID, onPrepared) +} + +func (g *Gateway) createRotationEscrow(ctx context.Context, settings GatewaySettings, model EscrowRotationModelSettings, role string, epoch uint64) (*CreateDevshardEscrowResult, error) { + keyEnv := strings.TrimSpace(model.PrivateKeyEnv) + commitment := GatewayEscrowCommitment{ + Model: model.ModelID, + Role: role, + Epoch: epoch, + PrivateKeyEnv: keyEnv, + BlockHeight: g.currentBlockHeight(), + } + // Intent-first: persist the commitment (with the precomputed tx hash) before broadcast. + onPrepared := func(txHash string) error { + c := commitment + c.TxHash = txHash + return withDBRetry(func() error { return g.store.SaveCommitment(c) }) + } + result, err := gatewayCreateEscrowOnChain(g, ctx, settings, model, onPrepared) if err != nil { return nil, err } - record := newRotationDevshardState(result, model, role, epoch) - if _, err := g.addCreatedEscrowRuntime(record); err != nil { + if err := g.persistRotationEscrow(result.EscrowID, model.ModelID, role, epoch, keyEnv); err != nil { + // Escrow is on chain; commitment survives → reconcile recovers it by tx hash. + log.Printf("escrow_rotation_persist_failed escrow=%d tx=%s model=%q error=%v recover_via_commitment=true", result.EscrowID, result.TxHash, model.ModelID, err) return nil, err } + g.clearCommitment(result.TxHash) log.Printf("escrow_rotation_created role=%s epoch=%d model=%q escrow=%d tx_hash=%s", role, epoch, model.ModelID, result.EscrowID, result.TxHash) return result, nil } @@ -381,24 +416,173 @@ func (g *Gateway) saveRotationStatus(status GatewayRotationStatus) { } } -func (g *Gateway) rotationFailureKey(modelID, role string, epoch uint64) string { - return fmt.Sprintf("%s|%s|%d", strings.TrimSpace(modelID), role, epoch) +func rotationBreakerKey(modelID, role string) string { + return strings.TrimSpace(modelID) + "|" + role } -func (g *Gateway) recordRotationCreateFailure(modelID, role string, epoch uint64) { +// rotationCreateGated reports whether create is in backoff; decrements one tick. +func (g *Gateway) rotationCreateGated(modelID, role string) bool { g.mu.Lock() defer g.mu.Unlock() - if g.rotationFailures == nil { - g.rotationFailures = make(map[string]struct{}) + breaker := g.rotationBreakers[rotationBreakerKey(modelID, role)] + if breaker == nil || breaker.cooldownTicks <= 0 { + return false } - g.rotationFailures[g.rotationFailureKey(modelID, role, epoch)] = struct{}{} + breaker.cooldownTicks-- + return true } -func (g *Gateway) rotationCreateFailed(modelID, role string, epoch uint64) bool { +// recordRotationCreateFailure grows the per-(model,role) backoff after a failure. +func (g *Gateway) recordRotationCreateFailure(modelID, role string) { g.mu.Lock() defer g.mu.Unlock() - _, ok := g.rotationFailures[g.rotationFailureKey(modelID, role, epoch)] - return ok + if g.rotationBreakers == nil { + g.rotationBreakers = make(map[string]*rotationBreaker) + } + key := rotationBreakerKey(modelID, role) + breaker := g.rotationBreakers[key] + if breaker == nil { + breaker = &rotationBreaker{} + g.rotationBreakers[key] = breaker + } + breaker.consecutiveFailures++ + cooldown := 1 << (breaker.consecutiveFailures - 1) + if cooldown > rotationBreakerMaxCooldownTicks { + cooldown = rotationBreakerMaxCooldownTicks + } + breaker.cooldownTicks = cooldown +} + +// resetRotationBreaker clears the backoff for a (model, role) after success. +func (g *Gateway) resetRotationBreaker(modelID, role string) { + g.mu.Lock() + defer g.mu.Unlock() + delete(g.rotationBreakers, rotationBreakerKey(modelID, role)) +} + +func (g *Gateway) currentBlockHeight() uint64 { + if g == nil || g.phaseGate == nil { + return 0 + } + height := g.phaseGate.Snapshot().BlockHeight + if height < 0 { + return 0 + } + return uint64(height) +} + +// withDBRetry retries a DB write with backoff to ride out a transient lock. +func withDBRetry(fn func() error) error { + var err error + for attempt := 0; attempt < escrowWriteRetries; attempt++ { + if err = fn(); err == nil { + return nil + } + if attempt < escrowWriteRetries-1 { + time.Sleep(escrowWriteRetryBackoff) + } + } + return err +} + +// persistRotationEscrow persists + registers a created escrow ("already exists" = ok). +func (g *Gateway) persistRotationEscrow(escrowID uint64, modelID, role string, epoch uint64, keyEnv string) error { + record := GatewayDevshardState{ + RuntimeConfig: RuntimeConfig{ + ID: strconv.FormatUint(escrowID, 10), + PrivateKeyEnv: strings.TrimSpace(keyEnv), + Model: modelID, + }, + Active: true, + RotationRole: role, + RotationEpoch: epoch, + } + return withDBRetry(func() error { + if _, err := g.addCreatedEscrowRuntime(record); err != nil { + if errors.Is(err, errDevshardAlreadyExists) { + return nil + } + return err + } + return nil + }) +} + +func (g *Gateway) clearCommitment(txHash string) { + if g == nil || g.store == nil || strings.TrimSpace(txHash) == "" { + return + } + if err := withDBRetry(func() error { return g.store.DeleteCommitment(txHash) }); err != nil { + log.Printf("escrow_rotation_commitment_clear_failed tx=%s error=%v", txHash, err) + } +} + +// commitmentReconcileGrace is how long a not-found tx is given before the +// commitment is cleared: the unordered-tx TTL plus margin for index lag. Until +// it elapses, a 404 may be a pending/unindexed tx that still creates an escrow. +const commitmentReconcileGrace = defaultUnorderedTxTTL + 2*time.Minute + +// reconcileCommitments recovers escrows from pending commitments via tx hash: +// found → persist + clear; committed-failed → clear; not-found → clear only once +// the tx can no longer land; chain error → keep for next pass. +func (g *Gateway) reconcileCommitments(ctx context.Context, settings GatewaySettings) { + if g == nil || g.store == nil { + return + } + commitments, err := g.store.LoadCommitments() + if err != nil { + log.Printf("escrow_commitments_load_failed error=%v", err) + return + } + for _, c := range commitments { + escrowID, found, err := gatewayQueryTxEscrowID(ctx, settings, c.TxHash) + if errors.Is(err, errTxNotFound) { + // Tx not on chain. An unordered tx can still land until its TTL + // elapses, so a fresh 404 is likely mempool/index lag — keep it. + if commitmentTxMayStillLand(c) { + log.Printf("escrow_commitment_tx_pending tx=%s model=%q", c.TxHash, c.Model) + continue + } + log.Printf("escrow_commitment_no_escrow tx=%s model=%q clearing=true", c.TxHash, c.Model) + g.clearCommitment(c.TxHash) + continue + } + if err != nil { + log.Printf("escrow_commitment_tx_query_failed tx=%s model=%q error=%v", c.TxHash, c.Model, err) + continue // chain unreachable — retry next pass + } + if !found { + log.Printf("escrow_commitment_tx_failed tx=%s model=%q clearing=true", c.TxHash, c.Model) + g.clearCommitment(c.TxHash) + continue + } + if err := g.persistRotationEscrow(escrowID, c.Model, c.Role, c.Epoch, c.PrivateKeyEnv); err != nil { + log.Printf("escrow_commitment_persist_failed tx=%s escrow=%d model=%q error=%v", c.TxHash, escrowID, c.Model, err) + continue // keep commitment — retry next pass + } + g.clearCommitment(c.TxHash) + g.resetRotationBreaker(c.Model, c.Role) + log.Printf("escrow_commitment_recovered tx=%s escrow=%d model=%q role=%s", c.TxHash, escrowID, c.Model, c.Role) + } +} + +// commitmentTxMayStillLand reports whether a not-found tx could yet be committed +// (within the unordered-tx window) — if so, the commitment must be kept. An +// unparseable timestamp is treated as still-pending so we never clear too early. +func commitmentTxMayStillLand(c GatewayEscrowCommitment) bool { + createdAt, err := time.Parse(time.RFC3339Nano, strings.TrimSpace(c.CreatedAt)) + if err != nil { + return true + } + return time.Since(createdAt) <= commitmentReconcileGrace +} + +func defaultQueryTxEscrowID(ctx context.Context, settings GatewaySettings, txHash string) (uint64, bool, error) { + txClient, err := newGatewayRESTChainTxClient(settings, "", "", 0, 0) + if err != nil { + return 0, false, err + } + return txClient.GetTxEscrowID(ctx, txHash) } func (g *Gateway) settleDevshardOnChain(ctx context.Context, id string, req adminSettleEscrowRequest) (*SettleDevshardEscrowResult, error) { diff --git a/devshard/cmd/devshardctl/gateway.go b/devshard/cmd/devshardctl/gateway.go index 04b8dd4bbe..72fbf8642c 100644 --- a/devshard/cmd/devshardctl/gateway.go +++ b/devshard/cmd/devshardctl/gateway.go @@ -56,7 +56,7 @@ type Gateway struct { baseStorageDir string rotatorStop chan struct{} rotatorDone chan struct{} - rotationFailures map[string]struct{} + rotationBreakers map[string]*rotationBreaker finalizeMu sync.Mutex settlementMu sync.Mutex settlementInFlight map[string]struct{} @@ -509,7 +509,7 @@ func NewGateway(runtimes []*devshardRuntime, limiter *GatewayLimiter, defaultMod settings: GatewaySettings{ DefaultModel: defaultModel, }, - rotationFailures: make(map[string]struct{}), + rotationBreakers: make(map[string]*rotationBreaker), settlementInFlight: make(map[string]struct{}), } g.participantLimiter.SetMetrics(g.metrics) @@ -557,6 +557,7 @@ func NewManagedGateway(runtimes []*devshardRuntime, limiter *GatewayLimiter, set for _, rt := range g.runtimeOrder { g.attachEscrowChecker(rt) } + go g.reconcileCommitments(context.Background(), settings) g.startEscrowRotatorIfEnabled() // Settle escrows left pending by a pre-restart drain. Runs synchronously // so the store read completes before the gateway serves traffic; the @@ -2733,7 +2734,7 @@ func (g *Gateway) handleAdminEscrows(w http.ResponseWriter, r *http.Request) { http.Error(w, fmt.Sprintf(`{"error":{"message":%q}}`, err.Error()), http.StatusBadRequest) return } - result, err := txClient.CreateDevshardEscrow(r.Context(), signer, req.Amount, modelID) + result, err := txClient.CreateDevshardEscrow(r.Context(), signer, req.Amount, modelID, nil) if err != nil { http.Error(w, fmt.Sprintf(`{"error":{"message":%q}}`, err.Error()), http.StatusBadGateway) return @@ -2813,7 +2814,7 @@ func (g *Gateway) addCreatedEscrowRuntime(record GatewayDevshardState) (GatewayD return record, fmt.Errorf("gateway state is not initialized") } if _, exists := g.runtimes[record.ID]; exists { - return record, fmt.Errorf("devshard %s already exists", record.ID) + return record, fmt.Errorf("devshard %s: %w", record.ID, errDevshardAlreadyExists) } if record.Model == "" { record.Model = state.Settings.DefaultModel diff --git a/devshard/cmd/devshardctl/gateway_runtime_retire_test.go b/devshard/cmd/devshardctl/gateway_runtime_retire_test.go index 80c3f20818..6f440fbab3 100644 --- a/devshard/cmd/devshardctl/gateway_runtime_retire_test.go +++ b/devshard/cmd/devshardctl/gateway_runtime_retire_test.go @@ -17,7 +17,7 @@ func newRetireTestGateway(id string) (*Gateway, *devshardRuntime) { g := &Gateway{ runtimes: map[string]*devshardRuntime{id: rt}, runtimeOrder: []*devshardRuntime{rt}, - rotationFailures: make(map[string]struct{}), + rotationBreakers: make(map[string]*rotationBreaker), } return g, rt } diff --git a/devshard/cmd/devshardctl/gateway_store.go b/devshard/cmd/devshardctl/gateway_store.go index f7544402d5..2014a39d7e 100644 --- a/devshard/cmd/devshardctl/gateway_store.go +++ b/devshard/cmd/devshardctl/gateway_store.go @@ -314,6 +314,19 @@ func NewGatewayStore(path string) (*GatewayStore, error) { if err != nil { return nil, fmt.Errorf("open gateway store: %w", err) } + // Serialize access and wait on contention instead of failing with "database is + // locked". Mirrors storage/sqlite.go. + db.SetMaxOpenConns(1) + for _, pragma := range []string{ + "PRAGMA journal_mode=WAL", + "PRAGMA synchronous=NORMAL", + "PRAGMA busy_timeout=5000", + } { + if _, err := db.Exec(pragma); err != nil { + db.Close() + return nil, fmt.Errorf("apply gateway store pragma %q: %w", pragma, err) + } + } stmts := []string{ `CREATE TABLE IF NOT EXISTS gateway_settings ( id INTEGER PRIMARY KEY CHECK (id = 1), @@ -480,6 +493,19 @@ func NewGatewayStore(path string) (*GatewayStore, error) { db.Close() return nil, fmt.Errorf("init gateway suspicious hosts table: %w", err) } + // Write-ahead intent for an escrow create (written before the on-chain tx). + if _, err := db.Exec(`CREATE TABLE IF NOT EXISTS escrow_rotation_commitments ( + tx_hash TEXT PRIMARY KEY, + model TEXT NOT NULL, + role TEXT NOT NULL DEFAULT '', + epoch INTEGER NOT NULL DEFAULT 0, + private_key_env TEXT NOT NULL DEFAULT '', + block_height INTEGER NOT NULL DEFAULT 0, + created_at TEXT NOT NULL + )`); err != nil { + db.Close() + return nil, fmt.Errorf("init escrow rotation commitments table: %w", err) + } if err := ensureColumn(db, "participant_throttle_state", "quarantine_until_utc", "TEXT NOT NULL DEFAULT ''"); err != nil { db.Close() return nil, fmt.Errorf("migrate participant throttle: %w", err) @@ -993,6 +1019,76 @@ func (s *GatewayStore) LoadRotationStatuses(limit int) ([]GatewayRotationStatus, return statuses, nil } +// GatewayEscrowCommitment is the write-ahead intent for one escrow create, keyed by tx hash. +type GatewayEscrowCommitment struct { + TxHash string + Model string + Role string + Epoch uint64 + PrivateKeyEnv string + BlockHeight uint64 + CreatedAt string +} + +// SaveCommitment records a create intent, keyed by tx hash. +func (s *GatewayStore) SaveCommitment(c GatewayEscrowCommitment) error { + if s == nil || s.db == nil { + return fmt.Errorf("gateway store unavailable") + } + now := time.Now().UTC().Format(time.RFC3339Nano) + _, err := s.db.Exec(` + INSERT OR REPLACE INTO escrow_rotation_commitments ( + tx_hash, model, role, epoch, private_key_env, block_height, created_at + ) VALUES (?, ?, ?, ?, ?, ?, ?)`, + strings.TrimSpace(c.TxHash), + strings.TrimSpace(c.Model), + strings.TrimSpace(c.Role), + c.Epoch, + strings.TrimSpace(c.PrivateKeyEnv), + c.BlockHeight, + now, + ) + if err != nil { + return fmt.Errorf("save escrow commitment tx=%s: %w", c.TxHash, err) + } + return nil +} + +// LoadCommitments returns all pending commitments (oldest first). +func (s *GatewayStore) LoadCommitments() ([]GatewayEscrowCommitment, error) { + if s == nil || s.db == nil { + return nil, nil + } + rows, err := s.db.Query(` + SELECT tx_hash, model, role, epoch, private_key_env, block_height, created_at + FROM escrow_rotation_commitments + ORDER BY created_at ASC`) + if err != nil { + return nil, fmt.Errorf("load escrow commitments: %w", err) + } + defer rows.Close() + var commitments []GatewayEscrowCommitment + for rows.Next() { + var c GatewayEscrowCommitment + if err := rows.Scan(&c.TxHash, &c.Model, &c.Role, &c.Epoch, &c.PrivateKeyEnv, &c.BlockHeight, &c.CreatedAt); err != nil { + return nil, fmt.Errorf("scan escrow commitment: %w", err) + } + commitments = append(commitments, c) + } + return commitments, rows.Err() +} + +// DeleteCommitment clears a commitment once its escrow is persisted (or proven absent). +func (s *GatewayStore) DeleteCommitment(txHash string) error { + if s == nil || s.db == nil { + return fmt.Errorf("gateway store unavailable") + } + if _, err := s.db.Exec(`DELETE FROM escrow_rotation_commitments WHERE tx_hash = ?`, strings.TrimSpace(txHash)); err != nil { + return fmt.Errorf("delete escrow commitment tx=%s: %w", txHash, err) + } + return nil +} + func (s *GatewayStore) LoadSuspiciousHosts() ([]GatewaySuspiciousHost, error) { if s == nil || s.db == nil { return nil, nil diff --git a/devshard/cmd/devshardctl/gateway_store_test.go b/devshard/cmd/devshardctl/gateway_store_test.go index b2702d6aa2..7c84bc836f 100644 --- a/devshard/cmd/devshardctl/gateway_store_test.go +++ b/devshard/cmd/devshardctl/gateway_store_test.go @@ -359,7 +359,7 @@ func TestEscrowRotationPreparePromotesRegularEscrowsOnTempCreateFailure(t *testi gatewaySettleDevshardOnChain = oldSettle }) - g := &Gateway{store: store, rotationFailures: make(map[string]struct{})} + g := &Gateway{store: store} g.prepareBridgeEscrows(ChainPhaseSnapshot{EpochIndex: 10}, settings) g.prepareBridgeEscrows(ChainPhaseSnapshot{EpochIndex: 10}, settings) @@ -439,7 +439,7 @@ func TestEscrowRotationFinishDoesNotSettleTempWhenRegularCreateFails(t *testing. gatewaySettleDevshardOnChain = oldSettle }) - g := &Gateway{store: store, rotationFailures: make(map[string]struct{})} + g := &Gateway{store: store} g.finishBridgeEscrows(ChainPhaseSnapshot{EpochIndex: 11}, settings) g.finishBridgeEscrows(ChainPhaseSnapshot{EpochIndex: 11}, settings) @@ -509,7 +509,7 @@ func TestEscrowRotationSkipsCreateWhenModelAbsentFromNetwork(t *testing.T) { map[string]map[string]float64{"Qwen/Present": {"host": 1}}, ) - g := &Gateway{store: store, capacity: capacity, rotationFailures: make(map[string]struct{})} + g := &Gateway{store: store, capacity: capacity, rotationBreakers: make(map[string]*rotationBreaker)} g.finishBridgeEscrows(ChainPhaseSnapshot{EpochIndex: 11}, settings) require.Equal(t, 0, createAttempts, "must not create escrow for a model absent from the network") @@ -571,7 +571,7 @@ func TestEscrowRotationCreatesWhenModelPresentInNetwork(t *testing.T) { map[string]map[string]float64{"Qwen/Test": {"host": 1}}, ) - g := &Gateway{store: store, capacity: capacity, rotationFailures: make(map[string]struct{})} + g := &Gateway{store: store, capacity: capacity, rotationBreakers: make(map[string]*rotationBreaker)} g.finishBridgeEscrows(ChainPhaseSnapshot{EpochIndex: 11}, settings) require.Equal(t, 2, createAttempts, "must create escrows for a model the network serves") @@ -626,7 +626,7 @@ func TestEscrowRotationFinishSettlesTempFromCurrentLatestEpoch(t *testing.T) { gatewaySettleDevshardOnChain = oldSettle }) - g := &Gateway{store: store, rotationFailures: make(map[string]struct{})} + g := &Gateway{store: store} g.finishBridgeEscrows(ChainPhaseSnapshot{EpochIndex: 10}, settings) require.Equal(t, 2, createAttempts) @@ -691,7 +691,7 @@ func TestEscrowRotationPrepareDeactivatesRegularWithoutSettlementWhenSettlementD rt := &devshardRuntime{id: "12"} rt.active.Store(true) - g := &Gateway{store: store, runtimes: map[string]*devshardRuntime{"12": rt}, rotationFailures: make(map[string]struct{})} + g := &Gateway{store: store, runtimes: map[string]*devshardRuntime{"12": rt}} g.prepareBridgeEscrows(ChainPhaseSnapshot{EpochIndex: 10}, settings) require.Equal(t, 1, createAttempts) @@ -759,7 +759,7 @@ func TestEscrowRotationFinishDeactivatesTempWithoutSettlementWhenSettlementDisab rt := &devshardRuntime{id: "12"} rt.active.Store(true) - g := &Gateway{store: store, runtimes: map[string]*devshardRuntime{"12": rt}, rotationFailures: make(map[string]struct{})} + g := &Gateway{store: store, runtimes: map[string]*devshardRuntime{"12": rt}} g.finishBridgeEscrows(ChainPhaseSnapshot{EpochIndex: 10}, settings) require.Equal(t, 1, createAttempts) @@ -837,7 +837,7 @@ func TestEscrowRotationPrepareRotatesModelsIndependently(t *testing.T) { gatewaySettleDevshardOnChain = oldSettle }) - g := &Gateway{store: store, rotationFailures: make(map[string]struct{})} + g := &Gateway{store: store} g.prepareBridgeEscrows(ChainPhaseSnapshot{EpochIndex: 10}, settings) require.Equal(t, []string{"13"}, settled) @@ -900,10 +900,9 @@ func TestEscrowRotationUsesEpochSwitchHeightDuringPoC(t *testing.T) { }) g := &Gateway{ - store: store, - settings: settings, - phaseGate: &ChainPhaseGate{}, - rotationFailures: make(map[string]struct{}), + store: store, + settings: settings, + phaseGate: &ChainPhaseGate{}, } g.phaseGate.storeSnapshot(ChainPhaseSnapshot{ BlockHeight: 350, From 961b7b667b3e5533266a2cedc1bafdc58d179458 Mon Sep 17 00:00:00 2001 From: akup Date: Wed, 8 Jul 2026 07:56:50 +0300 Subject: [PATCH 10/13] feat(devshard): add Postgres backend for gateway store with SQLite fallback Enable hybrid persistence for devshardctl gateway state when PGHOST is set: Postgres primary, SQLite fallback, one-shot migration from gateway.db, and a SQLite sync journal that replays outage-era writes to Postgres before PG resumes as primary. Harden reconnect with dropPg, in-request write retries, per-operation query timeouts, and a shorter gateway reconnect interval. Also make PG_CONNECT_TIMEOUT configurable for decentralized-api payload storage. --- decentralized-api/payloadstorage/factory.go | 8 +- .../payloadstorage/factory_test.go | 48 + .../payloadstorage/hybrid_storage.go | 32 +- .../payloadstorage/postgres_storage_test.go | 4 +- .../cmd/devshardctl/escrow_recovery_test.go | 16 +- devshard/cmd/devshardctl/gateway.go | 4 +- devshard/cmd/devshardctl/gateway_store.go | 1387 +++-------------- .../cmd/devshardctl/gateway_store_factory.go | 58 + .../devshardctl/gateway_store_factory_test.go | 110 ++ .../cmd/devshardctl/gateway_store_hybrid.go | 646 ++++++++ .../devshardctl/gateway_store_hybrid_test.go | 164 ++ .../cmd/devshardctl/gateway_store_migrate.go | 131 ++ .../devshardctl/gateway_store_migrate_test.go | 335 ++++ .../cmd/devshardctl/gateway_store_postgres.go | 897 +++++++++++ .../gateway_store_postgres_test.go | 55 + ...ateway_store_postgres_test_helpers_test.go | 57 + .../cmd/devshardctl/gateway_store_sqlite.go | 934 +++++++++++ .../devshardctl/gateway_store_sync_journal.go | 828 ++++++++++ .../gateway_store_sync_journal_test.go | 449 ++++++ .../cmd/devshardctl/gateway_store_test.go | 88 +- .../gateway_store_test_helpers_test.go | 33 + devshard/cmd/devshardctl/gateway_test.go | 30 +- devshard/cmd/devshardctl/main.go | 21 +- devshard/cmd/devshardctl/main_test.go | 12 +- devshard/docs/proxy.md | 42 +- 25 files changed, 5184 insertions(+), 1205 deletions(-) create mode 100644 devshard/cmd/devshardctl/gateway_store_factory.go create mode 100644 devshard/cmd/devshardctl/gateway_store_factory_test.go create mode 100644 devshard/cmd/devshardctl/gateway_store_hybrid.go create mode 100644 devshard/cmd/devshardctl/gateway_store_hybrid_test.go create mode 100644 devshard/cmd/devshardctl/gateway_store_migrate.go create mode 100644 devshard/cmd/devshardctl/gateway_store_migrate_test.go create mode 100644 devshard/cmd/devshardctl/gateway_store_postgres.go create mode 100644 devshard/cmd/devshardctl/gateway_store_postgres_test.go create mode 100644 devshard/cmd/devshardctl/gateway_store_postgres_test_helpers_test.go create mode 100644 devshard/cmd/devshardctl/gateway_store_sqlite.go create mode 100644 devshard/cmd/devshardctl/gateway_store_sync_journal.go create mode 100644 devshard/cmd/devshardctl/gateway_store_sync_journal_test.go create mode 100644 devshard/cmd/devshardctl/gateway_store_test_helpers_test.go diff --git a/decentralized-api/payloadstorage/factory.go b/decentralized-api/payloadstorage/factory.go index d9b89586d6..eb0b6ac5b4 100644 --- a/decentralized-api/payloadstorage/factory.go +++ b/decentralized-api/payloadstorage/factory.go @@ -28,14 +28,18 @@ func NewPayloadStorage(ctx context.Context, fileBasePath string) PayloadStorage if err != nil || retryInterval <= 0 { retryInterval = 240 * time.Second } + connectTimeout, err := time.ParseDuration(os.Getenv("PG_CONNECT_TIMEOUT")) + if err != nil || connectTimeout <= 0 { + connectTimeout = defaultPGConnectTimeout + } pgStorage, err := NewPostgresStorage(ctx) if err != nil { logging.Warn("PostgreSQL connection failed, will retry lazily on Store", types.PayloadStorage, "host", pgHost, "error", err) - return NewHybridStorage(nil, fileStorage, retryInterval) + return NewHybridStorage(nil, fileStorage, retryInterval, connectTimeout) } logging.Info("Using PostgreSQL with file fallback", types.PayloadStorage, "host", pgHost) - return NewHybridStorage(pgStorage, fileStorage, retryInterval) + return NewHybridStorage(pgStorage, fileStorage, retryInterval, connectTimeout) } diff --git a/decentralized-api/payloadstorage/factory_test.go b/decentralized-api/payloadstorage/factory_test.go index c1c52bf1b4..79188f6f78 100644 --- a/decentralized-api/payloadstorage/factory_test.go +++ b/decentralized-api/payloadstorage/factory_test.go @@ -76,3 +76,51 @@ func TestNewPayloadStorage_RetryInterval(t *testing.T) { } } +func TestNewPayloadStorage_ConnectTimeout(t *testing.T) { + os.Setenv("PGHOST", "localhost") + defer os.Unsetenv("PGHOST") + + tests := []struct { + name string + envValue string + expected time.Duration + }{ + { + name: "Default (unset)", + envValue: "", + expected: defaultPGConnectTimeout, + }, + { + name: "Custom valid duration", + envValue: "500ms", + expected: 500 * time.Millisecond, + }, + { + name: "Invalid duration (fallback to default)", + envValue: "invalid", + expected: defaultPGConnectTimeout, + }, + { + name: "Zero duration (fallback to default)", + envValue: "0s", + expected: defaultPGConnectTimeout, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if tt.envValue != "" { + os.Setenv("PG_CONNECT_TIMEOUT", tt.envValue) + defer os.Unsetenv("PG_CONNECT_TIMEOUT") + } else { + os.Unsetenv("PG_CONNECT_TIMEOUT") + } + + s := NewPayloadStorage(context.Background(), t.TempDir()) + hs, ok := s.(*HybridStorage) + require.True(t, ok, "Expected *HybridStorage") + assert.Equal(t, tt.expected, hs.connectTimeout) + }) + } +} + diff --git a/decentralized-api/payloadstorage/hybrid_storage.go b/decentralized-api/payloadstorage/hybrid_storage.go index 0a6ebb7ed4..57727b482a 100644 --- a/decentralized-api/payloadstorage/hybrid_storage.go +++ b/decentralized-api/payloadstorage/hybrid_storage.go @@ -11,24 +11,34 @@ import ( "github.com/productscience/inference/x/inference/types" ) -const ( - pgConnectTimeout = 2 * time.Second -) +const defaultPGConnectTimeout = 2 * time.Second // HybridStorage uses PostgreSQL as primary storage with file-based fallback. // Store: tries PG first (with lazy reconnection), falls back to file on error. // Retrieve: tries PG first (no reconnection delay), on error OR not found also checks file. // PruneEpoch: prunes both (best effort, no reconnection delay). type HybridStorage struct { - pg *PostgresStorage - file *FileStorage - mu sync.Mutex - lastRetry time.Time - retryInterval time.Duration + pg *PostgresStorage + file *FileStorage + mu sync.Mutex + lastRetry time.Time + retryInterval time.Duration + connectTimeout time.Duration } -func NewHybridStorage(pg *PostgresStorage, file *FileStorage, retryInterval time.Duration) *HybridStorage { - return &HybridStorage{pg: pg, file: file, retryInterval: retryInterval} +func NewHybridStorage(pg *PostgresStorage, file *FileStorage, retryInterval, connectTimeout time.Duration) *HybridStorage { + if retryInterval <= 0 { + retryInterval = 240 * time.Second + } + if connectTimeout <= 0 { + connectTimeout = defaultPGConnectTimeout + } + return &HybridStorage{ + pg: pg, + file: file, + retryInterval: retryInterval, + connectTimeout: connectTimeout, + } } // shouldAttemptConnect checks if reconnection should be attempted. @@ -66,7 +76,7 @@ func (h *HybridStorage) getOrConnectPg(ctx context.Context) *PostgresStorage { return pg } - connectCtx, cancel := context.WithTimeout(ctx, pgConnectTimeout) + connectCtx, cancel := context.WithTimeout(ctx, h.connectTimeout) defer cancel() newPg, err := NewPostgresStorage(connectCtx) diff --git a/decentralized-api/payloadstorage/postgres_storage_test.go b/decentralized-api/payloadstorage/postgres_storage_test.go index e888984621..960b5b06c0 100644 --- a/decentralized-api/payloadstorage/postgres_storage_test.go +++ b/decentralized-api/payloadstorage/postgres_storage_test.go @@ -228,7 +228,7 @@ func TestHybridStorage_FallbackOnPGError(t *testing.T) { require.NoError(t, err) defer pgStorage.Close() - hybrid := NewHybridStorage(pgStorage, fileStorage, 240*time.Second) + hybrid := NewHybridStorage(pgStorage, fileStorage, 240*time.Second, defaultPGConnectTimeout) // Data not in PG, but is in file - should find it prompt, response, err := hybrid.Retrieve(ctx, "inf-001", 100) @@ -250,7 +250,7 @@ func TestHybridStorage_PGPrimary(t *testing.T) { defer pgStorage.Close() fileStorage := NewFileStorage(tempDir) - hybrid := NewHybridStorage(pgStorage, fileStorage, 240*time.Second) + hybrid := NewHybridStorage(pgStorage, fileStorage, 240*time.Second, defaultPGConnectTimeout) // Store via hybrid (should go to PG) storedPrompt := []byte(`{"pg": "data"}`) diff --git a/devshard/cmd/devshardctl/escrow_recovery_test.go b/devshard/cmd/devshardctl/escrow_recovery_test.go index fbe4a50f80..aaae771fc7 100644 --- a/devshard/cmd/devshardctl/escrow_recovery_test.go +++ b/devshard/cmd/devshardctl/escrow_recovery_test.go @@ -15,7 +15,7 @@ import ( // The root-cause layer: the gateway store must open in WAL with a busy timeout // so concurrent writes wait instead of failing with "database is locked". func TestGatewayStoreUsesWALAndBusyTimeout(t *testing.T) { - store, err := NewGatewayStore(filepath.Join(t.TempDir(), "gateway.db")) + store, err := NewSQLiteGatewayStore(filepath.Join(t.TempDir(), "gateway.db")) require.NoError(t, err) t.Cleanup(func() { require.NoError(t, store.Close()) }) @@ -84,9 +84,9 @@ func stubQueryTxEscrowID(t *testing.T, fn func(string) (uint64, bool, error)) { t.Cleanup(func() { gatewayQueryTxEscrowID = saved }) } -func newRecoveryGateway(t *testing.T) (*Gateway, *GatewayStore, GatewaySettings) { +func newRecoveryGateway(t *testing.T) (*Gateway, GatewayStore, GatewaySettings) { t.Helper() - store, err := NewGatewayStore(filepath.Join(t.TempDir(), "gateway.db")) + store, err := NewSQLiteGatewayStore(filepath.Join(t.TempDir(), "gateway.db")) require.NoError(t, err) t.Cleanup(func() { _ = store.Close() }) settings := recoveryTestSettings() @@ -96,7 +96,7 @@ func newRecoveryGateway(t *testing.T) (*Gateway, *GatewayStore, GatewaySettings) return g, store, settings } -func devshardIDs(t *testing.T, store *GatewayStore) map[string]GatewayDevshardState { +func devshardIDs(t *testing.T, store GatewayStore) map[string]GatewayDevshardState { t.Helper() state, ok, err := store.LoadState() require.NoError(t, err) @@ -128,13 +128,14 @@ func TestCreateRotationEscrowAbortsWhenCommitmentWriteFails(t *testing.T) { model := normalizedEscrowRotationModels(settings)[0] // Make every write fail, as a locked/read-only DB would. - _, err := store.db.Exec("PRAGMA query_only=ON") + sqliteStore := requireSQLiteGatewayStore(t, store) + _, err := sqliteStore.db.Exec("PRAGMA query_only=ON") require.NoError(t, err) _, err = g.createRotationEscrow(context.Background(), settings, model, rotationRoleTemp, 10) require.Error(t, err) - _, _ = store.db.Exec("PRAGMA query_only=OFF") + _, _ = sqliteStore.db.Exec("PRAGMA query_only=OFF") assert.NotContains(t, devshardIDs(t, store), "778", "no escrow created when intent write fails") commitments, _ := store.LoadCommitments() assert.Empty(t, commitments) @@ -223,7 +224,8 @@ func TestReconcileCommitmentsKeepsFreshTxNotFound(t *testing.T) { func TestReconcileCommitmentsClearsExpiredTxNotFound(t *testing.T) { g, store, settings := newRecoveryGateway(t) old := time.Now().UTC().Add(-1 * time.Hour).Format(time.RFC3339Nano) - _, err := store.db.Exec(` + sqliteStore := requireSQLiteGatewayStore(t, store) + _, err := sqliteStore.db.Exec(` INSERT INTO escrow_rotation_commitments (tx_hash, model, role, epoch, private_key_env, block_height, created_at) VALUES (?, ?, ?, ?, ?, ?, ?)`, "TXOLD", "Qwen/Test", rotationRoleTemp, 0, "", 0, old) diff --git a/devshard/cmd/devshardctl/gateway.go b/devshard/cmd/devshardctl/gateway.go index 72fbf8642c..62e3b3a11b 100644 --- a/devshard/cmd/devshardctl/gateway.go +++ b/devshard/cmd/devshardctl/gateway.go @@ -47,7 +47,7 @@ type Gateway struct { metrics *DevshardMetrics capacity *CapacityState settings GatewaySettings - store *GatewayStore + store GatewayStore perf *PerfTracker perfStore *PerfStore chatCache *chatResponseCache @@ -521,7 +521,7 @@ func NewGateway(runtimes []*devshardRuntime, limiter *GatewayLimiter, defaultMod return g } -func NewManagedGateway(runtimes []*devshardRuntime, limiter *GatewayLimiter, settings GatewaySettings, baseStorageDir string, store *GatewayStore, perfArgs ...*PerfTracker) *Gateway { +func NewManagedGateway(runtimes []*devshardRuntime, limiter *GatewayLimiter, settings GatewaySettings, baseStorageDir string, store GatewayStore, perfArgs ...*PerfTracker) *Gateway { settings = settings.WithTuningDefaults() applyGatewayTuningSettings(settings) g := NewGateway(runtimes, limiter, settings.DefaultModel) diff --git a/devshard/cmd/devshardctl/gateway_store.go b/devshard/cmd/devshardctl/gateway_store.go index 2014a39d7e..8d9490ccac 100644 --- a/devshard/cmd/devshardctl/gateway_store.go +++ b/devshard/cmd/devshardctl/gateway_store.go @@ -1,13 +1,10 @@ package main import ( - "database/sql" "encoding/json" "fmt" "strings" "time" - - _ "modernc.org/sqlite" ) type GatewaySettings struct { @@ -305,620 +302,27 @@ type GatewayState struct { SuspiciousHosts []GatewaySuspiciousHost `json:"suspicious_hosts,omitempty"` } -type GatewayStore struct { - db *sql.DB -} - -func NewGatewayStore(path string) (*GatewayStore, error) { - db, err := sql.Open("sqlite", path) - if err != nil { - return nil, fmt.Errorf("open gateway store: %w", err) - } - // Serialize access and wait on contention instead of failing with "database is - // locked". Mirrors storage/sqlite.go. - db.SetMaxOpenConns(1) - for _, pragma := range []string{ - "PRAGMA journal_mode=WAL", - "PRAGMA synchronous=NORMAL", - "PRAGMA busy_timeout=5000", - } { - if _, err := db.Exec(pragma); err != nil { - db.Close() - return nil, fmt.Errorf("apply gateway store pragma %q: %w", pragma, err) - } - } - stmts := []string{ - `CREATE TABLE IF NOT EXISTS gateway_settings ( - id INTEGER PRIMARY KEY CHECK (id = 1), - chain_rest TEXT NOT NULL, - public_api TEXT NOT NULL DEFAULT '', - default_model TEXT NOT NULL, - default_request_max_tokens INTEGER NOT NULL, - request_max_tokens_cap INTEGER NOT NULL DEFAULT 4096, - max_concurrent_requests INTEGER NOT NULL DEFAULT 512, - max_concurrent_requests_per_10000_weight REAL NOT NULL DEFAULT 5.0, - poc_max_concurrent_requests_per_10000_weight REAL NOT NULL DEFAULT 10.0, - max_input_tokens_in_flight INTEGER NOT NULL, - model_limits_json TEXT NOT NULL DEFAULT '', - model_access_json TEXT NOT NULL DEFAULT '', - tx_gas_limit INTEGER NOT NULL DEFAULT 0, - participant_request_burst INTEGER NOT NULL DEFAULT 600, - participant_recovery_per_minute INTEGER NOT NULL DEFAULT 10, - participant_http_quarantine_ms INTEGER NOT NULL DEFAULT 3600000, - participant_transport_failure_quarantine_ms INTEGER NOT NULL DEFAULT 1800000, - participant_empty_stream_quarantine_ms INTEGER NOT NULL DEFAULT 1800000, - participant_stalled_winner_quarantine_ms INTEGER NOT NULL DEFAULT 1800000, - participant_empty_stream_threshold INTEGER NOT NULL DEFAULT 3, - participant_eof_transport_failure_threshold INTEGER NOT NULL DEFAULT 3, - redundancy_receipt_timeout_ms INTEGER NOT NULL DEFAULT 5000, - redundancy_first_token_timeout_floor_ms INTEGER NOT NULL DEFAULT 1000, - redundancy_per_input_token_first_token_lag_ms INTEGER NOT NULL DEFAULT 10, - redundancy_inter_chunk_stall_timeout_ms INTEGER NOT NULL DEFAULT 60000, - redundancy_streaming_attempt_hard_timeout_ms INTEGER NOT NULL DEFAULT 1200000, - redundancy_non_stream_response_floor_ms INTEGER NOT NULL DEFAULT 20000, - redundancy_non_stream_no_content_timeout_ms INTEGER NOT NULL DEFAULT 1200000, - redundancy_non_stream_max_attempt_wait_ms INTEGER NOT NULL DEFAULT 1800000, - redundancy_per_input_token_response_lag_ms INTEGER NOT NULL DEFAULT 20, - redundancy_secondary_wait_after_winner_ms INTEGER NOT NULL DEFAULT 600000, - redundancy_parallel_advantage_threshold REAL NOT NULL DEFAULT 0.5, - redundancy_unresponsive_threshold REAL NOT NULL DEFAULT 1.0, - redundancy_speed_policy TEXT NOT NULL DEFAULT 'hybrid', - redundancy_pairwise_budget_percentile REAL NOT NULL DEFAULT 0.9, - redundancy_pairwise_max_proactive_attempts INTEGER NOT NULL DEFAULT 3, - redundancy_pairwise_min_direct_comparisons INTEGER NOT NULL DEFAULT 4, - redundancy_pairwise_winner_hold_ms INTEGER NOT NULL DEFAULT 500, - redundancy_pairwise_winner_hold_min_speedup REAL NOT NULL DEFAULT 0.1, - redundancy_pairwise_winner_hold_min_samples INTEGER NOT NULL DEFAULT 6, - perf_sample_size INTEGER NOT NULL DEFAULT 256, - perf_window_ms INTEGER NOT NULL DEFAULT 3600000, - escrow_rotation_enabled INTEGER NOT NULL DEFAULT 0, - escrow_rotation_settlement_enabled INTEGER NOT NULL DEFAULT 0, - escrow_rotation_pre_poc_blocks INTEGER NOT NULL DEFAULT 300, - escrow_rotation_models_json TEXT NOT NULL DEFAULT '', - gateway_disabled_enabled INTEGER NOT NULL DEFAULT 0, - gateway_disabled_message TEXT NOT NULL DEFAULT '', - gateway_disabled_new_url TEXT NOT NULL DEFAULT '', - updated_at TEXT NOT NULL - )`, - `CREATE TABLE IF NOT EXISTS gateway_devshards ( - id TEXT PRIMARY KEY, - private_key_hex TEXT NOT NULL, - private_key_env TEXT NOT NULL DEFAULT '', - model TEXT NOT NULL DEFAULT '', - storage_path TEXT NOT NULL DEFAULT '', - active INTEGER NOT NULL DEFAULT 1, - rotation_role TEXT NOT NULL DEFAULT '', - rotation_epoch INTEGER NOT NULL DEFAULT 0, - settlement_pending INTEGER NOT NULL DEFAULT 0, - created_at TEXT NOT NULL, - updated_at TEXT NOT NULL - )`, - } - for _, stmt := range stmts { - if _, err := db.Exec(stmt); err != nil { - db.Close() - return nil, fmt.Errorf("init gateway store: %w", err) - } - } - if err := ensureGatewaySettingsColumn(db, "public_api", "TEXT NOT NULL DEFAULT ''"); err != nil { - db.Close() - return nil, fmt.Errorf("migrate gateway store: %w", err) - } - if err := ensureGatewaySettingsColumn(db, "tx_gas_limit", "INTEGER NOT NULL DEFAULT 0"); err != nil { - db.Close() - return nil, fmt.Errorf("migrate gateway tx settings: %w", err) - } - if err := ensureGatewaySettingsColumn(db, "model_limits_json", "TEXT NOT NULL DEFAULT ''"); err != nil { - db.Close() - return nil, fmt.Errorf("migrate gateway model limits: %w", err) - } - if err := ensureGatewaySettingsColumn(db, "model_access_json", "TEXT NOT NULL DEFAULT ''"); err != nil { - db.Close() - return nil, fmt.Errorf("migrate gateway model access: %w", err) - } - if err := ensureGatewaySettingsColumn(db, "request_max_tokens_cap", "INTEGER NOT NULL DEFAULT 4096"); err != nil { - db.Close() - return nil, fmt.Errorf("migrate gateway max token cap: %w", err) - } - if err := ensureGatewaySettingsColumn(db, "max_concurrent_requests_per_10000_weight", "REAL NOT NULL DEFAULT 5.0"); err != nil { - db.Close() - return nil, fmt.Errorf("migrate gateway weight concurrency: %w", err) - } - if err := ensureGatewaySettingsColumn(db, "poc_max_concurrent_requests_per_10000_weight", "REAL NOT NULL DEFAULT 10.0"); err != nil { - db.Close() - return nil, fmt.Errorf("migrate gateway poc weight concurrency: %w", err) - } - if err := ensureGatewaySettingsTuningColumns(db); err != nil { - db.Close() - return nil, fmt.Errorf("migrate gateway tuning settings: %w", err) - } - if err := ensureGatewaySettingsRotationColumns(db); err != nil { - db.Close() - return nil, fmt.Errorf("migrate gateway rotation settings: %w", err) - } - if err := ensureGatewaySettingsDisabledColumns(db); err != nil { - db.Close() - return nil, fmt.Errorf("migrate gateway disabled settings: %w", err) - } - if err := ensureGatewayDevshardsColumn(db, "protocol_version", "TEXT NOT NULL DEFAULT ''"); err != nil { - db.Close() - return nil, fmt.Errorf("migrate gateway devshards: %w", err) - } - if err := ensureGatewayDevshardsColumn(db, "rotation_role", "TEXT NOT NULL DEFAULT ''"); err != nil { - db.Close() - return nil, fmt.Errorf("migrate gateway devshard role: %w", err) - } - if err := ensureGatewayDevshardsColumn(db, "rotation_epoch", "INTEGER NOT NULL DEFAULT 0"); err != nil { - db.Close() - return nil, fmt.Errorf("migrate gateway devshard epoch: %w", err) - } - if err := ensureGatewayDevshardsColumn(db, "settlement_pending", "INTEGER NOT NULL DEFAULT 0"); err != nil { - db.Close() - return nil, fmt.Errorf("migrate gateway devshard settlement_pending: %w", err) - } - if _, err := db.Exec(`CREATE TABLE IF NOT EXISTS participant_throttle_state ( - participant_key TEXT PRIMARY KEY, - tokens REAL NOT NULL DEFAULT 0, - last_refill_at TEXT NOT NULL, - last_throttle_status INTEGER NOT NULL DEFAULT 0, - empty_stream_streak INTEGER NOT NULL DEFAULT 0 - )`); err != nil { - db.Close() - return nil, fmt.Errorf("init participant throttle table: %w", err) - } - if _, err := db.Exec(`CREATE TABLE IF NOT EXISTS gateway_rotation_status ( - model_id TEXT NOT NULL, - stage TEXT NOT NULL, - epoch INTEGER NOT NULL, - role TEXT NOT NULL DEFAULT '', - target_count INTEGER NOT NULL DEFAULT 0, - existing_count INTEGER NOT NULL DEFAULT 0, - created_count INTEGER NOT NULL DEFAULT 0, - promoted_count INTEGER NOT NULL DEFAULT 0, - settled_count INTEGER NOT NULL DEFAULT 0, - settle_failed_count INTEGER NOT NULL DEFAULT 0, - create_error TEXT NOT NULL DEFAULT '', - completed INTEGER NOT NULL DEFAULT 0, - updated_at TEXT NOT NULL, - PRIMARY KEY (model_id, stage, epoch) - )`); err != nil { - db.Close() - return nil, fmt.Errorf("init gateway rotation status table: %w", err) - } - if _, err := db.Exec(`CREATE TABLE IF NOT EXISTS gateway_suspicious_hosts ( - participant_key TEXT PRIMARY KEY, - note TEXT NOT NULL DEFAULT '', - created_at TEXT NOT NULL - )`); err != nil { - db.Close() - return nil, fmt.Errorf("init gateway suspicious hosts table: %w", err) - } - // Write-ahead intent for an escrow create (written before the on-chain tx). - if _, err := db.Exec(`CREATE TABLE IF NOT EXISTS escrow_rotation_commitments ( - tx_hash TEXT PRIMARY KEY, - model TEXT NOT NULL, - role TEXT NOT NULL DEFAULT '', - epoch INTEGER NOT NULL DEFAULT 0, - private_key_env TEXT NOT NULL DEFAULT '', - block_height INTEGER NOT NULL DEFAULT 0, - created_at TEXT NOT NULL - )`); err != nil { - db.Close() - return nil, fmt.Errorf("init escrow rotation commitments table: %w", err) - } - if err := ensureColumn(db, "participant_throttle_state", "quarantine_until_utc", "TEXT NOT NULL DEFAULT ''"); err != nil { - db.Close() - return nil, fmt.Errorf("migrate participant throttle: %w", err) - } - if err := ensureColumn(db, "participant_throttle_state", "empty_stream_streak", "INTEGER NOT NULL DEFAULT 0"); err != nil { - db.Close() - return nil, fmt.Errorf("migrate participant throttle streak: %w", err) - } - if err := ensureColumn(db, "participant_throttle_state", "eof_transport_failure_streak", "INTEGER NOT NULL DEFAULT 0"); err != nil { - db.Close() - return nil, fmt.Errorf("migrate participant throttle eof streak: %w", err) - } - if err := ensureColumn(db, "participant_throttle_state", "model_ids", "TEXT NOT NULL DEFAULT ''"); err != nil { - db.Close() - return nil, fmt.Errorf("migrate participant throttle models: %w", err) - } - if err := ensureColumn(db, "participant_throttle_state", "failure_strikes", "INTEGER NOT NULL DEFAULT 0"); err != nil { - db.Close() - return nil, fmt.Errorf("migrate participant throttle strikes: %w", err) - } - if _, err := db.Exec(` - UPDATE participant_throttle_state - SET failure_strikes = MAX(IFNULL(empty_stream_streak, 0), IFNULL(eof_transport_failure_streak, 0)) - WHERE IFNULL(failure_strikes, 0) = 0 - AND (IFNULL(empty_stream_streak, 0) > 0 OR IFNULL(eof_transport_failure_streak, 0) > 0)`); err != nil { - db.Close() - return nil, fmt.Errorf("migrate participant throttle strike values: %w", err) - } - - return &GatewayStore{db: db}, nil -} - -func (s *GatewayStore) Close() error { - if s == nil || s.db == nil { - return nil - } - return s.db.Close() -} - -func (s *GatewayStore) LoadState() (GatewayState, bool, error) { - var state GatewayState - row := s.db.QueryRow(` - SELECT chain_rest, public_api, default_model, default_request_max_tokens, request_max_tokens_cap, - max_concurrent_requests, max_concurrent_requests_per_10000_weight, - poc_max_concurrent_requests_per_10000_weight, max_input_tokens_in_flight, - model_limits_json, model_access_json, tx_gas_limit, - participant_request_burst, participant_recovery_per_minute, - participant_http_quarantine_ms, participant_transport_failure_quarantine_ms, - participant_empty_stream_quarantine_ms, participant_stalled_winner_quarantine_ms, - participant_empty_stream_threshold, participant_eof_transport_failure_threshold, - redundancy_receipt_timeout_ms, redundancy_first_token_timeout_floor_ms, - redundancy_per_input_token_first_token_lag_ms, redundancy_inter_chunk_stall_timeout_ms, - redundancy_streaming_attempt_hard_timeout_ms, - redundancy_non_stream_response_floor_ms, redundancy_non_stream_no_content_timeout_ms, - redundancy_non_stream_max_attempt_wait_ms, redundancy_per_input_token_response_lag_ms, - redundancy_secondary_wait_after_winner_ms, redundancy_parallel_advantage_threshold, - redundancy_unresponsive_threshold, redundancy_speed_policy, redundancy_pairwise_budget_percentile, - redundancy_pairwise_max_proactive_attempts, redundancy_pairwise_min_direct_comparisons, - redundancy_pairwise_winner_hold_ms, redundancy_pairwise_winner_hold_min_speedup, - redundancy_pairwise_winner_hold_min_samples, - perf_sample_size, perf_window_ms, - escrow_rotation_enabled, escrow_rotation_settlement_enabled, - escrow_rotation_pre_poc_blocks, escrow_rotation_models_json, - gateway_disabled_enabled, gateway_disabled_message, gateway_disabled_new_url - FROM gateway_settings - WHERE id = 1`) - var rotationEnabled int - var rotationSettlementEnabled int - var disabledEnabled int - var rotationModelsJSON string - var modelLimitsJSON string - var modelAccessJSON string - err := row.Scan( - &state.Settings.ChainREST, - &state.Settings.PublicAPI, - &state.Settings.DefaultModel, - &state.Settings.DefaultRequestMaxTokens, - &state.Settings.RequestMaxTokensCap, - &state.Settings.MaxConcurrentRequests, - &state.Settings.MaxConcurrentPer10000Weight, - &state.Settings.PoCMaxConcurrentPer10000Weight, - &state.Settings.MaxInputTokensInFlight, - &modelLimitsJSON, - &modelAccessJSON, - &state.Settings.TxGasLimit, - &state.Settings.ParticipantThrottle.RequestBurst, - &state.Settings.ParticipantThrottle.RecoveryPerMinute, - &state.Settings.ParticipantThrottle.HTTPQuarantineMS, - &state.Settings.ParticipantThrottle.TransportFailureQuarantineMS, - &state.Settings.ParticipantThrottle.EmptyStreamQuarantineMS, - &state.Settings.ParticipantThrottle.StalledWinnerQuarantineMS, - &state.Settings.ParticipantThrottle.EmptyStreamQuarantineThreshold, - &state.Settings.ParticipantThrottle.EOFTransportFailureThreshold, - &state.Settings.Redundancy.ReceiptTimeoutMS, - &state.Settings.Redundancy.FirstTokenTimeoutFloorMS, - &state.Settings.Redundancy.PerInputTokenFirstTokenLagMS, - &state.Settings.Redundancy.InterChunkStallTimeoutMS, - &state.Settings.Redundancy.StreamingAttemptHardTimeoutMS, - &state.Settings.Redundancy.NonStreamResponseFloorMS, - &state.Settings.Redundancy.NonStreamNoContentTimeoutMS, - &state.Settings.Redundancy.NonStreamMaxAttemptWaitMS, - &state.Settings.Redundancy.PerInputTokenResponseLagMS, - &state.Settings.Redundancy.SecondaryWaitAfterWinnerMS, - &state.Settings.Redundancy.ParallelAdvantageThreshold, - &state.Settings.Redundancy.UnresponsiveThreshold, - &state.Settings.Redundancy.SpeedPolicy, - &state.Settings.Redundancy.PairwiseBudgetPercentile, - &state.Settings.Redundancy.PairwiseMaxProactiveAttempts, - &state.Settings.Redundancy.PairwiseMinDirectComparisons, - &state.Settings.Redundancy.PairwiseWinnerHoldMS, - &state.Settings.Redundancy.PairwiseWinnerHoldMinSpeedup, - &state.Settings.Redundancy.PairwiseWinnerHoldMinSamples, - &state.Settings.Perf.SampleSize, - &state.Settings.Perf.WindowMS, - &rotationEnabled, - &rotationSettlementEnabled, - &state.Settings.EscrowRotation.PrePoCBlocks, - &rotationModelsJSON, - &disabledEnabled, - &state.Settings.Disabled.Message, - &state.Settings.Disabled.NewURL, - ) - if err == sql.ErrNoRows { - return GatewayState{}, false, nil - } - if err != nil { - return GatewayState{}, false, fmt.Errorf("load gateway settings: %w", err) - } - state.Settings.EscrowRotation.Enabled = rotationEnabled != 0 - state.Settings.EscrowRotation.SettlementEnabled = rotationSettlementEnabled != 0 - if strings.TrimSpace(rotationModelsJSON) != "" { - if err := json.Unmarshal([]byte(rotationModelsJSON), &state.Settings.EscrowRotation.Models); err != nil { - return GatewayState{}, false, fmt.Errorf("load gateway rotation models: %w", err) - } - } - if strings.TrimSpace(modelLimitsJSON) != "" { - if err := json.Unmarshal([]byte(modelLimitsJSON), &state.Settings.ModelLimits); err != nil { - return GatewayState{}, false, fmt.Errorf("load gateway model limits: %w", err) - } - } - if strings.TrimSpace(modelAccessJSON) != "" { - var legacyModelAccess []GatewayModelAccessSettings - if err := json.Unmarshal([]byte(modelAccessJSON), &legacyModelAccess); err != nil { - return GatewayState{}, false, fmt.Errorf("load gateway model access: %w", err) - } - state.Settings.ModelLimits = applyLegacyModelAccessToLimits(state.Settings.ModelLimits, legacyModelAccess) - } - state.Settings.Disabled.Enabled = disabledEnabled != 0 - state.Settings = state.Settings.WithTuningDefaults() - - rows, err := s.db.Query(` - SELECT id, private_key_hex, private_key_env, model, storage_path, active, created_at, updated_at, protocol_version, - rotation_role, rotation_epoch, settlement_pending - FROM gateway_devshards - ORDER BY id`) - if err != nil { - return GatewayState{}, false, fmt.Errorf("load gateway devshards: %w", err) - } - defer rows.Close() - for rows.Next() { - var devshard GatewayDevshardState - var active int - var settlementPending int - if err := rows.Scan( - &devshard.ID, - &devshard.PrivateKeyHex, - &devshard.PrivateKeyEnv, - &devshard.Model, - &devshard.StoragePath, - &active, - &devshard.CreatedAt, - &devshard.UpdatedAt, - &devshard.ProtocolVersion, - &devshard.RotationRole, - &devshard.RotationEpoch, - &settlementPending, - ); err != nil { - return GatewayState{}, false, fmt.Errorf("scan gateway devshard: %w", err) - } - devshard.Active = active != 0 - devshard.SettlementPending = settlementPending != 0 - state.Devshards = append(state.Devshards, devshard) - } - if err := rows.Err(); err != nil { - return GatewayState{}, false, fmt.Errorf("iterate gateway devshards: %w", err) - } - suspiciousHosts, err := s.LoadSuspiciousHosts() - if err != nil { - return GatewayState{}, false, err - } - state.SuspiciousHosts = suspiciousHosts - return state, true, nil -} - -func (s *GatewayStore) Initialize(settings GatewaySettings, devshards []GatewayDevshardState) error { - settings = settings.WithTuningDefaults() - now := time.Now().UTC().Format(time.RFC3339Nano) - tx, err := s.db.Begin() - if err != nil { - return fmt.Errorf("begin gateway init: %w", err) - } - defer tx.Rollback() - - var count int - if err := tx.QueryRow(`SELECT COUNT(*) FROM gateway_settings WHERE id = 1`).Scan(&count); err != nil { - return fmt.Errorf("count gateway settings: %w", err) - } - if count > 0 { - return nil - } - - if _, err := tx.Exec(` - INSERT INTO gateway_settings ( - id, chain_rest, public_api, default_model, default_request_max_tokens, request_max_tokens_cap, - max_concurrent_requests, max_concurrent_requests_per_10000_weight, - poc_max_concurrent_requests_per_10000_weight, max_input_tokens_in_flight, - model_limits_json, model_access_json, tx_gas_limit, - participant_request_burst, participant_recovery_per_minute, - participant_http_quarantine_ms, participant_transport_failure_quarantine_ms, - participant_empty_stream_quarantine_ms, participant_stalled_winner_quarantine_ms, - participant_empty_stream_threshold, participant_eof_transport_failure_threshold, - redundancy_receipt_timeout_ms, redundancy_first_token_timeout_floor_ms, - redundancy_per_input_token_first_token_lag_ms, redundancy_inter_chunk_stall_timeout_ms, - redundancy_streaming_attempt_hard_timeout_ms, - redundancy_non_stream_response_floor_ms, redundancy_non_stream_no_content_timeout_ms, - redundancy_non_stream_max_attempt_wait_ms, redundancy_per_input_token_response_lag_ms, - redundancy_secondary_wait_after_winner_ms, redundancy_parallel_advantage_threshold, - redundancy_unresponsive_threshold, redundancy_speed_policy, redundancy_pairwise_budget_percentile, - redundancy_pairwise_max_proactive_attempts, redundancy_pairwise_min_direct_comparisons, - redundancy_pairwise_winner_hold_ms, redundancy_pairwise_winner_hold_min_speedup, - redundancy_pairwise_winner_hold_min_samples, - perf_sample_size, perf_window_ms, - escrow_rotation_enabled, escrow_rotation_settlement_enabled, - escrow_rotation_pre_poc_blocks, escrow_rotation_models_json, - gateway_disabled_enabled, gateway_disabled_message, gateway_disabled_new_url, - updated_at - ) VALUES (1, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, - strings.TrimSpace(settings.ChainREST), - strings.TrimSpace(settings.PublicAPI), - strings.TrimSpace(settings.DefaultModel), - settings.DefaultRequestMaxTokens, - settings.RequestMaxTokensCap, - settings.MaxConcurrentRequests, - settings.MaxConcurrentPer10000Weight, - settings.PoCMaxConcurrentPer10000Weight, - settings.MaxInputTokensInFlight, - mustMarshalGatewayModelLimits(settings.ModelLimits), - "", - settings.TxGasLimit, - settings.ParticipantThrottle.RequestBurst, - settings.ParticipantThrottle.RecoveryPerMinute, - settings.ParticipantThrottle.HTTPQuarantineMS, - settings.ParticipantThrottle.TransportFailureQuarantineMS, - settings.ParticipantThrottle.EmptyStreamQuarantineMS, - settings.ParticipantThrottle.StalledWinnerQuarantineMS, - settings.ParticipantThrottle.EmptyStreamQuarantineThreshold, - settings.ParticipantThrottle.EOFTransportFailureThreshold, - settings.Redundancy.ReceiptTimeoutMS, - settings.Redundancy.FirstTokenTimeoutFloorMS, - settings.Redundancy.PerInputTokenFirstTokenLagMS, - settings.Redundancy.InterChunkStallTimeoutMS, - settings.Redundancy.StreamingAttemptHardTimeoutMS, - settings.Redundancy.NonStreamResponseFloorMS, - settings.Redundancy.NonStreamNoContentTimeoutMS, - settings.Redundancy.NonStreamMaxAttemptWaitMS, - settings.Redundancy.PerInputTokenResponseLagMS, - settings.Redundancy.SecondaryWaitAfterWinnerMS, - settings.Redundancy.ParallelAdvantageThreshold, - settings.Redundancy.UnresponsiveThreshold, - settings.Redundancy.SpeedPolicy, - settings.Redundancy.PairwiseBudgetPercentile, - settings.Redundancy.PairwiseMaxProactiveAttempts, - settings.Redundancy.PairwiseMinDirectComparisons, - settings.Redundancy.PairwiseWinnerHoldMS, - settings.Redundancy.PairwiseWinnerHoldMinSpeedup, - settings.Redundancy.PairwiseWinnerHoldMinSamples, - settings.Perf.SampleSize, - settings.Perf.WindowMS, - gatewayBoolToInt(settings.EscrowRotation.Enabled), - gatewayBoolToInt(settings.EscrowRotation.SettlementEnabled), - settings.EscrowRotation.PrePoCBlocks, - mustMarshalEscrowRotationModels(settings.EscrowRotation.Models), - gatewayBoolToInt(settings.Disabled.Enabled), - strings.TrimSpace(settings.Disabled.Message), - strings.TrimSpace(settings.Disabled.NewURL), - now, - ); err != nil { - return fmt.Errorf("insert gateway settings: %w", err) - } - - for _, devshard := range devshards { - if err := s.upsertDevshardTx(tx, devshard, now); err != nil { - return err - } - } - return tx.Commit() -} - -func (s *GatewayStore) UpdateSettings(settings GatewaySettings) error { - settings = settings.WithTuningDefaults() - res, err := s.db.Exec(` - UPDATE gateway_settings - SET chain_rest = ?, - public_api = ?, - default_model = ?, - default_request_max_tokens = ?, - request_max_tokens_cap = ?, - max_concurrent_requests = ?, - max_concurrent_requests_per_10000_weight = ?, - poc_max_concurrent_requests_per_10000_weight = ?, - max_input_tokens_in_flight = ?, - model_limits_json = ?, - model_access_json = ?, - tx_gas_limit = ?, - participant_request_burst = ?, - participant_recovery_per_minute = ?, - participant_http_quarantine_ms = ?, - participant_transport_failure_quarantine_ms = ?, - participant_empty_stream_quarantine_ms = ?, - participant_stalled_winner_quarantine_ms = ?, - participant_empty_stream_threshold = ?, - participant_eof_transport_failure_threshold = ?, - redundancy_receipt_timeout_ms = ?, - redundancy_first_token_timeout_floor_ms = ?, - redundancy_per_input_token_first_token_lag_ms = ?, - redundancy_inter_chunk_stall_timeout_ms = ?, - redundancy_streaming_attempt_hard_timeout_ms = ?, - redundancy_non_stream_response_floor_ms = ?, - redundancy_non_stream_no_content_timeout_ms = ?, - redundancy_non_stream_max_attempt_wait_ms = ?, - redundancy_per_input_token_response_lag_ms = ?, - redundancy_secondary_wait_after_winner_ms = ?, - redundancy_parallel_advantage_threshold = ?, - redundancy_unresponsive_threshold = ?, - redundancy_speed_policy = ?, - redundancy_pairwise_budget_percentile = ?, - redundancy_pairwise_max_proactive_attempts = ?, - redundancy_pairwise_min_direct_comparisons = ?, - redundancy_pairwise_winner_hold_ms = ?, - redundancy_pairwise_winner_hold_min_speedup = ?, - redundancy_pairwise_winner_hold_min_samples = ?, - perf_sample_size = ?, - perf_window_ms = ?, - escrow_rotation_enabled = ?, - escrow_rotation_settlement_enabled = ?, - escrow_rotation_pre_poc_blocks = ?, - escrow_rotation_models_json = ?, - gateway_disabled_enabled = ?, - gateway_disabled_message = ?, - gateway_disabled_new_url = ?, - updated_at = ? - WHERE id = 1`, - strings.TrimSpace(settings.ChainREST), - strings.TrimSpace(settings.PublicAPI), - strings.TrimSpace(settings.DefaultModel), - settings.DefaultRequestMaxTokens, - settings.RequestMaxTokensCap, - settings.MaxConcurrentRequests, - settings.MaxConcurrentPer10000Weight, - settings.PoCMaxConcurrentPer10000Weight, - settings.MaxInputTokensInFlight, - mustMarshalGatewayModelLimits(settings.ModelLimits), - "", - settings.TxGasLimit, - settings.ParticipantThrottle.RequestBurst, - settings.ParticipantThrottle.RecoveryPerMinute, - settings.ParticipantThrottle.HTTPQuarantineMS, - settings.ParticipantThrottle.TransportFailureQuarantineMS, - settings.ParticipantThrottle.EmptyStreamQuarantineMS, - settings.ParticipantThrottle.StalledWinnerQuarantineMS, - settings.ParticipantThrottle.EmptyStreamQuarantineThreshold, - settings.ParticipantThrottle.EOFTransportFailureThreshold, - settings.Redundancy.ReceiptTimeoutMS, - settings.Redundancy.FirstTokenTimeoutFloorMS, - settings.Redundancy.PerInputTokenFirstTokenLagMS, - settings.Redundancy.InterChunkStallTimeoutMS, - settings.Redundancy.StreamingAttemptHardTimeoutMS, - settings.Redundancy.NonStreamResponseFloorMS, - settings.Redundancy.NonStreamNoContentTimeoutMS, - settings.Redundancy.NonStreamMaxAttemptWaitMS, - settings.Redundancy.PerInputTokenResponseLagMS, - settings.Redundancy.SecondaryWaitAfterWinnerMS, - settings.Redundancy.ParallelAdvantageThreshold, - settings.Redundancy.UnresponsiveThreshold, - settings.Redundancy.SpeedPolicy, - settings.Redundancy.PairwiseBudgetPercentile, - settings.Redundancy.PairwiseMaxProactiveAttempts, - settings.Redundancy.PairwiseMinDirectComparisons, - settings.Redundancy.PairwiseWinnerHoldMS, - settings.Redundancy.PairwiseWinnerHoldMinSpeedup, - settings.Redundancy.PairwiseWinnerHoldMinSamples, - settings.Perf.SampleSize, - settings.Perf.WindowMS, - gatewayBoolToInt(settings.EscrowRotation.Enabled), - gatewayBoolToInt(settings.EscrowRotation.SettlementEnabled), - settings.EscrowRotation.PrePoCBlocks, - mustMarshalEscrowRotationModels(settings.EscrowRotation.Models), - gatewayBoolToInt(settings.Disabled.Enabled), - strings.TrimSpace(settings.Disabled.Message), - strings.TrimSpace(settings.Disabled.NewURL), - time.Now().UTC().Format(time.RFC3339Nano), - ) - if err != nil { - return fmt.Errorf("update gateway settings: %w", err) - } - n, err := res.RowsAffected() - if err != nil { - return fmt.Errorf("rows affected for gateway settings update: %w", err) - } - if n == 0 { - return fmt.Errorf("gateway settings not initialized") - } - return nil +// GatewayStore persists gateway management state (settings, devshards, throttle, rotation). +type GatewayStore interface { + Close() error + LoadState() (GatewayState, bool, error) + Initialize(settings GatewaySettings, devshards []GatewayDevshardState) error + UpdateSettings(settings GatewaySettings) error + SaveRotationStatus(status GatewayRotationStatus) error + LoadRotationStatuses(limit int) ([]GatewayRotationStatus, error) + SaveCommitment(c GatewayEscrowCommitment) error + LoadCommitments() ([]GatewayEscrowCommitment, error) + DeleteCommitment(txHash string) error + LoadSuspiciousHosts() ([]GatewaySuspiciousHost, error) + UpsertSuspiciousHosts(participantKeys []string, note string) ([]GatewaySuspiciousHost, error) + DeleteSuspiciousHosts(participantKeys []string) ([]GatewaySuspiciousHost, error) + UpsertDevshard(devshard GatewayDevshardState) error + SetDevshardActive(id string, active bool) error + SetDevshardSettlementPending(id string, pending bool) error + DeleteDevshard(id string) error + SaveParticipantThrottle(key string, modelIDs []string, tokens float64, lastRefillAt time.Time, status int, quarantineUntil time.Time, failureStrikes int) error + DeleteParticipantThrottle(key string) error + LoadParticipantThrottles() ([]ParticipantThrottleRow, error) } type GatewayRotationStatus struct { @@ -937,88 +341,6 @@ type GatewayRotationStatus struct { UpdatedAt string `json:"updated_at"` } -func (s *GatewayStore) SaveRotationStatus(status GatewayRotationStatus) error { - if s == nil || s.db == nil { - return nil - } - now := time.Now().UTC().Format(time.RFC3339Nano) - if strings.TrimSpace(status.UpdatedAt) != "" { - now = strings.TrimSpace(status.UpdatedAt) - } - _, err := s.db.Exec(` - INSERT OR REPLACE INTO gateway_rotation_status ( - model_id, stage, epoch, role, target_count, existing_count, created_count, - promoted_count, settled_count, settle_failed_count, create_error, completed, updated_at - ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, - strings.TrimSpace(status.ModelID), - strings.TrimSpace(status.Stage), - status.Epoch, - strings.TrimSpace(status.Role), - status.TargetCount, - status.ExistingCount, - status.CreatedCount, - status.PromotedCount, - status.SettledCount, - status.SettleFailedCount, - strings.TrimSpace(status.CreateError), - gatewayBoolToInt(status.Completed), - now, - ) - if err != nil { - return fmt.Errorf("save gateway rotation status model=%q stage=%q epoch=%d: %w", status.ModelID, status.Stage, status.Epoch, err) - } - return nil -} - -func (s *GatewayStore) LoadRotationStatuses(limit int) ([]GatewayRotationStatus, error) { - if s == nil || s.db == nil { - return nil, nil - } - query := ` - SELECT model_id, stage, epoch, role, target_count, existing_count, created_count, - promoted_count, settled_count, settle_failed_count, create_error, completed, updated_at - FROM gateway_rotation_status - ORDER BY updated_at DESC` - args := []any{} - if limit > 0 { - query += ` LIMIT ?` - args = append(args, limit) - } - rows, err := s.db.Query(query, args...) - if err != nil { - return nil, fmt.Errorf("load gateway rotation statuses: %w", err) - } - defer rows.Close() - var statuses []GatewayRotationStatus - for rows.Next() { - var status GatewayRotationStatus - var completed int - if err := rows.Scan( - &status.ModelID, - &status.Stage, - &status.Epoch, - &status.Role, - &status.TargetCount, - &status.ExistingCount, - &status.CreatedCount, - &status.PromotedCount, - &status.SettledCount, - &status.SettleFailedCount, - &status.CreateError, - &completed, - &status.UpdatedAt, - ); err != nil { - return nil, fmt.Errorf("scan gateway rotation status: %w", err) - } - status.Completed = completed != 0 - statuses = append(statuses, status) - } - if err := rows.Err(); err != nil { - return nil, err - } - return statuses, nil -} - // GatewayEscrowCommitment is the write-ahead intent for one escrow create, keyed by tx hash. type GatewayEscrowCommitment struct { TxHash string @@ -1030,269 +352,6 @@ type GatewayEscrowCommitment struct { CreatedAt string } -// SaveCommitment records a create intent, keyed by tx hash. -func (s *GatewayStore) SaveCommitment(c GatewayEscrowCommitment) error { - if s == nil || s.db == nil { - return fmt.Errorf("gateway store unavailable") - } - now := time.Now().UTC().Format(time.RFC3339Nano) - _, err := s.db.Exec(` - INSERT OR REPLACE INTO escrow_rotation_commitments ( - tx_hash, model, role, epoch, private_key_env, block_height, created_at - ) VALUES (?, ?, ?, ?, ?, ?, ?)`, - strings.TrimSpace(c.TxHash), - strings.TrimSpace(c.Model), - strings.TrimSpace(c.Role), - c.Epoch, - strings.TrimSpace(c.PrivateKeyEnv), - c.BlockHeight, - now, - ) - if err != nil { - return fmt.Errorf("save escrow commitment tx=%s: %w", c.TxHash, err) - } - return nil -} - -// LoadCommitments returns all pending commitments (oldest first). -func (s *GatewayStore) LoadCommitments() ([]GatewayEscrowCommitment, error) { - if s == nil || s.db == nil { - return nil, nil - } - rows, err := s.db.Query(` - SELECT tx_hash, model, role, epoch, private_key_env, block_height, created_at - FROM escrow_rotation_commitments - ORDER BY created_at ASC`) - if err != nil { - return nil, fmt.Errorf("load escrow commitments: %w", err) - } - defer rows.Close() - var commitments []GatewayEscrowCommitment - for rows.Next() { - var c GatewayEscrowCommitment - if err := rows.Scan(&c.TxHash, &c.Model, &c.Role, &c.Epoch, &c.PrivateKeyEnv, &c.BlockHeight, &c.CreatedAt); err != nil { - return nil, fmt.Errorf("scan escrow commitment: %w", err) - } - commitments = append(commitments, c) - } - return commitments, rows.Err() -} - -// DeleteCommitment clears a commitment once its escrow is persisted (or proven absent). -func (s *GatewayStore) DeleteCommitment(txHash string) error { - if s == nil || s.db == nil { - return fmt.Errorf("gateway store unavailable") - } - if _, err := s.db.Exec(`DELETE FROM escrow_rotation_commitments WHERE tx_hash = ?`, strings.TrimSpace(txHash)); err != nil { - return fmt.Errorf("delete escrow commitment tx=%s: %w", txHash, err) - } - return nil -} - -func (s *GatewayStore) LoadSuspiciousHosts() ([]GatewaySuspiciousHost, error) { - if s == nil || s.db == nil { - return nil, nil - } - rows, err := s.db.Query(` - SELECT participant_key, note, created_at - FROM gateway_suspicious_hosts - ORDER BY participant_key`) - if err != nil { - return nil, fmt.Errorf("load gateway suspicious hosts: %w", err) - } - defer rows.Close() - - var hosts []GatewaySuspiciousHost - for rows.Next() { - var host GatewaySuspiciousHost - if err := rows.Scan(&host.ParticipantKey, &host.Note, &host.CreatedAt); err != nil { - return nil, fmt.Errorf("scan gateway suspicious host: %w", err) - } - hosts = append(hosts, host) - } - if err := rows.Err(); err != nil { - return nil, err - } - return hosts, nil -} - -func (s *GatewayStore) UpsertSuspiciousHosts(participantKeys []string, note string) ([]GatewaySuspiciousHost, error) { - if s == nil || s.db == nil { - return nil, nil - } - participantKeys = normalizeParticipantKeys(participantKeys) - if len(participantKeys) == 0 { - return nil, fmt.Errorf("participant_keys must contain at least one key") - } - tx, err := s.db.Begin() - if err != nil { - return nil, fmt.Errorf("begin suspicious host upsert: %w", err) - } - defer tx.Rollback() - now := time.Now().UTC().Format(time.RFC3339Nano) - note = strings.TrimSpace(note) - for _, key := range participantKeys { - if _, err := tx.Exec(` - INSERT INTO gateway_suspicious_hosts (participant_key, note, created_at) - VALUES (?, ?, ?) - ON CONFLICT(participant_key) DO UPDATE SET note = excluded.note`, - key, note, now); err != nil { - return nil, fmt.Errorf("upsert suspicious host %s: %w", key, err) - } - } - if err := tx.Commit(); err != nil { - return nil, fmt.Errorf("commit suspicious host upsert: %w", err) - } - return s.LoadSuspiciousHosts() -} - -func (s *GatewayStore) DeleteSuspiciousHosts(participantKeys []string) ([]GatewaySuspiciousHost, error) { - if s == nil || s.db == nil { - return nil, nil - } - participantKeys = normalizeParticipantKeys(participantKeys) - if len(participantKeys) == 0 { - return nil, fmt.Errorf("participant_keys must contain at least one key") - } - tx, err := s.db.Begin() - if err != nil { - return nil, fmt.Errorf("begin suspicious host delete: %w", err) - } - defer tx.Rollback() - for _, key := range participantKeys { - if _, err := tx.Exec(`DELETE FROM gateway_suspicious_hosts WHERE participant_key = ?`, key); err != nil { - return nil, fmt.Errorf("delete suspicious host %s: %w", key, err) - } - } - if err := tx.Commit(); err != nil { - return nil, fmt.Errorf("commit suspicious host delete: %w", err) - } - return s.LoadSuspiciousHosts() -} - -func (s *GatewayStore) UpsertDevshard(devshard GatewayDevshardState) error { - now := time.Now().UTC().Format(time.RFC3339Nano) - tx, err := s.db.Begin() - if err != nil { - return fmt.Errorf("begin devshard upsert: %w", err) - } - defer tx.Rollback() - if err := s.upsertDevshardTx(tx, devshard, now); err != nil { - return err - } - return tx.Commit() -} - -func (s *GatewayStore) upsertDevshardTx(tx *sql.Tx, devshard GatewayDevshardState, now string) error { - createdAt := now - _ = tx.QueryRow(`SELECT created_at FROM gateway_devshards WHERE id = ?`, devshard.ID).Scan(&createdAt) - // Preserve the existing settlement_pending marker so an unrelated upsert - // never silently clears a queued settlement; a brand-new row falls back - // to the value carried on devshard. - settlementPending := gatewayBoolToInt(devshard.SettlementPending) - _ = tx.QueryRow(`SELECT settlement_pending FROM gateway_devshards WHERE id = ?`, devshard.ID).Scan(&settlementPending) - if _, err := tx.Exec(` - INSERT OR REPLACE INTO gateway_devshards ( - id, private_key_hex, private_key_env, model, storage_path, active, created_at, updated_at, protocol_version, - rotation_role, rotation_epoch, settlement_pending - ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, - strings.TrimSpace(devshard.ID), - strings.TrimSpace(devshard.PrivateKeyHex), - strings.TrimSpace(devshard.PrivateKeyEnv), - strings.TrimSpace(devshard.Model), - strings.TrimSpace(devshard.StoragePath), - gatewayBoolToInt(devshard.Active), - createdAt, - now, - strings.TrimSpace(devshard.ProtocolVersion), - strings.TrimSpace(devshard.RotationRole), - devshard.RotationEpoch, - settlementPending, - ); err != nil { - return fmt.Errorf("upsert gateway devshard %s: %w", devshard.ID, err) - } - return nil -} - -func (s *GatewayStore) SetDevshardActive(id string, active bool) error { - res, err := s.db.Exec(` - UPDATE gateway_devshards - SET active = ?, updated_at = ? - WHERE id = ?`, - gatewayBoolToInt(active), - time.Now().UTC().Format(time.RFC3339Nano), - strings.TrimSpace(id), - ) - if err != nil { - return fmt.Errorf("update devshard %s active=%t: %w", id, active, err) - } - n, err := res.RowsAffected() - if err != nil { - return fmt.Errorf("rows affected for devshard %s: %w", id, err) - } - if n == 0 { - return fmt.Errorf("devshard %s not found", id) - } - return nil -} - -func (s *GatewayStore) SetDevshardSettlementPending(id string, pending bool) error { - res, err := s.db.Exec(` - UPDATE gateway_devshards - SET settlement_pending = ?, updated_at = ? - WHERE id = ?`, - gatewayBoolToInt(pending), - time.Now().UTC().Format(time.RFC3339Nano), - strings.TrimSpace(id), - ) - if err != nil { - return fmt.Errorf("update devshard %s settlement_pending=%t: %w", id, pending, err) - } - n, err := res.RowsAffected() - if err != nil { - return fmt.Errorf("rows affected for devshard %s: %w", id, err) - } - if n == 0 { - return fmt.Errorf("devshard %s not found", id) - } - return nil -} - -func (s *GatewayStore) DeleteDevshard(id string) error { - res, err := s.db.Exec(`DELETE FROM gateway_devshards WHERE id = ?`, strings.TrimSpace(id)) - if err != nil { - return fmt.Errorf("delete devshard %s: %w", id, err) - } - n, err := res.RowsAffected() - if err != nil { - return fmt.Errorf("rows affected for delete devshard %s: %w", id, err) - } - if n == 0 { - return fmt.Errorf("devshard %s not found", id) - } - return nil -} - -func normalizeParticipantKeys(keys []string) []string { - if len(keys) == 0 { - return nil - } - normalized := make([]string, 0, len(keys)) - seen := make(map[string]struct{}, len(keys)) - for _, key := range keys { - key = strings.TrimSpace(key) - if key == "" { - continue - } - if _, ok := seen[key]; ok { - continue - } - seen[key] = struct{}{} - normalized = append(normalized, key) - } - return normalized -} - // ParticipantThrottleRow represents a persisted reactive throttle state for one host. type ParticipantThrottleRow struct { Key string @@ -1300,79 +359,253 @@ type ParticipantThrottleRow struct { Tokens float64 LastRefillAt time.Time Status int - QuarantineUntil time.Time // wall-clock end of unified quarantine; zero if unset + QuarantineUntil time.Time FailureStrikes int } -func (s *GatewayStore) SaveParticipantThrottle(key string, modelIDs []string, tokens float64, lastRefillAt time.Time, status int, quarantineUntil time.Time, failureStrikes int) error { - if s == nil || s.db == nil { - return nil - } - quarStr := "" - if !quarantineUntil.IsZero() { - quarStr = quarantineUntil.UTC().Format(time.RFC3339Nano) - } - _, err := s.db.Exec(` - INSERT OR REPLACE INTO participant_throttle_state - (participant_key, tokens, last_refill_at, last_throttle_status, quarantine_until_utc, failure_strikes, model_ids) - VALUES (?, ?, ?, ?, ?, ?, ?)`, - key, tokens, lastRefillAt.UTC().Format(time.RFC3339Nano), status, quarStr, failureStrikes, strings.Join(normalizeModelIDs(modelIDs), ",")) - if err != nil { - return fmt.Errorf("save participant throttle %s: %w", key, err) - } - return nil +func gatewaySettingsSelectColumns() string { + return strings.Join(gatewaySettingsColumnNames(), ", ") } -func (s *GatewayStore) DeleteParticipantThrottle(key string) error { - if s == nil || s.db == nil { - return nil +func gatewaySettingsColumnNames() []string { + return []string{ + "chain_rest", "public_api", "default_model", "default_request_max_tokens", "request_max_tokens_cap", + "max_concurrent_requests", "max_concurrent_requests_per_10000_weight", + "poc_max_concurrent_requests_per_10000_weight", "max_input_tokens_in_flight", + "model_limits_json", "model_access_json", "tx_gas_limit", + "participant_request_burst", "participant_recovery_per_minute", + "participant_http_quarantine_ms", "participant_transport_failure_quarantine_ms", + "participant_empty_stream_quarantine_ms", "participant_stalled_winner_quarantine_ms", + "participant_empty_stream_threshold", "participant_eof_transport_failure_threshold", + "redundancy_receipt_timeout_ms", "redundancy_first_token_timeout_floor_ms", + "redundancy_per_input_token_first_token_lag_ms", "redundancy_inter_chunk_stall_timeout_ms", + "redundancy_streaming_attempt_hard_timeout_ms", + "redundancy_non_stream_response_floor_ms", "redundancy_non_stream_no_content_timeout_ms", + "redundancy_non_stream_max_attempt_wait_ms", "redundancy_per_input_token_response_lag_ms", + "redundancy_secondary_wait_after_winner_ms", "redundancy_parallel_advantage_threshold", + "redundancy_unresponsive_threshold", "redundancy_speed_policy", "redundancy_pairwise_budget_percentile", + "redundancy_pairwise_max_proactive_attempts", "redundancy_pairwise_min_direct_comparisons", + "redundancy_pairwise_winner_hold_ms", "redundancy_pairwise_winner_hold_min_speedup", + "redundancy_pairwise_winner_hold_min_samples", + "perf_sample_size", "perf_window_ms", + "escrow_rotation_enabled", "escrow_rotation_settlement_enabled", + "escrow_rotation_pre_poc_blocks", "escrow_rotation_models_json", + "gateway_disabled_enabled", "gateway_disabled_message", "gateway_disabled_new_url", } - _, err := s.db.Exec(`DELETE FROM participant_throttle_state WHERE participant_key = ?`, key) - if err != nil { - return fmt.Errorf("delete participant throttle %s: %w", key, err) - } - return nil } -func (s *GatewayStore) LoadParticipantThrottles() ([]ParticipantThrottleRow, error) { - if s == nil || s.db == nil { - return nil, nil - } - rows, err := s.db.Query(` - SELECT participant_key, tokens, last_refill_at, last_throttle_status, - IFNULL(quarantine_until_utc, '') AS quarantine_until_utc, - IFNULL(failure_strikes, 0) AS failure_strikes, - IFNULL(model_ids, '') AS model_ids - FROM participant_throttle_state`) - if err != nil { - return nil, fmt.Errorf("load participant throttles: %w", err) +func gatewaySettingsInsertColumnNames() string { + return "id, " + gatewaySettingsSelectColumns() + ", updated_at" +} + +func gatewaySettingsUpdateAssignments(param string) string { + cols := gatewaySettingsColumnNames() + parts := make([]string, len(cols)) + for i, col := range cols { + parts[i] = col + " = " + param } - defer rows.Close() + return strings.Join(parts, ",\n\t\t ") +} - var result []ParticipantThrottleRow - for rows.Next() { - var row ParticipantThrottleRow - var lastRefillStr, quarantineStr, modelIDsStr string - if err := rows.Scan(&row.Key, &row.Tokens, &lastRefillStr, &row.Status, &quarantineStr, &row.FailureStrikes, &modelIDsStr); err != nil { - return nil, fmt.Errorf("scan participant throttle: %w", err) +type gatewaySettingsRow struct { + settings GatewaySettings + rotationEnabled int + rotationSettlementEnabled int + rotationModelsJSON string + modelLimitsJSON string + modelAccessJSON string + disabledEnabled int +} + +func newGatewaySettingsRow(settings GatewaySettings) gatewaySettingsRow { + settings = settings.WithTuningDefaults() + return gatewaySettingsRow{ + settings: settings, + rotationEnabled: gatewayBoolToInt(settings.EscrowRotation.Enabled), + rotationSettlementEnabled: gatewayBoolToInt(settings.EscrowRotation.SettlementEnabled), + rotationModelsJSON: mustMarshalEscrowRotationModels(settings.EscrowRotation.Models), + modelLimitsJSON: mustMarshalGatewayModelLimits(settings.ModelLimits), + modelAccessJSON: "", + disabledEnabled: gatewayBoolToInt(settings.Disabled.Enabled), + } +} + +func (r *gatewaySettingsRow) scanDestinations() []any { + s := &r.settings + return []any{ + &s.ChainREST, + &s.PublicAPI, + &s.DefaultModel, + &s.DefaultRequestMaxTokens, + &s.RequestMaxTokensCap, + &s.MaxConcurrentRequests, + &s.MaxConcurrentPer10000Weight, + &s.PoCMaxConcurrentPer10000Weight, + &s.MaxInputTokensInFlight, + &r.modelLimitsJSON, + &r.modelAccessJSON, + &s.TxGasLimit, + &s.ParticipantThrottle.RequestBurst, + &s.ParticipantThrottle.RecoveryPerMinute, + &s.ParticipantThrottle.HTTPQuarantineMS, + &s.ParticipantThrottle.TransportFailureQuarantineMS, + &s.ParticipantThrottle.EmptyStreamQuarantineMS, + &s.ParticipantThrottle.StalledWinnerQuarantineMS, + &s.ParticipantThrottle.EmptyStreamQuarantineThreshold, + &s.ParticipantThrottle.EOFTransportFailureThreshold, + &s.Redundancy.ReceiptTimeoutMS, + &s.Redundancy.FirstTokenTimeoutFloorMS, + &s.Redundancy.PerInputTokenFirstTokenLagMS, + &s.Redundancy.InterChunkStallTimeoutMS, + &s.Redundancy.StreamingAttemptHardTimeoutMS, + &s.Redundancy.NonStreamResponseFloorMS, + &s.Redundancy.NonStreamNoContentTimeoutMS, + &s.Redundancy.NonStreamMaxAttemptWaitMS, + &s.Redundancy.PerInputTokenResponseLagMS, + &s.Redundancy.SecondaryWaitAfterWinnerMS, + &s.Redundancy.ParallelAdvantageThreshold, + &s.Redundancy.UnresponsiveThreshold, + &s.Redundancy.SpeedPolicy, + &s.Redundancy.PairwiseBudgetPercentile, + &s.Redundancy.PairwiseMaxProactiveAttempts, + &s.Redundancy.PairwiseMinDirectComparisons, + &s.Redundancy.PairwiseWinnerHoldMS, + &s.Redundancy.PairwiseWinnerHoldMinSpeedup, + &s.Redundancy.PairwiseWinnerHoldMinSamples, + &s.Perf.SampleSize, + &s.Perf.WindowMS, + &r.rotationEnabled, + &r.rotationSettlementEnabled, + &s.EscrowRotation.PrePoCBlocks, + &r.rotationModelsJSON, + &r.disabledEnabled, + &s.Disabled.Message, + &s.Disabled.NewURL, + } +} + +func (r *gatewaySettingsRow) finish() (GatewaySettings, error) { + settings := r.settings + settings.EscrowRotation.Enabled = r.rotationEnabled != 0 + settings.EscrowRotation.SettlementEnabled = r.rotationSettlementEnabled != 0 + if strings.TrimSpace(r.rotationModelsJSON) != "" { + if err := json.Unmarshal([]byte(r.rotationModelsJSON), &settings.EscrowRotation.Models); err != nil { + return GatewaySettings{}, fmt.Errorf("load gateway rotation models: %w", err) } - row.ModelIDs = splitModelIDs(modelIDsStr) - row.LastRefillAt, err = time.Parse(time.RFC3339Nano, lastRefillStr) - if err != nil { - return nil, fmt.Errorf("parse last_refill_at for %s: %w", row.Key, err) + } + if strings.TrimSpace(r.modelLimitsJSON) != "" { + if err := json.Unmarshal([]byte(r.modelLimitsJSON), &settings.ModelLimits); err != nil { + return GatewaySettings{}, fmt.Errorf("load gateway model limits: %w", err) } - if strings.TrimSpace(quarantineStr) != "" { - row.QuarantineUntil, err = time.Parse(time.RFC3339Nano, quarantineStr) - if err != nil { - return nil, fmt.Errorf("parse quarantine_until_utc for %s: %w", row.Key, err) - } + } + if strings.TrimSpace(r.modelAccessJSON) != "" { + var legacyModelAccess []GatewayModelAccessSettings + if err := json.Unmarshal([]byte(r.modelAccessJSON), &legacyModelAccess); err != nil { + return GatewaySettings{}, fmt.Errorf("load gateway model access: %w", err) } - result = append(result, row) + settings.ModelLimits = applyLegacyModelAccessToLimits(settings.ModelLimits, legacyModelAccess) + } + settings.Disabled.Enabled = r.disabledEnabled != 0 + return settings.WithTuningDefaults(), nil +} + +func (r *gatewaySettingsRow) valueArgs() []any { + s := r.settings + return []any{ + strings.TrimSpace(s.ChainREST), + strings.TrimSpace(s.PublicAPI), + strings.TrimSpace(s.DefaultModel), + s.DefaultRequestMaxTokens, + s.RequestMaxTokensCap, + s.MaxConcurrentRequests, + s.MaxConcurrentPer10000Weight, + s.PoCMaxConcurrentPer10000Weight, + s.MaxInputTokensInFlight, + r.modelLimitsJSON, + r.modelAccessJSON, + s.TxGasLimit, + s.ParticipantThrottle.RequestBurst, + s.ParticipantThrottle.RecoveryPerMinute, + s.ParticipantThrottle.HTTPQuarantineMS, + s.ParticipantThrottle.TransportFailureQuarantineMS, + s.ParticipantThrottle.EmptyStreamQuarantineMS, + s.ParticipantThrottle.StalledWinnerQuarantineMS, + s.ParticipantThrottle.EmptyStreamQuarantineThreshold, + s.ParticipantThrottle.EOFTransportFailureThreshold, + s.Redundancy.ReceiptTimeoutMS, + s.Redundancy.FirstTokenTimeoutFloorMS, + s.Redundancy.PerInputTokenFirstTokenLagMS, + s.Redundancy.InterChunkStallTimeoutMS, + s.Redundancy.StreamingAttemptHardTimeoutMS, + s.Redundancy.NonStreamResponseFloorMS, + s.Redundancy.NonStreamNoContentTimeoutMS, + s.Redundancy.NonStreamMaxAttemptWaitMS, + s.Redundancy.PerInputTokenResponseLagMS, + s.Redundancy.SecondaryWaitAfterWinnerMS, + s.Redundancy.ParallelAdvantageThreshold, + s.Redundancy.UnresponsiveThreshold, + s.Redundancy.SpeedPolicy, + s.Redundancy.PairwiseBudgetPercentile, + s.Redundancy.PairwiseMaxProactiveAttempts, + s.Redundancy.PairwiseMinDirectComparisons, + s.Redundancy.PairwiseWinnerHoldMS, + s.Redundancy.PairwiseWinnerHoldMinSpeedup, + s.Redundancy.PairwiseWinnerHoldMinSamples, + s.Perf.SampleSize, + s.Perf.WindowMS, + r.rotationEnabled, + r.rotationSettlementEnabled, + s.EscrowRotation.PrePoCBlocks, + r.rotationModelsJSON, + r.disabledEnabled, + strings.TrimSpace(s.Disabled.Message), + strings.TrimSpace(s.Disabled.NewURL), + } +} + +type gatewaySettingsScanner interface { + Scan(dest ...any) error +} + +func scanGatewaySettings(scanner gatewaySettingsScanner) (GatewaySettings, error) { + var row gatewaySettingsRow + if err := scanner.Scan(row.scanDestinations()...); err != nil { + return GatewaySettings{}, err + } + return row.finish() +} + +func settingsInsertArgs(settings GatewaySettings, updatedAt string) []any { + row := newGatewaySettingsRow(settings) + args := []any{1} + args = append(args, row.valueArgs()...) + return append(args, updatedAt) +} + +func settingsUpdateArgs(settings GatewaySettings, updatedAt string) []any { + row := newGatewaySettingsRow(settings) + args := row.valueArgs() + return append(args, updatedAt) +} + +func pgPlaceholderList(n int) string { + if n <= 0 { + return "" } - if err := rows.Err(); err != nil { - return nil, err + parts := make([]string, n) + for i := range parts { + parts[i] = fmt.Sprintf("$%d", i+1) + } + return strings.Join(parts, ", ") +} + +func gatewaySettingsUpdateAssignmentsPG() string { + cols := gatewaySettingsColumnNames() + parts := make([]string, len(cols)) + for i, col := range cols { + parts[i] = fmt.Sprintf("%s = $%d", col, i+1) } - return result, nil + return strings.Join(parts, ",\n\t\t ") } func gatewayBoolToInt(v bool) int { @@ -1405,117 +638,3 @@ func mustMarshalGatewayModelLimits(limits []GatewayModelLimitSettings) string { return string(b) } -func ensureGatewaySettingsColumn(db *sql.DB, columnName, columnDDL string) error { - return ensureColumn(db, "gateway_settings", columnName, columnDDL) -} - -func ensureGatewaySettingsTuningColumns(db *sql.DB) error { - columns := []struct { - name string - ddl string - }{ - {"participant_request_burst", "INTEGER NOT NULL DEFAULT 600"}, - {"participant_recovery_per_minute", "INTEGER NOT NULL DEFAULT 10"}, - {"participant_http_quarantine_ms", "INTEGER NOT NULL DEFAULT 3600000"}, - {"participant_transport_failure_quarantine_ms", "INTEGER NOT NULL DEFAULT 1800000"}, - {"participant_empty_stream_quarantine_ms", "INTEGER NOT NULL DEFAULT 1800000"}, - {"participant_stalled_winner_quarantine_ms", "INTEGER NOT NULL DEFAULT 1800000"}, - {"participant_empty_stream_threshold", "INTEGER NOT NULL DEFAULT 3"}, - {"participant_eof_transport_failure_threshold", "INTEGER NOT NULL DEFAULT 3"}, - {"redundancy_receipt_timeout_ms", "INTEGER NOT NULL DEFAULT 5000"}, - {"redundancy_first_token_timeout_floor_ms", "INTEGER NOT NULL DEFAULT 1000"}, - {"redundancy_per_input_token_first_token_lag_ms", "INTEGER NOT NULL DEFAULT 10"}, - {"redundancy_inter_chunk_stall_timeout_ms", "INTEGER NOT NULL DEFAULT 60000"}, - {"redundancy_streaming_attempt_hard_timeout_ms", "INTEGER NOT NULL DEFAULT 1200000"}, - {"redundancy_non_stream_response_floor_ms", "INTEGER NOT NULL DEFAULT 20000"}, - {"redundancy_non_stream_no_content_timeout_ms", "INTEGER NOT NULL DEFAULT 1200000"}, - {"redundancy_non_stream_max_attempt_wait_ms", "INTEGER NOT NULL DEFAULT 1800000"}, - {"redundancy_per_input_token_response_lag_ms", "INTEGER NOT NULL DEFAULT 20"}, - {"redundancy_secondary_wait_after_winner_ms", "INTEGER NOT NULL DEFAULT 600000"}, - {"redundancy_parallel_advantage_threshold", "REAL NOT NULL DEFAULT 0.5"}, - {"redundancy_unresponsive_threshold", "REAL NOT NULL DEFAULT 1.0"}, - {"redundancy_speed_policy", "TEXT NOT NULL DEFAULT 'hybrid'"}, - {"redundancy_pairwise_budget_percentile", "REAL NOT NULL DEFAULT 0.9"}, - {"redundancy_pairwise_max_proactive_attempts", "INTEGER NOT NULL DEFAULT 3"}, - {"redundancy_pairwise_min_direct_comparisons", "INTEGER NOT NULL DEFAULT 4"}, - {"redundancy_pairwise_winner_hold_ms", "INTEGER NOT NULL DEFAULT 500"}, - {"redundancy_pairwise_winner_hold_min_speedup", "REAL NOT NULL DEFAULT 0.1"}, - {"redundancy_pairwise_winner_hold_min_samples", "INTEGER NOT NULL DEFAULT 6"}, - {"perf_sample_size", "INTEGER NOT NULL DEFAULT 256"}, - {"perf_window_ms", "INTEGER NOT NULL DEFAULT 3600000"}, - } - for _, column := range columns { - if err := ensureGatewaySettingsColumn(db, column.name, column.ddl); err != nil { - return err - } - } - return nil -} - -func ensureGatewaySettingsRotationColumns(db *sql.DB) error { - columns := []struct { - name string - ddl string - }{ - {"escrow_rotation_enabled", "INTEGER NOT NULL DEFAULT 0"}, - {"escrow_rotation_settlement_enabled", "INTEGER NOT NULL DEFAULT 0"}, - {"escrow_rotation_pre_poc_blocks", "INTEGER NOT NULL DEFAULT 300"}, - {"escrow_rotation_models_json", "TEXT NOT NULL DEFAULT ''"}, - } - for _, column := range columns { - if err := ensureGatewaySettingsColumn(db, column.name, column.ddl); err != nil { - return err - } - } - return nil -} - -func ensureGatewaySettingsDisabledColumns(db *sql.DB) error { - columns := []struct { - name string - ddl string - }{ - {"gateway_disabled_enabled", "INTEGER NOT NULL DEFAULT 0"}, - {"gateway_disabled_message", "TEXT NOT NULL DEFAULT ''"}, - {"gateway_disabled_new_url", "TEXT NOT NULL DEFAULT ''"}, - } - for _, column := range columns { - if err := ensureGatewaySettingsColumn(db, column.name, column.ddl); err != nil { - return err - } - } - return nil -} - -func ensureGatewayDevshardsColumn(db *sql.DB, columnName, columnDDL string) error { - return ensureColumn(db, "gateway_devshards", columnName, columnDDL) -} - -func ensureColumn(db *sql.DB, table, columnName, columnDDL string) error { - rows, err := db.Query(fmt.Sprintf(`PRAGMA table_info(%s)`, table)) - if err != nil { - return err - } - defer rows.Close() - - for rows.Next() { - var cid int - var name string - var dataType string - var notNull int - var defaultValue sql.NullString - var pk int - if err := rows.Scan(&cid, &name, &dataType, ¬Null, &defaultValue, &pk); err != nil { - return err - } - if name == columnName { - return nil - } - } - if err := rows.Err(); err != nil { - return err - } - - _, err = db.Exec(fmt.Sprintf(`ALTER TABLE %s ADD COLUMN %s %s`, table, columnName, columnDDL)) - return err -} diff --git a/devshard/cmd/devshardctl/gateway_store_factory.go b/devshard/cmd/devshardctl/gateway_store_factory.go new file mode 100644 index 0000000000..daff39f8a8 --- /dev/null +++ b/devshard/cmd/devshardctl/gateway_store_factory.go @@ -0,0 +1,58 @@ +package main + +import ( + "context" + "log" + "os" + "path/filepath" + "time" +) + +const defaultGatewayPGConnectTimeout = 2 * time.Second + +// NewGatewayStore opens the gateway persistence layer based on environment configuration. +// If PGHOST is unset, returns SQLiteGatewayStore only. +// If PGHOST is set, runs one-shot migration from SQLite when Postgres is reachable, +// then returns HybridGatewayStore (Postgres primary + SQLite fallback). +func NewGatewayStore(ctx context.Context, baseStorageDir string) (GatewayStore, error) { + sqlite, err := NewSQLiteGatewayStore(filepath.Join(baseStorageDir, "gateway.db")) + if err != nil { + return nil, err + } + + pgHost := os.Getenv("PGHOST") + if pgHost == "" { + return sqlite, nil + } + + retryInterval, err := time.ParseDuration(os.Getenv("PG_RETRY_INTERVAL")) + if err != nil || retryInterval <= 0 { + retryInterval = defaultGatewayPGRetryInterval + } + connectTimeout, err := time.ParseDuration(os.Getenv("PG_CONNECT_TIMEOUT")) + if err != nil || connectTimeout <= 0 { + connectTimeout = defaultGatewayPGConnectTimeout + } + + pg, pgErr := NewPostgresGatewayStore(ctx) + if pgErr != nil { + log.Printf("gateway store: postgres connection failed, will retry lazily on write (host=%s): %v", pgHost, pgErr) + return NewHybridGatewayStore(nil, sqlite, retryInterval, connectTimeout), nil + } + + if err := MigrateGatewaySQLiteToPostgres(ctx, sqlite, pg); err != nil { + _ = pg.Close() + _ = sqlite.Close() + return nil, err + } + + hybrid := NewHybridGatewayStore(nil, sqlite, retryInterval, connectTimeout) + if err := hybrid.ReconcileSyncJournal(ctx, pg); err != nil { + _ = pg.Close() + log.Printf("gateway store: sync journal drain failed at startup, staying on sqlite fallback: %v", err) + return hybrid, nil + } + + log.Printf("gateway store: using postgres with sqlite fallback (host=%s)", pgHost) + return hybrid, nil +} diff --git a/devshard/cmd/devshardctl/gateway_store_factory_test.go b/devshard/cmd/devshardctl/gateway_store_factory_test.go new file mode 100644 index 0000000000..6c68988ee5 --- /dev/null +++ b/devshard/cmd/devshardctl/gateway_store_factory_test.go @@ -0,0 +1,110 @@ +package main + +import ( + "context" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestNewGatewayStorePGHOSTUnset(t *testing.T) { + t.Setenv("PGHOST", "") + + ctx := context.Background() + store, err := NewGatewayStore(ctx, t.TempDir()) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, store.Close()) }) + + _, ok := store.(*SQLiteGatewayStore) + require.True(t, ok, "expected *SQLiteGatewayStore when PGHOST is unset") +} + +func TestNewGatewayStorePGHOSTSetPGUp(t *testing.T) { + cleanup := setupPostgresContainer(t) + t.Cleanup(cleanup) + + ctx := context.Background() + store, err := NewGatewayStore(ctx, t.TempDir()) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, store.Close()) }) + + hybrid, ok := store.(*HybridGatewayStore) + require.True(t, ok) + require.NotNil(t, hybrid.currentPg()) + + settings := GatewaySettings{ + ChainREST: "http://node:1317", + PublicAPI: "http://api:9000", + DefaultModel: "Qwen/Test", + DefaultRequestMaxTokens: 2048, + MaxConcurrentRequests: 2, + MaxInputTokensInFlight: 200, + }.WithTuningDefaults() + require.NoError(t, store.Initialize(settings, nil)) + + pgState, has, err := hybrid.currentPg().LoadState() + require.NoError(t, err) + require.True(t, has) + require.EqualValues(t, 2048, pgState.Settings.DefaultRequestMaxTokens) +} + +func TestNewGatewayStorePGHOSTSetPGDown(t *testing.T) { + t.Setenv("PGHOST", "127.0.0.1") + t.Setenv("PGPORT", "1") + t.Setenv("PG_CONNECT_TIMEOUT", "100ms") + + ctx := context.Background() + store, err := NewGatewayStore(ctx, t.TempDir()) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, store.Close()) }) + + hybrid, ok := store.(*HybridGatewayStore) + require.True(t, ok) + require.Nil(t, hybrid.currentPg()) + + settings := GatewaySettings{ + ChainREST: "http://node:1317", + PublicAPI: "http://api:9000", + DefaultModel: "Qwen/Test", + DefaultRequestMaxTokens: 1111, + MaxConcurrentRequests: 1, + MaxInputTokensInFlight: 100, + }.WithTuningDefaults() + require.NoError(t, store.Initialize(settings, nil)) + + sqliteState, has, err := hybrid.sqlite.LoadState() + require.NoError(t, err) + require.True(t, has) + require.EqualValues(t, 1111, sqliteState.Settings.DefaultRequestMaxTokens) +} + +func TestNewGatewayStoreAutoMigration(t *testing.T) { + cleanup := setupPostgresContainer(t) + t.Cleanup(cleanup) + + baseDir := t.TempDir() + sqlite, err := NewSQLiteGatewayStore(filepath.Join(baseDir, "gateway.db")) + require.NoError(t, err) + seedGatewayStoreForMigration(t, sqlite) + require.NoError(t, sqlite.Close()) + + ctx := context.Background() + store, err := NewGatewayStore(ctx, baseDir) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, store.Close()) }) + + hybrid, ok := store.(*HybridGatewayStore) + require.True(t, ok) + require.NotNil(t, hybrid.currentPg()) + + _, err = os.Stat(filepath.Join(baseDir, "gateway.db")) + require.NoError(t, err) + + pgState, has, err := hybrid.currentPg().LoadState() + require.NoError(t, err) + require.True(t, has) + require.EqualValues(t, 1500, pgState.Settings.DefaultRequestMaxTokens) + require.Len(t, pgState.Devshards, 2) +} diff --git a/devshard/cmd/devshardctl/gateway_store_hybrid.go b/devshard/cmd/devshardctl/gateway_store_hybrid.go new file mode 100644 index 0000000000..bdda66ab7a --- /dev/null +++ b/devshard/cmd/devshardctl/gateway_store_hybrid.go @@ -0,0 +1,646 @@ +package main + +import ( + "context" + "database/sql" + "fmt" + "log" + "os" + "strings" + "sync" + "time" +) + +const ( + gatewayPGConnectTimeout = 2 * time.Second + // defaultGatewayPGRetryInterval is how long we stay on the SQLite fallback + // before probing Postgres again after giving up. Kept short (relative to the + // payload store) because gateway management state is low write volume, so a + // snappier recovery + journal drain outweighs the occasional reconnect probe. + defaultGatewayPGRetryInterval = 30 * time.Second + // defaultGatewayPGWriteRetryBudget bounds the in-request retry loop that lets + // a brief Postgres blip (reset/failover) recover without dropping to SQLite. + defaultGatewayPGWriteRetryBudget = 2 * time.Second +) + +// gatewayPGWriteRetryBaseBackoff is the first inter-attempt sleep; it doubles +// each retry, bounded by the remaining budget. A var so tests can shrink it. +var gatewayPGWriteRetryBaseBackoff = 500 * time.Millisecond + +// HybridGatewayStore uses Postgres as primary storage with SQLite fallback. +// Writes try Postgres first (with lazy reconnect), then SQLite on failure. +// Reads try Postgres first without reconnect delay; on error or empty state +// they also check SQLite. +type HybridGatewayStore struct { + pg *PostgresGatewayStore + sqlite *SQLiteGatewayStore + mu sync.Mutex + lastRetry time.Time + retryInterval time.Duration + connectTimeout time.Duration + writeRetryBudget time.Duration + syncJournalEnabled bool + connectPG func(context.Context) (*PostgresGatewayStore, error) +} + +func NewHybridGatewayStore(pg *PostgresGatewayStore, sqlite *SQLiteGatewayStore, retryInterval, connectTimeout time.Duration) *HybridGatewayStore { + if retryInterval <= 0 { + retryInterval = defaultGatewayPGRetryInterval + } + if connectTimeout <= 0 { + connectTimeout = gatewayPGConnectTimeout + } + return &HybridGatewayStore{ + pg: pg, + sqlite: sqlite, + retryInterval: retryInterval, + connectTimeout: connectTimeout, + writeRetryBudget: gatewayPGWriteRetryBudget(), + syncJournalEnabled: gatewaySyncJournalEnabled(), + connectPG: NewPostgresGatewayStore, + } +} + +// gatewayPGWriteRetryBudget reads PG_WRITE_RETRY_BUDGET (a Go duration). A value +// of 0 disables in-request retry (single attempt then fallback); an invalid or +// unset value uses the default. +func gatewayPGWriteRetryBudget() time.Duration { + v := strings.TrimSpace(os.Getenv("PG_WRITE_RETRY_BUDGET")) + if v == "" { + return defaultGatewayPGWriteRetryBudget + } + d, err := time.ParseDuration(v) + if err != nil || d < 0 { + return defaultGatewayPGWriteRetryBudget + } + return d +} + +func (h *HybridGatewayStore) shouldAttemptConnect() (bool, *PostgresGatewayStore) { + h.mu.Lock() + defer h.mu.Unlock() + + if h.pg != nil { + return false, h.pg + } + if time.Since(h.lastRetry) < h.retryInterval { + return false, nil + } + h.lastRetry = time.Now() + return true, nil +} + +func (h *HybridGatewayStore) getOrConnectPg(ctx context.Context) *PostgresGatewayStore { + shouldAttempt, pg := h.shouldAttemptConnect() + if !shouldAttempt { + return pg + } + + connectCtx, cancel := context.WithTimeout(ctx, h.connectTimeout) + defer cancel() + + newPg, err := h.connectPG(connectCtx) + if err != nil { + return nil + } + + if err := h.reconcileAndPublishPg(ctx, newPg); err != nil { + log.Printf("gateway hybrid: sync journal drain failed, staying on sqlite fallback: %v", err) + _ = newPg.Close() + return nil + } + return h.currentPg() +} + +// ReconcileSyncJournal drains pending SQLite outage writes into Postgres and +// publishes the connection as primary. Used by the factory at startup. +func (h *HybridGatewayStore) ReconcileSyncJournal(ctx context.Context, pg *PostgresGatewayStore) error { + h.mu.Lock() + defer h.mu.Unlock() + return h.reconcileAndPublishPgLocked(ctx, pg) +} + +func (h *HybridGatewayStore) reconcileAndPublishPg(ctx context.Context, pg *PostgresGatewayStore) error { + h.mu.Lock() + defer h.mu.Unlock() + return h.reconcileAndPublishPgLocked(ctx, pg) +} + +func (h *HybridGatewayStore) reconcileAndPublishPgLocked(ctx context.Context, pg *PostgresGatewayStore) error { + if pg == nil { + return fmt.Errorf("postgres store unavailable") + } + if !h.syncJournalEnabled || h.sqlite == nil { + h.pg = pg + log.Printf("gateway store: postgres connection established") + return nil + } + for { + empty, err := drainGatewaySyncJournal(ctx, h.sqlite, pg) + if err != nil { + return err + } + if empty { + h.pg = pg + log.Printf("gateway store: postgres connection established") + return nil + } + } +} + +func (h *HybridGatewayStore) write( + ctx context.Context, + pgFn func(*PostgresGatewayStore) error, + sqliteFn func(*SQLiteGatewayStore) error, + journal []gatewaySyncEntry, + sqliteTxFn func(*sql.Tx) error, +) error { + if pg := h.getOrConnectPg(ctx); pg != nil { + if err := h.attemptPGWrite(ctx, pg, pgFn); err == nil { + return nil + } else { + log.Printf("gateway hybrid: postgres write failed after retries, falling back to sqlite: %v", err) + // Drop the cached connection so subsequent writes during the outage + // get journaled, and the next reconnect replays the journal before + // republishing Postgres as primary. Without this, pgxpool would + // transparently reconnect and serve PG writes again while stale journal + // entries linger, which a later restart-time drain would replay on top + // of newer PG rows. + h.dropPg(pg) + } + } + if h.sqlite == nil { + return errGatewayStoreUnavailable + } + if h.syncJournalEnabled && len(journal) > 0 && sqliteTxFn != nil { + return h.sqlite.writeWithSyncJournal(journal, sqliteTxFn) + } + return sqliteFn(h.sqlite) +} + +// attemptPGWrite runs pgFn, retrying transient failures within writeRetryBudget +// so a brief Postgres blip (connection reset, failover) recovers on the next +// attempt — pgxpool hands out a fresh connection — instead of dropping to +// SQLite for a whole retry interval. Only the first write after Postgres truly +// goes down pays the full budget; once dropPg clears the cached connection the +// remaining outage writes skip Postgres entirely. +func (h *HybridGatewayStore) attemptPGWrite(ctx context.Context, pg *PostgresGatewayStore, pgFn func(*PostgresGatewayStore) error) error { + deadline := time.Now().Add(h.writeRetryBudget) + backoff := gatewayPGWriteRetryBaseBackoff + for { + err := pgFn(pg) + if err == nil { + return nil + } + remaining := time.Until(deadline) + if remaining <= 0 { + return err + } + wait := backoff + if wait > remaining { + wait = remaining + } + log.Printf("gateway hybrid: postgres write failed, retrying in %s: %v", wait, err) + select { + case <-time.After(wait): + case <-ctx.Done(): + return err + } + backoff *= 2 + } +} + +func (h *HybridGatewayStore) currentPg() *PostgresGatewayStore { + h.mu.Lock() + defer h.mu.Unlock() + return h.pg +} + +// dropPg clears the cached primary connection if it still points at failed, so +// subsequent operations route to the SQLite fallback (and journal) until a +// reconnect re-drains the journal. It is a no-op if another goroutine already +// swapped the connection, which keeps the pool close idempotent. +func (h *HybridGatewayStore) dropPg(failed *PostgresGatewayStore) { + h.mu.Lock() + shouldClose := h.pg != nil && h.pg == failed + if shouldClose { + h.pg = nil + } + h.mu.Unlock() + if shouldClose { + _ = failed.Close() + } +} + +func (h *HybridGatewayStore) loadState(ctx context.Context) (GatewayState, bool, error) { + if pg := h.currentPg(); pg != nil { + state, has, err := pg.LoadState() + if err == nil && has { + return state, true, nil + } + if err != nil { + log.Printf("gateway hybrid: postgres load state failed, checking sqlite: %v", err) + } + if h.sqlite != nil { + sqliteState, sqliteHas, sqliteErr := h.sqlite.LoadState() + if sqliteErr == nil && sqliteHas { + return sqliteState, true, nil + } + if err != nil { + return GatewayState{}, false, err + } + return sqliteState, sqliteHas, sqliteErr + } + if err != nil { + return GatewayState{}, false, err + } + return state, has, nil + } + + if h.sqlite != nil { + state, has, err := h.sqlite.LoadState() + if err == nil && has { + return state, true, nil + } + if pg := h.getOrConnectPg(ctx); pg != nil { + return pg.LoadState() + } + return state, has, err + } + + if pg := h.getOrConnectPg(ctx); pg != nil { + return pg.LoadState() + } + return GatewayState{}, false, errGatewayStoreUnavailable +} + +func (h *HybridGatewayStore) Close() error { + var pgErr error + if h.pg != nil { + pgErr = h.pg.Close() + } + var sqliteErr error + if h.sqlite != nil { + sqliteErr = h.sqlite.Close() + } + if pgErr != nil { + return pgErr + } + return sqliteErr +} + +func (h *HybridGatewayStore) LoadState() (GatewayState, bool, error) { + return h.loadState(context.Background()) +} + +func (h *HybridGatewayStore) Initialize(settings GatewaySettings, devshards []GatewayDevshardState) error { + journal := []gatewaySyncEntry{{tableName: gatewayTableSettings, rowKey: gatewaySettingsRowKey, op: gatewaySyncOpUpsert}} + for _, d := range devshards { + journal = append(journal, gatewaySyncEntry{tableName: gatewayTableDevshards, rowKey: strings.TrimSpace(d.ID), op: gatewaySyncOpUpsert}) + } + return h.write(context.Background(), + func(pg *PostgresGatewayStore) error { return pg.Initialize(settings, devshards) }, + func(sqlite *SQLiteGatewayStore) error { return sqlite.Initialize(settings, devshards) }, + journal, + func(tx *sql.Tx) error { return h.sqlite.initializeFallbackTx(tx, settings, devshards) }, + ) +} + +func (h *HybridGatewayStore) UpdateSettings(settings GatewaySettings) error { + return h.write(context.Background(), + func(pg *PostgresGatewayStore) error { return pg.UpdateSettings(settings) }, + func(sqlite *SQLiteGatewayStore) error { return sqlite.UpdateSettings(settings) }, + []gatewaySyncEntry{{tableName: gatewayTableSettings, rowKey: gatewaySettingsRowKey, op: gatewaySyncOpUpsert}}, + func(tx *sql.Tx) error { return h.sqlite.updateSettingsTx(tx, settings) }, + ) +} + +func (h *HybridGatewayStore) SaveRotationStatus(status GatewayRotationStatus) error { + key := gatewayRotationStatusRowKey(status.ModelID, status.Stage, status.Epoch) + return h.write(context.Background(), + func(pg *PostgresGatewayStore) error { return pg.SaveRotationStatus(status) }, + func(sqlite *SQLiteGatewayStore) error { return sqlite.SaveRotationStatus(status) }, + []gatewaySyncEntry{{tableName: gatewayTableRotationStatus, rowKey: key, op: gatewaySyncOpUpsert}}, + func(tx *sql.Tx) error { return h.sqlite.saveRotationStatusTx(tx, status) }, + ) +} + +func (h *HybridGatewayStore) LoadRotationStatuses(limit int) ([]GatewayRotationStatus, error) { + if pg := h.currentPg(); pg != nil { + statuses, err := pg.LoadRotationStatuses(limit) + if err == nil && len(statuses) > 0 { + return statuses, nil + } + if err != nil { + log.Printf("gateway hybrid: postgres load rotation statuses failed, checking sqlite: %v", err) + } + if h.sqlite != nil { + sqliteStatuses, sqliteErr := h.sqlite.LoadRotationStatuses(limit) + if sqliteErr == nil && len(sqliteStatuses) > 0 { + return sqliteStatuses, nil + } + if err != nil { + return nil, err + } + return sqliteStatuses, sqliteErr + } + if err != nil { + return nil, err + } + return statuses, nil + } + + if h.sqlite != nil { + statuses, err := h.sqlite.LoadRotationStatuses(limit) + if err == nil && len(statuses) > 0 { + return statuses, nil + } + if pg := h.getOrConnectPg(context.Background()); pg != nil { + return pg.LoadRotationStatuses(limit) + } + return statuses, err + } + + if pg := h.getOrConnectPg(context.Background()); pg != nil { + return pg.LoadRotationStatuses(limit) + } + return nil, errGatewayStoreUnavailable +} + +func (h *HybridGatewayStore) SaveCommitment(c GatewayEscrowCommitment) error { + txHash := strings.TrimSpace(c.TxHash) + return h.write(context.Background(), + func(pg *PostgresGatewayStore) error { return pg.SaveCommitment(c) }, + func(sqlite *SQLiteGatewayStore) error { return sqlite.SaveCommitment(c) }, + []gatewaySyncEntry{{tableName: gatewayTableCommitments, rowKey: txHash, op: gatewaySyncOpUpsert}}, + func(tx *sql.Tx) error { return h.sqlite.saveCommitmentTx(tx, c) }, + ) +} + +func (h *HybridGatewayStore) LoadCommitments() ([]GatewayEscrowCommitment, error) { + if pg := h.currentPg(); pg != nil { + commitments, err := pg.LoadCommitments() + if err == nil && len(commitments) > 0 { + return commitments, nil + } + if err != nil { + log.Printf("gateway hybrid: postgres load commitments failed, checking sqlite: %v", err) + } + if h.sqlite != nil { + sqliteCommitments, sqliteErr := h.sqlite.LoadCommitments() + if sqliteErr == nil && len(sqliteCommitments) > 0 { + return sqliteCommitments, nil + } + if err != nil { + return nil, err + } + return sqliteCommitments, sqliteErr + } + if err != nil { + return nil, err + } + return commitments, nil + } + + if h.sqlite != nil { + commitments, err := h.sqlite.LoadCommitments() + if err == nil && len(commitments) > 0 { + return commitments, nil + } + if pg := h.getOrConnectPg(context.Background()); pg != nil { + return pg.LoadCommitments() + } + return commitments, err + } + + if pg := h.getOrConnectPg(context.Background()); pg != nil { + return pg.LoadCommitments() + } + return nil, errGatewayStoreUnavailable +} + +func (h *HybridGatewayStore) DeleteCommitment(txHash string) error { + txHash = strings.TrimSpace(txHash) + return h.write(context.Background(), + func(pg *PostgresGatewayStore) error { return pg.DeleteCommitment(txHash) }, + func(sqlite *SQLiteGatewayStore) error { return sqlite.DeleteCommitment(txHash) }, + []gatewaySyncEntry{{tableName: gatewayTableCommitments, rowKey: txHash, op: gatewaySyncOpDelete}}, + func(tx *sql.Tx) error { return h.sqlite.deleteCommitmentTx(tx, txHash) }, + ) +} + +func (h *HybridGatewayStore) LoadSuspiciousHosts() ([]GatewaySuspiciousHost, error) { + if pg := h.currentPg(); pg != nil { + hosts, err := pg.LoadSuspiciousHosts() + if err == nil && len(hosts) > 0 { + return hosts, nil + } + if err != nil { + log.Printf("gateway hybrid: postgres load suspicious hosts failed, checking sqlite: %v", err) + } + if h.sqlite != nil { + sqliteHosts, sqliteErr := h.sqlite.LoadSuspiciousHosts() + if sqliteErr == nil && len(sqliteHosts) > 0 { + return sqliteHosts, nil + } + if err != nil { + return nil, err + } + return sqliteHosts, sqliteErr + } + if err != nil { + return nil, err + } + return hosts, nil + } + + if h.sqlite != nil { + hosts, err := h.sqlite.LoadSuspiciousHosts() + if err == nil && len(hosts) > 0 { + return hosts, nil + } + if pg := h.getOrConnectPg(context.Background()); pg != nil { + return pg.LoadSuspiciousHosts() + } + return hosts, err + } + + if pg := h.getOrConnectPg(context.Background()); pg != nil { + return pg.LoadSuspiciousHosts() + } + return nil, errGatewayStoreUnavailable +} + +func (h *HybridGatewayStore) UpsertSuspiciousHosts(participantKeys []string, note string) ([]GatewaySuspiciousHost, error) { + var result []GatewaySuspiciousHost + keys := normalizeParticipantKeys(participantKeys) + journal := make([]gatewaySyncEntry, 0, len(keys)) + for _, key := range keys { + journal = append(journal, gatewaySyncEntry{tableName: gatewayTableSuspiciousHosts, rowKey: key, op: gatewaySyncOpUpsert}) + } + err := h.write(context.Background(), + func(pg *PostgresGatewayStore) error { + var err error + result, err = pg.UpsertSuspiciousHosts(participantKeys, note) + return err + }, + func(sqlite *SQLiteGatewayStore) error { + var err error + result, err = sqlite.UpsertSuspiciousHosts(participantKeys, note) + return err + }, + journal, + func(tx *sql.Tx) error { + if err := h.sqlite.upsertSuspiciousHostsTx(tx, participantKeys, note); err != nil { + return err + } + var loadErr error + result, loadErr = h.sqlite.loadSuspiciousHostsTx(tx) + return loadErr + }, + ) + return result, err +} + +func (h *HybridGatewayStore) DeleteSuspiciousHosts(participantKeys []string) ([]GatewaySuspiciousHost, error) { + var result []GatewaySuspiciousHost + keys := normalizeParticipantKeys(participantKeys) + journal := make([]gatewaySyncEntry, 0, len(keys)) + for _, key := range keys { + journal = append(journal, gatewaySyncEntry{tableName: gatewayTableSuspiciousHosts, rowKey: key, op: gatewaySyncOpDelete}) + } + err := h.write(context.Background(), + func(pg *PostgresGatewayStore) error { + var err error + result, err = pg.DeleteSuspiciousHosts(participantKeys) + return err + }, + func(sqlite *SQLiteGatewayStore) error { + var err error + result, err = sqlite.DeleteSuspiciousHosts(participantKeys) + return err + }, + journal, + func(tx *sql.Tx) error { + if err := h.sqlite.deleteSuspiciousHostsTx(tx, participantKeys); err != nil { + return err + } + var loadErr error + result, loadErr = h.sqlite.loadSuspiciousHostsTx(tx) + return loadErr + }, + ) + return result, err +} + +func (h *HybridGatewayStore) UpsertDevshard(devshard GatewayDevshardState) error { + id := strings.TrimSpace(devshard.ID) + return h.write(context.Background(), + func(pg *PostgresGatewayStore) error { return pg.UpsertDevshard(devshard) }, + func(sqlite *SQLiteGatewayStore) error { return sqlite.UpsertDevshard(devshard) }, + []gatewaySyncEntry{{tableName: gatewayTableDevshards, rowKey: id, op: gatewaySyncOpUpsert}}, + func(tx *sql.Tx) error { return h.sqlite.upsertDevshardTx(tx, devshard, time.Now().UTC().Format(time.RFC3339Nano)) }, + ) +} + +func (h *HybridGatewayStore) SetDevshardActive(id string, active bool) error { + id = strings.TrimSpace(id) + return h.write(context.Background(), + func(pg *PostgresGatewayStore) error { return pg.SetDevshardActive(id, active) }, + func(sqlite *SQLiteGatewayStore) error { return sqlite.SetDevshardActive(id, active) }, + []gatewaySyncEntry{{tableName: gatewayTableDevshards, rowKey: id, op: gatewaySyncOpUpsert}}, + func(tx *sql.Tx) error { return h.sqlite.setDevshardActiveTx(tx, id, active) }, + ) +} + +func (h *HybridGatewayStore) SetDevshardSettlementPending(id string, pending bool) error { + id = strings.TrimSpace(id) + return h.write(context.Background(), + func(pg *PostgresGatewayStore) error { return pg.SetDevshardSettlementPending(id, pending) }, + func(sqlite *SQLiteGatewayStore) error { return sqlite.SetDevshardSettlementPending(id, pending) }, + []gatewaySyncEntry{{tableName: gatewayTableDevshards, rowKey: id, op: gatewaySyncOpUpsert}}, + func(tx *sql.Tx) error { return h.sqlite.setDevshardSettlementPendingTx(tx, id, pending) }, + ) +} + +func (h *HybridGatewayStore) DeleteDevshard(id string) error { + id = strings.TrimSpace(id) + return h.write(context.Background(), + func(pg *PostgresGatewayStore) error { return pg.DeleteDevshard(id) }, + func(sqlite *SQLiteGatewayStore) error { return sqlite.DeleteDevshard(id) }, + []gatewaySyncEntry{{tableName: gatewayTableDevshards, rowKey: id, op: gatewaySyncOpDelete}}, + func(tx *sql.Tx) error { return h.sqlite.deleteDevshardTx(tx, id) }, + ) +} + +func (h *HybridGatewayStore) SaveParticipantThrottle(key string, modelIDs []string, tokens float64, lastRefillAt time.Time, status int, quarantineUntil time.Time, failureStrikes int) error { + return h.write(context.Background(), + func(pg *PostgresGatewayStore) error { + return pg.SaveParticipantThrottle(key, modelIDs, tokens, lastRefillAt, status, quarantineUntil, failureStrikes) + }, + func(sqlite *SQLiteGatewayStore) error { + return sqlite.SaveParticipantThrottle(key, modelIDs, tokens, lastRefillAt, status, quarantineUntil, failureStrikes) + }, + []gatewaySyncEntry{{tableName: gatewayTableThrottle, rowKey: key, op: gatewaySyncOpUpsert}}, + func(tx *sql.Tx) error { + return h.sqlite.saveParticipantThrottleTx(tx, key, modelIDs, tokens, lastRefillAt, status, quarantineUntil, failureStrikes) + }, + ) +} + +func (h *HybridGatewayStore) DeleteParticipantThrottle(key string) error { + return h.write(context.Background(), + func(pg *PostgresGatewayStore) error { return pg.DeleteParticipantThrottle(key) }, + func(sqlite *SQLiteGatewayStore) error { return sqlite.DeleteParticipantThrottle(key) }, + []gatewaySyncEntry{{tableName: gatewayTableThrottle, rowKey: key, op: gatewaySyncOpDelete}}, + func(tx *sql.Tx) error { return h.sqlite.deleteParticipantThrottleTx(tx, key) }, + ) +} + +func (h *HybridGatewayStore) LoadParticipantThrottles() ([]ParticipantThrottleRow, error) { + if pg := h.currentPg(); pg != nil { + rows, err := pg.LoadParticipantThrottles() + if err == nil && len(rows) > 0 { + return rows, nil + } + if err != nil { + log.Printf("gateway hybrid: postgres load participant throttles failed, checking sqlite: %v", err) + } + if h.sqlite != nil { + sqliteRows, sqliteErr := h.sqlite.LoadParticipantThrottles() + if sqliteErr == nil && len(sqliteRows) > 0 { + return sqliteRows, nil + } + if err != nil { + return nil, err + } + return sqliteRows, sqliteErr + } + if err != nil { + return nil, err + } + return rows, nil + } + + if h.sqlite != nil { + rows, err := h.sqlite.LoadParticipantThrottles() + if err == nil && len(rows) > 0 { + return rows, nil + } + if pg := h.getOrConnectPg(context.Background()); pg != nil { + return pg.LoadParticipantThrottles() + } + return rows, err + } + + if pg := h.getOrConnectPg(context.Background()); pg != nil { + return pg.LoadParticipantThrottles() + } + return nil, errGatewayStoreUnavailable +} + +var ( + errGatewayStoreUnavailable = fmt.Errorf("gateway store unavailable") +) + +var _ GatewayStore = (*HybridGatewayStore)(nil) diff --git a/devshard/cmd/devshardctl/gateway_store_hybrid_test.go b/devshard/cmd/devshardctl/gateway_store_hybrid_test.go new file mode 100644 index 0000000000..87c15da6ee --- /dev/null +++ b/devshard/cmd/devshardctl/gateway_store_hybrid_test.go @@ -0,0 +1,164 @@ +package main + +import ( + "context" + "path/filepath" + "sync/atomic" + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +func newTestSQLiteGatewayStoreOnly(t *testing.T) *SQLiteGatewayStore { + t.Helper() + store, err := NewSQLiteGatewayStore(filepath.Join(t.TempDir(), "gateway.db")) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, store.Close()) }) + return store +} + +func TestHybridGatewayStoreImplementsInterface(t *testing.T) { + var _ GatewayStore = (*HybridGatewayStore)(nil) +} + +func TestHybridGatewayStorePGPrimaryWriteRead(t *testing.T) { + cleanup := setupPostgresContainer(t) + t.Cleanup(cleanup) + + ctx := context.Background() + pg, err := NewPostgresGatewayStore(ctx) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, pg.Close()) }) + + sqlite := newTestSQLiteGatewayStoreOnly(t) + hybrid := NewHybridGatewayStore(pg, sqlite, time.Second, gatewayPGConnectTimeout) + t.Cleanup(func() { require.NoError(t, hybrid.Close()) }) + + settings := GatewaySettings{ + ChainREST: "http://node:1317", + PublicAPI: "http://api:9000", + DefaultModel: "Qwen/Test", + DefaultRequestMaxTokens: 2048, + MaxConcurrentRequests: 4, + MaxInputTokensInFlight: 400, + }.WithTuningDefaults() + require.NoError(t, hybrid.Initialize(settings, nil)) + + updated := settings + updated.DefaultRequestMaxTokens = 4096 + require.NoError(t, hybrid.UpdateSettings(updated)) + + pgState, pgHas, err := pg.LoadState() + require.NoError(t, err) + require.True(t, pgHas) + require.EqualValues(t, 4096, pgState.Settings.DefaultRequestMaxTokens) + + hybridState, hybridHas, err := hybrid.LoadState() + require.NoError(t, err) + require.True(t, hybridHas) + require.Equal(t, pgState.Settings, hybridState.Settings) +} + +func TestHybridGatewayStoreWriteFallbackOnPGDown(t *testing.T) { + sqlite := newTestSQLiteGatewayStoreOnly(t) + hybrid := NewHybridGatewayStore(nil, sqlite, time.Hour, gatewayPGConnectTimeout) + hybrid.connectPG = func(context.Context) (*PostgresGatewayStore, error) { + return nil, errGatewayStoreUnavailable + } + t.Cleanup(func() { require.NoError(t, hybrid.Close()) }) + + settings := GatewaySettings{ + ChainREST: "http://node:1317", + PublicAPI: "http://api:9000", + DefaultModel: "Qwen/Test", + DefaultRequestMaxTokens: 1000, + MaxConcurrentRequests: 2, + MaxInputTokensInFlight: 200, + }.WithTuningDefaults() + require.NoError(t, hybrid.Initialize(settings, nil)) + + sqliteState, sqliteHas, err := sqlite.LoadState() + require.NoError(t, err) + require.True(t, sqliteHas) + require.Equal(t, settings.DefaultRequestMaxTokens, sqliteState.Settings.DefaultRequestMaxTokens) +} + +func TestHybridGatewayStoreReadFallbackFromSQLite(t *testing.T) { + sqlite := newTestSQLiteGatewayStoreOnly(t) + settings := GatewaySettings{ + ChainREST: "http://node:1317", + PublicAPI: "http://api:9000", + DefaultModel: "Qwen/Test", + DefaultRequestMaxTokens: 777, + MaxConcurrentRequests: 3, + MaxInputTokensInFlight: 300, + }.WithTuningDefaults() + require.NoError(t, sqlite.Initialize(settings, nil)) + + hybrid := NewHybridGatewayStore(nil, sqlite, time.Hour, gatewayPGConnectTimeout) + hybrid.connectPG = func(context.Context) (*PostgresGatewayStore, error) { + return nil, errGatewayStoreUnavailable + } + t.Cleanup(func() { require.NoError(t, hybrid.Close()) }) + + state, has, err := hybrid.LoadState() + require.NoError(t, err) + require.True(t, has) + require.EqualValues(t, 777, state.Settings.DefaultRequestMaxTokens) +} + +func TestHybridGatewayStoreLazyReconnect(t *testing.T) { + cleanup := setupPostgresContainer(t) + t.Cleanup(cleanup) + + sqlite := newTestSQLiteGatewayStoreOnly(t) + hybrid := NewHybridGatewayStore(nil, sqlite, time.Millisecond, gatewayPGConnectTimeout) + t.Cleanup(func() { require.NoError(t, hybrid.Close()) }) + + settings := GatewaySettings{ + ChainREST: "http://node:1317", + PublicAPI: "http://api:9000", + DefaultModel: "Qwen/Test", + DefaultRequestMaxTokens: 3333, + MaxConcurrentRequests: 1, + MaxInputTokensInFlight: 100, + }.WithTuningDefaults() + require.NoError(t, hybrid.Initialize(settings, nil)) + + require.NotNil(t, hybrid.currentPg()) + pgState, pgHas, err := hybrid.currentPg().LoadState() + require.NoError(t, err) + require.True(t, pgHas) + require.EqualValues(t, 3333, pgState.Settings.DefaultRequestMaxTokens) +} + +func TestHybridGatewayStoreReconnectRateLimit(t *testing.T) { + sqlite := newTestSQLiteGatewayStoreOnly(t) + hybrid := NewHybridGatewayStore(nil, sqlite, time.Hour, gatewayPGConnectTimeout) + var attempts atomic.Int32 + hybrid.connectPG = func(context.Context) (*PostgresGatewayStore, error) { + attempts.Add(1) + return nil, errGatewayStoreUnavailable + } + t.Cleanup(func() { require.NoError(t, hybrid.Close()) }) + + settings := GatewaySettings{ + ChainREST: "http://node:1317", + PublicAPI: "http://api:9000", + DefaultModel: "Qwen/Test", + DefaultRequestMaxTokens: 1000, + MaxConcurrentRequests: 1, + MaxInputTokensInFlight: 100, + }.WithTuningDefaults() + + require.NoError(t, sqlite.Initialize(settings, nil)) + + require.NoError(t, hybrid.SaveRotationStatus(GatewayRotationStatus{ + ModelID: "Qwen/Test", Stage: "prepare_temp", Epoch: 1, Completed: true, + })) + require.NoError(t, hybrid.SaveRotationStatus(GatewayRotationStatus{ + ModelID: "Kimi/Rotate", Stage: "prepare_temp", Epoch: 2, Completed: true, + })) + require.EqualValues(t, 1, attempts.Load()) +} diff --git a/devshard/cmd/devshardctl/gateway_store_migrate.go b/devshard/cmd/devshardctl/gateway_store_migrate.go new file mode 100644 index 0000000000..869f06bcb7 --- /dev/null +++ b/devshard/cmd/devshardctl/gateway_store_migrate.go @@ -0,0 +1,131 @@ +package main + +import ( + "context" + "fmt" + "time" + + "github.com/jackc/pgx/v5" +) + +const gatewaySQLiteImportMarker = "sqlite_import" + +// MigrateGatewaySQLiteToPostgres copies existing SQLite gateway data into Postgres +// before hybrid serving. It is idempotent: a marker row prevents re-import. +func MigrateGatewaySQLiteToPostgres(ctx context.Context, src GatewayStore, dst *PostgresGatewayStore) error { + if src == nil || dst == nil || dst.pool == nil { + return nil + } + + if err := dst.ensureMigrationTable(ctx); err != nil { + return err + } + migrated, err := dst.hasMigrationMarker(ctx, gatewaySQLiteImportMarker) + if err != nil { + return err + } + if migrated { + return nil + } + + _, hasDst, err := dst.LoadState() + if err != nil { + return fmt.Errorf("check destination gateway settings: %w", err) + } + if hasDst { + return nil + } + + srcState, hasSrc, err := src.LoadState() + if err != nil { + return fmt.Errorf("load source gateway state: %w", err) + } + if !hasSrc { + return nil + } + + rotationStatuses, err := src.LoadRotationStatuses(0) + if err != nil { + return fmt.Errorf("load source rotation statuses: %w", err) + } + commitments, err := src.LoadCommitments() + if err != nil { + return fmt.Errorf("load source commitments: %w", err) + } + throttles, err := src.LoadParticipantThrottles() + if err != nil { + return fmt.Errorf("load source participant throttles: %w", err) + } + + tx, err := dst.pool.Begin(ctx) + if err != nil { + return fmt.Errorf("begin gateway migration: %w", err) + } + defer tx.Rollback(ctx) + + now := time.Now().UTC().Format(time.RFC3339Nano) + if err := applyGatewaySettingsUpsertToPG(ctx, tx, srcState.Settings, now); err != nil { + return fmt.Errorf("migrate gateway settings: %w", err) + } + + for _, devshard := range srcState.Devshards { + if err := applyGatewayDevshardToPG(ctx, tx, devshard, now); err != nil { + return err + } + } + for _, host := range srcState.SuspiciousHosts { + if err := applyGatewaySuspiciousHostToPG(ctx, tx, host); err != nil { + return fmt.Errorf("migrate suspicious host %s: %w", host.ParticipantKey, err) + } + } + for _, status := range rotationStatuses { + if err := applyGatewayRotationStatusToPG(ctx, tx, status); err != nil { + return fmt.Errorf("migrate rotation status model=%q stage=%q epoch=%d: %w", status.ModelID, status.Stage, status.Epoch, err) + } + } + for _, c := range commitments { + if err := applyGatewayCommitmentToPG(ctx, tx, c); err != nil { + return fmt.Errorf("migrate escrow commitment tx=%s: %w", c.TxHash, err) + } + } + for _, row := range throttles { + if err := applyGatewayThrottleToPG(ctx, tx, row); err != nil { + return err + } + } + + if _, err := tx.Exec(ctx, ` + INSERT INTO gateway_migration (name, completed_at) + VALUES ($1, $2) + ON CONFLICT (name) DO UPDATE SET completed_at = EXCLUDED.completed_at`, + gatewaySQLiteImportMarker, now, + ); err != nil { + return fmt.Errorf("write gateway migration marker: %w", err) + } + + if err := tx.Commit(ctx); err != nil { + return fmt.Errorf("commit gateway migration: %w", err) + } + return nil +} + +func (s *PostgresGatewayStore) ensureMigrationTable(ctx context.Context) error { + _, err := s.pool.Exec(ctx, ` + CREATE TABLE IF NOT EXISTS gateway_migration ( + name TEXT PRIMARY KEY, + completed_at TEXT NOT NULL + )`) + return err +} + +func (s *PostgresGatewayStore) hasMigrationMarker(ctx context.Context, name string) (bool, error) { + var completedAt string + err := s.pool.QueryRow(ctx, `SELECT completed_at FROM gateway_migration WHERE name = $1`, name).Scan(&completedAt) + if err == pgx.ErrNoRows { + return false, nil + } + if err != nil { + return false, err + } + return true, nil +} diff --git a/devshard/cmd/devshardctl/gateway_store_migrate_test.go b/devshard/cmd/devshardctl/gateway_store_migrate_test.go new file mode 100644 index 0000000000..40b93c8daa --- /dev/null +++ b/devshard/cmd/devshardctl/gateway_store_migrate_test.go @@ -0,0 +1,335 @@ +package main + +import ( + "context" + "path/filepath" + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +func seedGatewayStoreForMigration(t *testing.T, store GatewayStore) GatewayState { + t.Helper() + + settings := GatewaySettings{ + ChainREST: "http://node:1317", + PublicAPI: "http://api:9000", + DefaultModel: "Qwen/Test", + DefaultRequestMaxTokens: 1500, + MaxConcurrentRequests: 4, + MaxInputTokensInFlight: 400, + Disabled: GatewayDisabledSettings{ + Enabled: true, + Message: "migrating", + NewURL: "https://example.com/v1", + }, + EscrowRotation: EscrowRotationSettings{ + Enabled: true, + SettlementEnabled: true, + PrePoCBlocks: 99, + Models: []EscrowRotationModelSettings{{ + ModelID: "Kimi/Rotate", + TempCount: 1, + TargetCount: 4, + Amount: 500, + PrivateKeyEnv: "ROTATION_KEY", + }}, + }, + }.WithTuningDefaults() + + devshards := []GatewayDevshardState{ + { + RuntimeConfig: RuntimeConfig{ + ID: "ds-1", + PrivateKeyHex: "aaa", + Model: "Qwen/Test", + StoragePath: "/data/ds-1", + }, + Active: true, + RotationRole: rotationRoleRegular, + RotationEpoch: 5, + SettlementPending: false, + CreatedAt: "2026-01-01T00:00:00Z", + UpdatedAt: "2026-01-02T00:00:00Z", + }, + { + RuntimeConfig: RuntimeConfig{ + ID: "ds-2", + PrivateKeyHex: "bbb", + Model: "Kimi/Rotate", + StoragePath: "/data/ds-2", + }, + Active: false, + RotationRole: rotationRoleTemp, + RotationEpoch: 6, + SettlementPending: true, + CreatedAt: "2026-01-03T00:00:00Z", + UpdatedAt: "2026-01-04T00:00:00Z", + }, + } + + require.NoError(t, store.Initialize(settings, devshards)) + _, err := store.UpsertSuspiciousHosts([]string{"host-a", "host-b"}, "suspicious") + require.NoError(t, err) + require.NoError(t, store.SaveRotationStatus(GatewayRotationStatus{ + ModelID: "Qwen/Test", + Stage: "prepare_temp", + Epoch: 10, + Role: rotationRoleTemp, + TargetCount: 4, + ExistingCount: 1, + CreatedCount: 2, + Completed: true, + UpdatedAt: "2026-05-04T00:00:00Z", + })) + require.NoError(t, store.SaveCommitment(GatewayEscrowCommitment{ + TxHash: "TX-MIGRATE-1", + Model: "Qwen/Test", + Role: rotationRoleTemp, + Epoch: 10, + PrivateKeyEnv: "KEY_ENV", + BlockHeight: 12345, + CreatedAt: "2026-05-05T00:00:00Z", + })) + require.NoError(t, store.SaveCommitment(GatewayEscrowCommitment{ + TxHash: "TX-MIGRATE-2", + Model: "Kimi/Rotate", + Role: rotationRoleRegular, + Epoch: 11, + PrivateKeyEnv: "KEY_ENV_2", + BlockHeight: 12346, + CreatedAt: "2026-05-06T00:00:00Z", + })) + quarantine := time.Date(2026, 5, 7, 12, 0, 0, 0, time.UTC) + require.NoError(t, store.SaveParticipantThrottle( + "participant-1", + []string{"Qwen/Test", "Kimi/Rotate"}, + 12.5, + time.Date(2026, 5, 7, 10, 0, 0, 0, time.UTC), + 503, + quarantine, + 2, + )) + require.NoError(t, store.SaveParticipantThrottle( + "participant-2", + nil, + 0, + time.Date(2026, 5, 7, 11, 0, 0, 0, time.UTC), + 0, + time.Time{}, + 0, + )) + + state, has, err := store.LoadState() + require.NoError(t, err) + require.True(t, has) + return state +} + +func assertGatewayStoresEqual(t *testing.T, want GatewayStore, got GatewayStore) { + t.Helper() + + wantState, wantHas, err := want.LoadState() + require.NoError(t, err) + require.True(t, wantHas) + + gotState, gotHas, err := got.LoadState() + require.NoError(t, err) + require.True(t, gotHas) + require.Equal(t, wantState.Settings, gotState.Settings) + require.ElementsMatch(t, wantState.Devshards, gotState.Devshards) + require.ElementsMatch(t, wantState.SuspiciousHosts, gotState.SuspiciousHosts) + + wantStatuses, err := want.LoadRotationStatuses(0) + require.NoError(t, err) + gotStatuses, err := got.LoadRotationStatuses(0) + require.NoError(t, err) + require.ElementsMatch(t, wantStatuses, gotStatuses) + + wantCommitments, err := want.LoadCommitments() + require.NoError(t, err) + gotCommitments, err := got.LoadCommitments() + require.NoError(t, err) + require.ElementsMatch(t, wantCommitments, gotCommitments) + + wantThrottles, err := want.LoadParticipantThrottles() + require.NoError(t, err) + gotThrottles, err := got.LoadParticipantThrottles() + require.NoError(t, err) + require.ElementsMatch(t, wantThrottles, gotThrottles) +} + +func TestMigrateGatewaySQLiteToPostgres_FullFidelity(t *testing.T) { + ctx := context.Background() + + sqlite, err := NewSQLiteGatewayStore(filepath.Join(t.TempDir(), "gateway.db")) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, sqlite.Close()) }) + seedGatewayStoreForMigration(t, sqlite) + + pg := newTestPostgresGatewayStore(t) + + require.NoError(t, MigrateGatewaySQLiteToPostgres(ctx, sqlite, pg)) + assertGatewayStoresEqual(t, sqlite, pg) + + commitments, err := pg.LoadCommitments() + require.NoError(t, err) + require.Len(t, commitments, 2) + require.ElementsMatch(t, []string{"TX-MIGRATE-1", "TX-MIGRATE-2"}, []string{commitments[0].TxHash, commitments[1].TxHash}) +} + +func TestMigrateGatewaySQLiteToPostgres_Idempotent(t *testing.T) { + ctx := context.Background() + + sqlite, err := NewSQLiteGatewayStore(filepath.Join(t.TempDir(), "gateway.db")) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, sqlite.Close()) }) + seedGatewayStoreForMigration(t, sqlite) + + pg := newTestPostgresGatewayStore(t) + + require.NoError(t, MigrateGatewaySQLiteToPostgres(ctx, sqlite, pg)) + firstState, _, err := pg.LoadState() + require.NoError(t, err) + + require.NoError(t, MigrateGatewaySQLiteToPostgres(ctx, sqlite, pg)) + secondState, _, err := pg.LoadState() + require.NoError(t, err) + require.Equal(t, firstState, secondState) + + migrated, err := pg.hasMigrationMarker(ctx, gatewaySQLiteImportMarker) + require.NoError(t, err) + require.True(t, migrated) +} + +func TestMigrateGatewaySQLiteToPostgres_EmptinessGuard(t *testing.T) { + ctx := context.Background() + + sqlite, err := NewSQLiteGatewayStore(filepath.Join(t.TempDir(), "gateway.db")) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, sqlite.Close()) }) + seedGatewayStoreForMigration(t, sqlite) + + pg := newTestPostgresGatewayStore(t) + existing := GatewaySettings{ + ChainREST: "http://existing:1317", + PublicAPI: "http://existing:9000", + DefaultModel: "Existing/Model", + DefaultRequestMaxTokens: 999, + MaxConcurrentRequests: 1, + MaxInputTokensInFlight: 50, + }.WithTuningDefaults() + require.NoError(t, pg.Initialize(existing, nil)) + + require.NoError(t, MigrateGatewaySQLiteToPostgres(ctx, sqlite, pg)) + + state, has, err := pg.LoadState() + require.NoError(t, err) + require.True(t, has) + require.Equal(t, existing, state.Settings) +} + +func TestMigrateGatewaySQLiteToPostgres_EmptySource(t *testing.T) { + ctx := context.Background() + + sqlite, err := NewSQLiteGatewayStore(filepath.Join(t.TempDir(), "gateway.db")) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, sqlite.Close()) }) + + pg := newTestPostgresGatewayStore(t) + + require.NoError(t, MigrateGatewaySQLiteToPostgres(ctx, sqlite, pg)) + + _, has, err := pg.LoadState() + require.NoError(t, err) + require.False(t, has) + + migrated, err := pg.hasMigrationMarker(ctx, gatewaySQLiteImportMarker) + require.NoError(t, err) + require.False(t, migrated) +} + +func TestMigrateGatewaySQLiteToPostgres_RollbackLeavesNoPartialRows(t *testing.T) { + ctx := context.Background() + + sqlite, err := NewSQLiteGatewayStore(filepath.Join(t.TempDir(), "gateway.db")) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, sqlite.Close()) }) + seedGatewayStoreForMigration(t, sqlite) + + pg := newTestPostgresGatewayStore(t) + + srcState, _, err := sqlite.LoadState() + require.NoError(t, err) + + tx, err := pg.pool.Begin(ctx) + require.NoError(t, err) + now := time.Now().UTC().Format(time.RFC3339Nano) + insertArgs := settingsInsertArgs(srcState.Settings, now) + _, err = tx.Exec(ctx, ` + INSERT INTO gateway_settings (`+gatewaySettingsInsertColumnNames()+`) + VALUES (`+pgPlaceholderList(len(insertArgs))+`)`, insertArgs...) + require.NoError(t, err) + require.NoError(t, tx.Rollback(ctx)) + + _, has, err := pg.LoadState() + require.NoError(t, err) + require.False(t, has) + + require.NoError(t, MigrateGatewaySQLiteToPostgres(ctx, sqlite, pg)) + assertGatewayStoresEqual(t, sqlite, pg) +} + +func TestMigrateGatewaySQLiteToPostgres_CommitmentsPreserved(t *testing.T) { + ctx := context.Background() + + sqlite, err := NewSQLiteGatewayStore(filepath.Join(t.TempDir(), "gateway.db")) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, sqlite.Close()) }) + + settings := GatewaySettings{ + ChainREST: "http://node:1317", + PublicAPI: "http://api:9000", + DefaultModel: "Qwen/Test", + DefaultRequestMaxTokens: 1000, + MaxConcurrentRequests: 1, + MaxInputTokensInFlight: 100, + }.WithTuningDefaults() + require.NoError(t, sqlite.Initialize(settings, nil)) + + seed := []GatewayEscrowCommitment{ + { + TxHash: "TX-AAA", + Model: "Qwen/Test", + Role: rotationRoleTemp, + Epoch: 7, + PrivateKeyEnv: "ENV_A", + BlockHeight: 100, + CreatedAt: "2026-06-01T00:00:00Z", + }, + { + TxHash: "TX-BBB", + Model: "Kimi/Rotate", + Role: rotationRoleRegular, + Epoch: 8, + PrivateKeyEnv: "ENV_B", + BlockHeight: 200, + CreatedAt: "2026-06-02T00:00:00Z", + }, + } + for _, c := range seed { + require.NoError(t, sqlite.SaveCommitment(c)) + } + + pg := newTestPostgresGatewayStore(t) + require.NoError(t, MigrateGatewaySQLiteToPostgres(ctx, sqlite, pg)) + + want, err := sqlite.LoadCommitments() + require.NoError(t, err) + got, err := pg.LoadCommitments() + require.NoError(t, err) + require.ElementsMatch(t, want, got) + require.Len(t, got, 2) + require.ElementsMatch(t, []string{"TX-AAA", "TX-BBB"}, []string{got[0].TxHash, got[1].TxHash}) +} diff --git a/devshard/cmd/devshardctl/gateway_store_postgres.go b/devshard/cmd/devshardctl/gateway_store_postgres.go new file mode 100644 index 0000000000..fb5e24f627 --- /dev/null +++ b/devshard/cmd/devshardctl/gateway_store_postgres.go @@ -0,0 +1,897 @@ +package main + +import ( + "context" + "fmt" + "os" + "strings" + "time" + + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgxpool" +) + +const defaultGatewayPGOpTimeout = 2 * time.Second + +type PostgresGatewayStore struct { + pool *pgxpool.Pool + opTimeout time.Duration +} + +// gatewayPGOpTimeout reads PG_OPERATION_TIMEOUT (a Go duration). An invalid or +// unset value uses the default; 0 disables per-operation deadlines. +func gatewayPGOpTimeout() time.Duration { + v := strings.TrimSpace(os.Getenv("PG_OPERATION_TIMEOUT")) + if v == "" { + return defaultGatewayPGOpTimeout + } + d, err := time.ParseDuration(v) + if err != nil || d < 0 { + return defaultGatewayPGOpTimeout + } + return d +} + +func (s *PostgresGatewayStore) opCtx() (context.Context, context.CancelFunc) { + if s == nil || s.opTimeout <= 0 { + return context.Background(), func() {} + } + return context.WithTimeout(context.Background(), s.opTimeout) +} + +func NewPostgresGatewayStore(ctx context.Context) (*PostgresGatewayStore, error) { + pool, err := pgxpool.New(ctx, "") + if err != nil { + return nil, fmt.Errorf("connect gateway postgres store: %w", err) + } + if err := pool.Ping(ctx); err != nil { + pool.Close() + return nil, fmt.Errorf("ping gateway postgres store: %w", err) + } + s := &PostgresGatewayStore{pool: pool, opTimeout: gatewayPGOpTimeout()} + if err := s.ensureSchema(ctx); err != nil { + pool.Close() + return nil, err + } + return s, nil +} + +func (s *PostgresGatewayStore) ensureSchema(ctx context.Context) error { + stmts := []string{ + `CREATE TABLE IF NOT EXISTS gateway_settings ( + id SMALLINT PRIMARY KEY CHECK (id = 1), + chain_rest TEXT NOT NULL, + public_api TEXT NOT NULL DEFAULT '', + default_model TEXT NOT NULL, + default_request_max_tokens BIGINT NOT NULL, + request_max_tokens_cap BIGINT NOT NULL DEFAULT 4096, + max_concurrent_requests BIGINT NOT NULL DEFAULT 512, + max_concurrent_requests_per_10000_weight DOUBLE PRECISION NOT NULL DEFAULT 5.0, + poc_max_concurrent_requests_per_10000_weight DOUBLE PRECISION NOT NULL DEFAULT 10.0, + max_input_tokens_in_flight BIGINT NOT NULL, + model_limits_json TEXT NOT NULL DEFAULT '', + model_access_json TEXT NOT NULL DEFAULT '', + tx_gas_limit BIGINT NOT NULL DEFAULT 0, + participant_request_burst BIGINT NOT NULL DEFAULT 600, + participant_recovery_per_minute BIGINT NOT NULL DEFAULT 10, + participant_http_quarantine_ms BIGINT NOT NULL DEFAULT 3600000, + participant_transport_failure_quarantine_ms BIGINT NOT NULL DEFAULT 1800000, + participant_empty_stream_quarantine_ms BIGINT NOT NULL DEFAULT 1800000, + participant_stalled_winner_quarantine_ms BIGINT NOT NULL DEFAULT 1800000, + participant_empty_stream_threshold BIGINT NOT NULL DEFAULT 3, + participant_eof_transport_failure_threshold BIGINT NOT NULL DEFAULT 3, + redundancy_receipt_timeout_ms BIGINT NOT NULL DEFAULT 5000, + redundancy_first_token_timeout_floor_ms BIGINT NOT NULL DEFAULT 1000, + redundancy_per_input_token_first_token_lag_ms BIGINT NOT NULL DEFAULT 10, + redundancy_inter_chunk_stall_timeout_ms BIGINT NOT NULL DEFAULT 60000, + redundancy_streaming_attempt_hard_timeout_ms BIGINT NOT NULL DEFAULT 1200000, + redundancy_non_stream_response_floor_ms BIGINT NOT NULL DEFAULT 20000, + redundancy_non_stream_no_content_timeout_ms BIGINT NOT NULL DEFAULT 1200000, + redundancy_non_stream_max_attempt_wait_ms BIGINT NOT NULL DEFAULT 1800000, + redundancy_per_input_token_response_lag_ms BIGINT NOT NULL DEFAULT 20, + redundancy_secondary_wait_after_winner_ms BIGINT NOT NULL DEFAULT 600000, + redundancy_parallel_advantage_threshold DOUBLE PRECISION NOT NULL DEFAULT 0.5, + redundancy_unresponsive_threshold DOUBLE PRECISION NOT NULL DEFAULT 1.0, + redundancy_speed_policy TEXT NOT NULL DEFAULT 'hybrid', + redundancy_pairwise_budget_percentile DOUBLE PRECISION NOT NULL DEFAULT 0.9, + redundancy_pairwise_max_proactive_attempts BIGINT NOT NULL DEFAULT 3, + redundancy_pairwise_min_direct_comparisons BIGINT NOT NULL DEFAULT 4, + redundancy_pairwise_winner_hold_ms BIGINT NOT NULL DEFAULT 500, + redundancy_pairwise_winner_hold_min_speedup DOUBLE PRECISION NOT NULL DEFAULT 0.1, + redundancy_pairwise_winner_hold_min_samples BIGINT NOT NULL DEFAULT 6, + perf_sample_size BIGINT NOT NULL DEFAULT 256, + perf_window_ms BIGINT NOT NULL DEFAULT 3600000, + escrow_rotation_enabled SMALLINT NOT NULL DEFAULT 0, + escrow_rotation_settlement_enabled SMALLINT NOT NULL DEFAULT 0, + escrow_rotation_pre_poc_blocks BIGINT NOT NULL DEFAULT 300, + escrow_rotation_models_json TEXT NOT NULL DEFAULT '', + gateway_disabled_enabled SMALLINT NOT NULL DEFAULT 0, + gateway_disabled_message TEXT NOT NULL DEFAULT '', + gateway_disabled_new_url TEXT NOT NULL DEFAULT '', + updated_at TEXT NOT NULL + )`, + `CREATE TABLE IF NOT EXISTS gateway_devshards ( + id TEXT PRIMARY KEY, + private_key_hex TEXT NOT NULL, + private_key_env TEXT NOT NULL DEFAULT '', + model TEXT NOT NULL DEFAULT '', + storage_path TEXT NOT NULL DEFAULT '', + active SMALLINT NOT NULL DEFAULT 1, + rotation_role TEXT NOT NULL DEFAULT '', + rotation_epoch BIGINT NOT NULL DEFAULT 0, + settlement_pending SMALLINT NOT NULL DEFAULT 0, + protocol_version TEXT NOT NULL DEFAULT '', + created_at TEXT NOT NULL, + updated_at TEXT NOT NULL + )`, + `CREATE TABLE IF NOT EXISTS participant_throttle_state ( + participant_key TEXT PRIMARY KEY, + tokens DOUBLE PRECISION NOT NULL DEFAULT 0, + last_refill_at TEXT NOT NULL, + last_throttle_status BIGINT NOT NULL DEFAULT 0, + quarantine_until_utc TEXT NOT NULL DEFAULT '', + failure_strikes BIGINT NOT NULL DEFAULT 0, + model_ids TEXT NOT NULL DEFAULT '', + empty_stream_streak BIGINT NOT NULL DEFAULT 0, + eof_transport_failure_streak BIGINT NOT NULL DEFAULT 0 + )`, + `CREATE TABLE IF NOT EXISTS gateway_rotation_status ( + model_id TEXT NOT NULL, + stage TEXT NOT NULL, + epoch BIGINT NOT NULL, + role TEXT NOT NULL DEFAULT '', + target_count BIGINT NOT NULL DEFAULT 0, + existing_count BIGINT NOT NULL DEFAULT 0, + created_count BIGINT NOT NULL DEFAULT 0, + promoted_count BIGINT NOT NULL DEFAULT 0, + settled_count BIGINT NOT NULL DEFAULT 0, + settle_failed_count BIGINT NOT NULL DEFAULT 0, + create_error TEXT NOT NULL DEFAULT '', + completed SMALLINT NOT NULL DEFAULT 0, + updated_at TEXT NOT NULL, + PRIMARY KEY (model_id, stage, epoch) + )`, + `CREATE TABLE IF NOT EXISTS gateway_suspicious_hosts ( + participant_key TEXT PRIMARY KEY, + note TEXT NOT NULL DEFAULT '', + created_at TEXT NOT NULL + )`, + `CREATE TABLE IF NOT EXISTS escrow_rotation_commitments ( + tx_hash TEXT PRIMARY KEY, + model TEXT NOT NULL, + role TEXT NOT NULL DEFAULT '', + epoch BIGINT NOT NULL DEFAULT 0, + private_key_env TEXT NOT NULL DEFAULT '', + block_height BIGINT NOT NULL DEFAULT 0, + created_at TEXT NOT NULL + )`, + } + for _, stmt := range stmts { + if _, err := s.pool.Exec(ctx, stmt); err != nil { + return fmt.Errorf("init gateway postgres store: %w", err) + } + } + + if err := ensurePGGatewaySettingsColumn(ctx, s.pool, "public_api", "TEXT NOT NULL DEFAULT ''"); err != nil { + return fmt.Errorf("migrate gateway store: %w", err) + } + if err := ensurePGGatewaySettingsColumn(ctx, s.pool, "tx_gas_limit", "BIGINT NOT NULL DEFAULT 0"); err != nil { + return fmt.Errorf("migrate gateway tx settings: %w", err) + } + if err := ensurePGGatewaySettingsColumn(ctx, s.pool, "model_limits_json", "TEXT NOT NULL DEFAULT ''"); err != nil { + return fmt.Errorf("migrate gateway model limits: %w", err) + } + if err := ensurePGGatewaySettingsColumn(ctx, s.pool, "model_access_json", "TEXT NOT NULL DEFAULT ''"); err != nil { + return fmt.Errorf("migrate gateway model access: %w", err) + } + if err := ensurePGGatewaySettingsColumn(ctx, s.pool, "request_max_tokens_cap", "BIGINT NOT NULL DEFAULT 4096"); err != nil { + return fmt.Errorf("migrate gateway max token cap: %w", err) + } + if err := ensurePGGatewaySettingsColumn(ctx, s.pool, "max_concurrent_requests_per_10000_weight", "DOUBLE PRECISION NOT NULL DEFAULT 5.0"); err != nil { + return fmt.Errorf("migrate gateway weight concurrency: %w", err) + } + if err := ensurePGGatewaySettingsColumn(ctx, s.pool, "poc_max_concurrent_requests_per_10000_weight", "DOUBLE PRECISION NOT NULL DEFAULT 10.0"); err != nil { + return fmt.Errorf("migrate gateway poc weight concurrency: %w", err) + } + if err := ensurePGGatewaySettingsTuningColumns(ctx, s.pool); err != nil { + return fmt.Errorf("migrate gateway tuning settings: %w", err) + } + if err := ensurePGGatewaySettingsRotationColumns(ctx, s.pool); err != nil { + return fmt.Errorf("migrate gateway rotation settings: %w", err) + } + if err := ensurePGGatewaySettingsDisabledColumns(ctx, s.pool); err != nil { + return fmt.Errorf("migrate gateway disabled settings: %w", err) + } + if err := ensurePGGatewayDevshardsColumn(ctx, s.pool, "protocol_version", "TEXT NOT NULL DEFAULT ''"); err != nil { + return fmt.Errorf("migrate gateway devshards: %w", err) + } + if err := ensurePGGatewayDevshardsColumn(ctx, s.pool, "rotation_role", "TEXT NOT NULL DEFAULT ''"); err != nil { + return fmt.Errorf("migrate gateway devshard role: %w", err) + } + if err := ensurePGGatewayDevshardsColumn(ctx, s.pool, "rotation_epoch", "BIGINT NOT NULL DEFAULT 0"); err != nil { + return fmt.Errorf("migrate gateway devshard epoch: %w", err) + } + if err := ensurePGGatewayDevshardsColumn(ctx, s.pool, "settlement_pending", "SMALLINT NOT NULL DEFAULT 0"); err != nil { + return fmt.Errorf("migrate gateway devshard settlement_pending: %w", err) + } + + if _, err := s.pool.Exec(ctx, ` + UPDATE participant_throttle_state + SET failure_strikes = GREATEST(COALESCE(empty_stream_streak, 0), COALESCE(eof_transport_failure_streak, 0)) + WHERE COALESCE(failure_strikes, 0) = 0 + AND (COALESCE(empty_stream_streak, 0) > 0 OR COALESCE(eof_transport_failure_streak, 0) > 0)`); err != nil { + return fmt.Errorf("migrate participant throttle strike values: %w", err) + } + return nil +} + +func (s *PostgresGatewayStore) Close() error { + if s == nil || s.pool == nil { + return nil + } + s.pool.Close() + return nil +} + +func (s *PostgresGatewayStore) LoadState() (GatewayState, bool, error) { + ctx, cancel := s.opCtx() + defer cancel() + var state GatewayState + row := s.pool.QueryRow(ctx, ` + SELECT `+gatewaySettingsSelectColumns()+` + FROM gateway_settings + WHERE id = 1`) + settings, err := scanGatewaySettings(row) + if err == pgx.ErrNoRows { + return GatewayState{}, false, nil + } + if err != nil { + return GatewayState{}, false, fmt.Errorf("load gateway settings: %w", err) + } + state.Settings = settings + + rows, err := s.pool.Query(ctx, ` + SELECT id, private_key_hex, private_key_env, model, storage_path, active, created_at, updated_at, protocol_version, + rotation_role, rotation_epoch, settlement_pending + FROM gateway_devshards + ORDER BY id`) + if err != nil { + return GatewayState{}, false, fmt.Errorf("load gateway devshards: %w", err) + } + defer rows.Close() + for rows.Next() { + var devshard GatewayDevshardState + var active int + var settlementPending int + if err := rows.Scan( + &devshard.ID, + &devshard.PrivateKeyHex, + &devshard.PrivateKeyEnv, + &devshard.Model, + &devshard.StoragePath, + &active, + &devshard.CreatedAt, + &devshard.UpdatedAt, + &devshard.ProtocolVersion, + &devshard.RotationRole, + &devshard.RotationEpoch, + &settlementPending, + ); err != nil { + return GatewayState{}, false, fmt.Errorf("scan gateway devshard: %w", err) + } + devshard.Active = active != 0 + devshard.SettlementPending = settlementPending != 0 + state.Devshards = append(state.Devshards, devshard) + } + if err := rows.Err(); err != nil { + return GatewayState{}, false, fmt.Errorf("iterate gateway devshards: %w", err) + } + suspiciousHosts, err := s.LoadSuspiciousHosts() + if err != nil { + return GatewayState{}, false, err + } + state.SuspiciousHosts = suspiciousHosts + return state, true, nil +} + +func (s *PostgresGatewayStore) Initialize(settings GatewaySettings, devshards []GatewayDevshardState) error { + ctx, cancel := s.opCtx() + defer cancel() + settings = settings.WithTuningDefaults() + now := time.Now().UTC().Format(time.RFC3339Nano) + tx, err := s.pool.Begin(ctx) + if err != nil { + return fmt.Errorf("begin gateway init: %w", err) + } + defer tx.Rollback(ctx) + + var count int + if err := tx.QueryRow(ctx, `SELECT COUNT(*) FROM gateway_settings WHERE id = 1`).Scan(&count); err != nil { + return fmt.Errorf("count gateway settings: %w", err) + } + if count > 0 { + return nil + } + + insertArgs := settingsInsertArgs(settings, now) + if _, err := tx.Exec(ctx, fmt.Sprintf(` + INSERT INTO gateway_settings (%s) + VALUES (%s)`, + gatewaySettingsInsertColumnNames(), + pgPlaceholderList(len(insertArgs)), + ), insertArgs...); err != nil { + return fmt.Errorf("insert gateway settings: %w", err) + } + + for _, devshard := range devshards { + if err := s.upsertDevshardTx(ctx, tx, devshard, now); err != nil { + return err + } + } + return tx.Commit(ctx) +} + +func (s *PostgresGatewayStore) UpdateSettings(settings GatewaySettings) error { + ctx, cancel := s.opCtx() + defer cancel() + updatedAt := time.Now().UTC().Format(time.RFC3339Nano) + colCount := len(gatewaySettingsColumnNames()) + args := settingsUpdateArgs(settings, updatedAt) + args = append(args, 1) + cmdTag, err := s.pool.Exec(ctx, fmt.Sprintf(` + UPDATE gateway_settings + SET %s, + updated_at = $%d + WHERE id = $%d`, gatewaySettingsUpdateAssignmentsPG(), colCount+1, colCount+2), args...) + if err != nil { + return fmt.Errorf("update gateway settings: %w", err) + } + if cmdTag.RowsAffected() == 0 { + return fmt.Errorf("gateway settings not initialized") + } + return nil +} + +func (s *PostgresGatewayStore) SaveRotationStatus(status GatewayRotationStatus) error { + if s == nil || s.pool == nil { + return nil + } + ctx, cancel := s.opCtx() + defer cancel() + now := time.Now().UTC().Format(time.RFC3339Nano) + if strings.TrimSpace(status.UpdatedAt) != "" { + now = strings.TrimSpace(status.UpdatedAt) + } + _, err := s.pool.Exec(ctx, ` + INSERT INTO gateway_rotation_status ( + model_id, stage, epoch, role, target_count, existing_count, created_count, + promoted_count, settled_count, settle_failed_count, create_error, completed, updated_at + ) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13) + ON CONFLICT (model_id, stage, epoch) DO UPDATE SET + role = EXCLUDED.role, + target_count = EXCLUDED.target_count, + existing_count = EXCLUDED.existing_count, + created_count = EXCLUDED.created_count, + promoted_count = EXCLUDED.promoted_count, + settled_count = EXCLUDED.settled_count, + settle_failed_count = EXCLUDED.settle_failed_count, + create_error = EXCLUDED.create_error, + completed = EXCLUDED.completed, + updated_at = EXCLUDED.updated_at`, + strings.TrimSpace(status.ModelID), + strings.TrimSpace(status.Stage), + status.Epoch, + strings.TrimSpace(status.Role), + status.TargetCount, + status.ExistingCount, + status.CreatedCount, + status.PromotedCount, + status.SettledCount, + status.SettleFailedCount, + strings.TrimSpace(status.CreateError), + gatewayBoolToInt(status.Completed), + now, + ) + if err != nil { + return fmt.Errorf("save gateway rotation status model=%q stage=%q epoch=%d: %w", status.ModelID, status.Stage, status.Epoch, err) + } + return nil +} + +func (s *PostgresGatewayStore) LoadRotationStatuses(limit int) ([]GatewayRotationStatus, error) { + if s == nil || s.pool == nil { + return nil, nil + } + ctx, cancel := s.opCtx() + defer cancel() + query := ` + SELECT model_id, stage, epoch, role, target_count, existing_count, created_count, + promoted_count, settled_count, settle_failed_count, create_error, completed, updated_at + FROM gateway_rotation_status + ORDER BY updated_at DESC` + args := []any{} + if limit > 0 { + query += ` LIMIT $1` + args = append(args, limit) + } + rows, err := s.pool.Query(ctx, query, args...) + if err != nil { + return nil, fmt.Errorf("load gateway rotation statuses: %w", err) + } + defer rows.Close() + var statuses []GatewayRotationStatus + for rows.Next() { + var status GatewayRotationStatus + var completed int + if err := rows.Scan( + &status.ModelID, + &status.Stage, + &status.Epoch, + &status.Role, + &status.TargetCount, + &status.ExistingCount, + &status.CreatedCount, + &status.PromotedCount, + &status.SettledCount, + &status.SettleFailedCount, + &status.CreateError, + &completed, + &status.UpdatedAt, + ); err != nil { + return nil, fmt.Errorf("scan gateway rotation status: %w", err) + } + status.Completed = completed != 0 + statuses = append(statuses, status) + } + if err := rows.Err(); err != nil { + return nil, err + } + return statuses, nil +} + +func (s *PostgresGatewayStore) SaveCommitment(c GatewayEscrowCommitment) error { + if s == nil || s.pool == nil { + return fmt.Errorf("gateway store unavailable") + } + ctx, cancel := s.opCtx() + defer cancel() + createdAt := time.Now().UTC().Format(time.RFC3339Nano) + if strings.TrimSpace(c.CreatedAt) != "" { + createdAt = strings.TrimSpace(c.CreatedAt) + } + _, err := s.pool.Exec(ctx, ` + INSERT INTO escrow_rotation_commitments ( + tx_hash, model, role, epoch, private_key_env, block_height, created_at + ) VALUES ($1, $2, $3, $4, $5, $6, $7) + ON CONFLICT (tx_hash) DO UPDATE SET + model = EXCLUDED.model, + role = EXCLUDED.role, + epoch = EXCLUDED.epoch, + private_key_env = EXCLUDED.private_key_env, + block_height = EXCLUDED.block_height, + created_at = EXCLUDED.created_at`, + strings.TrimSpace(c.TxHash), + strings.TrimSpace(c.Model), + strings.TrimSpace(c.Role), + c.Epoch, + strings.TrimSpace(c.PrivateKeyEnv), + c.BlockHeight, + createdAt, + ) + if err != nil { + return fmt.Errorf("save escrow commitment tx=%s: %w", c.TxHash, err) + } + return nil +} + +func (s *PostgresGatewayStore) LoadCommitments() ([]GatewayEscrowCommitment, error) { + if s == nil || s.pool == nil { + return nil, nil + } + ctx, cancel := s.opCtx() + defer cancel() + rows, err := s.pool.Query(ctx, ` + SELECT tx_hash, model, role, epoch, private_key_env, block_height, created_at + FROM escrow_rotation_commitments + ORDER BY created_at ASC`) + if err != nil { + return nil, fmt.Errorf("load escrow commitments: %w", err) + } + defer rows.Close() + var commitments []GatewayEscrowCommitment + for rows.Next() { + var c GatewayEscrowCommitment + if err := rows.Scan(&c.TxHash, &c.Model, &c.Role, &c.Epoch, &c.PrivateKeyEnv, &c.BlockHeight, &c.CreatedAt); err != nil { + return nil, fmt.Errorf("scan escrow commitment: %w", err) + } + commitments = append(commitments, c) + } + return commitments, rows.Err() +} + +func (s *PostgresGatewayStore) DeleteCommitment(txHash string) error { + if s == nil || s.pool == nil { + return fmt.Errorf("gateway store unavailable") + } + ctx, cancel := s.opCtx() + defer cancel() + if _, err := s.pool.Exec(ctx, `DELETE FROM escrow_rotation_commitments WHERE tx_hash = $1`, strings.TrimSpace(txHash)); err != nil { + return fmt.Errorf("delete escrow commitment tx=%s: %w", txHash, err) + } + return nil +} + +func (s *PostgresGatewayStore) LoadSuspiciousHosts() ([]GatewaySuspiciousHost, error) { + if s == nil || s.pool == nil { + return nil, nil + } + ctx, cancel := s.opCtx() + defer cancel() + rows, err := s.pool.Query(ctx, ` + SELECT participant_key, note, created_at + FROM gateway_suspicious_hosts + ORDER BY participant_key`) + if err != nil { + return nil, fmt.Errorf("load gateway suspicious hosts: %w", err) + } + defer rows.Close() + + var hosts []GatewaySuspiciousHost + for rows.Next() { + var host GatewaySuspiciousHost + if err := rows.Scan(&host.ParticipantKey, &host.Note, &host.CreatedAt); err != nil { + return nil, fmt.Errorf("scan gateway suspicious host: %w", err) + } + hosts = append(hosts, host) + } + if err := rows.Err(); err != nil { + return nil, err + } + return hosts, nil +} + +func (s *PostgresGatewayStore) UpsertSuspiciousHosts(participantKeys []string, note string) ([]GatewaySuspiciousHost, error) { + if s == nil || s.pool == nil { + return nil, nil + } + ctx, cancel := s.opCtx() + defer cancel() + participantKeys = normalizeParticipantKeys(participantKeys) + if len(participantKeys) == 0 { + return nil, fmt.Errorf("participant_keys must contain at least one key") + } + tx, err := s.pool.Begin(ctx) + if err != nil { + return nil, fmt.Errorf("begin suspicious host upsert: %w", err) + } + defer tx.Rollback(ctx) + now := time.Now().UTC().Format(time.RFC3339Nano) + note = strings.TrimSpace(note) + for _, key := range participantKeys { + if _, err := tx.Exec(ctx, ` + INSERT INTO gateway_suspicious_hosts (participant_key, note, created_at) + VALUES ($1, $2, $3) + ON CONFLICT (participant_key) DO UPDATE SET note = EXCLUDED.note`, + key, note, now); err != nil { + return nil, fmt.Errorf("upsert suspicious host %s: %w", key, err) + } + } + if err := tx.Commit(ctx); err != nil { + return nil, fmt.Errorf("commit suspicious host upsert: %w", err) + } + return s.LoadSuspiciousHosts() +} + +func (s *PostgresGatewayStore) DeleteSuspiciousHosts(participantKeys []string) ([]GatewaySuspiciousHost, error) { + if s == nil || s.pool == nil { + return nil, nil + } + ctx, cancel := s.opCtx() + defer cancel() + participantKeys = normalizeParticipantKeys(participantKeys) + if len(participantKeys) == 0 { + return nil, fmt.Errorf("participant_keys must contain at least one key") + } + tx, err := s.pool.Begin(ctx) + if err != nil { + return nil, fmt.Errorf("begin suspicious host delete: %w", err) + } + defer tx.Rollback(ctx) + for _, key := range participantKeys { + if _, err := tx.Exec(ctx, `DELETE FROM gateway_suspicious_hosts WHERE participant_key = $1`, key); err != nil { + return nil, fmt.Errorf("delete suspicious host %s: %w", key, err) + } + } + if err := tx.Commit(ctx); err != nil { + return nil, fmt.Errorf("commit suspicious host delete: %w", err) + } + return s.LoadSuspiciousHosts() +} + +func (s *PostgresGatewayStore) UpsertDevshard(devshard GatewayDevshardState) error { + ctx, cancel := s.opCtx() + defer cancel() + now := time.Now().UTC().Format(time.RFC3339Nano) + tx, err := s.pool.Begin(ctx) + if err != nil { + return fmt.Errorf("begin devshard upsert: %w", err) + } + defer tx.Rollback(ctx) + if err := s.upsertDevshardTx(ctx, tx, devshard, now); err != nil { + return err + } + return tx.Commit(ctx) +} + +func (s *PostgresGatewayStore) upsertDevshardTx(ctx context.Context, tx pgx.Tx, devshard GatewayDevshardState, now string) error { + createdAt := now + _ = tx.QueryRow(ctx, `SELECT created_at FROM gateway_devshards WHERE id = $1`, devshard.ID).Scan(&createdAt) + // Preserve the existing settlement_pending marker so an unrelated upsert + // never silently clears a queued settlement; a brand-new row falls back + // to the value carried on devshard. + settlementPending := gatewayBoolToInt(devshard.SettlementPending) + _ = tx.QueryRow(ctx, `SELECT settlement_pending FROM gateway_devshards WHERE id = $1`, devshard.ID).Scan(&settlementPending) + if _, err := tx.Exec(ctx, ` + INSERT INTO gateway_devshards ( + id, private_key_hex, private_key_env, model, storage_path, active, created_at, updated_at, protocol_version, + rotation_role, rotation_epoch, settlement_pending + ) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12) + ON CONFLICT (id) DO UPDATE SET + private_key_hex = EXCLUDED.private_key_hex, + private_key_env = EXCLUDED.private_key_env, + model = EXCLUDED.model, + storage_path = EXCLUDED.storage_path, + active = EXCLUDED.active, + created_at = EXCLUDED.created_at, + updated_at = EXCLUDED.updated_at, + protocol_version = EXCLUDED.protocol_version, + rotation_role = EXCLUDED.rotation_role, + rotation_epoch = EXCLUDED.rotation_epoch, + settlement_pending = EXCLUDED.settlement_pending`, + strings.TrimSpace(devshard.ID), + strings.TrimSpace(devshard.PrivateKeyHex), + strings.TrimSpace(devshard.PrivateKeyEnv), + strings.TrimSpace(devshard.Model), + strings.TrimSpace(devshard.StoragePath), + gatewayBoolToInt(devshard.Active), + createdAt, + now, + strings.TrimSpace(devshard.ProtocolVersion), + strings.TrimSpace(devshard.RotationRole), + devshard.RotationEpoch, + settlementPending, + ); err != nil { + return fmt.Errorf("upsert gateway devshard %s: %w", devshard.ID, err) + } + return nil +} + +func (s *PostgresGatewayStore) SetDevshardActive(id string, active bool) error { + ctx, cancel := s.opCtx() + defer cancel() + cmdTag, err := s.pool.Exec(ctx, ` + UPDATE gateway_devshards + SET active = $1, updated_at = $2 + WHERE id = $3`, + gatewayBoolToInt(active), + time.Now().UTC().Format(time.RFC3339Nano), + strings.TrimSpace(id), + ) + if err != nil { + return fmt.Errorf("update devshard %s active=%t: %w", id, active, err) + } + if cmdTag.RowsAffected() == 0 { + return fmt.Errorf("devshard %s not found", id) + } + return nil +} + +func (s *PostgresGatewayStore) SetDevshardSettlementPending(id string, pending bool) error { + ctx, cancel := s.opCtx() + defer cancel() + cmdTag, err := s.pool.Exec(ctx, ` + UPDATE gateway_devshards + SET settlement_pending = $1, updated_at = $2 + WHERE id = $3`, + gatewayBoolToInt(pending), + time.Now().UTC().Format(time.RFC3339Nano), + strings.TrimSpace(id), + ) + if err != nil { + return fmt.Errorf("update devshard %s settlement_pending=%t: %w", id, pending, err) + } + if cmdTag.RowsAffected() == 0 { + return fmt.Errorf("devshard %s not found", id) + } + return nil +} + +func (s *PostgresGatewayStore) DeleteDevshard(id string) error { + ctx, cancel := s.opCtx() + defer cancel() + cmdTag, err := s.pool.Exec(ctx, `DELETE FROM gateway_devshards WHERE id = $1`, strings.TrimSpace(id)) + if err != nil { + return fmt.Errorf("delete devshard %s: %w", id, err) + } + if cmdTag.RowsAffected() == 0 { + return fmt.Errorf("devshard %s not found", id) + } + return nil +} + +func (s *PostgresGatewayStore) SaveParticipantThrottle(key string, modelIDs []string, tokens float64, lastRefillAt time.Time, status int, quarantineUntil time.Time, failureStrikes int) error { + if s == nil || s.pool == nil { + return nil + } + ctx, cancel := s.opCtx() + defer cancel() + quarStr := "" + if !quarantineUntil.IsZero() { + quarStr = quarantineUntil.UTC().Format(time.RFC3339Nano) + } + _, err := s.pool.Exec(ctx, ` + INSERT INTO participant_throttle_state + (participant_key, tokens, last_refill_at, last_throttle_status, quarantine_until_utc, failure_strikes, model_ids) + VALUES ($1, $2, $3, $4, $5, $6, $7) + ON CONFLICT (participant_key) DO UPDATE SET + tokens = EXCLUDED.tokens, + last_refill_at = EXCLUDED.last_refill_at, + last_throttle_status = EXCLUDED.last_throttle_status, + quarantine_until_utc = EXCLUDED.quarantine_until_utc, + failure_strikes = EXCLUDED.failure_strikes, + model_ids = EXCLUDED.model_ids`, + key, tokens, lastRefillAt.UTC().Format(time.RFC3339Nano), status, quarStr, failureStrikes, strings.Join(normalizeModelIDs(modelIDs), ",")) + if err != nil { + return fmt.Errorf("save participant throttle %s: %w", key, err) + } + return nil +} + +func (s *PostgresGatewayStore) DeleteParticipantThrottle(key string) error { + if s == nil || s.pool == nil { + return nil + } + ctx, cancel := s.opCtx() + defer cancel() + if _, err := s.pool.Exec(ctx, `DELETE FROM participant_throttle_state WHERE participant_key = $1`, key); err != nil { + return fmt.Errorf("delete participant throttle %s: %w", key, err) + } + return nil +} + +func (s *PostgresGatewayStore) LoadParticipantThrottles() ([]ParticipantThrottleRow, error) { + if s == nil || s.pool == nil { + return nil, nil + } + ctx, cancel := s.opCtx() + defer cancel() + rows, err := s.pool.Query(ctx, ` + SELECT participant_key, tokens, last_refill_at, last_throttle_status, + COALESCE(quarantine_until_utc, '') AS quarantine_until_utc, + COALESCE(failure_strikes, 0) AS failure_strikes, + COALESCE(model_ids, '') AS model_ids + FROM participant_throttle_state`) + if err != nil { + return nil, fmt.Errorf("load participant throttles: %w", err) + } + defer rows.Close() + + var result []ParticipantThrottleRow + for rows.Next() { + var row ParticipantThrottleRow + var lastRefillStr, quarantineStr, modelIDsStr string + if err := rows.Scan(&row.Key, &row.Tokens, &lastRefillStr, &row.Status, &quarantineStr, &row.FailureStrikes, &modelIDsStr); err != nil { + return nil, fmt.Errorf("scan participant throttle: %w", err) + } + row.ModelIDs = splitModelIDs(modelIDsStr) + row.LastRefillAt, err = time.Parse(time.RFC3339Nano, lastRefillStr) + if err != nil { + return nil, fmt.Errorf("parse last_refill_at for %s: %w", row.Key, err) + } + if strings.TrimSpace(quarantineStr) != "" { + row.QuarantineUntil, err = time.Parse(time.RFC3339Nano, quarantineStr) + if err != nil { + return nil, fmt.Errorf("parse quarantine_until_utc for %s: %w", row.Key, err) + } + } + result = append(result, row) + } + if err := rows.Err(); err != nil { + return nil, err + } + return result, nil +} + +func ensurePGColumn(ctx context.Context, pool *pgxpool.Pool, table, columnName, columnDDL string) error { + _, err := pool.Exec(ctx, fmt.Sprintf( + `ALTER TABLE %s ADD COLUMN IF NOT EXISTS %s %s`, table, columnName, columnDDL)) + return err +} + +func ensurePGGatewaySettingsColumn(ctx context.Context, pool *pgxpool.Pool, columnName, columnDDL string) error { + return ensurePGColumn(ctx, pool, "gateway_settings", columnName, columnDDL) +} + +func ensurePGGatewaySettingsTuningColumns(ctx context.Context, pool *pgxpool.Pool) error { + columns := []struct { + name string + ddl string + }{ + {"participant_request_burst", "BIGINT NOT NULL DEFAULT 600"}, + {"participant_recovery_per_minute", "BIGINT NOT NULL DEFAULT 10"}, + {"participant_http_quarantine_ms", "BIGINT NOT NULL DEFAULT 3600000"}, + {"participant_transport_failure_quarantine_ms", "BIGINT NOT NULL DEFAULT 1800000"}, + {"participant_empty_stream_quarantine_ms", "BIGINT NOT NULL DEFAULT 1800000"}, + {"participant_stalled_winner_quarantine_ms", "BIGINT NOT NULL DEFAULT 1800000"}, + {"participant_empty_stream_threshold", "BIGINT NOT NULL DEFAULT 3"}, + {"participant_eof_transport_failure_threshold", "BIGINT NOT NULL DEFAULT 3"}, + {"redundancy_receipt_timeout_ms", "BIGINT NOT NULL DEFAULT 5000"}, + {"redundancy_first_token_timeout_floor_ms", "BIGINT NOT NULL DEFAULT 1000"}, + {"redundancy_per_input_token_first_token_lag_ms", "BIGINT NOT NULL DEFAULT 10"}, + {"redundancy_inter_chunk_stall_timeout_ms", "BIGINT NOT NULL DEFAULT 60000"}, + {"redundancy_streaming_attempt_hard_timeout_ms", "BIGINT NOT NULL DEFAULT 1200000"}, + {"redundancy_non_stream_response_floor_ms", "BIGINT NOT NULL DEFAULT 20000"}, + {"redundancy_non_stream_no_content_timeout_ms", "BIGINT NOT NULL DEFAULT 1200000"}, + {"redundancy_non_stream_max_attempt_wait_ms", "BIGINT NOT NULL DEFAULT 1800000"}, + {"redundancy_per_input_token_response_lag_ms", "BIGINT NOT NULL DEFAULT 20"}, + {"redundancy_secondary_wait_after_winner_ms", "BIGINT NOT NULL DEFAULT 600000"}, + {"redundancy_parallel_advantage_threshold", "DOUBLE PRECISION NOT NULL DEFAULT 0.5"}, + {"redundancy_unresponsive_threshold", "DOUBLE PRECISION NOT NULL DEFAULT 1.0"}, + {"redundancy_speed_policy", "TEXT NOT NULL DEFAULT 'hybrid'"}, + {"redundancy_pairwise_budget_percentile", "DOUBLE PRECISION NOT NULL DEFAULT 0.9"}, + {"redundancy_pairwise_max_proactive_attempts", "BIGINT NOT NULL DEFAULT 3"}, + {"redundancy_pairwise_min_direct_comparisons", "BIGINT NOT NULL DEFAULT 4"}, + {"redundancy_pairwise_winner_hold_ms", "BIGINT NOT NULL DEFAULT 500"}, + {"redundancy_pairwise_winner_hold_min_speedup", "DOUBLE PRECISION NOT NULL DEFAULT 0.1"}, + {"redundancy_pairwise_winner_hold_min_samples", "BIGINT NOT NULL DEFAULT 6"}, + {"perf_sample_size", "BIGINT NOT NULL DEFAULT 256"}, + {"perf_window_ms", "BIGINT NOT NULL DEFAULT 3600000"}, + } + for _, column := range columns { + if err := ensurePGGatewaySettingsColumn(ctx, pool, column.name, column.ddl); err != nil { + return err + } + } + return nil +} + +func ensurePGGatewaySettingsRotationColumns(ctx context.Context, pool *pgxpool.Pool) error { + columns := []struct { + name string + ddl string + }{ + {"escrow_rotation_enabled", "SMALLINT NOT NULL DEFAULT 0"}, + {"escrow_rotation_settlement_enabled", "SMALLINT NOT NULL DEFAULT 0"}, + {"escrow_rotation_pre_poc_blocks", "BIGINT NOT NULL DEFAULT 300"}, + {"escrow_rotation_models_json", "TEXT NOT NULL DEFAULT ''"}, + } + for _, column := range columns { + if err := ensurePGGatewaySettingsColumn(ctx, pool, column.name, column.ddl); err != nil { + return err + } + } + return nil +} + +func ensurePGGatewaySettingsDisabledColumns(ctx context.Context, pool *pgxpool.Pool) error { + columns := []struct { + name string + ddl string + }{ + {"gateway_disabled_enabled", "SMALLINT NOT NULL DEFAULT 0"}, + {"gateway_disabled_message", "TEXT NOT NULL DEFAULT ''"}, + {"gateway_disabled_new_url", "TEXT NOT NULL DEFAULT ''"}, + } + for _, column := range columns { + if err := ensurePGGatewaySettingsColumn(ctx, pool, column.name, column.ddl); err != nil { + return err + } + } + return nil +} + +func ensurePGGatewayDevshardsColumn(ctx context.Context, pool *pgxpool.Pool, columnName, columnDDL string) error { + return ensurePGColumn(ctx, pool, "gateway_devshards", columnName, columnDDL) +} + +var _ GatewayStore = (*PostgresGatewayStore)(nil) diff --git a/devshard/cmd/devshardctl/gateway_store_postgres_test.go b/devshard/cmd/devshardctl/gateway_store_postgres_test.go new file mode 100644 index 0000000000..c182f54106 --- /dev/null +++ b/devshard/cmd/devshardctl/gateway_store_postgres_test.go @@ -0,0 +1,55 @@ +package main + +import ( + "context" + "errors" + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +func TestGatewayPGOpTimeoutEnv(t *testing.T) { + t.Setenv("PG_OPERATION_TIMEOUT", "1500ms") + require.Equal(t, 1500*time.Millisecond, gatewayPGOpTimeout()) + + t.Setenv("PG_OPERATION_TIMEOUT", "0") + require.Equal(t, time.Duration(0), gatewayPGOpTimeout()) + + t.Setenv("PG_OPERATION_TIMEOUT", "not-a-duration") + require.Equal(t, defaultGatewayPGOpTimeout, gatewayPGOpTimeout()) + + t.Setenv("PG_OPERATION_TIMEOUT", "") + require.Equal(t, defaultGatewayPGOpTimeout, gatewayPGOpTimeout()) +} + +func TestPostgresGatewayStoreOpCtx(t *testing.T) { + s := &PostgresGatewayStore{opTimeout: 50 * time.Millisecond} + ctx, cancel := s.opCtx() + defer cancel() + deadline, ok := ctx.Deadline() + require.True(t, ok) + require.InDelta(t, time.Now().Add(50*time.Millisecond).UnixNano(), deadline.UnixNano(), float64(25*time.Millisecond)) + + disabled := &PostgresGatewayStore{opTimeout: 0} + ctx2, cancel2 := disabled.opCtx() + defer cancel2() + _, ok = ctx2.Deadline() + require.False(t, ok) +} + +func TestPostgresGatewayStoreOpTimeoutCancelsSlowQuery(t *testing.T) { + cleanup := setupPostgresContainer(t) + t.Cleanup(cleanup) + + pg, err := NewPostgresGatewayStore(context.Background()) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, pg.Close()) }) + pg.opTimeout = 100 * time.Millisecond + + ctx, cancel := pg.opCtx() + defer cancel() + _, err = pg.pool.Exec(ctx, `SELECT pg_sleep(5)`) + require.Error(t, err) + require.True(t, errors.Is(err, context.DeadlineExceeded), "expected deadline exceeded, got: %v", err) +} diff --git a/devshard/cmd/devshardctl/gateway_store_postgres_test_helpers_test.go b/devshard/cmd/devshardctl/gateway_store_postgres_test_helpers_test.go new file mode 100644 index 0000000000..48be926f64 --- /dev/null +++ b/devshard/cmd/devshardctl/gateway_store_postgres_test_helpers_test.go @@ -0,0 +1,57 @@ +package main + +import ( + "context" + "testing" + "time" + + "github.com/stretchr/testify/require" + "github.com/testcontainers/testcontainers-go" + "github.com/testcontainers/testcontainers-go/modules/postgres" + "github.com/testcontainers/testcontainers-go/wait" +) + +func setupPostgresContainer(t *testing.T) func() { + t.Helper() + if testing.Short() { + t.Skip("skipping postgres testcontainers tests in -short mode (requires Docker)") + } + + ctx := context.Background() + container, err := postgres.Run(ctx, + "postgres:18.1-bookworm", + postgres.WithDatabase("testdb"), + postgres.WithUsername("testuser"), + postgres.WithPassword("testpass"), + testcontainers.WithWaitStrategy( + wait.ForLog("database system is ready to accept connections"). + WithOccurrence(2). + WithStartupTimeout(60*time.Second), + ), + ) + require.NoError(t, err) + + host, err := container.Host(ctx) + require.NoError(t, err) + port, err := container.MappedPort(ctx, "5432") + require.NoError(t, err) + + t.Setenv("PGHOST", host) + t.Setenv("PGPORT", port.Port()) + t.Setenv("PGDATABASE", "testdb") + t.Setenv("PGUSER", "testuser") + t.Setenv("PGPASSWORD", "testpass") + + return func() { _ = container.Terminate(ctx) } +} + +func newTestPostgresGatewayStore(t *testing.T) *PostgresGatewayStore { + t.Helper() + cleanup := setupPostgresContainer(t) + t.Cleanup(cleanup) + + store, err := NewPostgresGatewayStore(context.Background()) + require.NoError(t, err) + t.Cleanup(func() { _ = store.Close() }) + return store +} diff --git a/devshard/cmd/devshardctl/gateway_store_sqlite.go b/devshard/cmd/devshardctl/gateway_store_sqlite.go new file mode 100644 index 0000000000..e6d52a7d0d --- /dev/null +++ b/devshard/cmd/devshardctl/gateway_store_sqlite.go @@ -0,0 +1,934 @@ +package main + +import ( + "database/sql" + "fmt" + "strings" + "time" + + _ "modernc.org/sqlite" +) + +type SQLiteGatewayStore struct { + db *sql.DB +} + +func NewSQLiteGatewayStore(path string) (*SQLiteGatewayStore, error) { + db, err := sql.Open("sqlite", path) + if err != nil { + return nil, fmt.Errorf("open gateway store: %w", err) + } + // Serialize access and wait on contention instead of failing with "database is + // locked". Mirrors storage/sqlite.go. + db.SetMaxOpenConns(1) + for _, pragma := range []string{ + "PRAGMA journal_mode=WAL", + "PRAGMA synchronous=NORMAL", + "PRAGMA busy_timeout=5000", + } { + if _, err := db.Exec(pragma); err != nil { + db.Close() + return nil, fmt.Errorf("apply gateway store pragma %q: %w", pragma, err) + } + } + stmts := []string{ + `CREATE TABLE IF NOT EXISTS gateway_settings ( + id INTEGER PRIMARY KEY CHECK (id = 1), + chain_rest TEXT NOT NULL, + public_api TEXT NOT NULL DEFAULT '', + default_model TEXT NOT NULL, + default_request_max_tokens INTEGER NOT NULL, + request_max_tokens_cap INTEGER NOT NULL DEFAULT 4096, + max_concurrent_requests INTEGER NOT NULL DEFAULT 512, + max_concurrent_requests_per_10000_weight REAL NOT NULL DEFAULT 5.0, + poc_max_concurrent_requests_per_10000_weight REAL NOT NULL DEFAULT 10.0, + max_input_tokens_in_flight INTEGER NOT NULL, + model_limits_json TEXT NOT NULL DEFAULT '', + model_access_json TEXT NOT NULL DEFAULT '', + tx_gas_limit INTEGER NOT NULL DEFAULT 0, + participant_request_burst INTEGER NOT NULL DEFAULT 600, + participant_recovery_per_minute INTEGER NOT NULL DEFAULT 10, + participant_http_quarantine_ms INTEGER NOT NULL DEFAULT 3600000, + participant_transport_failure_quarantine_ms INTEGER NOT NULL DEFAULT 1800000, + participant_empty_stream_quarantine_ms INTEGER NOT NULL DEFAULT 1800000, + participant_stalled_winner_quarantine_ms INTEGER NOT NULL DEFAULT 1800000, + participant_empty_stream_threshold INTEGER NOT NULL DEFAULT 3, + participant_eof_transport_failure_threshold INTEGER NOT NULL DEFAULT 3, + redundancy_receipt_timeout_ms INTEGER NOT NULL DEFAULT 5000, + redundancy_first_token_timeout_floor_ms INTEGER NOT NULL DEFAULT 1000, + redundancy_per_input_token_first_token_lag_ms INTEGER NOT NULL DEFAULT 10, + redundancy_inter_chunk_stall_timeout_ms INTEGER NOT NULL DEFAULT 60000, + redundancy_streaming_attempt_hard_timeout_ms INTEGER NOT NULL DEFAULT 1200000, + redundancy_non_stream_response_floor_ms INTEGER NOT NULL DEFAULT 20000, + redundancy_non_stream_no_content_timeout_ms INTEGER NOT NULL DEFAULT 1200000, + redundancy_non_stream_max_attempt_wait_ms INTEGER NOT NULL DEFAULT 1800000, + redundancy_per_input_token_response_lag_ms INTEGER NOT NULL DEFAULT 20, + redundancy_secondary_wait_after_winner_ms INTEGER NOT NULL DEFAULT 600000, + redundancy_parallel_advantage_threshold REAL NOT NULL DEFAULT 0.5, + redundancy_unresponsive_threshold REAL NOT NULL DEFAULT 1.0, + redundancy_speed_policy TEXT NOT NULL DEFAULT 'hybrid', + redundancy_pairwise_budget_percentile REAL NOT NULL DEFAULT 0.9, + redundancy_pairwise_max_proactive_attempts INTEGER NOT NULL DEFAULT 3, + redundancy_pairwise_min_direct_comparisons INTEGER NOT NULL DEFAULT 4, + redundancy_pairwise_winner_hold_ms INTEGER NOT NULL DEFAULT 500, + redundancy_pairwise_winner_hold_min_speedup REAL NOT NULL DEFAULT 0.1, + redundancy_pairwise_winner_hold_min_samples INTEGER NOT NULL DEFAULT 6, + perf_sample_size INTEGER NOT NULL DEFAULT 256, + perf_window_ms INTEGER NOT NULL DEFAULT 3600000, + escrow_rotation_enabled INTEGER NOT NULL DEFAULT 0, + escrow_rotation_settlement_enabled INTEGER NOT NULL DEFAULT 0, + escrow_rotation_pre_poc_blocks INTEGER NOT NULL DEFAULT 300, + escrow_rotation_models_json TEXT NOT NULL DEFAULT '', + gateway_disabled_enabled INTEGER NOT NULL DEFAULT 0, + gateway_disabled_message TEXT NOT NULL DEFAULT '', + gateway_disabled_new_url TEXT NOT NULL DEFAULT '', + updated_at TEXT NOT NULL + )`, + `CREATE TABLE IF NOT EXISTS gateway_devshards ( + id TEXT PRIMARY KEY, + private_key_hex TEXT NOT NULL, + private_key_env TEXT NOT NULL DEFAULT '', + model TEXT NOT NULL DEFAULT '', + storage_path TEXT NOT NULL DEFAULT '', + active INTEGER NOT NULL DEFAULT 1, + rotation_role TEXT NOT NULL DEFAULT '', + rotation_epoch INTEGER NOT NULL DEFAULT 0, + settlement_pending INTEGER NOT NULL DEFAULT 0, + created_at TEXT NOT NULL, + updated_at TEXT NOT NULL + )`, + } + for _, stmt := range stmts { + if _, err := db.Exec(stmt); err != nil { + db.Close() + return nil, fmt.Errorf("init gateway store: %w", err) + } + } + if err := ensureGatewaySettingsColumn(db, "public_api", "TEXT NOT NULL DEFAULT ''"); err != nil { + db.Close() + return nil, fmt.Errorf("migrate gateway store: %w", err) + } + if err := ensureGatewaySettingsColumn(db, "tx_gas_limit", "INTEGER NOT NULL DEFAULT 0"); err != nil { + db.Close() + return nil, fmt.Errorf("migrate gateway tx settings: %w", err) + } + if err := ensureGatewaySettingsColumn(db, "model_limits_json", "TEXT NOT NULL DEFAULT ''"); err != nil { + db.Close() + return nil, fmt.Errorf("migrate gateway model limits: %w", err) + } + if err := ensureGatewaySettingsColumn(db, "model_access_json", "TEXT NOT NULL DEFAULT ''"); err != nil { + db.Close() + return nil, fmt.Errorf("migrate gateway model access: %w", err) + } + if err := ensureGatewaySettingsColumn(db, "request_max_tokens_cap", "INTEGER NOT NULL DEFAULT 4096"); err != nil { + db.Close() + return nil, fmt.Errorf("migrate gateway max token cap: %w", err) + } + if err := ensureGatewaySettingsColumn(db, "max_concurrent_requests_per_10000_weight", "REAL NOT NULL DEFAULT 5.0"); err != nil { + db.Close() + return nil, fmt.Errorf("migrate gateway weight concurrency: %w", err) + } + if err := ensureGatewaySettingsColumn(db, "poc_max_concurrent_requests_per_10000_weight", "REAL NOT NULL DEFAULT 10.0"); err != nil { + db.Close() + return nil, fmt.Errorf("migrate gateway poc weight concurrency: %w", err) + } + if err := ensureGatewaySettingsTuningColumns(db); err != nil { + db.Close() + return nil, fmt.Errorf("migrate gateway tuning settings: %w", err) + } + if err := ensureGatewaySettingsRotationColumns(db); err != nil { + db.Close() + return nil, fmt.Errorf("migrate gateway rotation settings: %w", err) + } + if err := ensureGatewaySettingsDisabledColumns(db); err != nil { + db.Close() + return nil, fmt.Errorf("migrate gateway disabled settings: %w", err) + } + if err := ensureGatewayDevshardsColumn(db, "protocol_version", "TEXT NOT NULL DEFAULT ''"); err != nil { + db.Close() + return nil, fmt.Errorf("migrate gateway devshards: %w", err) + } + if err := ensureGatewayDevshardsColumn(db, "rotation_role", "TEXT NOT NULL DEFAULT ''"); err != nil { + db.Close() + return nil, fmt.Errorf("migrate gateway devshard role: %w", err) + } + if err := ensureGatewayDevshardsColumn(db, "rotation_epoch", "INTEGER NOT NULL DEFAULT 0"); err != nil { + db.Close() + return nil, fmt.Errorf("migrate gateway devshard epoch: %w", err) + } + if err := ensureGatewayDevshardsColumn(db, "settlement_pending", "INTEGER NOT NULL DEFAULT 0"); err != nil { + db.Close() + return nil, fmt.Errorf("migrate gateway devshard settlement_pending: %w", err) + } + if _, err := db.Exec(`CREATE TABLE IF NOT EXISTS participant_throttle_state ( + participant_key TEXT PRIMARY KEY, + tokens REAL NOT NULL DEFAULT 0, + last_refill_at TEXT NOT NULL, + last_throttle_status INTEGER NOT NULL DEFAULT 0, + empty_stream_streak INTEGER NOT NULL DEFAULT 0 + )`); err != nil { + db.Close() + return nil, fmt.Errorf("init participant throttle table: %w", err) + } + if _, err := db.Exec(`CREATE TABLE IF NOT EXISTS gateway_rotation_status ( + model_id TEXT NOT NULL, + stage TEXT NOT NULL, + epoch INTEGER NOT NULL, + role TEXT NOT NULL DEFAULT '', + target_count INTEGER NOT NULL DEFAULT 0, + existing_count INTEGER NOT NULL DEFAULT 0, + created_count INTEGER NOT NULL DEFAULT 0, + promoted_count INTEGER NOT NULL DEFAULT 0, + settled_count INTEGER NOT NULL DEFAULT 0, + settle_failed_count INTEGER NOT NULL DEFAULT 0, + create_error TEXT NOT NULL DEFAULT '', + completed INTEGER NOT NULL DEFAULT 0, + updated_at TEXT NOT NULL, + PRIMARY KEY (model_id, stage, epoch) + )`); err != nil { + db.Close() + return nil, fmt.Errorf("init gateway rotation status table: %w", err) + } + if _, err := db.Exec(`CREATE TABLE IF NOT EXISTS gateway_suspicious_hosts ( + participant_key TEXT PRIMARY KEY, + note TEXT NOT NULL DEFAULT '', + created_at TEXT NOT NULL + )`); err != nil { + db.Close() + return nil, fmt.Errorf("init gateway suspicious hosts table: %w", err) + } + // Write-ahead intent for an escrow create (written before the on-chain tx). + if _, err := db.Exec(`CREATE TABLE IF NOT EXISTS escrow_rotation_commitments ( + tx_hash TEXT PRIMARY KEY, + model TEXT NOT NULL, + role TEXT NOT NULL DEFAULT '', + epoch INTEGER NOT NULL DEFAULT 0, + private_key_env TEXT NOT NULL DEFAULT '', + block_height INTEGER NOT NULL DEFAULT 0, + created_at TEXT NOT NULL + )`); err != nil { + db.Close() + return nil, fmt.Errorf("init escrow rotation commitments table: %w", err) + } + if err := ensureColumn(db, "participant_throttle_state", "quarantine_until_utc", "TEXT NOT NULL DEFAULT ''"); err != nil { + db.Close() + return nil, fmt.Errorf("migrate participant throttle: %w", err) + } + if err := ensureColumn(db, "participant_throttle_state", "empty_stream_streak", "INTEGER NOT NULL DEFAULT 0"); err != nil { + db.Close() + return nil, fmt.Errorf("migrate participant throttle streak: %w", err) + } + if err := ensureColumn(db, "participant_throttle_state", "eof_transport_failure_streak", "INTEGER NOT NULL DEFAULT 0"); err != nil { + db.Close() + return nil, fmt.Errorf("migrate participant throttle eof streak: %w", err) + } + if err := ensureColumn(db, "participant_throttle_state", "model_ids", "TEXT NOT NULL DEFAULT ''"); err != nil { + db.Close() + return nil, fmt.Errorf("migrate participant throttle models: %w", err) + } + if err := ensureColumn(db, "participant_throttle_state", "failure_strikes", "INTEGER NOT NULL DEFAULT 0"); err != nil { + db.Close() + return nil, fmt.Errorf("migrate participant throttle strikes: %w", err) + } + if _, err := db.Exec(` + UPDATE participant_throttle_state + SET failure_strikes = MAX(IFNULL(empty_stream_streak, 0), IFNULL(eof_transport_failure_streak, 0)) + WHERE IFNULL(failure_strikes, 0) = 0 + AND (IFNULL(empty_stream_streak, 0) > 0 OR IFNULL(eof_transport_failure_streak, 0) > 0)`); err != nil { + db.Close() + return nil, fmt.Errorf("migrate participant throttle strike values: %w", err) + } + if _, err := db.Exec(`CREATE TABLE IF NOT EXISTS gateway_pg_sync_journal ( + seq INTEGER PRIMARY KEY AUTOINCREMENT, + table_name TEXT NOT NULL, + row_key TEXT NOT NULL, + op TEXT NOT NULL, + enqueued_at TEXT NOT NULL + )`); err != nil { + db.Close() + return nil, fmt.Errorf("init gateway sync journal: %w", err) + } + + return &SQLiteGatewayStore{db: db}, nil +} + +func (s *SQLiteGatewayStore) Close() error { + if s == nil || s.db == nil { + return nil + } + return s.db.Close() +} + +func (s *SQLiteGatewayStore) LoadState() (GatewayState, bool, error) { + var state GatewayState + row := s.db.QueryRow(` + SELECT ` + gatewaySettingsSelectColumns() + ` + FROM gateway_settings + WHERE id = 1`) + settings, err := scanGatewaySettings(row) + if err == sql.ErrNoRows { + return GatewayState{}, false, nil + } + if err != nil { + return GatewayState{}, false, fmt.Errorf("load gateway settings: %w", err) + } + state.Settings = settings + + rows, err := s.db.Query(` + SELECT id, private_key_hex, private_key_env, model, storage_path, active, created_at, updated_at, protocol_version, + rotation_role, rotation_epoch, settlement_pending + FROM gateway_devshards + ORDER BY id`) + if err != nil { + return GatewayState{}, false, fmt.Errorf("load gateway devshards: %w", err) + } + defer rows.Close() + for rows.Next() { + var devshard GatewayDevshardState + var active int + var settlementPending int + if err := rows.Scan( + &devshard.ID, + &devshard.PrivateKeyHex, + &devshard.PrivateKeyEnv, + &devshard.Model, + &devshard.StoragePath, + &active, + &devshard.CreatedAt, + &devshard.UpdatedAt, + &devshard.ProtocolVersion, + &devshard.RotationRole, + &devshard.RotationEpoch, + &settlementPending, + ); err != nil { + return GatewayState{}, false, fmt.Errorf("scan gateway devshard: %w", err) + } + devshard.Active = active != 0 + devshard.SettlementPending = settlementPending != 0 + state.Devshards = append(state.Devshards, devshard) + } + if err := rows.Err(); err != nil { + return GatewayState{}, false, fmt.Errorf("iterate gateway devshards: %w", err) + } + suspiciousHosts, err := s.LoadSuspiciousHosts() + if err != nil { + return GatewayState{}, false, err + } + state.SuspiciousHosts = suspiciousHosts + return state, true, nil +} + +func (s *SQLiteGatewayStore) Initialize(settings GatewaySettings, devshards []GatewayDevshardState) error { + settings = settings.WithTuningDefaults() + now := time.Now().UTC().Format(time.RFC3339Nano) + tx, err := s.db.Begin() + if err != nil { + return fmt.Errorf("begin gateway init: %w", err) + } + defer tx.Rollback() + + var count int + if err := tx.QueryRow(`SELECT COUNT(*) FROM gateway_settings WHERE id = 1`).Scan(&count); err != nil { + return fmt.Errorf("count gateway settings: %w", err) + } + if count > 0 { + return nil + } + + if _, err := tx.Exec(fmt.Sprintf(` + INSERT INTO gateway_settings (%s) + VALUES (%s)`, + gatewaySettingsInsertColumnNames(), + sqlitePlaceholderList(len(settingsInsertArgs(settings, now))), + ), settingsInsertArgs(settings, now)...); err != nil { + return fmt.Errorf("insert gateway settings: %w", err) + } + + for _, devshard := range devshards { + if err := s.upsertDevshardTx(tx, devshard, now); err != nil { + return err + } + } + return tx.Commit() +} + +func (s *SQLiteGatewayStore) UpdateSettings(settings GatewaySettings) error { + updatedAt := time.Now().UTC().Format(time.RFC3339Nano) + res, err := s.db.Exec(fmt.Sprintf(` + UPDATE gateway_settings + SET %s, + updated_at = ? + WHERE id = 1`, gatewaySettingsUpdateAssignments("?")), + settingsUpdateArgs(settings, updatedAt)..., + ) + if err != nil { + return fmt.Errorf("update gateway settings: %w", err) + } + n, err := res.RowsAffected() + if err != nil { + return fmt.Errorf("rows affected for gateway settings update: %w", err) + } + if n == 0 { + return fmt.Errorf("gateway settings not initialized") + } + return nil +} + +func (s *SQLiteGatewayStore) SaveRotationStatus(status GatewayRotationStatus) error { + if s == nil || s.db == nil { + return nil + } + now := time.Now().UTC().Format(time.RFC3339Nano) + if strings.TrimSpace(status.UpdatedAt) != "" { + now = strings.TrimSpace(status.UpdatedAt) + } + _, err := s.db.Exec(` + INSERT OR REPLACE INTO gateway_rotation_status ( + model_id, stage, epoch, role, target_count, existing_count, created_count, + promoted_count, settled_count, settle_failed_count, create_error, completed, updated_at + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, + strings.TrimSpace(status.ModelID), + strings.TrimSpace(status.Stage), + status.Epoch, + strings.TrimSpace(status.Role), + status.TargetCount, + status.ExistingCount, + status.CreatedCount, + status.PromotedCount, + status.SettledCount, + status.SettleFailedCount, + strings.TrimSpace(status.CreateError), + gatewayBoolToInt(status.Completed), + now, + ) + if err != nil { + return fmt.Errorf("save gateway rotation status model=%q stage=%q epoch=%d: %w", status.ModelID, status.Stage, status.Epoch, err) + } + return nil +} + +func (s *SQLiteGatewayStore) LoadRotationStatuses(limit int) ([]GatewayRotationStatus, error) { + if s == nil || s.db == nil { + return nil, nil + } + query := ` + SELECT model_id, stage, epoch, role, target_count, existing_count, created_count, + promoted_count, settled_count, settle_failed_count, create_error, completed, updated_at + FROM gateway_rotation_status + ORDER BY updated_at DESC` + args := []any{} + if limit > 0 { + query += ` LIMIT ?` + args = append(args, limit) + } + rows, err := s.db.Query(query, args...) + if err != nil { + return nil, fmt.Errorf("load gateway rotation statuses: %w", err) + } + defer rows.Close() + var statuses []GatewayRotationStatus + for rows.Next() { + var status GatewayRotationStatus + var completed int + if err := rows.Scan( + &status.ModelID, + &status.Stage, + &status.Epoch, + &status.Role, + &status.TargetCount, + &status.ExistingCount, + &status.CreatedCount, + &status.PromotedCount, + &status.SettledCount, + &status.SettleFailedCount, + &status.CreateError, + &completed, + &status.UpdatedAt, + ); err != nil { + return nil, fmt.Errorf("scan gateway rotation status: %w", err) + } + status.Completed = completed != 0 + statuses = append(statuses, status) + } + if err := rows.Err(); err != nil { + return nil, err + } + return statuses, nil +} + +// SaveCommitment records a create intent, keyed by tx hash. +func (s *SQLiteGatewayStore) SaveCommitment(c GatewayEscrowCommitment) error { + if s == nil || s.db == nil { + return fmt.Errorf("gateway store unavailable") + } + now := time.Now().UTC().Format(time.RFC3339Nano) + _, err := s.db.Exec(` + INSERT OR REPLACE INTO escrow_rotation_commitments ( + tx_hash, model, role, epoch, private_key_env, block_height, created_at + ) VALUES (?, ?, ?, ?, ?, ?, ?)`, + strings.TrimSpace(c.TxHash), + strings.TrimSpace(c.Model), + strings.TrimSpace(c.Role), + c.Epoch, + strings.TrimSpace(c.PrivateKeyEnv), + c.BlockHeight, + now, + ) + if err != nil { + return fmt.Errorf("save escrow commitment tx=%s: %w", c.TxHash, err) + } + return nil +} + +// LoadCommitments returns all pending commitments (oldest first). +func (s *SQLiteGatewayStore) LoadCommitments() ([]GatewayEscrowCommitment, error) { + if s == nil || s.db == nil { + return nil, nil + } + rows, err := s.db.Query(` + SELECT tx_hash, model, role, epoch, private_key_env, block_height, created_at + FROM escrow_rotation_commitments + ORDER BY created_at ASC`) + if err != nil { + return nil, fmt.Errorf("load escrow commitments: %w", err) + } + defer rows.Close() + var commitments []GatewayEscrowCommitment + for rows.Next() { + var c GatewayEscrowCommitment + if err := rows.Scan(&c.TxHash, &c.Model, &c.Role, &c.Epoch, &c.PrivateKeyEnv, &c.BlockHeight, &c.CreatedAt); err != nil { + return nil, fmt.Errorf("scan escrow commitment: %w", err) + } + commitments = append(commitments, c) + } + return commitments, rows.Err() +} + +// DeleteCommitment clears a commitment once its escrow is persisted (or proven absent). +func (s *SQLiteGatewayStore) DeleteCommitment(txHash string) error { + if s == nil || s.db == nil { + return fmt.Errorf("gateway store unavailable") + } + if _, err := s.db.Exec(`DELETE FROM escrow_rotation_commitments WHERE tx_hash = ?`, strings.TrimSpace(txHash)); err != nil { + return fmt.Errorf("delete escrow commitment tx=%s: %w", txHash, err) + } + return nil +} + +func (s *SQLiteGatewayStore) LoadSuspiciousHosts() ([]GatewaySuspiciousHost, error) { + if s == nil || s.db == nil { + return nil, nil + } + return scanGatewaySuspiciousHosts(s.db) +} + +// loadSuspiciousHostsTx reads the suspicious hosts within an open transaction. +// It must be used instead of LoadSuspiciousHosts inside writeWithSyncJournal: +// the SQLite handle is capped at one connection (SetMaxOpenConns(1)), so a +// separate s.db.Query while the tx holds that connection would deadlock. +func (s *SQLiteGatewayStore) loadSuspiciousHostsTx(tx *sql.Tx) ([]GatewaySuspiciousHost, error) { + if tx == nil { + return nil, fmt.Errorf("nil transaction") + } + return scanGatewaySuspiciousHosts(tx) +} + +type gatewayRowQueryer interface { + Query(query string, args ...any) (*sql.Rows, error) +} + +func scanGatewaySuspiciousHosts(q gatewayRowQueryer) ([]GatewaySuspiciousHost, error) { + rows, err := q.Query(` + SELECT participant_key, note, created_at + FROM gateway_suspicious_hosts + ORDER BY participant_key`) + if err != nil { + return nil, fmt.Errorf("load gateway suspicious hosts: %w", err) + } + defer rows.Close() + + var hosts []GatewaySuspiciousHost + for rows.Next() { + var host GatewaySuspiciousHost + if err := rows.Scan(&host.ParticipantKey, &host.Note, &host.CreatedAt); err != nil { + return nil, fmt.Errorf("scan gateway suspicious host: %w", err) + } + hosts = append(hosts, host) + } + if err := rows.Err(); err != nil { + return nil, err + } + return hosts, nil +} + +func (s *SQLiteGatewayStore) UpsertSuspiciousHosts(participantKeys []string, note string) ([]GatewaySuspiciousHost, error) { + if s == nil || s.db == nil { + return nil, nil + } + participantKeys = normalizeParticipantKeys(participantKeys) + if len(participantKeys) == 0 { + return nil, fmt.Errorf("participant_keys must contain at least one key") + } + tx, err := s.db.Begin() + if err != nil { + return nil, fmt.Errorf("begin suspicious host upsert: %w", err) + } + defer tx.Rollback() + now := time.Now().UTC().Format(time.RFC3339Nano) + note = strings.TrimSpace(note) + for _, key := range participantKeys { + if _, err := tx.Exec(` + INSERT INTO gateway_suspicious_hosts (participant_key, note, created_at) + VALUES (?, ?, ?) + ON CONFLICT(participant_key) DO UPDATE SET note = excluded.note`, + key, note, now); err != nil { + return nil, fmt.Errorf("upsert suspicious host %s: %w", key, err) + } + } + if err := tx.Commit(); err != nil { + return nil, fmt.Errorf("commit suspicious host upsert: %w", err) + } + return s.LoadSuspiciousHosts() +} + +func (s *SQLiteGatewayStore) DeleteSuspiciousHosts(participantKeys []string) ([]GatewaySuspiciousHost, error) { + if s == nil || s.db == nil { + return nil, nil + } + participantKeys = normalizeParticipantKeys(participantKeys) + if len(participantKeys) == 0 { + return nil, fmt.Errorf("participant_keys must contain at least one key") + } + tx, err := s.db.Begin() + if err != nil { + return nil, fmt.Errorf("begin suspicious host delete: %w", err) + } + defer tx.Rollback() + for _, key := range participantKeys { + if _, err := tx.Exec(`DELETE FROM gateway_suspicious_hosts WHERE participant_key = ?`, key); err != nil { + return nil, fmt.Errorf("delete suspicious host %s: %w", key, err) + } + } + if err := tx.Commit(); err != nil { + return nil, fmt.Errorf("commit suspicious host delete: %w", err) + } + return s.LoadSuspiciousHosts() +} + +func (s *SQLiteGatewayStore) UpsertDevshard(devshard GatewayDevshardState) error { + now := time.Now().UTC().Format(time.RFC3339Nano) + tx, err := s.db.Begin() + if err != nil { + return fmt.Errorf("begin devshard upsert: %w", err) + } + defer tx.Rollback() + if err := s.upsertDevshardTx(tx, devshard, now); err != nil { + return err + } + return tx.Commit() +} + +func (s *SQLiteGatewayStore) upsertDevshardTx(tx *sql.Tx, devshard GatewayDevshardState, now string) error { + createdAt := now + _ = tx.QueryRow(`SELECT created_at FROM gateway_devshards WHERE id = ?`, devshard.ID).Scan(&createdAt) + // Preserve the existing settlement_pending marker so an unrelated upsert + // never silently clears a queued settlement; a brand-new row falls back + // to the value carried on devshard. + settlementPending := gatewayBoolToInt(devshard.SettlementPending) + _ = tx.QueryRow(`SELECT settlement_pending FROM gateway_devshards WHERE id = ?`, devshard.ID).Scan(&settlementPending) + if _, err := tx.Exec(` + INSERT OR REPLACE INTO gateway_devshards ( + id, private_key_hex, private_key_env, model, storage_path, active, created_at, updated_at, protocol_version, + rotation_role, rotation_epoch, settlement_pending + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, + strings.TrimSpace(devshard.ID), + strings.TrimSpace(devshard.PrivateKeyHex), + strings.TrimSpace(devshard.PrivateKeyEnv), + strings.TrimSpace(devshard.Model), + strings.TrimSpace(devshard.StoragePath), + gatewayBoolToInt(devshard.Active), + createdAt, + now, + strings.TrimSpace(devshard.ProtocolVersion), + strings.TrimSpace(devshard.RotationRole), + devshard.RotationEpoch, + settlementPending, + ); err != nil { + return fmt.Errorf("upsert gateway devshard %s: %w", devshard.ID, err) + } + return nil +} + +func (s *SQLiteGatewayStore) SetDevshardActive(id string, active bool) error { + res, err := s.db.Exec(` + UPDATE gateway_devshards + SET active = ?, updated_at = ? + WHERE id = ?`, + gatewayBoolToInt(active), + time.Now().UTC().Format(time.RFC3339Nano), + strings.TrimSpace(id), + ) + if err != nil { + return fmt.Errorf("update devshard %s active=%t: %w", id, active, err) + } + n, err := res.RowsAffected() + if err != nil { + return fmt.Errorf("rows affected for devshard %s: %w", id, err) + } + if n == 0 { + return fmt.Errorf("devshard %s not found", id) + } + return nil +} + +func (s *SQLiteGatewayStore) SetDevshardSettlementPending(id string, pending bool) error { + res, err := s.db.Exec(` + UPDATE gateway_devshards + SET settlement_pending = ?, updated_at = ? + WHERE id = ?`, + gatewayBoolToInt(pending), + time.Now().UTC().Format(time.RFC3339Nano), + strings.TrimSpace(id), + ) + if err != nil { + return fmt.Errorf("update devshard %s settlement_pending=%t: %w", id, pending, err) + } + n, err := res.RowsAffected() + if err != nil { + return fmt.Errorf("rows affected for devshard %s: %w", id, err) + } + if n == 0 { + return fmt.Errorf("devshard %s not found", id) + } + return nil +} + +func (s *SQLiteGatewayStore) DeleteDevshard(id string) error { + res, err := s.db.Exec(`DELETE FROM gateway_devshards WHERE id = ?`, strings.TrimSpace(id)) + if err != nil { + return fmt.Errorf("delete devshard %s: %w", id, err) + } + n, err := res.RowsAffected() + if err != nil { + return fmt.Errorf("rows affected for delete devshard %s: %w", id, err) + } + if n == 0 { + return fmt.Errorf("devshard %s not found", id) + } + return nil +} + +func normalizeParticipantKeys(keys []string) []string { + if len(keys) == 0 { + return nil + } + normalized := make([]string, 0, len(keys)) + seen := make(map[string]struct{}, len(keys)) + for _, key := range keys { + key = strings.TrimSpace(key) + if key == "" { + continue + } + if _, ok := seen[key]; ok { + continue + } + seen[key] = struct{}{} + normalized = append(normalized, key) + } + return normalized +} + +func (s *SQLiteGatewayStore) SaveParticipantThrottle(key string, modelIDs []string, tokens float64, lastRefillAt time.Time, status int, quarantineUntil time.Time, failureStrikes int) error { + if s == nil || s.db == nil { + return nil + } + quarStr := "" + if !quarantineUntil.IsZero() { + quarStr = quarantineUntil.UTC().Format(time.RFC3339Nano) + } + _, err := s.db.Exec(` + INSERT OR REPLACE INTO participant_throttle_state + (participant_key, tokens, last_refill_at, last_throttle_status, quarantine_until_utc, failure_strikes, model_ids) + VALUES (?, ?, ?, ?, ?, ?, ?)`, + key, tokens, lastRefillAt.UTC().Format(time.RFC3339Nano), status, quarStr, failureStrikes, strings.Join(normalizeModelIDs(modelIDs), ",")) + if err != nil { + return fmt.Errorf("save participant throttle %s: %w", key, err) + } + return nil +} + +func (s *SQLiteGatewayStore) DeleteParticipantThrottle(key string) error { + if s == nil || s.db == nil { + return nil + } + _, err := s.db.Exec(`DELETE FROM participant_throttle_state WHERE participant_key = ?`, key) + if err != nil { + return fmt.Errorf("delete participant throttle %s: %w", key, err) + } + return nil +} + +func (s *SQLiteGatewayStore) LoadParticipantThrottles() ([]ParticipantThrottleRow, error) { + if s == nil || s.db == nil { + return nil, nil + } + rows, err := s.db.Query(` + SELECT participant_key, tokens, last_refill_at, last_throttle_status, + IFNULL(quarantine_until_utc, '') AS quarantine_until_utc, + IFNULL(failure_strikes, 0) AS failure_strikes, + IFNULL(model_ids, '') AS model_ids + FROM participant_throttle_state`) + if err != nil { + return nil, fmt.Errorf("load participant throttles: %w", err) + } + defer rows.Close() + + var result []ParticipantThrottleRow + for rows.Next() { + var row ParticipantThrottleRow + var lastRefillStr, quarantineStr, modelIDsStr string + if err := rows.Scan(&row.Key, &row.Tokens, &lastRefillStr, &row.Status, &quarantineStr, &row.FailureStrikes, &modelIDsStr); err != nil { + return nil, fmt.Errorf("scan participant throttle: %w", err) + } + row.ModelIDs = splitModelIDs(modelIDsStr) + row.LastRefillAt, err = time.Parse(time.RFC3339Nano, lastRefillStr) + if err != nil { + return nil, fmt.Errorf("parse last_refill_at for %s: %w", row.Key, err) + } + if strings.TrimSpace(quarantineStr) != "" { + row.QuarantineUntil, err = time.Parse(time.RFC3339Nano, quarantineStr) + if err != nil { + return nil, fmt.Errorf("parse quarantine_until_utc for %s: %w", row.Key, err) + } + } + result = append(result, row) + } + if err := rows.Err(); err != nil { + return nil, err + } + return result, nil +} + +func ensureGatewaySettingsColumn(db *sql.DB, columnName, columnDDL string) error { + return ensureColumn(db, "gateway_settings", columnName, columnDDL) +} + +func ensureGatewaySettingsTuningColumns(db *sql.DB) error { + columns := []struct { + name string + ddl string + }{ + {"participant_request_burst", "INTEGER NOT NULL DEFAULT 600"}, + {"participant_recovery_per_minute", "INTEGER NOT NULL DEFAULT 10"}, + {"participant_http_quarantine_ms", "INTEGER NOT NULL DEFAULT 3600000"}, + {"participant_transport_failure_quarantine_ms", "INTEGER NOT NULL DEFAULT 1800000"}, + {"participant_empty_stream_quarantine_ms", "INTEGER NOT NULL DEFAULT 1800000"}, + {"participant_stalled_winner_quarantine_ms", "INTEGER NOT NULL DEFAULT 1800000"}, + {"participant_empty_stream_threshold", "INTEGER NOT NULL DEFAULT 3"}, + {"participant_eof_transport_failure_threshold", "INTEGER NOT NULL DEFAULT 3"}, + {"redundancy_receipt_timeout_ms", "INTEGER NOT NULL DEFAULT 5000"}, + {"redundancy_first_token_timeout_floor_ms", "INTEGER NOT NULL DEFAULT 1000"}, + {"redundancy_per_input_token_first_token_lag_ms", "INTEGER NOT NULL DEFAULT 10"}, + {"redundancy_inter_chunk_stall_timeout_ms", "INTEGER NOT NULL DEFAULT 60000"}, + {"redundancy_streaming_attempt_hard_timeout_ms", "INTEGER NOT NULL DEFAULT 1200000"}, + {"redundancy_non_stream_response_floor_ms", "INTEGER NOT NULL DEFAULT 20000"}, + {"redundancy_non_stream_no_content_timeout_ms", "INTEGER NOT NULL DEFAULT 1200000"}, + {"redundancy_non_stream_max_attempt_wait_ms", "INTEGER NOT NULL DEFAULT 1800000"}, + {"redundancy_per_input_token_response_lag_ms", "INTEGER NOT NULL DEFAULT 20"}, + {"redundancy_secondary_wait_after_winner_ms", "INTEGER NOT NULL DEFAULT 600000"}, + {"redundancy_parallel_advantage_threshold", "REAL NOT NULL DEFAULT 0.5"}, + {"redundancy_unresponsive_threshold", "REAL NOT NULL DEFAULT 1.0"}, + {"redundancy_speed_policy", "TEXT NOT NULL DEFAULT 'hybrid'"}, + {"redundancy_pairwise_budget_percentile", "REAL NOT NULL DEFAULT 0.9"}, + {"redundancy_pairwise_max_proactive_attempts", "INTEGER NOT NULL DEFAULT 3"}, + {"redundancy_pairwise_min_direct_comparisons", "INTEGER NOT NULL DEFAULT 4"}, + {"redundancy_pairwise_winner_hold_ms", "INTEGER NOT NULL DEFAULT 500"}, + {"redundancy_pairwise_winner_hold_min_speedup", "REAL NOT NULL DEFAULT 0.1"}, + {"redundancy_pairwise_winner_hold_min_samples", "INTEGER NOT NULL DEFAULT 6"}, + {"perf_sample_size", "INTEGER NOT NULL DEFAULT 256"}, + {"perf_window_ms", "INTEGER NOT NULL DEFAULT 3600000"}, + } + for _, column := range columns { + if err := ensureGatewaySettingsColumn(db, column.name, column.ddl); err != nil { + return err + } + } + return nil +} + +func ensureGatewaySettingsRotationColumns(db *sql.DB) error { + columns := []struct { + name string + ddl string + }{ + {"escrow_rotation_enabled", "INTEGER NOT NULL DEFAULT 0"}, + {"escrow_rotation_settlement_enabled", "INTEGER NOT NULL DEFAULT 0"}, + {"escrow_rotation_pre_poc_blocks", "INTEGER NOT NULL DEFAULT 300"}, + {"escrow_rotation_models_json", "TEXT NOT NULL DEFAULT ''"}, + } + for _, column := range columns { + if err := ensureGatewaySettingsColumn(db, column.name, column.ddl); err != nil { + return err + } + } + return nil +} + +func ensureGatewaySettingsDisabledColumns(db *sql.DB) error { + columns := []struct { + name string + ddl string + }{ + {"gateway_disabled_enabled", "INTEGER NOT NULL DEFAULT 0"}, + {"gateway_disabled_message", "TEXT NOT NULL DEFAULT ''"}, + {"gateway_disabled_new_url", "TEXT NOT NULL DEFAULT ''"}, + } + for _, column := range columns { + if err := ensureGatewaySettingsColumn(db, column.name, column.ddl); err != nil { + return err + } + } + return nil +} + +func ensureGatewayDevshardsColumn(db *sql.DB, columnName, columnDDL string) error { + return ensureColumn(db, "gateway_devshards", columnName, columnDDL) +} + +func ensureColumn(db *sql.DB, table, columnName, columnDDL string) error { + rows, err := db.Query(fmt.Sprintf(`PRAGMA table_info(%s)`, table)) + if err != nil { + return err + } + defer rows.Close() + + for rows.Next() { + var cid int + var name string + var dataType string + var notNull int + var defaultValue sql.NullString + var pk int + if err := rows.Scan(&cid, &name, &dataType, ¬Null, &defaultValue, &pk); err != nil { + return err + } + if name == columnName { + return nil + } + } + if err := rows.Err(); err != nil { + return err + } + + _, err = db.Exec(fmt.Sprintf(`ALTER TABLE %s ADD COLUMN %s %s`, table, columnName, columnDDL)) + return err +} + +func sqlitePlaceholderList(n int) string { + if n <= 0 { + return "" + } + return strings.TrimRight(strings.Repeat("?, ", n), ", ") +} + +var _ GatewayStore = (*SQLiteGatewayStore)(nil) diff --git a/devshard/cmd/devshardctl/gateway_store_sync_journal.go b/devshard/cmd/devshardctl/gateway_store_sync_journal.go new file mode 100644 index 0000000000..eb1321303b --- /dev/null +++ b/devshard/cmd/devshardctl/gateway_store_sync_journal.go @@ -0,0 +1,828 @@ +package main + +import ( + "context" + "database/sql" + "fmt" + "os" + "strconv" + "strings" + "time" + + "github.com/jackc/pgx/v5" +) + +const ( + gatewayTableSettings = "gateway_settings" + gatewayTableDevshards = "gateway_devshards" + gatewayTableThrottle = "participant_throttle_state" + gatewayTableRotationStatus = "gateway_rotation_status" + gatewayTableSuspiciousHosts = "gateway_suspicious_hosts" + gatewayTableCommitments = "escrow_rotation_commitments" + gatewaySettingsRowKey = "1" + gatewaySyncOpUpsert = "upsert" + gatewaySyncOpDelete = "delete" +) + +type gatewaySyncEntry struct { + tableName string + rowKey string + op string +} + +type gatewaySyncKey struct { + tableName string + rowKey string +} + +type gatewaySyncJournalRow struct { + seq int64 + tableName string + rowKey string + op string +} + +func gatewaySyncJournalEnabled() bool { + raw := strings.TrimSpace(os.Getenv("GATEWAY_PG_SYNC_JOURNAL")) + if raw == "" { + return true + } + switch strings.ToLower(raw) { + case "0", "false", "no", "off": + return false + default: + return true + } +} + +func gatewayRotationStatusRowKey(modelID, stage string, epoch uint64) string { + return fmt.Sprintf("%s|%s|%d", strings.TrimSpace(modelID), strings.TrimSpace(stage), epoch) +} + +func parseRotationStatusRowKey(rowKey string) (string, string, uint64, error) { + parts := strings.Split(rowKey, "|") + if len(parts) != 3 { + return "", "", 0, fmt.Errorf("invalid rotation status row key %q", rowKey) + } + epoch, err := strconv.ParseUint(parts[2], 10, 64) + if err != nil { + return "", "", 0, fmt.Errorf("parse rotation status epoch in %q: %w", rowKey, err) + } + return parts[0], parts[1], epoch, nil +} + +func coalesceSyncJournalRows(rows []gatewaySyncJournalRow) map[gatewaySyncKey]string { + out := make(map[gatewaySyncKey]string, len(rows)) + for _, row := range rows { + out[gatewaySyncKey{tableName: row.tableName, rowKey: row.rowKey}] = row.op + } + return out +} + +func (s *SQLiteGatewayStore) writeWithSyncJournal(entries []gatewaySyncEntry, fn func(*sql.Tx) error) error { + if s == nil || s.db == nil { + return fmt.Errorf("gateway store unavailable") + } + if len(entries) == 0 { + return fmt.Errorf("sync journal entries required") + } + tx, err := s.db.Begin() + if err != nil { + return fmt.Errorf("begin sync journal write: %w", err) + } + defer tx.Rollback() + + if err := fn(tx); err != nil { + return err + } + now := time.Now().UTC().Format(time.RFC3339Nano) + for _, entry := range entries { + if _, err := tx.Exec(` + INSERT INTO gateway_pg_sync_journal (table_name, row_key, op, enqueued_at) + VALUES (?, ?, ?, ?)`, + entry.tableName, entry.rowKey, entry.op, now, + ); err != nil { + return fmt.Errorf("enqueue sync journal %s/%s: %w", entry.tableName, entry.rowKey, err) + } + } + return tx.Commit() +} + +func (s *SQLiteGatewayStore) maxSyncJournalSeq() (int64, error) { + if s == nil || s.db == nil { + return 0, nil + } + var maxSeq sql.NullInt64 + if err := s.db.QueryRow(`SELECT MAX(seq) FROM gateway_pg_sync_journal`).Scan(&maxSeq); err != nil { + return 0, fmt.Errorf("max sync journal seq: %w", err) + } + if !maxSeq.Valid { + return 0, nil + } + return maxSeq.Int64, nil +} + +func (s *SQLiteGatewayStore) countSyncJournalEntries() (int, error) { + if s == nil || s.db == nil { + return 0, nil + } + var count int + if err := s.db.QueryRow(`SELECT COUNT(*) FROM gateway_pg_sync_journal`).Scan(&count); err != nil { + return 0, fmt.Errorf("count sync journal entries: %w", err) + } + return count, nil +} + +func (s *SQLiteGatewayStore) loadSyncJournalUpTo(maxSeq int64) ([]gatewaySyncJournalRow, error) { + rows, err := s.db.Query(` + SELECT seq, table_name, row_key, op + FROM gateway_pg_sync_journal + WHERE seq <= ? + ORDER BY seq ASC`, maxSeq) + if err != nil { + return nil, fmt.Errorf("load sync journal up to %d: %w", maxSeq, err) + } + defer rows.Close() + + var result []gatewaySyncJournalRow + for rows.Next() { + var row gatewaySyncJournalRow + if err := rows.Scan(&row.seq, &row.tableName, &row.rowKey, &row.op); err != nil { + return nil, fmt.Errorf("scan sync journal row: %w", err) + } + result = append(result, row) + } + return result, rows.Err() +} + +func (s *SQLiteGatewayStore) deleteSyncJournalUpTo(maxSeq int64) error { + if _, err := s.db.Exec(`DELETE FROM gateway_pg_sync_journal WHERE seq <= ?`, maxSeq); err != nil { + return fmt.Errorf("delete sync journal up to %d: %w", maxSeq, err) + } + return nil +} + +func drainGatewaySyncJournal(ctx context.Context, sqlite *SQLiteGatewayStore, pg *PostgresGatewayStore) (empty bool, err error) { + if sqlite == nil || pg == nil || pg.pool == nil { + return true, nil + } + count, err := sqlite.countSyncJournalEntries() + if err != nil { + return false, err + } + if count == 0 { + return true, nil + } + + maxSeq, err := sqlite.maxSyncJournalSeq() + if err != nil { + return false, err + } + if maxSeq == 0 { + return true, nil + } + + rows, err := sqlite.loadSyncJournalUpTo(maxSeq) + if err != nil { + return false, err + } + coalesced := coalesceSyncJournalRows(rows) + + tx, err := pg.pool.Begin(ctx) + if err != nil { + return false, fmt.Errorf("begin sync journal drain: %w", err) + } + defer tx.Rollback(ctx) + + for key, op := range coalesced { + switch op { + case gatewaySyncOpDelete: + if err := deleteGatewayRowFromPG(ctx, tx, key.tableName, key.rowKey); err != nil { + return false, err + } + case gatewaySyncOpUpsert: + if err := applyGatewaySyncUpsertToPG(ctx, tx, sqlite, key.tableName, key.rowKey); err != nil { + return false, err + } + default: + return false, fmt.Errorf("unknown sync journal op %q for %s/%s", op, key.tableName, key.rowKey) + } + } + if err := tx.Commit(ctx); err != nil { + return false, fmt.Errorf("commit sync journal drain: %w", err) + } + if err := sqlite.deleteSyncJournalUpTo(maxSeq); err != nil { + return false, err + } + + remaining, err := sqlite.countSyncJournalEntries() + if err != nil { + return false, err + } + return remaining == 0, nil +} + +func applyGatewaySyncUpsertToPG(ctx context.Context, tx pgx.Tx, sqlite *SQLiteGatewayStore, tableName, rowKey string) error { + switch tableName { + case gatewayTableSettings: + settings, updatedAt, ok, err := sqlite.loadGatewaySettingsRow() + if err != nil { + return err + } + if !ok { + return fmt.Errorf("sync journal upsert settings: row missing in sqlite") + } + return applyGatewaySettingsUpsertToPG(ctx, tx, settings, updatedAt) + case gatewayTableDevshards: + devshard, ok, err := sqlite.loadGatewayDevshardRow(rowKey) + if err != nil { + return err + } + if !ok { + return fmt.Errorf("sync journal upsert devshard %s: row missing in sqlite", rowKey) + } + return applyGatewayDevshardToPG(ctx, tx, devshard, time.Now().UTC().Format(time.RFC3339Nano)) + case gatewayTableThrottle: + row, ok, err := sqlite.loadParticipantThrottleRow(rowKey) + if err != nil { + return err + } + if !ok { + return fmt.Errorf("sync journal upsert throttle %s: row missing in sqlite", rowKey) + } + return applyGatewayThrottleToPG(ctx, tx, row) + case gatewayTableRotationStatus: + modelID, stage, epoch, err := parseRotationStatusRowKey(rowKey) + if err != nil { + return err + } + status, ok, err := sqlite.loadGatewayRotationStatusRow(modelID, stage, epoch) + if err != nil { + return err + } + if !ok { + return fmt.Errorf("sync journal upsert rotation status %s: row missing in sqlite", rowKey) + } + return applyGatewayRotationStatusToPG(ctx, tx, status) + case gatewayTableSuspiciousHosts: + host, ok, err := sqlite.loadGatewaySuspiciousHostRow(rowKey) + if err != nil { + return err + } + if !ok { + return fmt.Errorf("sync journal upsert suspicious host %s: row missing in sqlite", rowKey) + } + return applyGatewaySuspiciousHostToPG(ctx, tx, host) + case gatewayTableCommitments: + commitment, ok, err := sqlite.loadGatewayCommitmentRow(rowKey) + if err != nil { + return err + } + if !ok { + return fmt.Errorf("sync journal upsert commitment %s: row missing in sqlite", rowKey) + } + return applyGatewayCommitmentToPG(ctx, tx, commitment) + default: + return fmt.Errorf("unknown sync journal table %q", tableName) + } +} + +func deleteGatewayRowFromPG(ctx context.Context, tx pgx.Tx, tableName, rowKey string) error { + switch tableName { + case gatewayTableSettings: + _, err := tx.Exec(ctx, `DELETE FROM gateway_settings WHERE id = 1`) + return err + case gatewayTableDevshards: + _, err := tx.Exec(ctx, `DELETE FROM gateway_devshards WHERE id = $1`, rowKey) + return err + case gatewayTableThrottle: + _, err := tx.Exec(ctx, `DELETE FROM participant_throttle_state WHERE participant_key = $1`, rowKey) + return err + case gatewayTableRotationStatus: + modelID, stage, epoch, err := parseRotationStatusRowKey(rowKey) + if err != nil { + return err + } + _, err = tx.Exec(ctx, ` + DELETE FROM gateway_rotation_status + WHERE model_id = $1 AND stage = $2 AND epoch = $3`, + modelID, stage, epoch) + return err + case gatewayTableSuspiciousHosts: + _, err := tx.Exec(ctx, `DELETE FROM gateway_suspicious_hosts WHERE participant_key = $1`, rowKey) + return err + case gatewayTableCommitments: + _, err := tx.Exec(ctx, `DELETE FROM escrow_rotation_commitments WHERE tx_hash = $1`, rowKey) + return err + default: + return fmt.Errorf("unknown sync journal table %q", tableName) + } +} + +func gatewaySettingsUpsertAssignmentsPG() string { + cols := gatewaySettingsColumnNames() + parts := make([]string, len(cols)) + for i, col := range cols { + parts[i] = fmt.Sprintf("%s = EXCLUDED.%s", col, col) + } + return strings.Join(parts, ",\n\t\t ") +} + +func applyGatewaySettingsUpsertToPG(ctx context.Context, tx pgx.Tx, settings GatewaySettings, updatedAt string) error { + insertArgs := settingsInsertArgs(settings, updatedAt) + _, err := tx.Exec(ctx, fmt.Sprintf(` + INSERT INTO gateway_settings (%s) + VALUES (%s) + ON CONFLICT (id) DO UPDATE SET %s, + updated_at = EXCLUDED.updated_at`, + gatewaySettingsInsertColumnNames(), + pgPlaceholderList(len(insertArgs)), + gatewaySettingsUpsertAssignmentsPG(), + ), insertArgs...) + if err != nil { + return fmt.Errorf("apply gateway settings to postgres: %w", err) + } + return nil +} + +func applyGatewayDevshardToPG(ctx context.Context, tx pgx.Tx, devshard GatewayDevshardState, now string) error { + createdAt := strings.TrimSpace(devshard.CreatedAt) + if createdAt == "" { + createdAt = now + } + updatedAt := strings.TrimSpace(devshard.UpdatedAt) + if updatedAt == "" { + updatedAt = now + } + _, err := tx.Exec(ctx, ` + INSERT INTO gateway_devshards ( + id, private_key_hex, private_key_env, model, storage_path, active, created_at, updated_at, protocol_version, + rotation_role, rotation_epoch, settlement_pending + ) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12) + ON CONFLICT (id) DO UPDATE SET + private_key_hex = EXCLUDED.private_key_hex, + private_key_env = EXCLUDED.private_key_env, + model = EXCLUDED.model, + storage_path = EXCLUDED.storage_path, + active = EXCLUDED.active, + created_at = EXCLUDED.created_at, + updated_at = EXCLUDED.updated_at, + protocol_version = EXCLUDED.protocol_version, + rotation_role = EXCLUDED.rotation_role, + rotation_epoch = EXCLUDED.rotation_epoch, + settlement_pending = EXCLUDED.settlement_pending`, + strings.TrimSpace(devshard.ID), + strings.TrimSpace(devshard.PrivateKeyHex), + strings.TrimSpace(devshard.PrivateKeyEnv), + strings.TrimSpace(devshard.Model), + strings.TrimSpace(devshard.StoragePath), + gatewayBoolToInt(devshard.Active), + createdAt, + updatedAt, + strings.TrimSpace(devshard.ProtocolVersion), + strings.TrimSpace(devshard.RotationRole), + devshard.RotationEpoch, + gatewayBoolToInt(devshard.SettlementPending), + ) + if err != nil { + return fmt.Errorf("apply gateway devshard %s to postgres: %w", devshard.ID, err) + } + return nil +} + +func applyGatewayThrottleToPG(ctx context.Context, tx pgx.Tx, row ParticipantThrottleRow) error { + quarStr := "" + if !row.QuarantineUntil.IsZero() { + quarStr = row.QuarantineUntil.UTC().Format(time.RFC3339Nano) + } + _, err := tx.Exec(ctx, ` + INSERT INTO participant_throttle_state + (participant_key, tokens, last_refill_at, last_throttle_status, quarantine_until_utc, failure_strikes, model_ids) + VALUES ($1, $2, $3, $4, $5, $6, $7) + ON CONFLICT (participant_key) DO UPDATE SET + tokens = EXCLUDED.tokens, + last_refill_at = EXCLUDED.last_refill_at, + last_throttle_status = EXCLUDED.last_throttle_status, + quarantine_until_utc = EXCLUDED.quarantine_until_utc, + failure_strikes = EXCLUDED.failure_strikes, + model_ids = EXCLUDED.model_ids`, + row.Key, + row.Tokens, + row.LastRefillAt.UTC().Format(time.RFC3339Nano), + row.Status, + quarStr, + row.FailureStrikes, + strings.Join(normalizeModelIDs(row.ModelIDs), ","), + ) + if err != nil { + return fmt.Errorf("apply participant throttle %s to postgres: %w", row.Key, err) + } + return nil +} + +func applyGatewayRotationStatusToPG(ctx context.Context, tx pgx.Tx, status GatewayRotationStatus) error { + updatedAt := strings.TrimSpace(status.UpdatedAt) + if updatedAt == "" { + updatedAt = time.Now().UTC().Format(time.RFC3339Nano) + } + _, err := tx.Exec(ctx, ` + INSERT INTO gateway_rotation_status ( + model_id, stage, epoch, role, target_count, existing_count, created_count, + promoted_count, settled_count, settle_failed_count, create_error, completed, updated_at + ) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13) + ON CONFLICT (model_id, stage, epoch) DO UPDATE SET + role = EXCLUDED.role, + target_count = EXCLUDED.target_count, + existing_count = EXCLUDED.existing_count, + created_count = EXCLUDED.created_count, + promoted_count = EXCLUDED.promoted_count, + settled_count = EXCLUDED.settled_count, + settle_failed_count = EXCLUDED.settle_failed_count, + create_error = EXCLUDED.create_error, + completed = EXCLUDED.completed, + updated_at = EXCLUDED.updated_at`, + strings.TrimSpace(status.ModelID), + strings.TrimSpace(status.Stage), + status.Epoch, + strings.TrimSpace(status.Role), + status.TargetCount, + status.ExistingCount, + status.CreatedCount, + status.PromotedCount, + status.SettledCount, + status.SettleFailedCount, + strings.TrimSpace(status.CreateError), + gatewayBoolToInt(status.Completed), + updatedAt, + ) + if err != nil { + return fmt.Errorf("apply rotation status model=%q stage=%q epoch=%d to postgres: %w", status.ModelID, status.Stage, status.Epoch, err) + } + return nil +} + +func applyGatewaySuspiciousHostToPG(ctx context.Context, tx pgx.Tx, host GatewaySuspiciousHost) error { + createdAt := strings.TrimSpace(host.CreatedAt) + if createdAt == "" { + createdAt = time.Now().UTC().Format(time.RFC3339Nano) + } + _, err := tx.Exec(ctx, ` + INSERT INTO gateway_suspicious_hosts (participant_key, note, created_at) + VALUES ($1, $2, $3) + ON CONFLICT (participant_key) DO UPDATE + SET note = EXCLUDED.note, created_at = EXCLUDED.created_at`, + host.ParticipantKey, host.Note, createdAt, + ) + if err != nil { + return fmt.Errorf("apply suspicious host %s to postgres: %w", host.ParticipantKey, err) + } + return nil +} + +func applyGatewayCommitmentToPG(ctx context.Context, tx pgx.Tx, c GatewayEscrowCommitment) error { + createdAt := strings.TrimSpace(c.CreatedAt) + if createdAt == "" { + createdAt = time.Now().UTC().Format(time.RFC3339Nano) + } + _, err := tx.Exec(ctx, ` + INSERT INTO escrow_rotation_commitments ( + tx_hash, model, role, epoch, private_key_env, block_height, created_at + ) VALUES ($1, $2, $3, $4, $5, $6, $7) + ON CONFLICT (tx_hash) DO UPDATE SET + model = EXCLUDED.model, + role = EXCLUDED.role, + epoch = EXCLUDED.epoch, + private_key_env = EXCLUDED.private_key_env, + block_height = EXCLUDED.block_height, + created_at = EXCLUDED.created_at`, + strings.TrimSpace(c.TxHash), + strings.TrimSpace(c.Model), + strings.TrimSpace(c.Role), + c.Epoch, + strings.TrimSpace(c.PrivateKeyEnv), + c.BlockHeight, + createdAt, + ) + if err != nil { + return fmt.Errorf("apply escrow commitment tx=%s to postgres: %w", c.TxHash, err) + } + return nil +} + +func (s *SQLiteGatewayStore) loadGatewaySettingsRow() (GatewaySettings, string, bool, error) { + if s == nil || s.db == nil { + return GatewaySettings{}, "", false, nil + } + row := s.db.QueryRow(` + SELECT ` + gatewaySettingsSelectColumns() + ` + FROM gateway_settings + WHERE id = 1`) + settings, err := scanGatewaySettings(row) + if err == sql.ErrNoRows { + return GatewaySettings{}, "", false, nil + } + if err != nil { + return GatewaySettings{}, "", false, fmt.Errorf("load gateway settings row: %w", err) + } + var updatedAt string + if err := s.db.QueryRow(`SELECT updated_at FROM gateway_settings WHERE id = 1`).Scan(&updatedAt); err != nil { + return GatewaySettings{}, "", false, fmt.Errorf("load gateway settings updated_at: %w", err) + } + return settings, updatedAt, true, nil +} + +func (s *SQLiteGatewayStore) loadGatewayDevshardRow(id string) (GatewayDevshardState, bool, error) { + if s == nil || s.db == nil { + return GatewayDevshardState{}, false, nil + } + var devshard GatewayDevshardState + var active, settlementPending int + err := s.db.QueryRow(` + SELECT id, private_key_hex, private_key_env, model, storage_path, active, created_at, updated_at, protocol_version, + rotation_role, rotation_epoch, settlement_pending + FROM gateway_devshards + WHERE id = ?`, strings.TrimSpace(id)).Scan( + &devshard.ID, &devshard.PrivateKeyHex, &devshard.PrivateKeyEnv, &devshard.Model, &devshard.StoragePath, + &active, &devshard.CreatedAt, &devshard.UpdatedAt, &devshard.ProtocolVersion, + &devshard.RotationRole, &devshard.RotationEpoch, &settlementPending, + ) + if err == sql.ErrNoRows { + return GatewayDevshardState{}, false, nil + } + if err != nil { + return GatewayDevshardState{}, false, fmt.Errorf("load gateway devshard %s: %w", id, err) + } + devshard.Active = active != 0 + devshard.SettlementPending = settlementPending != 0 + return devshard, true, nil +} + +func (s *SQLiteGatewayStore) loadParticipantThrottleRow(key string) (ParticipantThrottleRow, bool, error) { + if s == nil || s.db == nil { + return ParticipantThrottleRow{}, false, nil + } + var row ParticipantThrottleRow + var lastRefillStr, quarantineStr, modelIDsStr string + err := s.db.QueryRow(` + SELECT participant_key, tokens, last_refill_at, last_throttle_status, + IFNULL(quarantine_until_utc, '') AS quarantine_until_utc, + IFNULL(failure_strikes, 0) AS failure_strikes, + IFNULL(model_ids, '') AS model_ids + FROM participant_throttle_state + WHERE participant_key = ?`, key).Scan( + &row.Key, &row.Tokens, &lastRefillStr, &row.Status, &quarantineStr, &row.FailureStrikes, &modelIDsStr, + ) + if err == sql.ErrNoRows { + return ParticipantThrottleRow{}, false, nil + } + if err != nil { + return ParticipantThrottleRow{}, false, fmt.Errorf("load participant throttle %s: %w", key, err) + } + row.ModelIDs = splitModelIDs(modelIDsStr) + row.LastRefillAt, err = time.Parse(time.RFC3339Nano, lastRefillStr) + if err != nil { + return ParticipantThrottleRow{}, false, fmt.Errorf("parse last_refill_at for %s: %w", key, err) + } + if strings.TrimSpace(quarantineStr) != "" { + row.QuarantineUntil, err = time.Parse(time.RFC3339Nano, quarantineStr) + if err != nil { + return ParticipantThrottleRow{}, false, fmt.Errorf("parse quarantine_until_utc for %s: %w", key, err) + } + } + return row, true, nil +} + +func (s *SQLiteGatewayStore) loadGatewayRotationStatusRow(modelID, stage string, epoch uint64) (GatewayRotationStatus, bool, error) { + if s == nil || s.db == nil { + return GatewayRotationStatus{}, false, nil + } + var status GatewayRotationStatus + var completed int + err := s.db.QueryRow(` + SELECT model_id, stage, epoch, role, target_count, existing_count, created_count, + promoted_count, settled_count, settle_failed_count, create_error, completed, updated_at + FROM gateway_rotation_status + WHERE model_id = ? AND stage = ? AND epoch = ?`, + strings.TrimSpace(modelID), strings.TrimSpace(stage), epoch, + ).Scan( + &status.ModelID, &status.Stage, &status.Epoch, &status.Role, + &status.TargetCount, &status.ExistingCount, &status.CreatedCount, + &status.PromotedCount, &status.SettledCount, &status.SettleFailedCount, + &status.CreateError, &completed, &status.UpdatedAt, + ) + if err == sql.ErrNoRows { + return GatewayRotationStatus{}, false, nil + } + if err != nil { + return GatewayRotationStatus{}, false, fmt.Errorf("load rotation status: %w", err) + } + status.Completed = completed != 0 + return status, true, nil +} + +func (s *SQLiteGatewayStore) loadGatewaySuspiciousHostRow(participantKey string) (GatewaySuspiciousHost, bool, error) { + if s == nil || s.db == nil { + return GatewaySuspiciousHost{}, false, nil + } + var host GatewaySuspiciousHost + err := s.db.QueryRow(` + SELECT participant_key, note, created_at + FROM gateway_suspicious_hosts + WHERE participant_key = ?`, strings.TrimSpace(participantKey), + ).Scan(&host.ParticipantKey, &host.Note, &host.CreatedAt) + if err == sql.ErrNoRows { + return GatewaySuspiciousHost{}, false, nil + } + if err != nil { + return GatewaySuspiciousHost{}, false, fmt.Errorf("load suspicious host %s: %w", participantKey, err) + } + return host, true, nil +} + +func (s *SQLiteGatewayStore) loadGatewayCommitmentRow(txHash string) (GatewayEscrowCommitment, bool, error) { + if s == nil || s.db == nil { + return GatewayEscrowCommitment{}, false, nil + } + var c GatewayEscrowCommitment + err := s.db.QueryRow(` + SELECT tx_hash, model, role, epoch, private_key_env, block_height, created_at + FROM escrow_rotation_commitments + WHERE tx_hash = ?`, strings.TrimSpace(txHash), + ).Scan(&c.TxHash, &c.Model, &c.Role, &c.Epoch, &c.PrivateKeyEnv, &c.BlockHeight, &c.CreatedAt) + if err == sql.ErrNoRows { + return GatewayEscrowCommitment{}, false, nil + } + if err != nil { + return GatewayEscrowCommitment{}, false, fmt.Errorf("load commitment %s: %w", txHash, err) + } + return c, true, nil +} + +func (s *SQLiteGatewayStore) updateSettingsTx(tx *sql.Tx, settings GatewaySettings) error { + updatedAt := time.Now().UTC().Format(time.RFC3339Nano) + res, err := tx.Exec(fmt.Sprintf(` + UPDATE gateway_settings + SET %s, + updated_at = ? + WHERE id = 1`, gatewaySettingsUpdateAssignments("?")), + settingsUpdateArgs(settings, updatedAt)..., + ) + if err != nil { + return fmt.Errorf("update gateway settings: %w", err) + } + n, err := res.RowsAffected() + if err != nil { + return fmt.Errorf("rows affected for gateway settings update: %w", err) + } + if n == 0 { + return fmt.Errorf("gateway settings not initialized") + } + return nil +} + +func (s *SQLiteGatewayStore) saveRotationStatusTx(tx *sql.Tx, status GatewayRotationStatus) error { + now := time.Now().UTC().Format(time.RFC3339Nano) + if strings.TrimSpace(status.UpdatedAt) != "" { + now = strings.TrimSpace(status.UpdatedAt) + } + _, err := tx.Exec(` + INSERT OR REPLACE INTO gateway_rotation_status ( + model_id, stage, epoch, role, target_count, existing_count, created_count, + promoted_count, settled_count, settle_failed_count, create_error, completed, updated_at + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, + strings.TrimSpace(status.ModelID), strings.TrimSpace(status.Stage), status.Epoch, + strings.TrimSpace(status.Role), status.TargetCount, status.ExistingCount, status.CreatedCount, + status.PromotedCount, status.SettledCount, status.SettleFailedCount, + strings.TrimSpace(status.CreateError), gatewayBoolToInt(status.Completed), now, + ) + return err +} + +func (s *SQLiteGatewayStore) saveCommitmentTx(tx *sql.Tx, c GatewayEscrowCommitment) error { + createdAt := time.Now().UTC().Format(time.RFC3339Nano) + if strings.TrimSpace(c.CreatedAt) != "" { + createdAt = strings.TrimSpace(c.CreatedAt) + } + _, err := tx.Exec(` + INSERT OR REPLACE INTO escrow_rotation_commitments ( + tx_hash, model, role, epoch, private_key_env, block_height, created_at + ) VALUES (?, ?, ?, ?, ?, ?, ?)`, + strings.TrimSpace(c.TxHash), strings.TrimSpace(c.Model), strings.TrimSpace(c.Role), + c.Epoch, strings.TrimSpace(c.PrivateKeyEnv), c.BlockHeight, createdAt, + ) + return err +} + +func (s *SQLiteGatewayStore) deleteCommitmentTx(tx *sql.Tx, txHash string) error { + _, err := tx.Exec(`DELETE FROM escrow_rotation_commitments WHERE tx_hash = ?`, strings.TrimSpace(txHash)) + return err +} + +func (s *SQLiteGatewayStore) upsertSuspiciousHostsTx(tx *sql.Tx, participantKeys []string, note string) error { + participantKeys = normalizeParticipantKeys(participantKeys) + if len(participantKeys) == 0 { + return fmt.Errorf("participant_keys must contain at least one key") + } + now := time.Now().UTC().Format(time.RFC3339Nano) + note = strings.TrimSpace(note) + for _, key := range participantKeys { + if _, err := tx.Exec(` + INSERT INTO gateway_suspicious_hosts (participant_key, note, created_at) + VALUES (?, ?, ?) + ON CONFLICT(participant_key) DO UPDATE SET note = excluded.note`, + key, note, now); err != nil { + return fmt.Errorf("upsert suspicious host %s: %w", key, err) + } + } + return nil +} + +func (s *SQLiteGatewayStore) deleteSuspiciousHostsTx(tx *sql.Tx, participantKeys []string) error { + for _, key := range normalizeParticipantKeys(participantKeys) { + if _, err := tx.Exec(`DELETE FROM gateway_suspicious_hosts WHERE participant_key = ?`, key); err != nil { + return fmt.Errorf("delete suspicious host %s: %w", key, err) + } + } + return nil +} + +func (s *SQLiteGatewayStore) setDevshardActiveTx(tx *sql.Tx, id string, active bool) error { + res, err := tx.Exec(`UPDATE gateway_devshards SET active = ?, updated_at = ? WHERE id = ?`, + gatewayBoolToInt(active), time.Now().UTC().Format(time.RFC3339Nano), strings.TrimSpace(id)) + if err != nil { + return err + } + n, _ := res.RowsAffected() + if n == 0 { + return fmt.Errorf("devshard %s not found", id) + } + return nil +} + +func (s *SQLiteGatewayStore) setDevshardSettlementPendingTx(tx *sql.Tx, id string, pending bool) error { + res, err := tx.Exec(`UPDATE gateway_devshards SET settlement_pending = ?, updated_at = ? WHERE id = ?`, + gatewayBoolToInt(pending), time.Now().UTC().Format(time.RFC3339Nano), strings.TrimSpace(id)) + if err != nil { + return err + } + n, _ := res.RowsAffected() + if n == 0 { + return fmt.Errorf("devshard %s not found", id) + } + return nil +} + +func (s *SQLiteGatewayStore) deleteDevshardTx(tx *sql.Tx, id string) error { + res, err := tx.Exec(`DELETE FROM gateway_devshards WHERE id = ?`, strings.TrimSpace(id)) + if err != nil { + return err + } + n, _ := res.RowsAffected() + if n == 0 { + return fmt.Errorf("devshard %s not found", id) + } + return nil +} + +func (s *SQLiteGatewayStore) saveParticipantThrottleTx(tx *sql.Tx, key string, modelIDs []string, tokens float64, lastRefillAt time.Time, status int, quarantineUntil time.Time, failureStrikes int) error { + quarStr := "" + if !quarantineUntil.IsZero() { + quarStr = quarantineUntil.UTC().Format(time.RFC3339Nano) + } + _, err := tx.Exec(` + INSERT OR REPLACE INTO participant_throttle_state + (participant_key, tokens, last_refill_at, last_throttle_status, quarantine_until_utc, failure_strikes, model_ids) + VALUES (?, ?, ?, ?, ?, ?, ?)`, + key, tokens, lastRefillAt.UTC().Format(time.RFC3339Nano), status, quarStr, failureStrikes, + strings.Join(normalizeModelIDs(modelIDs), ","), + ) + return err +} + +func (s *SQLiteGatewayStore) deleteParticipantThrottleTx(tx *sql.Tx, key string) error { + _, err := tx.Exec(`DELETE FROM participant_throttle_state WHERE participant_key = ?`, key) + return err +} + +func (s *SQLiteGatewayStore) initializeFallbackTx(tx *sql.Tx, settings GatewaySettings, devshards []GatewayDevshardState) error { + settings = settings.WithTuningDefaults() + now := time.Now().UTC().Format(time.RFC3339Nano) + var count int + if err := tx.QueryRow(`SELECT COUNT(*) FROM gateway_settings WHERE id = 1`).Scan(&count); err != nil { + return err + } + if count == 0 { + if _, err := tx.Exec(fmt.Sprintf(`INSERT INTO gateway_settings (%s) VALUES (%s)`, + gatewaySettingsInsertColumnNames(), + sqlitePlaceholderList(len(settingsInsertArgs(settings, now))), + ), settingsInsertArgs(settings, now)...); err != nil { + return err + } + } + for _, devshard := range devshards { + if err := s.upsertDevshardTx(tx, devshard, now); err != nil { + return err + } + } + return nil +} diff --git a/devshard/cmd/devshardctl/gateway_store_sync_journal_test.go b/devshard/cmd/devshardctl/gateway_store_sync_journal_test.go new file mode 100644 index 0000000000..c882c4d5dc --- /dev/null +++ b/devshard/cmd/devshardctl/gateway_store_sync_journal_test.go @@ -0,0 +1,449 @@ +package main + +import ( + "context" + "database/sql" + "fmt" + "path/filepath" + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +func TestCoalesceSyncJournalRows(t *testing.T) { + rows := []gatewaySyncJournalRow{ + {seq: 1, tableName: gatewayTableDevshards, rowKey: "a", op: gatewaySyncOpUpsert}, + {seq: 2, tableName: gatewayTableDevshards, rowKey: "a", op: gatewaySyncOpUpsert}, + {seq: 3, tableName: gatewayTableDevshards, rowKey: "a", op: gatewaySyncOpDelete}, + {seq: 4, tableName: gatewayTableDevshards, rowKey: "b", op: gatewaySyncOpDelete}, + {seq: 5, tableName: gatewayTableDevshards, rowKey: "b", op: gatewaySyncOpUpsert}, + } + coalesced := coalesceSyncJournalRows(rows) + require.Equal(t, gatewaySyncOpDelete, coalesced[gatewaySyncKey{tableName: gatewayTableDevshards, rowKey: "a"}]) + require.Equal(t, gatewaySyncOpUpsert, coalesced[gatewaySyncKey{tableName: gatewayTableDevshards, rowKey: "b"}]) +} + +func TestHybridSyncJournalOnFallback(t *testing.T) { + sqlite := newTestSQLiteGatewayStoreOnly(t) + settings := GatewaySettings{ + ChainREST: "http://node:1317", + PublicAPI: "http://api:9000", + DefaultModel: "Qwen/Test", + DefaultRequestMaxTokens: 1000, + MaxConcurrentRequests: 2, + MaxInputTokensInFlight: 200, + }.WithTuningDefaults() + require.NoError(t, sqlite.Initialize(settings, nil)) + + hybrid := NewHybridGatewayStore(nil, sqlite, timeHour, gatewayPGConnectTimeout) + hybrid.connectPG = func(context.Context) (*PostgresGatewayStore, error) { + return nil, errGatewayStoreUnavailable + } + t.Cleanup(func() { require.NoError(t, hybrid.Close()) }) + + updated := settings + updated.DefaultRequestMaxTokens = 2222 + require.NoError(t, hybrid.UpdateSettings(updated)) + + count, err := sqlite.countSyncJournalEntries() + require.NoError(t, err) + require.Equal(t, 1, count) + + rows, err := sqlite.loadSyncJournalUpTo(10) + require.NoError(t, err) + require.Len(t, rows, 1) + require.Equal(t, gatewayTableSettings, rows[0].tableName) + require.Equal(t, gatewaySettingsRowKey, rows[0].rowKey) + require.Equal(t, gatewaySyncOpUpsert, rows[0].op) +} + +func TestHybridSyncJournalAtomicWrite(t *testing.T) { + sqlite := newTestSQLiteGatewayStoreOnly(t) + settings := GatewaySettings{ + ChainREST: "http://node:1317", + PublicAPI: "http://api:9000", + DefaultModel: "Qwen/Test", + DefaultRequestMaxTokens: 1000, + MaxConcurrentRequests: 2, + MaxInputTokensInFlight: 200, + }.WithTuningDefaults() + require.NoError(t, sqlite.Initialize(settings, nil)) + + err := sqlite.writeWithSyncJournal( + []gatewaySyncEntry{{tableName: gatewayTableSettings, rowKey: gatewaySettingsRowKey, op: gatewaySyncOpUpsert}}, + func(tx *sql.Tx) error { return fmt.Errorf("forced sqlite write failure") }, + ) + require.Error(t, err) + + count, err := sqlite.countSyncJournalEntries() + require.NoError(t, err) + require.Equal(t, 0, count) +} + +func TestHybridSyncJournalKillSwitch(t *testing.T) { + t.Setenv("GATEWAY_PG_SYNC_JOURNAL", "false") + + sqlite := newTestSQLiteGatewayStoreOnly(t) + settings := GatewaySettings{ + ChainREST: "http://node:1317", + PublicAPI: "http://api:9000", + DefaultModel: "Qwen/Test", + DefaultRequestMaxTokens: 1000, + MaxConcurrentRequests: 2, + MaxInputTokensInFlight: 200, + }.WithTuningDefaults() + require.NoError(t, sqlite.Initialize(settings, nil)) + + hybrid := NewHybridGatewayStore(nil, sqlite, timeHour, gatewayPGConnectTimeout) + hybrid.connectPG = func(context.Context) (*PostgresGatewayStore, error) { + return nil, errGatewayStoreUnavailable + } + t.Cleanup(func() { require.NoError(t, hybrid.Close()) }) + + updated := settings + updated.DefaultRequestMaxTokens = 3333 + require.NoError(t, hybrid.UpdateSettings(updated)) + + count, err := sqlite.countSyncJournalEntries() + require.NoError(t, err) + require.Equal(t, 0, count) +} + +func TestSyncJournalRestartPersistence(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "gateway.db") + + sqlite, err := NewSQLiteGatewayStore(path) + require.NoError(t, err) + settings := GatewaySettings{ + ChainREST: "http://node:1317", + PublicAPI: "http://api:9000", + DefaultModel: "Qwen/Test", + DefaultRequestMaxTokens: 1000, + MaxConcurrentRequests: 1, + MaxInputTokensInFlight: 100, + }.WithTuningDefaults() + require.NoError(t, sqlite.Initialize(settings, nil)) + + hybrid := NewHybridGatewayStore(nil, sqlite, timeHour, gatewayPGConnectTimeout) + hybrid.connectPG = func(context.Context) (*PostgresGatewayStore, error) { + return nil, errGatewayStoreUnavailable + } + require.NoError(t, hybrid.UpdateSettings(settings)) + require.NoError(t, hybrid.Close()) + require.NoError(t, sqlite.Close()) + + reopened, err := NewSQLiteGatewayStore(path) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, reopened.Close()) }) + + count, err := reopened.countSyncJournalEntries() + require.NoError(t, err) + require.Equal(t, 1, count) +} + +func TestSyncJournalDrainRestoresOutageDeltas(t *testing.T) { + cleanup := setupPostgresContainer(t) + t.Cleanup(cleanup) + + ctx := context.Background() + pg, err := NewPostgresGatewayStore(ctx) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, pg.Close()) }) + + sqlite := newTestSQLiteGatewayStoreOnly(t) + base := GatewaySettings{ + ChainREST: "http://node:1317", + PublicAPI: "http://api:9000", + DefaultModel: "Qwen/Test", + DefaultRequestMaxTokens: 1000, + MaxConcurrentRequests: 2, + MaxInputTokensInFlight: 200, + }.WithTuningDefaults() + require.NoError(t, pg.Initialize(base, []GatewayDevshardState{{ + RuntimeConfig: RuntimeConfig{ID: "keep-me", Model: "Qwen/Test", StoragePath: "/data/keep"}, + Active: true, + }})) + + hybrid := NewHybridGatewayStore(nil, sqlite, timeHour, gatewayPGConnectTimeout) + hybrid.connectPG = func(context.Context) (*PostgresGatewayStore, error) { + return nil, errGatewayStoreUnavailable + } + require.NoError(t, sqlite.Initialize(base, []GatewayDevshardState{{ + RuntimeConfig: RuntimeConfig{ID: "keep-me", Model: "Qwen/Test", StoragePath: "/data/keep"}, + Active: true, + }, { + RuntimeConfig: RuntimeConfig{ID: "outage-only", Model: "Kimi/Rotate", StoragePath: "/data/outage"}, + Active: true, + }})) + outageSettings := base + outageSettings.DefaultRequestMaxTokens = 5555 + require.NoError(t, hybrid.UpdateSettings(outageSettings)) + require.NoError(t, hybrid.UpsertDevshard(GatewayDevshardState{ + RuntimeConfig: RuntimeConfig{ID: "outage-only", Model: "Kimi/Rotate", StoragePath: "/data/outage-new"}, + Active: true, + })) + require.NoError(t, hybrid.DeleteDevshard("keep-me")) + + drainHybrid := NewHybridGatewayStore(nil, sqlite, timeHour, gatewayPGConnectTimeout) + require.NoError(t, drainHybrid.ReconcileSyncJournal(ctx, pg)) + + pgState, has, err := pg.LoadState() + require.NoError(t, err) + require.True(t, has) + require.EqualValues(t, 5555, pgState.Settings.DefaultRequestMaxTokens) + require.Len(t, pgState.Devshards, 1) + require.Equal(t, "outage-only", pgState.Devshards[0].ID) + require.Equal(t, "/data/outage-new", pgState.Devshards[0].StoragePath) + + count, err := sqlite.countSyncJournalEntries() + require.NoError(t, err) + require.Equal(t, 0, count) +} + +func TestSyncJournalDrainNoClobberPGOnlyRows(t *testing.T) { + cleanup := setupPostgresContainer(t) + t.Cleanup(cleanup) + + ctx := context.Background() + pg, err := NewPostgresGatewayStore(ctx) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, pg.Close()) }) + + sqlite := newTestSQLiteGatewayStoreOnly(t) + pgOnly := GatewaySettings{ + ChainREST: "http://pg-only:1317", + PublicAPI: "http://pg-only:9000", + DefaultModel: "PG/Only", + DefaultRequestMaxTokens: 9999, + MaxConcurrentRequests: 9, + MaxInputTokensInFlight: 900, + }.WithTuningDefaults() + require.NoError(t, pg.Initialize(pgOnly, nil)) + + staleSQLite := GatewaySettings{ + ChainREST: "http://stale:1317", + PublicAPI: "http://stale:9000", + DefaultModel: "Stale/Model", + DefaultRequestMaxTokens: 1, + MaxConcurrentRequests: 1, + MaxInputTokensInFlight: 1, + }.WithTuningDefaults() + require.NoError(t, sqlite.Initialize(staleSQLite, nil)) + + hybrid := NewHybridGatewayStore(nil, sqlite, timeHour, gatewayPGConnectTimeout) + hybrid.connectPG = func(context.Context) (*PostgresGatewayStore, error) { + return nil, errGatewayStoreUnavailable + } + require.NoError(t, hybrid.SaveRotationStatus(GatewayRotationStatus{ + ModelID: "Qwen/Test", Stage: "prepare_temp", Epoch: 3, Completed: true, + })) + + drainHybrid := NewHybridGatewayStore(nil, sqlite, timeHour, gatewayPGConnectTimeout) + require.NoError(t, drainHybrid.ReconcileSyncJournal(ctx, pg)) + + pgState, has, err := pg.LoadState() + require.NoError(t, err) + require.True(t, has) + require.Equal(t, pgOnly, pgState.Settings) + + statuses, err := pg.LoadRotationStatuses(0) + require.NoError(t, err) + require.Len(t, statuses, 1) + require.Equal(t, "prepare_temp", statuses[0].Stage) +} + +func TestSyncJournalDrainIdempotent(t *testing.T) { + cleanup := setupPostgresContainer(t) + t.Cleanup(cleanup) + + ctx := context.Background() + pg, err := NewPostgresGatewayStore(ctx) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, pg.Close()) }) + + sqlite := newTestSQLiteGatewayStoreOnly(t) + settings := GatewaySettings{ + ChainREST: "http://node:1317", + PublicAPI: "http://api:9000", + DefaultModel: "Qwen/Test", + DefaultRequestMaxTokens: 1000, + MaxConcurrentRequests: 1, + MaxInputTokensInFlight: 100, + }.WithTuningDefaults() + require.NoError(t, sqlite.Initialize(settings, nil)) + + hybrid := NewHybridGatewayStore(nil, sqlite, timeHour, gatewayPGConnectTimeout) + hybrid.connectPG = func(context.Context) (*PostgresGatewayStore, error) { + return nil, errGatewayStoreUnavailable + } + require.NoError(t, hybrid.UpdateSettings(settings)) + + drainHybrid := NewHybridGatewayStore(nil, sqlite, timeHour, gatewayPGConnectTimeout) + require.NoError(t, drainHybrid.ReconcileSyncJournal(ctx, pg)) + require.NoError(t, drainHybrid.ReconcileSyncJournal(ctx, pg)) + + count, err := sqlite.countSyncJournalEntries() + require.NoError(t, err) + require.Equal(t, 0, count) +} + +// TestHybridSuspiciousHostsFallbackNoDeadlock guards against reading the +// suspicious hosts via a fresh s.db query inside writeWithSyncJournal's open +// transaction. SQLite is capped at one connection, so that would deadlock. +func TestHybridSuspiciousHostsFallbackNoDeadlock(t *testing.T) { + sqlite := newTestSQLiteGatewayStoreOnly(t) + settings := GatewaySettings{ + ChainREST: "http://node:1317", + PublicAPI: "http://api:9000", + DefaultModel: "Qwen/Test", + DefaultRequestMaxTokens: 1000, + MaxConcurrentRequests: 1, + MaxInputTokensInFlight: 100, + }.WithTuningDefaults() + require.NoError(t, sqlite.Initialize(settings, nil)) + + hybrid := NewHybridGatewayStore(nil, sqlite, timeHour, gatewayPGConnectTimeout) + hybrid.connectPG = func(context.Context) (*PostgresGatewayStore, error) { + return nil, errGatewayStoreUnavailable + } + t.Cleanup(func() { require.NoError(t, hybrid.Close()) }) + + runWithTimeout(t, "UpsertSuspiciousHosts", func() { + hosts, err := hybrid.UpsertSuspiciousHosts([]string{"pk1", "pk2"}, "flagged") + require.NoError(t, err) + require.Len(t, hosts, 2) + }) + + count, err := sqlite.countSyncJournalEntries() + require.NoError(t, err) + require.Equal(t, 2, count) + + runWithTimeout(t, "DeleteSuspiciousHosts", func() { + hosts, err := hybrid.DeleteSuspiciousHosts([]string{"pk1"}) + require.NoError(t, err) + require.Len(t, hosts, 1) + }) +} + +func runWithTimeout(t *testing.T, name string, fn func()) { + t.Helper() + done := make(chan struct{}) + go func() { + defer close(done) + fn() + }() + select { + case <-done: + case <-time.After(10 * time.Second): + t.Fatalf("%s deadlocked in SQLite fallback", name) + } +} + +// TestHybridDropPgClearsCachedConnection verifies a failed Postgres write drops +// the cached connection so subsequent writes journal and the next reconnect +// re-drains the journal before republishing Postgres as primary. +func TestHybridDropPgClearsCachedConnection(t *testing.T) { + sqlite := newTestSQLiteGatewayStoreOnly(t) + hybrid := NewHybridGatewayStore(nil, sqlite, timeHour, gatewayPGConnectTimeout) + t.Cleanup(func() { require.NoError(t, hybrid.Close()) }) + + pg := &PostgresGatewayStore{} + hybrid.mu.Lock() + hybrid.pg = pg + hybrid.mu.Unlock() + + // Dropping a stale pointer must not clear a newer connection. + hybrid.dropPg(&PostgresGatewayStore{}) + require.Equal(t, pg, hybrid.currentPg()) + + // Dropping the live connection clears it and is idempotent. + hybrid.dropPg(pg) + require.Nil(t, hybrid.currentPg()) + hybrid.dropPg(pg) + require.Nil(t, hybrid.currentPg()) +} + +// TestHybridPGWriteRetrySurvivesBlip verifies a transient Postgres failure is +// retried within the budget and succeeds without dropping to SQLite. +func TestHybridPGWriteRetrySurvivesBlip(t *testing.T) { + prev := gatewayPGWriteRetryBaseBackoff + gatewayPGWriteRetryBaseBackoff = time.Millisecond + t.Cleanup(func() { gatewayPGWriteRetryBaseBackoff = prev }) + + hybrid := NewHybridGatewayStore(nil, nil, timeHour, gatewayPGConnectTimeout) + hybrid.writeRetryBudget = time.Second + + pg := &PostgresGatewayStore{} + attempts := 0 + err := hybrid.attemptPGWrite(context.Background(), pg, func(*PostgresGatewayStore) error { + attempts++ + if attempts < 3 { + return fmt.Errorf("blip") + } + return nil + }) + require.NoError(t, err) + require.Equal(t, 3, attempts) +} + +// TestHybridPGWriteRetryBudgetDisabled verifies a zero budget makes a single +// attempt with no retry. +func TestHybridPGWriteRetryBudgetDisabled(t *testing.T) { + hybrid := NewHybridGatewayStore(nil, nil, timeHour, gatewayPGConnectTimeout) + hybrid.writeRetryBudget = 0 + + pg := &PostgresGatewayStore{} + attempts := 0 + err := hybrid.attemptPGWrite(context.Background(), pg, func(*PostgresGatewayStore) error { + attempts++ + return fmt.Errorf("down") + }) + require.Error(t, err) + require.Equal(t, 1, attempts) +} + +// TestHybridWriteFallsBackAfterRetryBudget verifies that once the retry budget +// is exhausted the write drops Postgres, routes to SQLite, and journals. +func TestHybridWriteFallsBackAfterRetryBudget(t *testing.T) { + sqlite := newTestSQLiteGatewayStoreOnly(t) + settings := GatewaySettings{ + ChainREST: "http://node:1317", + PublicAPI: "http://api:9000", + DefaultModel: "Qwen/Test", + DefaultRequestMaxTokens: 1000, + MaxConcurrentRequests: 1, + MaxInputTokensInFlight: 100, + }.WithTuningDefaults() + require.NoError(t, sqlite.Initialize(settings, nil)) + + hybrid := NewHybridGatewayStore(nil, sqlite, timeHour, gatewayPGConnectTimeout) + hybrid.writeRetryBudget = 0 + t.Cleanup(func() { require.NoError(t, hybrid.Close()) }) + + pgAttempts := 0 + failingPG := &PostgresGatewayStore{} + hybrid.mu.Lock() + hybrid.pg = failingPG + hybrid.mu.Unlock() + + err := hybrid.write(context.Background(), + func(*PostgresGatewayStore) error { + pgAttempts++ + return fmt.Errorf("pg down") + }, + func(sqlite *SQLiteGatewayStore) error { return sqlite.UpdateSettings(settings) }, + []gatewaySyncEntry{{tableName: gatewayTableSettings, rowKey: gatewaySettingsRowKey, op: gatewaySyncOpUpsert}}, + func(tx *sql.Tx) error { return sqlite.updateSettingsTx(tx, settings) }, + ) + require.NoError(t, err) + require.Equal(t, 1, pgAttempts) + require.Nil(t, hybrid.currentPg()) + + count, err := sqlite.countSyncJournalEntries() + require.NoError(t, err) + require.Equal(t, 1, count) +} + +const timeHour = time.Hour diff --git a/devshard/cmd/devshardctl/gateway_store_test.go b/devshard/cmd/devshardctl/gateway_store_test.go index 7c84bc836f..3174243101 100644 --- a/devshard/cmd/devshardctl/gateway_store_test.go +++ b/devshard/cmd/devshardctl/gateway_store_test.go @@ -8,6 +8,7 @@ import ( "net/http/httptest" "path/filepath" "testing" + "time" "github.com/stretchr/testify/require" @@ -15,7 +16,7 @@ import ( ) func TestGatewayStoreInitializeAndLoadState(t *testing.T) { - store, err := NewGatewayStore(filepath.Join(t.TempDir(), "gateway.db")) + store, err := NewSQLiteGatewayStore(filepath.Join(t.TempDir(), "gateway.db")) require.NoError(t, err) t.Cleanup(func() { require.NoError(t, store.Close()) @@ -86,7 +87,7 @@ func TestAdminAuthMiddlewareRequiresAdminKey(t *testing.T) { } func TestGatewayStoreUpdateSettings(t *testing.T) { - store, err := NewGatewayStore(filepath.Join(t.TempDir(), "gateway.db")) + store, err := NewSQLiteGatewayStore(filepath.Join(t.TempDir(), "gateway.db")) require.NoError(t, err) t.Cleanup(func() { require.NoError(t, store.Close()) @@ -189,7 +190,7 @@ func TestGatewayStoreUpdateSettings(t *testing.T) { } func TestGatewayStoreLoadsLegacyModelAccessIntoModelLimits(t *testing.T) { - store, err := NewGatewayStore(filepath.Join(t.TempDir(), "gateway.db")) + store, err := NewSQLiteGatewayStore(filepath.Join(t.TempDir(), "gateway.db")) require.NoError(t, err) t.Cleanup(func() { require.NoError(t, store.Close()) @@ -213,7 +214,7 @@ func TestGatewayStoreLoadsLegacyModelAccessIntoModelLimits(t *testing.T) { Message: "Qwen temporarily unavailable", }}) require.NoError(t, err) - _, err = store.db.Exec(`UPDATE gateway_settings SET model_access_json = ? WHERE id = 1`, string(legacyAccess)) + _, err = requireSQLiteGatewayStore(t, store).db.Exec(`UPDATE gateway_settings SET model_access_json = ? WHERE id = 1`, string(legacyAccess)) require.NoError(t, err) state, ok, err := store.LoadState() @@ -228,7 +229,7 @@ func TestGatewayStoreLoadsLegacyModelAccessIntoModelLimits(t *testing.T) { } func TestGatewayStorePersistsSuspiciousHosts(t *testing.T) { - store, err := NewGatewayStore(filepath.Join(t.TempDir(), "gateway.db")) + store, err := NewSQLiteGatewayStore(filepath.Join(t.TempDir(), "gateway.db")) require.NoError(t, err) t.Cleanup(func() { require.NoError(t, store.Close()) @@ -306,7 +307,7 @@ func TestValidateGatewaySettingsRequiresRotationModels(t *testing.T) { } func TestEscrowRotationPreparePromotesRegularEscrowsOnTempCreateFailure(t *testing.T) { - store, err := NewGatewayStore(filepath.Join(t.TempDir(), "gateway.db")) + store, err := NewSQLiteGatewayStore(filepath.Join(t.TempDir(), "gateway.db")) require.NoError(t, err) t.Cleanup(func() { require.NoError(t, store.Close()) @@ -391,7 +392,7 @@ func TestNewRotationDevshardStatePersistsProtocolV2(t *testing.T) { } func TestEscrowRotationFinishDoesNotSettleTempWhenRegularCreateFails(t *testing.T) { - store, err := NewGatewayStore(filepath.Join(t.TempDir(), "gateway.db")) + store, err := NewSQLiteGatewayStore(filepath.Join(t.TempDir(), "gateway.db")) require.NoError(t, err) t.Cleanup(func() { require.NoError(t, store.Close()) @@ -448,7 +449,7 @@ func TestEscrowRotationFinishDoesNotSettleTempWhenRegularCreateFails(t *testing. } func TestEscrowRotationSkipsCreateWhenModelAbsentFromNetwork(t *testing.T) { - store, err := NewGatewayStore(filepath.Join(t.TempDir(), "gateway.db")) + store, err := NewSQLiteGatewayStore(filepath.Join(t.TempDir(), "gateway.db")) require.NoError(t, err) t.Cleanup(func() { require.NoError(t, store.Close()) @@ -517,7 +518,7 @@ func TestEscrowRotationSkipsCreateWhenModelAbsentFromNetwork(t *testing.T) { } func TestEscrowRotationCreatesWhenModelPresentInNetwork(t *testing.T) { - store, err := NewGatewayStore(filepath.Join(t.TempDir(), "gateway.db")) + store, err := NewSQLiteGatewayStore(filepath.Join(t.TempDir(), "gateway.db")) require.NoError(t, err) t.Cleanup(func() { require.NoError(t, store.Close()) @@ -578,7 +579,7 @@ func TestEscrowRotationCreatesWhenModelPresentInNetwork(t *testing.T) { } func TestEscrowRotationFinishSettlesTempFromCurrentLatestEpoch(t *testing.T) { - store, err := NewGatewayStore(filepath.Join(t.TempDir(), "gateway.db")) + store, err := NewSQLiteGatewayStore(filepath.Join(t.TempDir(), "gateway.db")) require.NoError(t, err) t.Cleanup(func() { require.NoError(t, store.Close()) @@ -641,7 +642,7 @@ func TestEscrowRotationFinishSettlesTempFromCurrentLatestEpoch(t *testing.T) { } func TestEscrowRotationPrepareDeactivatesRegularWithoutSettlementWhenSettlementDisabled(t *testing.T) { - store, err := NewGatewayStore(filepath.Join(t.TempDir(), "gateway.db")) + store, err := NewSQLiteGatewayStore(filepath.Join(t.TempDir(), "gateway.db")) require.NoError(t, err) t.Cleanup(func() { require.NoError(t, store.Close()) @@ -709,7 +710,7 @@ func TestEscrowRotationPrepareDeactivatesRegularWithoutSettlementWhenSettlementD } func TestEscrowRotationFinishDeactivatesTempWithoutSettlementWhenSettlementDisabled(t *testing.T) { - store, err := NewGatewayStore(filepath.Join(t.TempDir(), "gateway.db")) + store, err := NewSQLiteGatewayStore(filepath.Join(t.TempDir(), "gateway.db")) require.NoError(t, err) t.Cleanup(func() { require.NoError(t, store.Close()) @@ -777,7 +778,7 @@ func TestEscrowRotationFinishDeactivatesTempWithoutSettlementWhenSettlementDisab } func TestEscrowRotationPrepareRotatesModelsIndependently(t *testing.T) { - store, err := NewGatewayStore(filepath.Join(t.TempDir(), "gateway.db")) + store, err := NewSQLiteGatewayStore(filepath.Join(t.TempDir(), "gateway.db")) require.NoError(t, err) t.Cleanup(func() { require.NoError(t, store.Close()) @@ -850,7 +851,7 @@ func TestEscrowRotationPrepareRotatesModelsIndependently(t *testing.T) { } func TestEscrowRotationUsesEpochSwitchHeightDuringPoC(t *testing.T) { - store, err := NewGatewayStore(filepath.Join(t.TempDir(), "gateway.db")) + store, err := NewSQLiteGatewayStore(filepath.Join(t.TempDir(), "gateway.db")) require.NoError(t, err) t.Cleanup(func() { require.NoError(t, store.Close()) @@ -919,7 +920,7 @@ func TestEscrowRotationUsesEpochSwitchHeightDuringPoC(t *testing.T) { } func TestGatewayStoreSetDevshardSettlementPending(t *testing.T) { - store, err := NewGatewayStore(filepath.Join(t.TempDir(), "gateway.db")) + store, err := NewSQLiteGatewayStore(filepath.Join(t.TempDir(), "gateway.db")) require.NoError(t, err) t.Cleanup(func() { require.NoError(t, store.Close()) }) @@ -968,3 +969,60 @@ func gatewayDevshardsByID(devshards []GatewayDevshardState) map[string]GatewayDe } return byID } + +func TestSQLiteGatewayStoreImplementsInterface(t *testing.T) { + var _ GatewayStore = (*SQLiteGatewayStore)(nil) +} + +func TestGatewaySettingsColumnsRoundTrip(t *testing.T) { + store, err := NewSQLiteGatewayStore(filepath.Join(t.TempDir(), "gateway.db")) + require.NoError(t, err) + t.Cleanup(func() { + require.NoError(t, store.Close()) + }) + + settings := GatewaySettings{ + ChainREST: "http://node:1317", + PublicAPI: "http://api:9000", + DefaultModel: "Qwen/Test", + DefaultRequestMaxTokens: 2048, + RequestMaxTokensCap: 4096, + MaxConcurrentRequests: 8, + MaxInputTokensInFlight: 1200, + TxGasLimit: 500000, + ModelLimits: []GatewayModelLimitSettings{{ + ModelID: "Qwen/Test", + AccessMode: string(gatewayAccessModeOpen), + }}, + EscrowRotation: EscrowRotationSettings{ + Enabled: true, + SettlementEnabled: true, + PrePoCBlocks: 400, + Models: []EscrowRotationModelSettings{{ + ModelID: "Qwen/Test", + TargetCount: 3, + Amount: 1000, + }}, + }, + Disabled: GatewayDisabledSettings{ + Enabled: true, + Message: "maintenance", + NewURL: "https://gateway.example/new", + }, + }.WithTuningDefaults() + + now := time.Now().UTC().Format(time.RFC3339Nano) + args := settingsInsertArgs(settings, now) + _, err = store.db.Exec(fmt.Sprintf(` + INSERT INTO gateway_settings (%s) + VALUES (%s)`, + gatewaySettingsInsertColumnNames(), + sqlitePlaceholderList(len(args)), + ), args...) + require.NoError(t, err) + + row := store.db.QueryRow(`SELECT ` + gatewaySettingsSelectColumns() + ` FROM gateway_settings WHERE id = 1`) + loaded, err := scanGatewaySettings(row) + require.NoError(t, err) + require.Equal(t, settings, loaded) +} diff --git a/devshard/cmd/devshardctl/gateway_store_test_helpers_test.go b/devshard/cmd/devshardctl/gateway_store_test_helpers_test.go new file mode 100644 index 0000000000..5d4fc7db7c --- /dev/null +++ b/devshard/cmd/devshardctl/gateway_store_test_helpers_test.go @@ -0,0 +1,33 @@ +package main + +import ( + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" +) + +func newTestGatewayStore(t *testing.T, backend string) GatewayStore { + t.Helper() + switch backend { + case "sqlite": + store, err := NewSQLiteGatewayStore(filepath.Join(t.TempDir(), "gateway.db")) + require.NoError(t, err) + t.Cleanup(func() { + require.NoError(t, store.Close()) + }) + return store + case "postgres": + return newTestPostgresGatewayStore(t) + default: + t.Fatalf("unsupported test gateway backend %q", backend) + return nil + } +} + +func requireSQLiteGatewayStore(t *testing.T, store GatewayStore) *SQLiteGatewayStore { + t.Helper() + sqliteStore, ok := store.(*SQLiteGatewayStore) + require.True(t, ok, "expected *SQLiteGatewayStore") + return sqliteStore +} diff --git a/devshard/cmd/devshardctl/gateway_test.go b/devshard/cmd/devshardctl/gateway_test.go index 001b693cea..140c7ad147 100644 --- a/devshard/cmd/devshardctl/gateway_test.go +++ b/devshard/cmd/devshardctl/gateway_test.go @@ -96,7 +96,7 @@ func seedGatewayTestCapacity(g *Gateway, weights map[string]float64) { func gatewayTestDepletionGateway(t *testing.T, rt *devshardRuntime, modifySettings ...func(*GatewaySettings)) (*Gateway, *atomic.Int32, *atomic.Int32) { t.Helper() - store, err := NewGatewayStore(filepath.Join(t.TempDir(), "gateway.db")) + store, err := NewSQLiteGatewayStore(filepath.Join(t.TempDir(), "gateway.db")) require.NoError(t, err) t.Cleanup(func() { require.NoError(t, store.Close()) }) @@ -782,7 +782,7 @@ func TestNewRESTBridgeForProtocolUsesDevshardEscrowEndpointByDefault(t *testing. } func TestAdminDeactivateDevshardAllowsActiveRequestsAndStopsNewChat(t *testing.T) { - store, err := NewGatewayStore(filepath.Join(t.TempDir(), "gateway.db")) + store, err := NewSQLiteGatewayStore(filepath.Join(t.TempDir(), "gateway.db")) require.NoError(t, err) t.Cleanup(func() { require.NoError(t, store.Close()) @@ -902,7 +902,7 @@ func TestAdminDevshardParticipantsShowsQuarantineState(t *testing.T) { } func TestAdminAddDevshardWiresSharedPhaseGate(t *testing.T) { - store, err := NewGatewayStore(filepath.Join(t.TempDir(), "gateway.db")) + store, err := NewSQLiteGatewayStore(filepath.Join(t.TempDir(), "gateway.db")) require.NoError(t, err) t.Cleanup(func() { require.NoError(t, store.Close()) @@ -1007,7 +1007,7 @@ func TestBuildRuntimeRejectsExplicitProtocolV1(t *testing.T) { } func TestAdminImportDevshardLoadsInactiveRuntimeAndAccounting(t *testing.T) { - store, err := NewGatewayStore(filepath.Join(t.TempDir(), "gateway.db")) + store, err := NewSQLiteGatewayStore(filepath.Join(t.TempDir(), "gateway.db")) require.NoError(t, err) t.Cleanup(func() { require.NoError(t, store.Close()) @@ -1119,7 +1119,7 @@ func TestAdminImportDevshardLoadsInactiveRuntimeAndAccounting(t *testing.T) { } func TestGatewayHandleDevshardFinalizeRequiresNoActiveRequests(t *testing.T) { - store, err := NewGatewayStore(filepath.Join(t.TempDir(), "gateway.db")) + store, err := NewSQLiteGatewayStore(filepath.Join(t.TempDir(), "gateway.db")) require.NoError(t, err) t.Cleanup(func() { require.NoError(t, store.Close()) @@ -1172,7 +1172,7 @@ func TestGatewayHandleDevshardFinalizeRequiresNoActiveRequests(t *testing.T) { } func TestAdminSuspiciousHostsEndpointPersistsAndUpdatesRuntime(t *testing.T) { - store, err := NewGatewayStore(filepath.Join(t.TempDir(), "gateway.db")) + store, err := NewSQLiteGatewayStore(filepath.Join(t.TempDir(), "gateway.db")) require.NoError(t, err) t.Cleanup(func() { require.NoError(t, store.Close()) @@ -2265,7 +2265,7 @@ func TestParticipantRequestLimiterClearQuarantineStartsProbation(t *testing.T) { } func TestParticipantRequestLimiterPersistsThrottleState(t *testing.T) { - store, err := NewGatewayStore(filepath.Join(t.TempDir(), "gateway.db")) + store, err := NewSQLiteGatewayStore(filepath.Join(t.TempDir(), "gateway.db")) require.NoError(t, err) t.Cleanup(func() { store.Close() }) @@ -2284,7 +2284,7 @@ func TestParticipantRequestLimiterPersistsThrottleState(t *testing.T) { } func TestParticipantRequestLimiterPersistsModelScopedThrottleState(t *testing.T) { - store, err := NewGatewayStore(filepath.Join(t.TempDir(), "gateway.db")) + store, err := NewSQLiteGatewayStore(filepath.Join(t.TempDir(), "gateway.db")) require.NoError(t, err) t.Cleanup(func() { store.Close() }) @@ -2304,7 +2304,7 @@ func TestParticipantRequestLimiterPersistsModelScopedThrottleState(t *testing.T) } func TestParticipantRequestLimiterPersistsFailureStrikes(t *testing.T) { - store, err := NewGatewayStore(filepath.Join(t.TempDir(), "gateway.db")) + store, err := NewSQLiteGatewayStore(filepath.Join(t.TempDir(), "gateway.db")) require.NoError(t, err) t.Cleanup(func() { store.Close() }) @@ -2330,7 +2330,7 @@ func TestParticipantRequestLimiterLoadStateRecoversTokens(t *testing.T) { } func TestParticipantRequestLimiterLoadStateDeletesFullyRecovered(t *testing.T) { - store, err := NewGatewayStore(filepath.Join(t.TempDir(), "gateway.db")) + store, err := NewSQLiteGatewayStore(filepath.Join(t.TempDir(), "gateway.db")) require.NoError(t, err) t.Cleanup(func() { store.Close() }) @@ -2348,7 +2348,7 @@ func TestParticipantRequestLimiterLoadStateDeletesFullyRecovered(t *testing.T) { } func TestParticipantRequestLimiterPersistsProbationOnExpiry(t *testing.T) { - store, err := NewGatewayStore(filepath.Join(t.TempDir(), "gateway.db")) + store, err := NewSQLiteGatewayStore(filepath.Join(t.TempDir(), "gateway.db")) require.NoError(t, err) t.Cleanup(func() { store.Close() }) @@ -2595,7 +2595,7 @@ func writeGatewayLegacyStateDB(t *testing.T, path, escrowID string, latestNonce } func TestAdminSettingsUpdatesLimiterAndDefaultTokens(t *testing.T) { - store, err := NewGatewayStore(filepath.Join(t.TempDir(), "gateway.db")) + store, err := NewSQLiteGatewayStore(filepath.Join(t.TempDir(), "gateway.db")) require.NoError(t, err) t.Cleanup(func() { require.NoError(t, store.Close()) @@ -2685,7 +2685,7 @@ func TestAdminSettingsUpdatesLimiterAndDefaultTokens(t *testing.T) { } func TestAdminSettingsRejectsInvalidTuning(t *testing.T) { - store, err := NewGatewayStore(filepath.Join(t.TempDir(), "gateway.db")) + store, err := NewSQLiteGatewayStore(filepath.Join(t.TempDir(), "gateway.db")) require.NoError(t, err) t.Cleanup(func() { require.NoError(t, store.Close()) @@ -2720,7 +2720,7 @@ func TestAdminSettingsRejectsInvalidTuning(t *testing.T) { } func TestAdminSettingsUpdatesEscrowRotationSettlementEnabled(t *testing.T) { - store, err := NewGatewayStore(filepath.Join(t.TempDir(), "gateway.db")) + store, err := NewSQLiteGatewayStore(filepath.Join(t.TempDir(), "gateway.db")) require.NoError(t, err) t.Cleanup(func() { require.NoError(t, store.Close()) @@ -2756,7 +2756,7 @@ func TestAdminSettingsUpdatesEscrowRotationSettlementEnabled(t *testing.T) { } func TestDebugRotationReportsCountdownAndLatestStatus(t *testing.T) { - store, err := NewGatewayStore(filepath.Join(t.TempDir(), "gateway.db")) + store, err := NewSQLiteGatewayStore(filepath.Join(t.TempDir(), "gateway.db")) require.NoError(t, err) t.Cleanup(func() { require.NoError(t, store.Close()) diff --git a/devshard/cmd/devshardctl/main.go b/devshard/cmd/devshardctl/main.go index bcc7eca441..a80cee26b5 100644 --- a/devshard/cmd/devshardctl/main.go +++ b/devshard/cmd/devshardctl/main.go @@ -102,12 +102,13 @@ type bootstrapOptions struct { var gatewayRuntimeBuilder = buildRuntime func main() { + ctx := context.Background() ConfigurePoCRequestMode(os.Getenv("DEVSHARD_POC_REQUEST_MODE")) ConfigureCapacityAwareLimits(os.Getenv("DEVSHARD_CAPACITY_AWARE_LIMITS")) configureClassifyCapsFromEnv() flags := parseCLIFlags() runtimeOpts := mustLoadRuntimeOptions(flags) - gatewayStore := mustOpenGatewayStore(runtimeOpts.baseStorageDir) + gatewayStore := mustOpenGatewayStore(ctx, runtimeOpts.baseStorageDir) defer func() { if err := gatewayStore.Close(); err != nil { log.Printf("close gateway state: %v", err) @@ -245,7 +246,7 @@ func resolveBaseStorageDir(flagStorageDir, storagePath string) string { return baseStorageDir } -func mustLoadParticipantThrottleState(store *GatewayStore) { +func mustLoadParticipantThrottleState(store GatewayStore) { sharedParticipantRequestLimiter.SetStore(store) throttles, err := store.LoadParticipantThrottles() if err != nil { @@ -260,15 +261,15 @@ func mustLoadParticipantThrottleState(store *GatewayStore) { } } -func mustOpenGatewayStore(baseStorageDir string) *GatewayStore { - gatewayStore, err := NewGatewayStore(filepath.Join(baseStorageDir, "gateway.db")) +func mustOpenGatewayStore(ctx context.Context, baseStorageDir string) GatewayStore { + gatewayStore, err := NewGatewayStore(ctx, baseStorageDir) if err != nil { log.Fatalf("open gateway state: %v", err) } return gatewayStore } -func mustLoadPersistedGatewayState(gatewayStore *GatewayStore) (GatewayState, bool) { +func mustLoadPersistedGatewayState(gatewayStore GatewayStore) (GatewayState, bool) { gatewayState, hasState, err := gatewayStore.LoadState() if err != nil { log.Fatalf("load gateway state: %v", err) @@ -276,7 +277,7 @@ func mustLoadPersistedGatewayState(gatewayStore *GatewayStore) (GatewayState, bo return gatewayState, hasState } -func mustReloadGatewayState(gatewayStore *GatewayStore) GatewayState { +func mustReloadGatewayState(gatewayStore GatewayStore) GatewayState { gatewayState, hasState, err := gatewayStore.LoadState() if err != nil { log.Fatalf("reload gateway state: %v", err) @@ -287,7 +288,7 @@ func mustReloadGatewayState(gatewayStore *GatewayStore) GatewayState { return gatewayState } -func mustRepairPersistedGatewayEndpointSettings(gatewayStore *GatewayStore, gatewayState *GatewayState, flags cliFlags) { +func mustRepairPersistedGatewayEndpointSettings(gatewayStore GatewayStore, gatewayState *GatewayState, flags cliFlags) { if gatewayStore == nil || gatewayState == nil { return } @@ -311,7 +312,7 @@ func mustRepairPersistedGatewayEndpointSettings(gatewayStore *GatewayStore, gate log.Printf("repaired persisted gateway endpoint settings chain_rest=%q public_api=%q", settings.ChainREST, settings.PublicAPI) } -func mustBootstrapGatewayState(gatewayStore *GatewayStore, opts bootstrapOptions) { +func mustBootstrapGatewayState(gatewayStore GatewayStore, opts bootstrapOptions) { runtimeCfgs, err := resolveRuntimeConfigs(opts.escrowID, opts.privateKeyHex, opts.defaultModel, opts.storagePath) if err != nil { log.Fatal(err) @@ -332,7 +333,7 @@ func mustBootstrapGatewayState(gatewayStore *GatewayStore, opts bootstrapOptions } } -func mustBuildGateway(gatewayStore *GatewayStore, gatewayState GatewayState, baseStorageDir string) *Gateway { +func mustBuildGateway(gatewayStore GatewayStore, gatewayState GatewayState, baseStorageDir string) *Gateway { gatewayState.Settings = gatewayState.Settings.WithTuningDefaults() DefaultRequestMaxTokens = gatewayState.Settings.DefaultRequestMaxTokens RequestMaxTokensCap = gatewayState.Settings.RequestMaxTokensCap @@ -363,7 +364,7 @@ func mustBuildGateway(gatewayStore *GatewayStore, gatewayState GatewayState, bas return gateway } -func buildGatewayRuntimes(gatewayStore *GatewayStore, gatewayState *GatewayState, baseStorageDir string, perf *PerfTracker) ([]*devshardRuntime, error) { +func buildGatewayRuntimes(gatewayStore GatewayStore, gatewayState *GatewayState, baseStorageDir string, perf *PerfTracker) ([]*devshardRuntime, error) { // Load ALL devshards (active and inactive) so that inactive ones // remain accessible for finalization, debug, and settlement retrieval. // Inactive runtimes are loaded with active=false and excluded from diff --git a/devshard/cmd/devshardctl/main_test.go b/devshard/cmd/devshardctl/main_test.go index 6cd7c1567b..1e6155ed4b 100644 --- a/devshard/cmd/devshardctl/main_test.go +++ b/devshard/cmd/devshardctl/main_test.go @@ -26,7 +26,7 @@ func TestBootstrapEscrowRotationSettlementDefaultsDisabled(t *testing.T) { } func TestBuildGatewayRuntimesDeactivatesMissingEscrow(t *testing.T) { - store, err := NewGatewayStore(filepath.Join(t.TempDir(), "gateway.db")) + store, err := NewSQLiteGatewayStore(filepath.Join(t.TempDir(), "gateway.db")) require.NoError(t, err) t.Cleanup(func() { require.NoError(t, store.Close()) @@ -78,7 +78,7 @@ func TestBuildGatewayRuntimesDeactivatesMissingEscrow(t *testing.T) { } func TestBuildGatewayRuntimesDeactivatesMissingPrivateKey(t *testing.T) { - store, err := NewGatewayStore(filepath.Join(t.TempDir(), "gateway.db")) + store, err := NewSQLiteGatewayStore(filepath.Join(t.TempDir(), "gateway.db")) require.NoError(t, err) t.Cleanup(func() { require.NoError(t, store.Close()) @@ -130,7 +130,7 @@ func TestBuildGatewayRuntimesDeactivatesMissingPrivateKey(t *testing.T) { } func TestBuildGatewayRuntimesPreservesActiveOnOtherErrors(t *testing.T) { - store, err := NewGatewayStore(filepath.Join(t.TempDir(), "gateway.db")) + store, err := NewSQLiteGatewayStore(filepath.Join(t.TempDir(), "gateway.db")) require.NoError(t, err) t.Cleanup(func() { require.NoError(t, store.Close()) @@ -169,7 +169,7 @@ func TestBuildGatewayRuntimesPreservesActiveOnOtherErrors(t *testing.T) { } func TestRepairPersistedGatewayEndpointSettingsBackfillsBlankPublicAPI(t *testing.T) { - store, err := NewGatewayStore(filepath.Join(t.TempDir(), "gateway.db")) + store, err := NewSQLiteGatewayStore(filepath.Join(t.TempDir(), "gateway.db")) require.NoError(t, err) t.Cleanup(func() { require.NoError(t, store.Close()) @@ -202,7 +202,7 @@ func TestRepairPersistedGatewayEndpointSettingsBackfillsBlankPublicAPI(t *testin } func TestRepairPersistedGatewayEndpointSettingsPreservesConfiguredPublicAPI(t *testing.T) { - store, err := NewGatewayStore(filepath.Join(t.TempDir(), "gateway.db")) + store, err := NewSQLiteGatewayStore(filepath.Join(t.TempDir(), "gateway.db")) require.NoError(t, err) t.Cleanup(func() { require.NoError(t, store.Close()) @@ -233,7 +233,7 @@ func TestRepairPersistedGatewayEndpointSettingsPreservesConfiguredPublicAPI(t *t require.Equal(t, "http://configured-api:9000", reloaded.Settings.PublicAPI) } -func reloadGatewayStateForTest(t *testing.T, store *GatewayStore) (GatewayState, bool) { +func reloadGatewayStateForTest(t *testing.T, store GatewayStore) (GatewayState, bool) { t.Helper() state, ok, err := store.LoadState() require.NoError(t, err) diff --git a/devshard/docs/proxy.md b/devshard/docs/proxy.md index 19c70224f6..475cf73b6c 100644 --- a/devshard/docs/proxy.md +++ b/devshard/docs/proxy.md @@ -36,8 +36,48 @@ All settings can be passed as flags or environment variables. Flags take precede | - | `DEVSHARD_ESCROW_ROTATION_PRE_POC_BLOCKS` | no | `300` | Blocks before the next epoch switch at `set_new_validators` to create temp bridge escrows | | - | `DEVSHARD_ESCROW_ROTATION_MODELS_JSON` | when rotation enabled | - | JSON array of per-model rotation configs: `model_id`, `temp_count`, `target_count`, `amount`, `private_key_env` | | - | `DEVSHARD_META_DRAIN_TIMEOUT_SECONDS` | no | `30` | After client disconnect, keep draining host SSE for protocol completion (`devshard_meta`, `ProcessResponse`, `MsgFinishInference`) up to this many seconds | +| - | `PGHOST` | no | - | When set, gateway settings use Postgres as primary storage with SQLite fallback (`{baseStorageDir}/gateway.db`) | +| - | `PGPORT` | when `PGHOST` set | `5432` | Postgres port | +| - | `PGDATABASE` | when `PGHOST` set | - | Postgres database name | +| - | `PGUSER` | when `PGHOST` set | - | Postgres user | +| - | `PGPASSWORD` | when `PGHOST` set | - | Postgres password | +| - | `PG_RETRY_INTERVAL` | when `PGHOST` set | `240s` | Minimum interval between lazy Postgres reconnect attempts on writes | +| - | `PG_CONNECT_TIMEOUT` | when `PGHOST` set | `2s` | Timeout for each Postgres connect attempt | +| - | `GATEWAY_PG_SYNC_JOURNAL` | when `PGHOST` set | `true` | When enabled, SQLite fallback writes are journaled and replayed into Postgres on reconnect before PG resumes as primary | + +### Gateway persistence backend + +Gateway settings, devshard topology, rotation status, escrow commitments, suspicious +hosts, and participant throttle state are persisted under `{baseStorageDir}/gateway.db` +(SQLite). This file is always opened. + +When `PGHOST` is set: + +- On startup, if Postgres is reachable, existing SQLite data is imported into + Postgres once (idempotent; skipped when Postgres already has settings or a + migration marker exists). +- The gateway then uses **hybrid storage**: Postgres is primary; SQLite is a + fallback when Postgres is unavailable. +- Writes try Postgres first (with lazy reconnect, rate-limited by + `PG_RETRY_INTERVAL` and bounded by `PG_CONNECT_TIMEOUT`). On failure they + fall back to SQLite. +- Reads try Postgres first; on error or empty state they also check SQLite. + +When `PGHOST` is unset, only SQLite is used (same behavior as before hybrid +support). + +**Rollback:** unset `PGHOST` to run SQLite-only again. Writes that landed in +Postgres after migration are not automatically copied back to SQLite. + +**Postgres outage:** writes during an outage are stored in SQLite only and recorded in +`gateway_pg_sync_journal`. When Postgres reconnects, those outage deltas are +replayed into Postgres before it resumes as primary (disable with +`GATEWAY_PG_SYNC_JOURNAL=false` to revert to payload-style no-backfill behavior). + +Gateway tables (`gateway_*`, `escrow_rotation_*`, `participant_throttle_state`) +can share the same Postgres database as devshard session or payload tables; table +names do not collide. -## Quick start ```bash devshardctl \ From 5c2b21d90974092b7de7eb291ab2bbf272f06039 Mon Sep 17 00:00:00 2001 From: akup Date: Wed, 8 Jul 2026 08:05:44 +0300 Subject: [PATCH 11/13] PG_TO_SQLITE_FALLBACK env switch, fallback can be disabled --- .../cmd/devshardctl/gateway_store_factory.go | 15 +++- .../devshardctl/gateway_store_factory_test.go | 13 +++ .../cmd/devshardctl/gateway_store_hybrid.go | 88 ++++++++++++++----- .../devshardctl/gateway_store_sync_journal.go | 14 --- .../gateway_store_sync_journal_test.go | 14 ++- devshard/docs/proxy.md | 14 +-- 6 files changed, 110 insertions(+), 48 deletions(-) diff --git a/devshard/cmd/devshardctl/gateway_store_factory.go b/devshard/cmd/devshardctl/gateway_store_factory.go index daff39f8a8..d2c0481e9c 100644 --- a/devshard/cmd/devshardctl/gateway_store_factory.go +++ b/devshard/cmd/devshardctl/gateway_store_factory.go @@ -2,6 +2,7 @@ package main import ( "context" + "fmt" "log" "os" "path/filepath" @@ -36,6 +37,10 @@ func NewGatewayStore(ctx context.Context, baseStorageDir string) (GatewayStore, pg, pgErr := NewPostgresGatewayStore(ctx) if pgErr != nil { + if !gatewayPGToSQLiteFallback() { + _ = sqlite.Close() + return nil, fmt.Errorf("gateway store: postgres required when PG_TO_SQLITE_FALLBACK=false (host=%s): %w", pgHost, pgErr) + } log.Printf("gateway store: postgres connection failed, will retry lazily on write (host=%s): %v", pgHost, pgErr) return NewHybridGatewayStore(nil, sqlite, retryInterval, connectTimeout), nil } @@ -49,10 +54,18 @@ func NewGatewayStore(ctx context.Context, baseStorageDir string) (GatewayStore, hybrid := NewHybridGatewayStore(nil, sqlite, retryInterval, connectTimeout) if err := hybrid.ReconcileSyncJournal(ctx, pg); err != nil { _ = pg.Close() + if !gatewayPGToSQLiteFallback() { + _ = sqlite.Close() + return nil, fmt.Errorf("gateway store: sync journal drain failed: %w", err) + } log.Printf("gateway store: sync journal drain failed at startup, staying on sqlite fallback: %v", err) return hybrid, nil } - log.Printf("gateway store: using postgres with sqlite fallback (host=%s)", pgHost) + if gatewayPGToSQLiteFallback() { + log.Printf("gateway store: using postgres with sqlite fallback (host=%s)", pgHost) + } else { + log.Printf("gateway store: using postgres only (host=%s, PG_TO_SQLITE_FALLBACK=false)", pgHost) + } return hybrid, nil } diff --git a/devshard/cmd/devshardctl/gateway_store_factory_test.go b/devshard/cmd/devshardctl/gateway_store_factory_test.go index 6c68988ee5..a92b737071 100644 --- a/devshard/cmd/devshardctl/gateway_store_factory_test.go +++ b/devshard/cmd/devshardctl/gateway_store_factory_test.go @@ -80,6 +80,19 @@ func TestNewGatewayStorePGHOSTSetPGDown(t *testing.T) { require.EqualValues(t, 1111, sqliteState.Settings.DefaultRequestMaxTokens) } +func TestNewGatewayStorePGHOSTSetPGDownNoSQLiteFallback(t *testing.T) { + t.Setenv("PGHOST", "127.0.0.1") + t.Setenv("PGPORT", "1") + t.Setenv("PG_CONNECT_TIMEOUT", "100ms") + t.Setenv("PG_TO_SQLITE_FALLBACK", "false") + + ctx := context.Background() + store, err := NewGatewayStore(ctx, t.TempDir()) + require.Error(t, err) + require.Nil(t, store) + require.Contains(t, err.Error(), "PG_TO_SQLITE_FALLBACK=false") +} + func TestNewGatewayStoreAutoMigration(t *testing.T) { cleanup := setupPostgresContainer(t) t.Cleanup(cleanup) diff --git a/devshard/cmd/devshardctl/gateway_store_hybrid.go b/devshard/cmd/devshardctl/gateway_store_hybrid.go index bdda66ab7a..7499397822 100644 --- a/devshard/cmd/devshardctl/gateway_store_hybrid.go +++ b/devshard/cmd/devshardctl/gateway_store_hybrid.go @@ -17,7 +17,7 @@ const ( // before probing Postgres again after giving up. Kept short (relative to the // payload store) because gateway management state is low write volume, so a // snappier recovery + journal drain outweighs the occasional reconnect probe. - defaultGatewayPGRetryInterval = 30 * time.Second + defaultGatewayPGRetryInterval = 15 * time.Second // defaultGatewayPGWriteRetryBudget bounds the in-request retry loop that lets // a brief Postgres blip (reset/failover) recover without dropping to SQLite. defaultGatewayPGWriteRetryBudget = 2 * time.Second @@ -38,9 +38,9 @@ type HybridGatewayStore struct { lastRetry time.Time retryInterval time.Duration connectTimeout time.Duration - writeRetryBudget time.Duration - syncJournalEnabled bool - connectPG func(context.Context) (*PostgresGatewayStore, error) + writeRetryBudget time.Duration + sqliteFallbackEnabled bool + connectPG func(context.Context) (*PostgresGatewayStore, error) } func NewHybridGatewayStore(pg *PostgresGatewayStore, sqlite *SQLiteGatewayStore, retryInterval, connectTimeout time.Duration) *HybridGatewayStore { @@ -51,16 +51,36 @@ func NewHybridGatewayStore(pg *PostgresGatewayStore, sqlite *SQLiteGatewayStore, connectTimeout = gatewayPGConnectTimeout } return &HybridGatewayStore{ - pg: pg, - sqlite: sqlite, - retryInterval: retryInterval, - connectTimeout: connectTimeout, - writeRetryBudget: gatewayPGWriteRetryBudget(), - syncJournalEnabled: gatewaySyncJournalEnabled(), - connectPG: NewPostgresGatewayStore, + pg: pg, + sqlite: sqlite, + retryInterval: retryInterval, + connectTimeout: connectTimeout, + writeRetryBudget: gatewayPGWriteRetryBudget(), + sqliteFallbackEnabled: gatewayPGToSQLiteFallback(), + connectPG: NewPostgresGatewayStore, } } +// gatewayPGToSQLiteFallback reads PG_TO_SQLITE_FALLBACK (default true). When +// false, Postgres failures return errors instead of falling back to SQLite; the +// sync journal is only used while fallback is enabled. +func gatewayPGToSQLiteFallback() bool { + raw := strings.TrimSpace(os.Getenv("PG_TO_SQLITE_FALLBACK")) + if raw == "" { + return true + } + switch strings.ToLower(raw) { + case "0", "false", "no", "off": + return false + default: + return true + } +} + +func (h *HybridGatewayStore) sqliteFallbackAllowed() bool { + return h.sqliteFallbackEnabled && h.sqlite != nil +} + // gatewayPGWriteRetryBudget reads PG_WRITE_RETRY_BUDGET (a Go duration). A value // of 0 disables in-request retry (single attempt then fallback); an invalid or // unset value uses the default. @@ -130,7 +150,7 @@ func (h *HybridGatewayStore) reconcileAndPublishPgLocked(ctx context.Context, pg if pg == nil { return fmt.Errorf("postgres store unavailable") } - if !h.syncJournalEnabled || h.sqlite == nil { + if !h.sqliteFallbackEnabled || h.sqlite == nil { h.pg = pg log.Printf("gateway store: postgres connection established") return nil @@ -159,7 +179,7 @@ func (h *HybridGatewayStore) write( if err := h.attemptPGWrite(ctx, pg, pgFn); err == nil { return nil } else { - log.Printf("gateway hybrid: postgres write failed after retries, falling back to sqlite: %v", err) + log.Printf("gateway hybrid: postgres write failed after retries: %v", err) // Drop the cached connection so subsequent writes during the outage // get journaled, and the next reconnect replays the journal before // republishing Postgres as primary. Without this, pgxpool would @@ -167,12 +187,17 @@ func (h *HybridGatewayStore) write( // entries linger, which a later restart-time drain would replay on top // of newer PG rows. h.dropPg(pg) + if !h.sqliteFallbackEnabled { + return err + } } + } else if !h.sqliteFallbackEnabled { + return errGatewayStoreUnavailable } if h.sqlite == nil { return errGatewayStoreUnavailable } - if h.syncJournalEnabled && len(journal) > 0 && sqliteTxFn != nil { + if h.sqliteFallbackEnabled && len(journal) > 0 && sqliteTxFn != nil { return h.sqlite.writeWithSyncJournal(journal, sqliteTxFn) } return sqliteFn(h.sqlite) @@ -240,8 +265,11 @@ func (h *HybridGatewayStore) loadState(ctx context.Context) (GatewayState, bool, } if err != nil { log.Printf("gateway hybrid: postgres load state failed, checking sqlite: %v", err) + if !h.sqliteFallbackEnabled { + return GatewayState{}, false, err + } } - if h.sqlite != nil { + if h.sqliteFallbackAllowed() { sqliteState, sqliteHas, sqliteErr := h.sqlite.LoadState() if sqliteErr == nil && sqliteHas { return sqliteState, true, nil @@ -257,7 +285,7 @@ func (h *HybridGatewayStore) loadState(ctx context.Context) (GatewayState, bool, return state, has, nil } - if h.sqlite != nil { + if h.sqliteFallbackAllowed() { state, has, err := h.sqlite.LoadState() if err == nil && has { return state, true, nil @@ -333,8 +361,11 @@ func (h *HybridGatewayStore) LoadRotationStatuses(limit int) ([]GatewayRotationS } if err != nil { log.Printf("gateway hybrid: postgres load rotation statuses failed, checking sqlite: %v", err) + if !h.sqliteFallbackEnabled { + return nil, err + } } - if h.sqlite != nil { + if h.sqliteFallbackAllowed() { sqliteStatuses, sqliteErr := h.sqlite.LoadRotationStatuses(limit) if sqliteErr == nil && len(sqliteStatuses) > 0 { return sqliteStatuses, nil @@ -350,7 +381,7 @@ func (h *HybridGatewayStore) LoadRotationStatuses(limit int) ([]GatewayRotationS return statuses, nil } - if h.sqlite != nil { + if h.sqliteFallbackAllowed() { statuses, err := h.sqlite.LoadRotationStatuses(limit) if err == nil && len(statuses) > 0 { return statuses, nil @@ -385,8 +416,11 @@ func (h *HybridGatewayStore) LoadCommitments() ([]GatewayEscrowCommitment, error } if err != nil { log.Printf("gateway hybrid: postgres load commitments failed, checking sqlite: %v", err) + if !h.sqliteFallbackEnabled { + return nil, err + } } - if h.sqlite != nil { + if h.sqliteFallbackAllowed() { sqliteCommitments, sqliteErr := h.sqlite.LoadCommitments() if sqliteErr == nil && len(sqliteCommitments) > 0 { return sqliteCommitments, nil @@ -402,7 +436,7 @@ func (h *HybridGatewayStore) LoadCommitments() ([]GatewayEscrowCommitment, error return commitments, nil } - if h.sqlite != nil { + if h.sqliteFallbackAllowed() { commitments, err := h.sqlite.LoadCommitments() if err == nil && len(commitments) > 0 { return commitments, nil @@ -437,8 +471,11 @@ func (h *HybridGatewayStore) LoadSuspiciousHosts() ([]GatewaySuspiciousHost, err } if err != nil { log.Printf("gateway hybrid: postgres load suspicious hosts failed, checking sqlite: %v", err) + if !h.sqliteFallbackEnabled { + return nil, err + } } - if h.sqlite != nil { + if h.sqliteFallbackAllowed() { sqliteHosts, sqliteErr := h.sqlite.LoadSuspiciousHosts() if sqliteErr == nil && len(sqliteHosts) > 0 { return sqliteHosts, nil @@ -454,7 +491,7 @@ func (h *HybridGatewayStore) LoadSuspiciousHosts() ([]GatewaySuspiciousHost, err return hosts, nil } - if h.sqlite != nil { + if h.sqliteFallbackAllowed() { hosts, err := h.sqlite.LoadSuspiciousHosts() if err == nil && len(hosts) > 0 { return hosts, nil @@ -605,8 +642,11 @@ func (h *HybridGatewayStore) LoadParticipantThrottles() ([]ParticipantThrottleRo } if err != nil { log.Printf("gateway hybrid: postgres load participant throttles failed, checking sqlite: %v", err) + if !h.sqliteFallbackEnabled { + return nil, err + } } - if h.sqlite != nil { + if h.sqliteFallbackAllowed() { sqliteRows, sqliteErr := h.sqlite.LoadParticipantThrottles() if sqliteErr == nil && len(sqliteRows) > 0 { return sqliteRows, nil @@ -622,7 +662,7 @@ func (h *HybridGatewayStore) LoadParticipantThrottles() ([]ParticipantThrottleRo return rows, nil } - if h.sqlite != nil { + if h.sqliteFallbackAllowed() { rows, err := h.sqlite.LoadParticipantThrottles() if err == nil && len(rows) > 0 { return rows, nil diff --git a/devshard/cmd/devshardctl/gateway_store_sync_journal.go b/devshard/cmd/devshardctl/gateway_store_sync_journal.go index eb1321303b..ae3a214915 100644 --- a/devshard/cmd/devshardctl/gateway_store_sync_journal.go +++ b/devshard/cmd/devshardctl/gateway_store_sync_journal.go @@ -4,7 +4,6 @@ import ( "context" "database/sql" "fmt" - "os" "strconv" "strings" "time" @@ -42,19 +41,6 @@ type gatewaySyncJournalRow struct { op string } -func gatewaySyncJournalEnabled() bool { - raw := strings.TrimSpace(os.Getenv("GATEWAY_PG_SYNC_JOURNAL")) - if raw == "" { - return true - } - switch strings.ToLower(raw) { - case "0", "false", "no", "off": - return false - default: - return true - } -} - func gatewayRotationStatusRowKey(modelID, stage string, epoch uint64) string { return fmt.Sprintf("%s|%s|%d", strings.TrimSpace(modelID), strings.TrimSpace(stage), epoch) } diff --git a/devshard/cmd/devshardctl/gateway_store_sync_journal_test.go b/devshard/cmd/devshardctl/gateway_store_sync_journal_test.go index c882c4d5dc..9d388963f3 100644 --- a/devshard/cmd/devshardctl/gateway_store_sync_journal_test.go +++ b/devshard/cmd/devshardctl/gateway_store_sync_journal_test.go @@ -81,8 +81,8 @@ func TestHybridSyncJournalAtomicWrite(t *testing.T) { require.Equal(t, 0, count) } -func TestHybridSyncJournalKillSwitch(t *testing.T) { - t.Setenv("GATEWAY_PG_SYNC_JOURNAL", "false") +func TestHybridPGToSQLiteFallbackDisabled(t *testing.T) { + t.Setenv("PG_TO_SQLITE_FALLBACK", "false") sqlite := newTestSQLiteGatewayStoreOnly(t) settings := GatewaySettings{ @@ -103,11 +103,19 @@ func TestHybridSyncJournalKillSwitch(t *testing.T) { updated := settings updated.DefaultRequestMaxTokens = 3333 - require.NoError(t, hybrid.UpdateSettings(updated)) + err := hybrid.UpdateSettings(updated) + require.Error(t, err) count, err := sqlite.countSyncJournalEntries() require.NoError(t, err) require.Equal(t, 0, count) + + _, has, err := sqlite.LoadState() + require.NoError(t, err) + require.True(t, has) + sqliteState, _, err := sqlite.LoadState() + require.NoError(t, err) + require.EqualValues(t, 1000, sqliteState.Settings.DefaultRequestMaxTokens) } func TestSyncJournalRestartPersistence(t *testing.T) { diff --git a/devshard/docs/proxy.md b/devshard/docs/proxy.md index 475cf73b6c..1844001264 100644 --- a/devshard/docs/proxy.md +++ b/devshard/docs/proxy.md @@ -41,9 +41,9 @@ All settings can be passed as flags or environment variables. Flags take precede | - | `PGDATABASE` | when `PGHOST` set | - | Postgres database name | | - | `PGUSER` | when `PGHOST` set | - | Postgres user | | - | `PGPASSWORD` | when `PGHOST` set | - | Postgres password | -| - | `PG_RETRY_INTERVAL` | when `PGHOST` set | `240s` | Minimum interval between lazy Postgres reconnect attempts on writes | +| - | `PG_RETRY_INTERVAL` | when `PGHOST` set | `15s` | Minimum interval between lazy Postgres reconnect attempts on writes (gateway default; dapi payload storage defaults to `240s` if unset) | | - | `PG_CONNECT_TIMEOUT` | when `PGHOST` set | `2s` | Timeout for each Postgres connect attempt | -| - | `GATEWAY_PG_SYNC_JOURNAL` | when `PGHOST` set | `true` | When enabled, SQLite fallback writes are journaled and replayed into Postgres on reconnect before PG resumes as primary | +| - | `PG_TO_SQLITE_FALLBACK` | when `PGHOST` set | `true` | When enabled, Postgres failures fall back to SQLite and outage writes are journaled for replay on reconnect. When `false`, Postgres is required — failures return errors | ### Gateway persistence backend @@ -69,10 +69,12 @@ support). **Rollback:** unset `PGHOST` to run SQLite-only again. Writes that landed in Postgres after migration are not automatically copied back to SQLite. -**Postgres outage:** writes during an outage are stored in SQLite only and recorded in -`gateway_pg_sync_journal`. When Postgres reconnects, those outage deltas are -replayed into Postgres before it resumes as primary (disable with -`GATEWAY_PG_SYNC_JOURNAL=false` to revert to payload-style no-backfill behavior). +**Postgres outage:** when `PG_TO_SQLITE_FALLBACK=true` (default), writes during an +outage are stored in SQLite and recorded in `gateway_pg_sync_journal`. When +Postgres reconnects, those outage deltas are replayed into Postgres before it +resumes as primary. Set `PG_TO_SQLITE_FALLBACK=false` for Postgres-only mode — +writes and reads error when Postgres is unavailable (no SQLite fallback, no +journal). Gateway tables (`gateway_*`, `escrow_rotation_*`, `participant_throttle_state`) can share the same Postgres database as devshard session or payload tables; table From 2aeed2c79ceaa6bd8503dc8da8b7a389707af18a Mon Sep 17 00:00:00 2001 From: akup Date: Wed, 8 Jul 2026 10:50:50 +0300 Subject: [PATCH 12/13] fix(devshard): drain gateway sync journal in bounded chunks --- .../devshardctl/gateway_store_sync_journal.go | 115 +++++---- .../gateway_store_sync_journal_test.go | 107 ++++++++ .../cmd/devshardctl/gateway_store_test.go | 238 +++++++++--------- .../gateway_store_test_helpers_test.go | 5 + 4 files changed, 307 insertions(+), 158 deletions(-) diff --git a/devshard/cmd/devshardctl/gateway_store_sync_journal.go b/devshard/cmd/devshardctl/gateway_store_sync_journal.go index ae3a214915..4d16a499f0 100644 --- a/devshard/cmd/devshardctl/gateway_store_sync_journal.go +++ b/devshard/cmd/devshardctl/gateway_store_sync_journal.go @@ -23,6 +23,10 @@ const ( gatewaySyncOpDelete = "delete" ) +// gatewaySyncJournalDrainChunkSize bounds how many raw journal rows are loaded +// and coalesced per drain round. A var so tests can shrink it. +var gatewaySyncJournalDrainChunkSize = 1000 + type gatewaySyncEntry struct { tableName string rowKey string @@ -94,20 +98,6 @@ func (s *SQLiteGatewayStore) writeWithSyncJournal(entries []gatewaySyncEntry, fn return tx.Commit() } -func (s *SQLiteGatewayStore) maxSyncJournalSeq() (int64, error) { - if s == nil || s.db == nil { - return 0, nil - } - var maxSeq sql.NullInt64 - if err := s.db.QueryRow(`SELECT MAX(seq) FROM gateway_pg_sync_journal`).Scan(&maxSeq); err != nil { - return 0, fmt.Errorf("max sync journal seq: %w", err) - } - if !maxSeq.Valid { - return 0, nil - } - return maxSeq.Int64, nil -} - func (s *SQLiteGatewayStore) countSyncJournalEntries() (int, error) { if s == nil || s.db == nil { return 0, nil @@ -119,6 +109,35 @@ func (s *SQLiteGatewayStore) countSyncJournalEntries() (int, error) { return count, nil } +func (s *SQLiteGatewayStore) loadSyncJournalChunk(afterSeq int64, limit int) ([]gatewaySyncJournalRow, error) { + if s == nil || s.db == nil { + return nil, nil + } + if limit <= 0 { + limit = gatewaySyncJournalDrainChunkSize + } + rows, err := s.db.Query(` + SELECT seq, table_name, row_key, op + FROM gateway_pg_sync_journal + WHERE seq > ? + ORDER BY seq ASC + LIMIT ?`, afterSeq, limit) + if err != nil { + return nil, fmt.Errorf("load sync journal chunk after %d: %w", afterSeq, err) + } + defer rows.Close() + + var result []gatewaySyncJournalRow + for rows.Next() { + var row gatewaySyncJournalRow + if err := rows.Scan(&row.seq, &row.tableName, &row.rowKey, &row.op); err != nil { + return nil, fmt.Errorf("scan sync journal row: %w", err) + } + result = append(result, row) + } + return result, rows.Err() +} + func (s *SQLiteGatewayStore) loadSyncJournalUpTo(maxSeq int64) ([]gatewaySyncJournalRow, error) { rows, err := s.db.Query(` SELECT seq, table_name, row_key, op @@ -160,23 +179,40 @@ func drainGatewaySyncJournal(ctx context.Context, sqlite *SQLiteGatewayStore, pg return true, nil } - maxSeq, err := sqlite.maxSyncJournalSeq() - if err != nil { - return false, err - } - if maxSeq == 0 { - return true, nil + var lastSeq int64 + for { + chunk, err := sqlite.loadSyncJournalChunk(lastSeq, gatewaySyncJournalDrainChunkSize) + if err != nil { + return false, err + } + if len(chunk) == 0 { + break + } + chunkMaxSeq := chunk[len(chunk)-1].seq + coalesced := coalesceSyncJournalRows(chunk) + if err := applyCoalescedSyncJournalToPG(ctx, pg, sqlite, coalesced); err != nil { + return false, err + } + if err := sqlite.deleteSyncJournalUpTo(chunkMaxSeq); err != nil { + return false, err + } + lastSeq = chunkMaxSeq } - rows, err := sqlite.loadSyncJournalUpTo(maxSeq) + remaining, err := sqlite.countSyncJournalEntries() if err != nil { return false, err } - coalesced := coalesceSyncJournalRows(rows) + return remaining == 0, nil +} +func applyCoalescedSyncJournalToPG(ctx context.Context, pg *PostgresGatewayStore, sqlite *SQLiteGatewayStore, coalesced map[gatewaySyncKey]string) error { + if len(coalesced) == 0 { + return nil + } tx, err := pg.pool.Begin(ctx) if err != nil { - return false, fmt.Errorf("begin sync journal drain: %w", err) + return fmt.Errorf("begin sync journal drain: %w", err) } defer tx.Rollback(ctx) @@ -184,31 +220,26 @@ func drainGatewaySyncJournal(ctx context.Context, sqlite *SQLiteGatewayStore, pg switch op { case gatewaySyncOpDelete: if err := deleteGatewayRowFromPG(ctx, tx, key.tableName, key.rowKey); err != nil { - return false, err + return err } case gatewaySyncOpUpsert: if err := applyGatewaySyncUpsertToPG(ctx, tx, sqlite, key.tableName, key.rowKey); err != nil { - return false, err + return err } default: - return false, fmt.Errorf("unknown sync journal op %q for %s/%s", op, key.tableName, key.rowKey) + return fmt.Errorf("unknown sync journal op %q for %s/%s", op, key.tableName, key.rowKey) } } if err := tx.Commit(ctx); err != nil { - return false, fmt.Errorf("commit sync journal drain: %w", err) + return fmt.Errorf("commit sync journal drain: %w", err) } - if err := sqlite.deleteSyncJournalUpTo(maxSeq); err != nil { - return false, err - } - - remaining, err := sqlite.countSyncJournalEntries() - if err != nil { - return false, err - } - return remaining == 0, nil + return nil } func applyGatewaySyncUpsertToPG(ctx context.Context, tx pgx.Tx, sqlite *SQLiteGatewayStore, tableName, rowKey string) error { + // When the SQLite row is gone, skip the upsert: per-chunk coalescing may replay + // a stale upsert from an earlier chunk while the matching delete lives in a + // later chunk. The delete removes the row from Postgres once that chunk runs. switch tableName { case gatewayTableSettings: settings, updatedAt, ok, err := sqlite.loadGatewaySettingsRow() @@ -216,7 +247,7 @@ func applyGatewaySyncUpsertToPG(ctx context.Context, tx pgx.Tx, sqlite *SQLiteGa return err } if !ok { - return fmt.Errorf("sync journal upsert settings: row missing in sqlite") + return nil } return applyGatewaySettingsUpsertToPG(ctx, tx, settings, updatedAt) case gatewayTableDevshards: @@ -225,7 +256,7 @@ func applyGatewaySyncUpsertToPG(ctx context.Context, tx pgx.Tx, sqlite *SQLiteGa return err } if !ok { - return fmt.Errorf("sync journal upsert devshard %s: row missing in sqlite", rowKey) + return nil } return applyGatewayDevshardToPG(ctx, tx, devshard, time.Now().UTC().Format(time.RFC3339Nano)) case gatewayTableThrottle: @@ -234,7 +265,7 @@ func applyGatewaySyncUpsertToPG(ctx context.Context, tx pgx.Tx, sqlite *SQLiteGa return err } if !ok { - return fmt.Errorf("sync journal upsert throttle %s: row missing in sqlite", rowKey) + return nil } return applyGatewayThrottleToPG(ctx, tx, row) case gatewayTableRotationStatus: @@ -247,7 +278,7 @@ func applyGatewaySyncUpsertToPG(ctx context.Context, tx pgx.Tx, sqlite *SQLiteGa return err } if !ok { - return fmt.Errorf("sync journal upsert rotation status %s: row missing in sqlite", rowKey) + return nil } return applyGatewayRotationStatusToPG(ctx, tx, status) case gatewayTableSuspiciousHosts: @@ -256,7 +287,7 @@ func applyGatewaySyncUpsertToPG(ctx context.Context, tx pgx.Tx, sqlite *SQLiteGa return err } if !ok { - return fmt.Errorf("sync journal upsert suspicious host %s: row missing in sqlite", rowKey) + return nil } return applyGatewaySuspiciousHostToPG(ctx, tx, host) case gatewayTableCommitments: @@ -265,7 +296,7 @@ func applyGatewaySyncUpsertToPG(ctx context.Context, tx pgx.Tx, sqlite *SQLiteGa return err } if !ok { - return fmt.Errorf("sync journal upsert commitment %s: row missing in sqlite", rowKey) + return nil } return applyGatewayCommitmentToPG(ctx, tx, commitment) default: diff --git a/devshard/cmd/devshardctl/gateway_store_sync_journal_test.go b/devshard/cmd/devshardctl/gateway_store_sync_journal_test.go index 9d388963f3..a1babf039a 100644 --- a/devshard/cmd/devshardctl/gateway_store_sync_journal_test.go +++ b/devshard/cmd/devshardctl/gateway_store_sync_journal_test.go @@ -24,6 +24,113 @@ func TestCoalesceSyncJournalRows(t *testing.T) { require.Equal(t, gatewaySyncOpUpsert, coalesced[gatewaySyncKey{tableName: gatewayTableDevshards, rowKey: "b"}]) } +func TestSyncJournalDrainProcessesInChunks(t *testing.T) { + prev := gatewaySyncJournalDrainChunkSize + gatewaySyncJournalDrainChunkSize = 2 + t.Cleanup(func() { gatewaySyncJournalDrainChunkSize = prev }) + + cleanup := setupPostgresContainer(t) + t.Cleanup(cleanup) + + ctx := context.Background() + pg, err := NewPostgresGatewayStore(ctx) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, pg.Close()) }) + + sqlite := newTestSQLiteGatewayStoreOnly(t) + base := GatewaySettings{ + ChainREST: "http://node:1317", + PublicAPI: "http://api:9000", + DefaultModel: "Qwen/Test", + DefaultRequestMaxTokens: 1000, + MaxConcurrentRequests: 1, + MaxInputTokensInFlight: 100, + }.WithTuningDefaults() + require.NoError(t, pg.Initialize(base, nil)) + require.NoError(t, sqlite.Initialize(base, nil)) + + hybrid := NewHybridGatewayStore(nil, sqlite, timeHour, gatewayPGConnectTimeout) + hybrid.connectPG = func(context.Context) (*PostgresGatewayStore, error) { + return nil, errGatewayStoreUnavailable + } + for i := 0; i < 5; i++ { + updated := base + updated.DefaultRequestMaxTokens = uint64(2000 + i) + require.NoError(t, hybrid.UpdateSettings(updated)) + } + + count, err := sqlite.countSyncJournalEntries() + require.NoError(t, err) + require.Equal(t, 5, count) + + drainHybrid := NewHybridGatewayStore(nil, sqlite, timeHour, gatewayPGConnectTimeout) + require.NoError(t, drainHybrid.ReconcileSyncJournal(ctx, pg)) + + pgState, has, err := pg.LoadState() + require.NoError(t, err) + require.True(t, has) + require.EqualValues(t, 2004, pgState.Settings.DefaultRequestMaxTokens) + + count, err = sqlite.countSyncJournalEntries() + require.NoError(t, err) + require.Equal(t, 0, count) +} + +func TestSyncJournalDrainUpsertDeleteAcrossChunks(t *testing.T) { + prev := gatewaySyncJournalDrainChunkSize + gatewaySyncJournalDrainChunkSize = 1 + t.Cleanup(func() { gatewaySyncJournalDrainChunkSize = prev }) + + cleanup := setupPostgresContainer(t) + t.Cleanup(cleanup) + + ctx := context.Background() + pg, err := NewPostgresGatewayStore(ctx) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, pg.Close()) }) + + sqlite := newTestSQLiteGatewayStoreOnly(t) + base := GatewaySettings{ + ChainREST: "http://node:1317", + PublicAPI: "http://api:9000", + DefaultModel: "Qwen/Test", + DefaultRequestMaxTokens: 1000, + MaxConcurrentRequests: 1, + MaxInputTokensInFlight: 100, + }.WithTuningDefaults() + require.NoError(t, pg.Initialize(base, nil)) + require.NoError(t, sqlite.Initialize(base, nil)) + + // Seed PG with a commitment that outage-era SQLite save+delete should remove. + require.NoError(t, pg.SaveCommitment(GatewayEscrowCommitment{ + TxHash: "TX-CHUNK-1", Model: "Qwen/Test", Role: rotationRoleTemp, Epoch: 7, + })) + + hybrid := NewHybridGatewayStore(nil, sqlite, timeHour, gatewayPGConnectTimeout) + hybrid.connectPG = func(context.Context) (*PostgresGatewayStore, error) { + return nil, errGatewayStoreUnavailable + } + require.NoError(t, hybrid.SaveCommitment(GatewayEscrowCommitment{ + TxHash: "TX-CHUNK-1", Model: "Qwen/Test", Role: rotationRoleTemp, Epoch: 7, + })) + require.NoError(t, hybrid.DeleteCommitment("TX-CHUNK-1")) + + count, err := sqlite.countSyncJournalEntries() + require.NoError(t, err) + require.Equal(t, 2, count) + + drainHybrid := NewHybridGatewayStore(nil, sqlite, timeHour, gatewayPGConnectTimeout) + require.NoError(t, drainHybrid.ReconcileSyncJournal(ctx, pg)) + + commitments, err := pg.LoadCommitments() + require.NoError(t, err) + require.Empty(t, commitments) + + count, err = sqlite.countSyncJournalEntries() + require.NoError(t, err) + require.Equal(t, 0, count) +} + func TestHybridSyncJournalOnFallback(t *testing.T) { sqlite := newTestSQLiteGatewayStoreOnly(t) settings := GatewaySettings{ diff --git a/devshard/cmd/devshardctl/gateway_store_test.go b/devshard/cmd/devshardctl/gateway_store_test.go index 3174243101..9fc28d0098 100644 --- a/devshard/cmd/devshardctl/gateway_store_test.go +++ b/devshard/cmd/devshardctl/gateway_store_test.go @@ -16,47 +16,47 @@ import ( ) func TestGatewayStoreInitializeAndLoadState(t *testing.T) { - store, err := NewSQLiteGatewayStore(filepath.Join(t.TempDir(), "gateway.db")) - require.NoError(t, err) - t.Cleanup(func() { - require.NoError(t, store.Close()) - }) - - settings := GatewaySettings{ - ChainREST: "http://node:1317", - PublicAPI: "http://api:9000", - DefaultModel: "Qwen/Test", - DefaultRequestMaxTokens: 1234, - MaxConcurrentRequests: 5, - MaxInputTokensInFlight: 999, - }.WithTuningDefaults() - devshards := []GatewayDevshardState{{ - RuntimeConfig: RuntimeConfig{ - ID: "12", - PrivateKeyHex: "secret", - Model: "Qwen/Test", - StoragePath: "/root/.devshardctl/escrow-12", - }, - Active: true, - RotationRole: rotationRoleRegular, - RotationEpoch: 7, - }} - - require.NoError(t, store.Initialize(settings, devshards)) - - state, ok, err := store.LoadState() - require.NoError(t, err) - require.True(t, ok) - require.Equal(t, settings, state.Settings) - require.Len(t, state.Devshards, 1) - require.Equal(t, "12", state.Devshards[0].ID) - require.True(t, state.Devshards[0].Active) - require.Equal(t, "/root/.devshardctl/escrow-12", state.Devshards[0].StoragePath) - require.Equal(t, rotationRoleRegular, state.Devshards[0].RotationRole) - require.EqualValues(t, 7, state.Devshards[0].RotationEpoch) - require.False(t, state.Settings.Disabled.Enabled) - require.Equal(t, defaultGatewayDisabledMessage, state.Settings.Disabled.Message) - require.Empty(t, state.Settings.Disabled.NewURL) + for _, backend := range gatewayStoreTestBackends { + t.Run(backend, func(t *testing.T) { + store := newTestGatewayStore(t, backend) + + settings := GatewaySettings{ + ChainREST: "http://node:1317", + PublicAPI: "http://api:9000", + DefaultModel: "Qwen/Test", + DefaultRequestMaxTokens: 1234, + MaxConcurrentRequests: 5, + MaxInputTokensInFlight: 999, + }.WithTuningDefaults() + devshards := []GatewayDevshardState{{ + RuntimeConfig: RuntimeConfig{ + ID: "12", + PrivateKeyHex: "secret", + Model: "Qwen/Test", + StoragePath: "/root/.devshardctl/escrow-12", + }, + Active: true, + RotationRole: rotationRoleRegular, + RotationEpoch: 7, + }} + + require.NoError(t, store.Initialize(settings, devshards)) + + state, ok, err := store.LoadState() + require.NoError(t, err) + require.True(t, ok) + require.Equal(t, settings, state.Settings) + require.Len(t, state.Devshards, 1) + require.Equal(t, "12", state.Devshards[0].ID) + require.True(t, state.Devshards[0].Active) + require.Equal(t, "/root/.devshardctl/escrow-12", state.Devshards[0].StoragePath) + require.Equal(t, rotationRoleRegular, state.Devshards[0].RotationRole) + require.EqualValues(t, 7, state.Devshards[0].RotationEpoch) + require.False(t, state.Settings.Disabled.Enabled) + require.Equal(t, defaultGatewayDisabledMessage, state.Settings.Disabled.Message) + require.Empty(t, state.Settings.Disabled.NewURL) + }) + } } func TestAdminAuthMiddlewareRequiresAdminKey(t *testing.T) { @@ -87,11 +87,15 @@ func TestAdminAuthMiddlewareRequiresAdminKey(t *testing.T) { } func TestGatewayStoreUpdateSettings(t *testing.T) { - store, err := NewSQLiteGatewayStore(filepath.Join(t.TempDir(), "gateway.db")) - require.NoError(t, err) - t.Cleanup(func() { - require.NoError(t, store.Close()) - }) + for _, backend := range gatewayStoreTestBackends { + t.Run(backend, func(t *testing.T) { + assertGatewayStoreUpdateSettings(t, newTestGatewayStore(t, backend)) + }) + } +} + +func assertGatewayStoreUpdateSettings(t *testing.T, store GatewayStore) { + t.Helper() require.NoError(t, store.Initialize(GatewaySettings{ ChainREST: "http://node:1317", @@ -229,36 +233,36 @@ func TestGatewayStoreLoadsLegacyModelAccessIntoModelLimits(t *testing.T) { } func TestGatewayStorePersistsSuspiciousHosts(t *testing.T) { - store, err := NewSQLiteGatewayStore(filepath.Join(t.TempDir(), "gateway.db")) - require.NoError(t, err) - t.Cleanup(func() { - require.NoError(t, store.Close()) - }) - require.NoError(t, store.Initialize(GatewaySettings{ - ChainREST: "http://node:1317", - PublicAPI: "http://api:9000", - DefaultModel: "Qwen/Test", - DefaultRequestMaxTokens: 1000, - MaxConcurrentRequests: 2, - MaxInputTokensInFlight: 200, - }, nil)) - - hosts, err := store.UpsertSuspiciousHosts([]string{" host-a ", "host-b", "host-a"}, "bad output") - require.NoError(t, err) - require.Len(t, hosts, 2) - require.Equal(t, "host-a", hosts[0].ParticipantKey) - require.Equal(t, "bad output", hosts[0].Note) - require.Equal(t, "host-b", hosts[1].ParticipantKey) - - state, ok, err := store.LoadState() - require.NoError(t, err) - require.True(t, ok) - require.Len(t, state.SuspiciousHosts, 2) - - hosts, err = store.DeleteSuspiciousHosts([]string{"host-a"}) - require.NoError(t, err) - require.Len(t, hosts, 1) - require.Equal(t, "host-b", hosts[0].ParticipantKey) + for _, backend := range gatewayStoreTestBackends { + t.Run(backend, func(t *testing.T) { + store := newTestGatewayStore(t, backend) + require.NoError(t, store.Initialize(GatewaySettings{ + ChainREST: "http://node:1317", + PublicAPI: "http://api:9000", + DefaultModel: "Qwen/Test", + DefaultRequestMaxTokens: 1000, + MaxConcurrentRequests: 2, + MaxInputTokensInFlight: 200, + }, nil)) + + hosts, err := store.UpsertSuspiciousHosts([]string{" host-a ", "host-b", "host-a"}, "bad output") + require.NoError(t, err) + require.Len(t, hosts, 2) + require.Equal(t, "host-a", hosts[0].ParticipantKey) + require.Equal(t, "bad output", hosts[0].Note) + require.Equal(t, "host-b", hosts[1].ParticipantKey) + + state, ok, err := store.LoadState() + require.NoError(t, err) + require.True(t, ok) + require.Len(t, state.SuspiciousHosts, 2) + + hosts, err = store.DeleteSuspiciousHosts([]string{"host-a"}) + require.NoError(t, err) + require.Len(t, hosts, 1) + require.Equal(t, "host-b", hosts[0].ParticipantKey) + }) + } } func TestValidateGatewaySettingsRequiresRotationModels(t *testing.T) { @@ -920,46 +924,48 @@ func TestEscrowRotationUsesEpochSwitchHeightDuringPoC(t *testing.T) { } func TestGatewayStoreSetDevshardSettlementPending(t *testing.T) { - store, err := NewSQLiteGatewayStore(filepath.Join(t.TempDir(), "gateway.db")) - require.NoError(t, err) - t.Cleanup(func() { require.NoError(t, store.Close()) }) - - require.NoError(t, store.Initialize(GatewaySettings{ - ChainREST: "http://node:1317", DefaultModel: "m", DefaultRequestMaxTokens: 1000, - }.WithTuningDefaults(), []GatewayDevshardState{{ - RuntimeConfig: RuntimeConfig{ID: "12", PrivateKeyHex: "secret", Model: "m"}, - Active: true, - }})) - - // Default is not pending. - state, ok, err := store.LoadState() - require.NoError(t, err) - require.True(t, ok) - require.False(t, gatewayDevshardsByID(state.Devshards)["12"].SettlementPending) - - // Set pending → persisted and survives reload. - require.NoError(t, store.SetDevshardSettlementPending("12", true)) - state, _, err = store.LoadState() - require.NoError(t, err) - require.True(t, gatewayDevshardsByID(state.Devshards)["12"].SettlementPending) - - // An unrelated upsert must NOT wipe the pending marker. - require.NoError(t, store.UpsertDevshard(GatewayDevshardState{ - RuntimeConfig: RuntimeConfig{ID: "12", PrivateKeyHex: "secret", Model: "m"}, - Active: false, - })) - state, _, err = store.LoadState() - require.NoError(t, err) - require.True(t, gatewayDevshardsByID(state.Devshards)["12"].SettlementPending) - - // Clear pending. - require.NoError(t, store.SetDevshardSettlementPending("12", false)) - state, _, err = store.LoadState() - require.NoError(t, err) - require.False(t, gatewayDevshardsByID(state.Devshards)["12"].SettlementPending) - - // Unknown id errors. - require.Error(t, store.SetDevshardSettlementPending("nope", true)) + for _, backend := range gatewayStoreTestBackends { + t.Run(backend, func(t *testing.T) { + store := newTestGatewayStore(t, backend) + + require.NoError(t, store.Initialize(GatewaySettings{ + ChainREST: "http://node:1317", DefaultModel: "m", DefaultRequestMaxTokens: 1000, + }.WithTuningDefaults(), []GatewayDevshardState{{ + RuntimeConfig: RuntimeConfig{ID: "12", PrivateKeyHex: "secret", Model: "m"}, + Active: true, + }})) + + // Default is not pending. + state, ok, err := store.LoadState() + require.NoError(t, err) + require.True(t, ok) + require.False(t, gatewayDevshardsByID(state.Devshards)["12"].SettlementPending) + + // Set pending → persisted and survives reload. + require.NoError(t, store.SetDevshardSettlementPending("12", true)) + state, _, err = store.LoadState() + require.NoError(t, err) + require.True(t, gatewayDevshardsByID(state.Devshards)["12"].SettlementPending) + + // An unrelated upsert must NOT wipe the pending marker. + require.NoError(t, store.UpsertDevshard(GatewayDevshardState{ + RuntimeConfig: RuntimeConfig{ID: "12", PrivateKeyHex: "secret", Model: "m"}, + Active: false, + })) + state, _, err = store.LoadState() + require.NoError(t, err) + require.True(t, gatewayDevshardsByID(state.Devshards)["12"].SettlementPending) + + // Clear pending. + require.NoError(t, store.SetDevshardSettlementPending("12", false)) + state, _, err = store.LoadState() + require.NoError(t, err) + require.False(t, gatewayDevshardsByID(state.Devshards)["12"].SettlementPending) + + // Unknown id errors. + require.Error(t, store.SetDevshardSettlementPending("nope", true)) + }) + } } func gatewayDevshardsByID(devshards []GatewayDevshardState) map[string]GatewayDevshardState { diff --git a/devshard/cmd/devshardctl/gateway_store_test_helpers_test.go b/devshard/cmd/devshardctl/gateway_store_test_helpers_test.go index 5d4fc7db7c..47fe0cce5c 100644 --- a/devshard/cmd/devshardctl/gateway_store_test_helpers_test.go +++ b/devshard/cmd/devshardctl/gateway_store_test_helpers_test.go @@ -7,6 +7,11 @@ import ( "github.com/stretchr/testify/require" ) +// gatewayStoreTestBackends lists the backends that backend-agnostic store +// contract tests run against. The "postgres" backend is skipped in -short mode +// (it needs Docker) via setupPostgresContainer. +var gatewayStoreTestBackends = []string{"sqlite", "postgres"} + func newTestGatewayStore(t *testing.T, backend string) GatewayStore { t.Helper() switch backend { From d8e5c498fa179adba42d1b12661b68c6c26415b8 Mon Sep 17 00:00:00 2001 From: akup Date: Sun, 12 Jul 2026 18:50:24 +0300 Subject: [PATCH 13/13] fix(devshard): embed Storage in session Close test fake --- devshard/user/session_close_test.go | 34 ++++++----------------------- 1 file changed, 7 insertions(+), 27 deletions(-) diff --git a/devshard/user/session_close_test.go b/devshard/user/session_close_test.go index b939b69986..a19538eaf1 100644 --- a/devshard/user/session_close_test.go +++ b/devshard/user/session_close_test.go @@ -6,40 +6,20 @@ import ( "github.com/stretchr/testify/require" "devshard/storage" - "devshard/types" ) // closeCountingStore is a storage.Storage that records how many times Close is -// called. Every other method is an inert stub: this fake exists only to prove -// that Session.Close releases the underlying store, which is the resource the -// per-runtime memory leak was failing to free. +// called. The embedded storage.Storage is intentionally nil: this fake exists +// only to prove that Session.Close releases the underlying store, which is the +// resource the per-runtime memory leak was failing to free. Embedding the +// interface keeps the fake conformant as storage.Storage grows, so the only +// method with real behavior is Close; any other call would nil-panic, which is +// the correct signal for an unexpected use of this inert fake. type closeCountingStore struct { + storage.Storage closeCalls int } -func (s *closeCountingStore) CreateSession(storage.CreateSessionParams) error { return nil } -func (s *closeCountingStore) MarkSettled(string) error { return nil } -func (s *closeCountingStore) ListActiveSessions() ([]storage.ActiveSession, error) { - return nil, nil -} -func (s *closeCountingStore) AppendDiff(string, types.DiffRecord) error { return nil } -func (s *closeCountingStore) GetDiffs(string, uint64, uint64) ([]types.DiffRecord, error) { - return nil, nil -} -func (s *closeCountingStore) AddSignature(string, uint64, uint32, []byte) error { return nil } -func (s *closeCountingStore) GetSignatures(string, uint64) (map[uint32][]byte, error) { - return nil, nil -} -func (s *closeCountingStore) GetSessionMeta(string) (*storage.SessionMeta, error) { - return nil, storage.ErrSessionNotFound -} -func (s *closeCountingStore) MarkFinalized(string, uint64) error { return nil } -func (s *closeCountingStore) LastFinalized(string) (uint64, error) { return 0, nil } -func (s *closeCountingStore) SaveSnapshot(string, uint64, []byte) error { return nil } -func (s *closeCountingStore) LoadSnapshot(string) (uint64, []byte, error) { - return 0, nil, storage.ErrSnapshotNotFound -} -func (s *closeCountingStore) PruneEpoch(uint64) error { return nil } func (s *closeCountingStore) Close() error { s.closeCalls++ return nil