Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
49 changes: 49 additions & 0 deletions devshard/cmd/devshardctl/escrow_recovery_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -138,6 +138,55 @@ func TestCreateRotationEscrowIntentFirstThenPersistAndClear(t *testing.T) {
assert.Empty(t, commitments, "commitment cleared after persist")
}

// Rotation-created escrows inherit the protocol version implied by the
// gateway-wide route prefix (DEVSHARD_ROUTE_PREFIX), both on the direct
// persist path and through commitment recovery.
func TestCreateRotationEscrowCarriesProtocolVersionFromRoutePrefix(t *testing.T) {
g, store, settings := newRecoveryGateway(t)
stubCreateOnChain(t, "TXPV1", 555)
t.Setenv("DEVSHARD_ROUTE_PREFIX", "/devshard/v3")
model := normalizedEscrowRotationModels(settings)[0]

_, err := g.createRotationEscrow(context.Background(), settings, model, rotationRoleTemp, 10)
require.NoError(t, err)

record := devshardIDs(t, store)["555"]
assert.Equal(t, "3", record.ProtocolVersion, "persisted escrow inherits protocol from route prefix")
}

// A route prefix whose version segment is not a protocol version (e.g. a named
// versiond runtime) keeps the empty/v1-default behavior; semver-like versions
// map by their major component.
Comment thread
DimaOrekhovPS marked this conversation as resolved.
func TestRotationEscrowProtocolVersionRouteMapping(t *testing.T) {
t.Setenv("DEVSHARD_ROUTE_PREFIX", "/devshard/mainnet-canary")
assert.Empty(t, rotationEscrowProtocolVersion())
t.Setenv("DEVSHARD_ROUTE_PREFIX", "/devshard/v3")
assert.Equal(t, "3", rotationEscrowProtocolVersion())
t.Setenv("DEVSHARD_ROUTE_PREFIX", "/devshard/v2.1.0")
assert.Equal(t, "2", rotationEscrowProtocolVersion())
t.Setenv("DEVSHARD_ROUTE_PREFIX", "/devshard/3")
assert.Equal(t, "3", rotationEscrowProtocolVersion())
}

func TestReconcileCommitmentsCarriesProtocolVersion(t *testing.T) {
g, store, settings := newRecoveryGateway(t)
require.NoError(t, store.SaveCommitment(GatewayEscrowCommitment{
TxHash: "TXPV2", Model: "Qwen/Test", Role: rotationRoleTemp, Epoch: 11,
PrivateKeyEnv: "DEVSHARD_PRIVATE_KEY", ProtocolVersion: "3",
}))

commitments, err := store.LoadCommitments()
require.NoError(t, err)
require.Len(t, commitments, 1)
assert.Equal(t, "3", commitments[0].ProtocolVersion, "commitment round-trips protocol version")

stubQueryTxEscrowID(t, func(string) (uint64, bool, error) { return 556, true, nil })
g.reconcileCommitments(context.Background(), settings)

record := devshardIDs(t, store)["556"]
assert.Equal(t, "3", record.ProtocolVersion, "recovered escrow keeps protocol version")
}

// If the intent commitment cannot be written, no chain tx is broadcast and no
// escrow is created — the safe failure.
func TestCreateRotationEscrowAbortsWhenCommitmentWriteFails(t *testing.T) {
Expand Down
64 changes: 50 additions & 14 deletions devshard/cmd/devshardctl/escrow_rotator.go
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ import (
"strings"
"time"

devshardpkg "devshard"
"devshard/types"
)

Expand Down Expand Up @@ -332,12 +333,14 @@ func (g *Gateway) createEscrowOnChain(ctx context.Context, settings GatewaySetti
}

func (g *Gateway) createRotationEscrow(ctx context.Context, settings GatewaySettings, model EscrowRotationModelSettings, role string, epoch uint64) (*CreateDevshardEscrowResult, error) {
protocolVersion := rotationEscrowProtocolVersion()
commitment := GatewayEscrowCommitment{
Model: model.ModelID,
Role: role,
Epoch: epoch,
PrivateKeyEnv: model.PrivateKeyEnv,
BlockHeight: g.currentBlockHeight(),
Model: model.ModelID,
Role: role,
Epoch: epoch,
PrivateKeyEnv: model.PrivateKeyEnv,
ProtocolVersion: protocolVersion,
BlockHeight: g.currentBlockHeight(),
}
// Intent-first: persist the commitment (with the precomputed tx hash) before broadcast.
onPrepared := func(txHash string) error {
Expand All @@ -349,7 +352,7 @@ func (g *Gateway) createRotationEscrow(ctx context.Context, settings GatewaySett
if err != nil {
return nil, err
}
if err := g.persistRotationEscrow(ctx, result.EscrowID, model.ModelID, role, epoch, model.PrivateKeyEnv); err != nil {
if err := g.persistRotationEscrow(ctx, result.EscrowID, model.ModelID, role, epoch, model.PrivateKeyEnv, protocolVersion); err != nil {
// Escrow is on chain; commitment survives -> reconcile recovers it by tx hash.
log.Printf("escrow_rotation_persist_failed escrow=%d tx=%s model=%q error=%v recover_via_commitment=true", result.EscrowID, result.TxHash, model.ModelID, err)
return nil, err
Expand All @@ -362,16 +365,48 @@ func (g *Gateway) createRotationEscrow(ctx context.Context, settings GatewaySett
func newRotationDevshardState(result *CreateDevshardEscrowResult, model EscrowRotationModelSettings, role string, epoch uint64) GatewayDevshardState {
return GatewayDevshardState{
RuntimeConfig: RuntimeConfig{
ID: strconv.FormatUint(result.EscrowID, 10),
PrivateKeyEnv: strings.TrimSpace(model.PrivateKeyEnv),
Model: strings.TrimSpace(model.ModelID),
ID: strconv.FormatUint(result.EscrowID, 10),
PrivateKeyEnv: strings.TrimSpace(model.PrivateKeyEnv),
Model: strings.TrimSpace(model.ModelID),
ProtocolVersion: rotationEscrowProtocolVersion(),
},
Active: true,
RotationRole: role,
RotationEpoch: epoch,
}
}

// rotationEscrowProtocolVersion is the protocol version stamped on escrows
// created by rotation/depletion. It is derived from the gateway-wide route
// prefix (DEVSHARD_ROUTE_PREFIX / build version), so a gateway serving
// /devshard/v3 mints protocol-v3 escrows. Semver-like route versions map by
// major (v2.1.0 -> v2), relying on the same naming convention that ties a
// route version to its protocol. An unparseable version segment (e.g. a
// named versiond runtime) falls back to the v1 default, matching the
// pre-existing behavior for explicit registrations without a protocol.
Comment thread
DimaOrekhovPS marked this conversation as resolved.
func rotationEscrowProtocolVersion() string {
routePrefix, err := resolveGatewayRoutePrefix()
if err != nil {
log.Printf("escrow_rotation_protocol_version_fallback reason=route_prefix_unresolved error=%v", err)
return ""
}
_, version, err := devshardpkg.ResolveRoutePrefix(routePrefix)
if err != nil {
log.Printf("escrow_rotation_protocol_version_fallback route_prefix=%q reason=version_segment_unresolved error=%v", routePrefix, err)
return ""
}
normalized := strings.TrimSpace(version)
if i := strings.IndexByte(normalized, '.'); i > 0 {
normalized = normalized[:i] // e.g. v2.1.0 -> v2
}
pv, err := types.ParseProtocolVersion(normalized)
if err != nil {
log.Printf("escrow_rotation_protocol_version_fallback route_prefix=%q version=%q reason=unparseable_protocol error=%v", routePrefix, version, err)
return ""
Comment thread
DimaOrekhovPS marked this conversation as resolved.
}
return string(pv)
}

func normalizedEscrowRotationModels(settings GatewaySettings) []EscrowRotationModelSettings {
models := make([]EscrowRotationModelSettings, 0, len(settings.EscrowRotation.Models))
for _, model := range settings.EscrowRotation.Models {
Expand Down Expand Up @@ -496,12 +531,13 @@ func withDBRetry(ctx context.Context, fn func() error) error {
}

// persistRotationEscrow persists + registers a created escrow ("already exists" = ok).
func (g *Gateway) persistRotationEscrow(ctx context.Context, escrowID uint64, modelID, role string, epoch uint64, keyEnv string) error {
func (g *Gateway) persistRotationEscrow(ctx context.Context, escrowID uint64, modelID, role string, epoch uint64, keyEnv, protocolVersion string) error {
record := GatewayDevshardState{
RuntimeConfig: RuntimeConfig{
ID: strconv.FormatUint(escrowID, 10),
PrivateKeyEnv: keyEnv,
Model: modelID,
ID: strconv.FormatUint(escrowID, 10),
PrivateKeyEnv: keyEnv,
Model: modelID,
ProtocolVersion: strings.TrimSpace(protocolVersion),
},
Active: true,
RotationRole: role,
Expand Down Expand Up @@ -566,7 +602,7 @@ func (g *Gateway) reconcileCommitments(ctx context.Context, settings GatewaySett
g.clearCommitment(ctx, c.TxHash)
continue
}
if err := g.persistRotationEscrow(ctx, escrowID, c.Model, c.Role, c.Epoch, c.PrivateKeyEnv); err != nil {
if err := g.persistRotationEscrow(ctx, escrowID, c.Model, c.Role, c.Epoch, c.PrivateKeyEnv, c.ProtocolVersion); err != nil {
log.Printf("escrow_commitment_persist_failed tx=%s escrow=%d model=%q error=%v", c.TxHash, escrowID, c.Model, err)
continue // keep commitment — retry next pass
}
Expand Down
28 changes: 17 additions & 11 deletions devshard/cmd/devshardctl/gateway_store.go
Original file line number Diff line number Diff line change
Expand Up @@ -506,6 +506,10 @@ func NewGatewayStore(path string) (*GatewayStore, error) {
db.Close()
return nil, fmt.Errorf("init escrow rotation commitments table: %w", err)
}
if err := ensureColumn(db, "escrow_rotation_commitments", "protocol_version", "TEXT NOT NULL DEFAULT ''"); err != nil {
db.Close()
return nil, fmt.Errorf("migrate escrow rotation commitments protocol version: %w", err)
}
if err := ensureColumn(db, "participant_throttle_state", "quarantine_until_utc", "TEXT NOT NULL DEFAULT ''"); err != nil {
db.Close()
return nil, fmt.Errorf("migrate participant throttle: %w", err)
Expand Down Expand Up @@ -1021,13 +1025,14 @@ func (s *GatewayStore) LoadRotationStatuses(limit int) ([]GatewayRotationStatus,

// GatewayEscrowCommitment is the write-ahead intent for one escrow create, keyed by tx hash.
type GatewayEscrowCommitment struct {
TxHash string
Model string
Role string
Epoch uint64
PrivateKeyEnv string
BlockHeight uint64
CreatedAt time.Time
TxHash string
Model string
Role string
Epoch uint64
PrivateKeyEnv string
ProtocolVersion string
BlockHeight uint64
CreatedAt time.Time
}

// SaveCommitment records a create intent, keyed by tx hash.
Expand All @@ -1043,13 +1048,14 @@ func (s *GatewayStore) SaveCommitment(c GatewayEscrowCommitment) error {
}
_, err := s.db.Exec(`
INSERT OR REPLACE INTO escrow_rotation_commitments (
tx_hash, model, role, epoch, private_key_env, block_height, created_at
) VALUES (?, ?, ?, ?, ?, ?, ?)`,
tx_hash, model, role, epoch, private_key_env, protocol_version, block_height, created_at
) VALUES (?, ?, ?, ?, ?, ?, ?, ?)`,
strings.TrimSpace(c.TxHash),
strings.TrimSpace(c.Model),
strings.TrimSpace(c.Role),
c.Epoch,
c.PrivateKeyEnv,
strings.TrimSpace(c.ProtocolVersion),
c.BlockHeight,
createdAt,
)
Expand Down Expand Up @@ -1079,7 +1085,7 @@ func (s *GatewayStore) LoadCommitments() ([]GatewayEscrowCommitment, error) {
return nil, nil
}
rows, err := s.db.Query(`
SELECT tx_hash, model, role, epoch, private_key_env, block_height, created_at
SELECT tx_hash, model, role, epoch, private_key_env, protocol_version, block_height, created_at
FROM escrow_rotation_commitments
ORDER BY created_at ASC`)
if err != nil {
Expand All @@ -1090,7 +1096,7 @@ func (s *GatewayStore) LoadCommitments() ([]GatewayEscrowCommitment, error) {
for rows.Next() {
var c GatewayEscrowCommitment
var createdAt string
if err := rows.Scan(&c.TxHash, &c.Model, &c.Role, &c.Epoch, &c.PrivateKeyEnv, &c.BlockHeight, &createdAt); err != nil {
if err := rows.Scan(&c.TxHash, &c.Model, &c.Role, &c.Epoch, &c.PrivateKeyEnv, &c.ProtocolVersion, &c.BlockHeight, &createdAt); err != nil {
return nil, fmt.Errorf("scan escrow commitment: %w", err)
}
c.CreatedAt = scanGatewayDBTime(createdAt)
Expand Down
Loading