Versionless devshard stats endpoints

deploy/join/observability/grafana/dashboards/gonka-devshard-observability.json

@@ -9,7 +9,7 @@
       },
       "id": 1,
       "options": {
-        "content": "Use this dashboard to answer one question quickly: where did a devshard inference stop? Start with Terminal Outcomes and Operator Interruptions, then use logs by request_id for the exact request and failure_where.",
+        "content": "Use this dashboard to answer one question quickly: where did a devshard inference stop? Start with Terminal Outcomes and Operator Interruptions, then use logs by request_id for the exact request and failure_where.\n\nHTTP shard debugging (not this dashboard): prefer versionless `GET /devshard/sessions/{id}/diffs` and `GET /devshard/stats/shards/{id}` — see versionless-observability-plan.md. Legacy `/devshard/{version}/…` obs URLs still work via join-proxy rewrite.",
         "mode": "markdown"
       },
       "title": "How to Read This",

devshard/cmd/devshardd/session/bind_obs_test.go

@@ -0,0 +1,230 @@
+package session
+
+import (
+	"bytes"
+	"encoding/hex"
+	"net/http"
+	"net/http/httptest"
+	"strconv"
+	"testing"
+	"time"
+
+	"github.com/labstack/echo/v4"
+	"github.com/stretchr/testify/require"
+
+	"devshard/bridge"
+	"devshard/internal/testutil"
+	"devshard/signing"
+	"devshard/storage"
+	"devshard/stub"
+	"devshard/transport"
+)
+
+func setupBindTestManager(t *testing.T, escrowID string) (*HostManager, *storage.SQLite, *signing.Secp256k1Signer, *signing.Secp256k1Signer) {
+	t.Helper()
+	store := newManagerTestStore(t)
+	hosts := make([]*signing.Secp256k1Signer, 3)
+	for i := range hosts {
+		hosts[i] = mustGenerateKey(t)
+	}
+	user := mustGenerateKey(t)
+	addresses := make([]string, len(hosts))
+	for i, h := range hosts {
+		addresses[i] = h.Address()
+	}
+	br := &mockBridge{
+		escrow: &bridge.EscrowInfo{
+			EscrowID:       escrowID,
+			EpochID:        7,
+			Amount:         100000,
+			CreatorAddress: user.Address(),
+			Slots:          addresses,
+			TokenPrice:     1,
+		},
+	}
+	mgr := NewHostManager(store, hosts[0], stub.NewInferenceEngine(), stub.NewValidationEngine(), nil, testutil.RuntimeTestVersion, br, nil, nil)
+	return mgr, store, user, hosts[0]
+}
+
+func signedPOST(t *testing.T, e *echo.Echo, signer *signing.Secp256k1Signer, path, escrowID string, body []byte) *httptest.ResponseRecorder {
+	t.Helper()
+	ts := time.Now().Unix()
+	sig, err := transport.SignRequest(signer, escrowID, body, ts)
+	require.NoError(t, err)
+	req := httptest.NewRequest(http.MethodPost, path, bytes.NewReader(body))
+	req.Header.Set(echo.HeaderContentType, "application/json")
+	req.Header.Set(transport.HeaderSignature, hex.EncodeToString(sig))
+	req.Header.Set(transport.HeaderTimestamp, strconv.FormatInt(ts, 10))
+	rec := httptest.NewRecorder()
+	e.ServeHTTP(rec, req)
+	return rec
+}
+
+func TestObsGET_DoesNotBindSession(t *testing.T) {
+	const escrowID = "obs-unbound"
+	mgr, store, _, _ := setupBindTestManager(t, escrowID)
+
+	e := echo.New()
+	mgr.Register(e.Group(""))
+
+	req := httptest.NewRequest(http.MethodGet, "/sessions/"+escrowID+"/diffs", nil)
+	rec := httptest.NewRecorder()
+	e.ServeHTTP(rec, req)
+	require.Equal(t, http.StatusNotFound, rec.Code, "body: %s", rec.Body.String())
+
+	_, err := store.GetSessionMeta(escrowID)
+	require.ErrorIs(t, err, storage.ErrSessionNotFound)
+
+	active, err := store.ListActiveSessions()
+	require.NoError(t, err)
+	require.Empty(t, active)
+}
+
+func TestObsMempoolSignatures_DoNotBindSession(t *testing.T) {
+	const escrowID = "obs-unbound-2"
+	mgr, store, _, _ := setupBindTestManager(t, escrowID)
+	e := echo.New()
+	mgr.Register(e.Group(""))
+
+	for _, path := range []string{
+		"/sessions/" + escrowID + "/mempool",
+		"/sessions/" + escrowID + "/signatures",
+	} {
+		req := httptest.NewRequest(http.MethodGet, path, nil)
+		rec := httptest.NewRecorder()
+		e.ServeHTTP(rec, req)
+		require.Equal(t, http.StatusNotFound, rec.Code, "path=%s body=%s", path, rec.Body.String())
+	}
+
+	_, err := store.GetSessionMeta(escrowID)
+	require.ErrorIs(t, err, storage.ErrSessionNotFound)
+}
+
+func TestGossipUnbound_DoesNotBindSession(t *testing.T) {
+	const escrowID = "gossip-unbound"
+	mgr, store, _, hostSigner := setupBindTestManager(t, escrowID)
+	e := echo.New()
+	mgr.Register(e.Group(""))
+
+	body := []byte(`{"nonce":1}`)
+	rec := signedPOST(t, e, hostSigner, "/sessions/"+escrowID+"/gossip/nonce", escrowID, body)
+	require.Equal(t, http.StatusNotFound, rec.Code, "body: %s", rec.Body.String())
+
+	_, err := store.GetSessionMeta(escrowID)
+	require.ErrorIs(t, err, storage.ErrSessionNotFound)
+}
+
+func TestOwnerChat_BindsSession(t *testing.T) {
+	const escrowID = "owner-bind"
+	mgr, store, user, _ := setupBindTestManager(t, escrowID)
+	e := echo.New()
+	mgr.Register(e.Group(""))
+
+	body := []byte(`{"model":"m","messages":[{"role":"user","content":"hi"}]}`)
+	rec := signedPOST(t, e, user, "/sessions/"+escrowID+"/chat/completions", escrowID, body)
+	// Inference may fail downstream; bind must have happened regardless.
+	meta, err := store.GetSessionMeta(escrowID)
+	require.NoError(t, err, "owner chat must CreateSession; http=%d body=%s", rec.Code, rec.Body.String())
+	require.Equal(t, testutil.RuntimeTestVersion, meta.Version)
+	require.Equal(t, user.Address(), meta.CreatorAddr)
+}
+
+type countingGetEscrowBridge struct {
+	bridge.MainnetBridge
+	calls int
+}
+
+func (b *countingGetEscrowBridge) GetEscrow(escrowID string) (*bridge.EscrowInfo, error) {
+	b.calls++
+	return b.MainnetBridge.GetEscrow(escrowID)
+}
+
+func TestOwnerChat_FirstBindSingleGetEscrow(t *testing.T) {
+	const escrowID = "owner-bind-once"
+	store := newManagerTestStore(t)
+	hosts := make([]*signing.Secp256k1Signer, 3)
+	for i := range hosts {
+		hosts[i] = mustGenerateKey(t)
+	}
+	user := mustGenerateKey(t)
+	addresses := make([]string, len(hosts))
+	for i, h := range hosts {
+		addresses[i] = h.Address()
+	}
+	inner := &mockBridge{
+		escrow: &bridge.EscrowInfo{
+			EscrowID:       escrowID,
+			EpochID:        7,
+			Amount:         100000,
+			CreatorAddress: user.Address(),
+			Slots:          addresses,
+			TokenPrice:     1,
+		},
+	}
+	br := &countingGetEscrowBridge{MainnetBridge: inner}
+	mgr := NewHostManager(store, hosts[0], stub.NewInferenceEngine(), stub.NewValidationEngine(), nil, testutil.RuntimeTestVersion, br, nil, nil)
+
+	e := echo.New()
+	mgr.Register(e.Group(""))
+	body := []byte(`{"model":"m","messages":[{"role":"user","content":"hi"}]}`)
+	_ = signedPOST(t, e, user, "/sessions/"+escrowID+"/chat/completions", escrowID, body)
+
+	meta, err := store.GetSessionMeta(escrowID)
+	require.NoError(t, err)
+	require.Equal(t, testutil.RuntimeTestVersion, meta.Version)
+	require.Equal(t, 1, br.calls, "first-bind path must GetEscrow once (reuse for BuildGroup + CreateSession)")
+}
+
+func TestNonOwnerChat_DoesNotBindSession(t *testing.T) {
+	const escrowID = "non-owner-bind"
+	mgr, store, _, hostSigner := setupBindTestManager(t, escrowID)
+	e := echo.New()
+	mgr.Register(e.Group(""))
+
+	body := []byte(`{"model":"m","messages":[{"role":"user","content":"hi"}]}`)
+	rec := signedPOST(t, e, hostSigner, "/sessions/"+escrowID+"/chat/completions", escrowID, body)
+	require.Equal(t, http.StatusForbidden, rec.Code, "body: %s", rec.Body.String())
+
+	_, err := store.GetSessionMeta(escrowID)
+	require.ErrorIs(t, err, storage.ErrSessionNotFound)
+}
+
+func TestSessionServerExisting_NoCreate(t *testing.T) {
+	const escrowID = "existing-miss"
+	mgr, store, _, _ := setupBindTestManager(t, escrowID)
+
+	_, err := mgr.SessionServerExisting(escrowID)
+	require.ErrorIs(t, err, storage.ErrSessionNotFound)
+
+	_, err = store.GetSessionMeta(escrowID)
+	require.ErrorIs(t, err, storage.ErrSessionNotFound)
+}
+
+func TestGetOrCreate_RecoversBeforeCreate(t *testing.T) {
+	store := newManagerTestStore(t)
+	_, user, hostSigner := populateStore(t, store, 2)
+
+	addresses := []string{hostSigner.Address()}
+	// populateStore uses 3 hosts; rebuild addresses from meta.
+	meta, err := store.GetSessionMeta("1")
+	require.NoError(t, err)
+	addresses = make([]string, len(meta.Group))
+	for i, s := range meta.Group {
+		addresses[i] = s.ValidatorAddress
+	}
+
+	br := &mockBridge{
+		escrow: &bridge.EscrowInfo{
+			EscrowID:       "1",
+			EpochID:        7,
+			Amount:         100000,
+			CreatorAddress: user.Address(),
+			Slots:          addresses,
+		},
+	}
+	mgr := NewHostManager(store, hostSigner, stub.NewInferenceEngine(), stub.NewValidationEngine(), nil, testutil.RuntimeTestVersion, br, nil, nil)
+
+	srv, err := mgr.getOrCreate("1", nil)
+	require.NoError(t, err)
+	require.Equal(t, uint64(2), srv.Host().SnapshotState().LatestNonce)
+}

devshard/cmd/devshardd/session/manager.go

@@ -56,10 +56,11 @@ type HostManager struct {
 	availability       devshardpkg.AvailabilityProvider
 	maxNonce           devshardpkg.MaxNonceProvider
 
-	statsMu           sync.Mutex
-	statsShardsCache  *statsShardsResponse
-	statsShardsCached time.Time
-	statsDetailsCache map[string]statsShardDetailCache
+	statsMu            sync.Mutex
+	statsShardsCache   *statsShardsResponse
+	statsShardsCached  time.Time
+	statsDetailsCache  map[string]statsShardDetailCache
+	statsNegativeCache map[string]statsNegativeCacheEntry
 }
 
 const (
@@ -98,6 +99,7 @@ func NewHostManager(
 		payloadStore:       ps,
 		recorder:           recorder,
 		statsDetailsCache:  make(map[string]statsShardDetailCache),
+		statsNegativeCache: make(map[string]statsNegativeCacheEntry),
 	}
 }
 
@@ -128,8 +130,65 @@ func (m *HostManager) Close() error {
 }
 
 // SessionServer resolves or creates the per-escrow transport server.
+// Prefer BindOwnerChat for HTTP chat; this remains for tests and explicit bind.
 func (m *HostManager) SessionServer(escrowID string) (*transport.Server, error) {
-	return m.getOrCreate(escrowID)
+	return m.getOrCreate(escrowID, nil)
+}
+
+// SessionServerExisting returns a live or recovered session server.
+// It never CreateSession / binds a protocol version. Missing sessions return
+// storage.ErrSessionNotFound.
+func (m *HostManager) SessionServerExisting(escrowID string) (*transport.Server, error) {
+	if srv, ok := m.existingServer(escrowID); ok {
+		return srv, nil
+	}
+	srv, err := m.recoverAndStoreSession(escrowID)
+	if err != nil {
+		return nil, err
+	}
+	return srv, nil
+}
+
+// BindOwnerChat verifies the request as the escrow owner, then returns an
+// existing session or binds a new one with this process's boundVersion.
+// Auth context (sender + body) is injected for HandleInference.
+func (m *HostManager) BindOwnerChat(c echo.Context) (*transport.Server, error) {
+	escrowID := c.Param("id")
+	addr, body, err := transport.VerifyPOSTAuth(c, m.verifier, escrowID, 0)
+	if err != nil {
+		return nil, err
+	}
+
+	srv, err := m.SessionServerExisting(escrowID)
+	if err == nil {
+		if !srv.IsOwner(addr) {
+			return nil, echo.NewHTTPError(http.StatusForbidden, "restricted to escrow owner")
+		}
+		transport.InjectAuthContext(c, addr, body)
+		return srv, nil
+	}
+	if !errors.Is(err, storage.ErrSessionNotFound) {
+		return nil, err
+	}
+
+	escrow, err := m.bridge.GetEscrow(escrowID)
+	if err != nil {
+		return nil, fmt.Errorf("get escrow: %w", err)
+	}
+	if escrow == nil || escrow.CreatorAddress == "" || addr != escrow.CreatorAddress {
+		return nil, echo.NewHTTPError(http.StatusForbidden, "restricted to escrow owner")
+	}
+
+	// Pass the already-fetched escrow so create() does not GetEscrow again.
+	srv, err = m.getOrCreate(escrowID, escrow)
+	if err != nil {
+		return nil, err
+	}
+	if !srv.IsOwner(addr) {
+		return nil, echo.NewHTTPError(http.StatusForbidden, "restricted to escrow owner")
+	}
+	transport.InjectAuthContext(c, addr, body)
+	return srv, nil
 }
 
 // HandleSettlementFinalized marks the session inactive and drops the live
@@ -153,7 +212,10 @@ func (m *HostManager) HandleSettlementFinalized(escrowID string) error {
 	return nil
 }
 
-func (m *HostManager) getOrCreate(escrowID string) (*transport.Server, error) {
+// getOrCreate returns a live session, recovering from store or creating.
+// When escrow is non-nil (BindOwnerChat first-bind path), create reuses it and
+// skips a second bridge.GetEscrow.
+func (m *HostManager) getOrCreate(escrowID string, escrow *bridge.EscrowInfo) (*transport.Server, error) {
 	if srv, ok := m.existingServer(escrowID); ok {
 		return srv, nil
 	}
@@ -169,7 +231,16 @@ func (m *HostManager) getOrCreate(escrowID string) (*transport.Server, error) {
 			return nil, err
 		}
 
-		srv, err := m.create(escrowID)
+		// Prefer recovering an already-bound session over CreateSession.
+		srv, err := m.recoverStoredSession(escrowID)
+		if err == nil {
+			return m.storeSessionIfAbsent(escrowID, srv), nil
+		}
+		if !errors.Is(err, storage.ErrSessionNotFound) {
+			return nil, err
+		}
+
+		srv, err = m.create(escrowID, escrow)
 		if err != nil {
 			return nil, err
 		}
@@ -265,15 +336,18 @@ func (m *HostManager) EvictBefore(cutoffEpoch uint64) int {
 	return len(evicted)
 }
 
-func (m *HostManager) create(escrowID string) (*transport.Server, error) {
-	group, err := bridge.BuildGroup(escrowID, m.bridge)
-	if err != nil {
-		return nil, fmt.Errorf("build group: %w", err)
+func (m *HostManager) create(escrowID string, escrow *bridge.EscrowInfo) (*transport.Server, error) {
+	if escrow == nil {
+		var err error
+		escrow, err = m.bridge.GetEscrow(escrowID)
+		if err != nil {
+			return nil, fmt.Errorf("get escrow: %w", err)
+		}
 	}
 
-	escrow, err := m.bridge.GetEscrow(escrowID)
+	group, err := bridge.BuildGroupFromEscrow(escrow)
 	if err != nil {
-		return nil, fmt.Errorf("get escrow: %w", err)
+		return nil, fmt.Errorf("build group: %w", err)
 	}
 
 	creatorAddr := escrow.CreatorAddress
@@ -440,7 +514,7 @@ func (m *HostManager) recoverStoredSession(escrowID string) (*transport.Server,
 func (m *HostManager) Register(g *echo.Group) {
 	g.GET("/stats/shards", m.handleStatsShards)
 	g.GET("/stats/shards/:escrow_id", m.handleStatsShard)
-	devshardserver.RegisterLazySessionRoutes(g, m, m)
+	devshardserver.RegisterLazySessionRoutes(g, m, m, m)
 }
 
 // HandlePayloads serves payloads to validators for devshard validation.