diff --git a/decentralized-api/go.mod b/decentralized-api/go.mod index 341a17e3a0..4b408f89d0 100644 --- a/decentralized-api/go.mod +++ b/decentralized-api/go.mod @@ -9,6 +9,7 @@ require ( cosmossdk.io/x/feegrant v0.1.1 cosmossdk.io/x/upgrade v0.1.4 devshard v0.0.0-00010101000000-000000000000 + github.com/cockroachdb/apd/v2 v2.0.2 github.com/cometbft/cometbft v0.38.21 github.com/consensys/gnark-crypto v0.18.1 github.com/cosmos/btcutil v1.0.5 @@ -103,7 +104,6 @@ require ( github.com/chzyer/readline v1.5.1 // indirect github.com/cloudwego/base64x v0.1.6 // indirect github.com/cncf/xds/go v0.0.0-20251210132809-ee656c7534f5 // indirect - github.com/cockroachdb/apd/v2 v2.0.2 // indirect github.com/cockroachdb/errors v1.12.0 // indirect github.com/cockroachdb/fifo v0.0.0-20240606204812-0bbfbd93a7ce // indirect github.com/cockroachdb/logtags v0.0.0-20241215232642-bb51bb14a506 // indirect diff --git a/decentralized-api/internal/validation/conformance_vectors_test.go b/decentralized-api/internal/validation/conformance_vectors_test.go new file mode 100644 index 0000000000..9541624eb8 --- /dev/null +++ b/decentralized-api/internal/validation/conformance_vectors_test.go @@ -0,0 +1,114 @@ +package validation + +// Cross-language conformance harness for deterministic sampling. +// +// testdata/conformance_vectors.json is generated from the vLLM reference +// implementation (see DETERMINISTIC_SAMPLING_CONTRACT.md) and is the executable +// form of the contract. The Go validator MUST reproduce every vector bit-for-bit +// before it can be trusted for cross-language sampling replay. +// +// Status (item ①): this harness loads and validates the vectors' invariants so +// the fixture is wired in and CI-checked. The actual Go decimal pipeline + RNG +// (item ②) is not implemented yet; TestConformanceReplay skips until it lands, +// at which point it becomes the gate for cross-language parity. + +import ( + "encoding/json" + "os" + "path/filepath" + "testing" +) + +type conformanceCase struct { + Name string `json:"name"` + Logprobs map[string]string `json:"logprobs"` + Temperature string `json:"temperature"` + TopP *string `json:"top_p"` + TopK *int `json:"top_k"` + MinP *string `json:"min_p"` + SeedStr string `json:"seed_str"` + ExpectedWeights map[string]int64 `json:"expected_weights"` + ExpectedWeightSum int64 `json:"expected_weight_sum"` + ExpectedToken string `json:"expected_token"` +} + +type conformanceVectors struct { + ContractVersion string `json:"contract_version"` + WeightScale int64 `json:"weight_scale"` + RNGReference struct { + Seed string `json:"seed"` + FirstU64 []uint64 `json:"first_u64"` + } `json:"rng_reference"` + Cases []conformanceCase `json:"cases"` +} + +func loadConformanceVectors(t *testing.T) conformanceVectors { + t.Helper() + path := filepath.Join("testdata", "conformance_vectors.json") + raw, err := os.ReadFile(path) + if err != nil { + t.Fatalf("read %s: %v", path, err) + } + var v conformanceVectors + if err := json.Unmarshal(raw, &v); err != nil { + t.Fatalf("unmarshal %s: %v", path, err) + } + return v +} + +// TestConformanceVectorsWellFormed locks the fixture in: it must parse, cover the +// pinned contract constants, and be internally consistent. This runs today. +func TestConformanceVectorsWellFormed(t *testing.T) { + v := loadConformanceVectors(t) + + if v.ContractVersion != "1.0.0" { + t.Errorf("contract_version = %q, want 1.0.0", v.ContractVersion) + } + if v.WeightScale != 65536 { + t.Errorf("weight_scale = %d, want 65536 (2^16)", v.WeightScale) + } + if len(v.Cases) == 0 { + t.Fatal("no cases in conformance vectors") + } + + // RNG reference vector pinned by the contract (§5). + if len(v.RNGReference.FirstU64) == 0 { + t.Fatal("missing rng_reference.first_u64") + } + const wantFirstU64 = uint64(4286832458236889005) + if got := v.RNGReference.FirstU64[0]; got != wantFirstU64 { + t.Errorf("rng_reference.first_u64[0] = %d, want %d", got, wantFirstU64) + } + + for _, c := range v.Cases { + var sum int64 + for _, w := range c.ExpectedWeights { + if w < 0 { + t.Errorf("%s: negative weight %d", c.Name, w) + } + sum += w + } + if sum != v.WeightScale { + t.Errorf("%s: weights sum to %d, want %d", c.Name, sum, v.WeightScale) + } + if c.ExpectedWeightSum != v.WeightScale { + t.Errorf("%s: expected_weight_sum = %d, want %d", + c.Name, c.ExpectedWeightSum, v.WeightScale) + } + if _, ok := c.ExpectedWeights[c.ExpectedToken]; !ok { + t.Errorf("%s: expected_token %q not among expected_weights", + c.Name, c.ExpectedToken) + } + } +} + +// TestConformanceReplay: the cross-language parity gate is IMPLEMENTED and +// PASSING in the ./detsample sub-package, which hosts the Go decimal pipeline + +// Sha256CounterRNG and is dependency-free so it runs offline. See +// detsample.TestPipelineWeightsMatch (bit-identical integer weights vs the vLLM +// reference) and detsample.TestPipelineTokenMatch (identical sampled token) over +// the same conformance_vectors.json. This placeholder stays skipped; the parent +// `validation` package invokes detsample once the validator is wired to it. +func TestConformanceReplay(t *testing.T) { + t.Skip("parity gate lives in ./detsample (TestPipelineWeightsMatch / TestPipelineTokenMatch)") +} diff --git a/decentralized-api/internal/validation/detsample/distance.go b/decentralized-api/internal/validation/detsample/distance.go new file mode 100644 index 0000000000..de9f48b271 --- /dev/null +++ b/decentralized-api/internal/validation/detsample/distance.go @@ -0,0 +1,72 @@ +package detsample + +// Stage-2 distance ("Check 1"). The validator recomputes logprobs on the model +// and compares them to the signed ones; a large distance means the signed +// distribution is not this model's. This mirrors validation_sampling.mae_distance +// (Python) so the chain and vLLM agree on the metric. +// +// The metric is the mean over positions of the mean absolute logprob difference +// over the executor's reported top-K support. GPU-evidence follow-up Experiment 4 +// (gonka-ai/gonka#1199): MAE over the top-K support separates honest from a wrong +// model by ~40-65x, while the sampled-token delta alone overlaps and cannot gate. + +import ( + "math" + "sort" +) + +// Stage2MAEFraudThreshold is the fraud cut on MAEDistance. PLACEHOLDER pending +// Decision D (#1199): Experiments 4/5 put the honest floor near ~0.01 MAE and a +// clearly different model near ~0.4-0.8, with int8 quantization a gray zone around +// ~0.1-0.2. 0.25 accepts int8 as "the model" while still catching int4 and cheaper +// models. The final value — and whether int8 counts as the model — is a policy +// decision, NOT a measurement. Not enforcing. +const Stage2MAEFraudThreshold = 0.25 + +// maeMissingPenalty is charged for a token present on the executor side but absent +// on the validator side, so a truncated/mismatched distribution reads as distant. +const maeMissingPenalty = 10.0 + +// MAEDistance returns the mean-over-positions MAE over the executor's reported +// support. Returns 10.0 (maximally distant) on a length mismatch, 0.0 on empty. +func MAEDistance(executor, validator []map[string]float64) float64 { + if len(executor) == 0 { + return 0.0 + } + if len(executor) != len(validator) { + return 10.0 // length mismatch -> maximally distant + } + + perPosition := make([]float64, 0, len(executor)) + for i, exec := range executor { + if len(exec) == 0 { + continue + } + val := validator[i] + // Sum over sorted token IDs so the float accumulation order matches the + // Python validator's (byte-exact cross-language distance). + tids := make([]string, 0, len(exec)) + for tid := range exec { + tids = append(tids, tid) + } + sort.Strings(tids) + var sum float64 + for _, tid := range tids { + if v, ok := val[tid]; ok { + sum += math.Abs(exec[tid] - v) + } else { + sum += maeMissingPenalty + } + } + perPosition = append(perPosition, sum/float64(len(exec))) + } + + if len(perPosition) == 0 { + return 0.0 + } + var total float64 + for _, d := range perPosition { + total += d + } + return total / float64(len(perPosition)) +} diff --git a/decentralized-api/internal/validation/detsample/distance_test.go b/decentralized-api/internal/validation/detsample/distance_test.go new file mode 100644 index 0000000000..f5ed851631 --- /dev/null +++ b/decentralized-api/internal/validation/detsample/distance_test.go @@ -0,0 +1,43 @@ +package detsample + +import ( + "math" + "testing" +) + +var distPos0 = map[string]float64{"5": -0.5, "10": -1.2, "2": -2.5, "100": -3.9, "7": -4.1} +var distPos1 = map[string]float64{"3": -0.3, "42": -1.1, "9": -2.2, "11": -3.0, "1": -4.5} + +func TestMAEDistanceIdenticalIsZero(t *testing.T) { + if d := MAEDistance([]map[string]float64{distPos0}, []map[string]float64{distPos0}); d != 0.0 { + t.Errorf("MAEDistance identical = %v, want 0", d) + } +} + +func TestMAEDistanceUniformShift(t *testing.T) { + shifted := make(map[string]float64, len(distPos0)) + for tid, v := range distPos0 { + shifted[tid] = v - 0.5 + } + d := MAEDistance([]map[string]float64{distPos0}, []map[string]float64{shifted}) + if math.Abs(d-0.5) > 1e-9 { + t.Errorf("MAEDistance uniform 0.5 shift = %v, want 0.5", d) + } +} + +func TestMAEDistanceLengthMismatchIsMax(t *testing.T) { + d := MAEDistance( + []map[string]float64{distPos0, distPos1}, + []map[string]float64{distPos0}) + if d != 10.0 { + t.Errorf("MAEDistance length mismatch = %v, want 10", d) + } +} + +func TestMAEDistanceMissingTokenPenalized(t *testing.T) { + partial := map[string]float64{"5": -0.5, "10": -1.2, "2": -2.5, "100": -3.9} // one token dropped + d := MAEDistance([]map[string]float64{distPos0}, []map[string]float64{partial}) + if d <= Stage2MAEFraudThreshold { + t.Errorf("MAEDistance missing token = %v, want > %v", d, Stage2MAEFraudThreshold) + } +} diff --git a/decentralized-api/internal/validation/detsample/pipeline.go b/decentralized-api/internal/validation/detsample/pipeline.go new file mode 100644 index 0000000000..d22568bb6a --- /dev/null +++ b/decentralized-api/internal/validation/detsample/pipeline.go @@ -0,0 +1,286 @@ +package detsample + +// Decimal pipeline: logprob strings -> integer weights (contract §4). This is a +// line-for-line port of the vLLM reference logprobs_to_weights, using +// cockroachdb/apd (General Decimal Arithmetic, same spec as CPython's libmpdec) +// under prec=10 / ROUND_HALF_EVEN. Bit-identical results are verified against +// the shared conformance vectors. + +import ( + "fmt" + "sort" + + "github.com/cockroachdb/apd/v2" +) + +const weightScale = 65536 + +// zeroDecimal is a shared apd zero for sign comparisons. +var zeroDecimal = apd.New(0, 0) + +// newCtx returns the pinned decimal context (contract §2): prec=10, HALF_EVEN. +func newCtx() *apd.Context { + c := apd.BaseContext.WithPrecision(10) + c.Rounding = apd.RoundHalfEven + return c +} + +func parseDec(s string) (*apd.Decimal, error) { + d, _, err := apd.NewFromString(s) + if err != nil { + return nil, fmt.Errorf("detsample: bad decimal %q: %w", s, err) + } + return d, nil +} + +func sortedKeys(m map[string]string) []string { + out := make([]string, 0, len(m)) + for k := range m { + out = append(out, k) + } + sort.Strings(out) + return out +} + +// stableSortByProbDesc mirrors Python's sorted(tids, key=probs, reverse=True): +// a stable descending sort. Input must already be in lexicographic order so +// ties keep that order. +func stableSortByProbDesc(tids []string, probs map[string]*apd.Decimal) []string { + out := append([]string(nil), tids...) + sort.SliceStable(out, func(i, j int) bool { + return probs[out[i]].Cmp(probs[out[j]]) > 0 + }) + return out +} + +// LogprobsToWeights reproduces the reference pipeline (contract §4). logprobs +// are canonical decimal strings (contract §1). temperature must be > 0. +func LogprobsToWeights( + logprobs map[string]string, + temperature string, + topP *string, + topK *int, + minP *string, +) (map[string]int64, error) { + c := newCtx() + + T, err := parseDec(temperature) + if err != nil { + return nil, err + } + + tids := sortedKeys(logprobs) + if len(tids) == 0 { + return nil, fmt.Errorf("detsample: empty logprobs") + } + + // Temperature scaling: scaled[t] = Decimal(logprob[t]) / T. + scaled := make(map[string]*apd.Decimal, len(tids)) + for _, tid := range tids { + lp, err := parseDec(logprobs[tid]) + if err != nil { + return nil, err + } + d := new(apd.Decimal) + if _, err := c.Quo(d, lp, T); err != nil { + return nil, err + } + scaled[tid] = d + } + + // Softmax with max-shift. + maxVal := scaled[tids[0]] + for _, tid := range tids[1:] { + if scaled[tid].Cmp(maxVal) > 0 { + maxVal = scaled[tid] + } + } + exps := make(map[string]*apd.Decimal, len(tids)) + for _, tid := range tids { + shifted := new(apd.Decimal) + if _, err := c.Sub(shifted, scaled[tid], maxVal); err != nil { + return nil, err + } + e := new(apd.Decimal) + if _, err := c.Exp(e, shifted); err != nil { + return nil, err + } + exps[tid] = e + } + totalExp := new(apd.Decimal) + for _, tid := range tids { + if _, err := c.Add(totalExp, totalExp, exps[tid]); err != nil { + return nil, err + } + } + probs := make(map[string]*apd.Decimal, len(tids)) + for _, tid := range tids { + p := new(apd.Decimal) + if _, err := c.Quo(p, exps[tid], totalExp); err != nil { + return nil, err + } + probs[tid] = p + } + + // top_k + if topK != nil && *topK < len(tids) { + keep := stableSortByProbDesc(tids, probs)[:*topK] + probs = subset(probs, keep) + tids = sortedStrings(keep) + } + + // top_p + if topP != nil { + tp, err := parseDec(*topP) + if err != nil { + return nil, err + } + order := stableSortByProbDesc(tids, probs) + cumsum := new(apd.Decimal) + var kept []string + for _, tid := range order { + if _, err := c.Add(cumsum, cumsum, probs[tid]); err != nil { + return nil, err + } + kept = append(kept, tid) + if cumsum.Cmp(tp) >= 0 { + break + } + } + probs = subset(probs, kept) + tids = sortedStrings(kept) + } + + // min_p + if minP != nil { + mp, err := parseDec(*minP) + if err != nil { + return nil, err + } + maxProb := probs[tids[0]] + for _, tid := range tids[1:] { + if probs[tid].Cmp(maxProb) > 0 { + maxProb = probs[tid] + } + } + threshold := new(apd.Decimal) + if _, err := c.Mul(threshold, maxProb, mp); err != nil { + return nil, err + } + var kept []string + for _, tid := range tids { + if probs[tid].Cmp(threshold) >= 0 { + kept = append(kept, tid) + } + } + if len(kept) == 0 { + best := tids[0] + for _, tid := range tids[1:] { + if probs[tid].Cmp(probs[best]) > 0 { + best = tid + } + } + kept = []string{best} + } + probs = subset(probs, kept) + tids = sortedStrings(kept) + } + + // Re-normalize over survivors. + keptTotal := new(apd.Decimal) + for _, tid := range tids { + if _, err := c.Add(keptTotal, keptTotal, probs[tid]); err != nil { + return nil, err + } + } + norm := make(map[string]*apd.Decimal, len(tids)) + for _, tid := range tids { + n := new(apd.Decimal) + if _, err := c.Quo(n, probs[tid], keptTotal); err != nil { + return nil, err + } + norm[tid] = n + } + + // Quantize to integer weights: int((norm * 65536).to_integral_value()). + scale := apd.New(weightScale, 0) + weights := make(map[string]int64, len(tids)) + for _, tid := range tids { + prod := new(apd.Decimal) + if _, err := c.Mul(prod, norm[tid], scale); err != nil { + return nil, err + } + q := new(apd.Decimal) + if _, err := c.Quantize(q, prod, 0); err != nil { + return nil, err + } + wi, err := q.Int64() + if err != nil { + return nil, fmt.Errorf("detsample: weight not an int64 for %q: %w", tid, err) + } + weights[tid] = wi + } + + // Residual fix: assign 65536 - sum to the token with the largest + // (weight, token_id_str) tuple. + var sum int64 + for _, tid := range tids { + sum += weights[tid] + } + residual := int64(weightScale) - sum + maxTid := tids[0] + for _, tid := range tids[1:] { + if weights[tid] > weights[maxTid] || + (weights[tid] == weights[maxTid] && tid > maxTid) { + maxTid = tid + } + } + weights[maxTid] += residual + + return weights, nil +} + +// DecimalSampleFromLogprobs runs the full pipeline and samples a token +// (contract §4 + §6). The weight list is built in lexicographic token-ID order; +// the returned index maps back through the same order. +func DecimalSampleFromLogprobs( + logprobs map[string]string, + rng *Sha256CounterRNG, + temperature string, + topP *string, + topK *int, + minP *string, +) (string, error) { + weights, err := LogprobsToWeights(logprobs, temperature, topP, topK, minP) + if err != nil { + return "", err + } + tids := make([]string, 0, len(weights)) + for tid := range weights { + tids = append(tids, tid) + } + sort.Strings(tids) + wl := make([]int64, len(tids)) + for i, tid := range tids { + wl[i] = weights[tid] + } + idx, err := SampleCategoricalWeights(wl, rng) + if err != nil { + return "", err + } + return tids[idx], nil +} + +func subset(m map[string]*apd.Decimal, keys []string) map[string]*apd.Decimal { + out := make(map[string]*apd.Decimal, len(keys)) + for _, k := range keys { + out[k] = m[k] + } + return out +} + +func sortedStrings(s []string) []string { + out := append([]string(nil), s...) + sort.Strings(out) + return out +} diff --git a/decentralized-api/internal/validation/detsample/pipeline_test.go b/decentralized-api/internal/validation/detsample/pipeline_test.go new file mode 100644 index 0000000000..d5f95c2147 --- /dev/null +++ b/decentralized-api/internal/validation/detsample/pipeline_test.go @@ -0,0 +1,109 @@ +package detsample + +import ( + "sort" + "testing" +) + +// pipelineCase adds the pipeline inputs to the shared vector fixture. +type pipelineCase struct { + Name string `json:"name"` + Logprobs map[string]string `json:"logprobs"` + Temperature string `json:"temperature"` + TopP *string `json:"top_p"` + TopK *int `json:"top_k"` + MinP *string `json:"min_p"` + SeedStr string `json:"seed_str"` + ExpectedWeights map[string]int64 `json:"expected_weights"` + ExpectedToken string `json:"expected_token"` +} + +// TestPipelineWeightsMatch is the cross-language parity gate: it runs the Go +// decimal pipeline over each vector's logprobs and asserts the integer weights +// are bit-identical to the vLLM reference (expected_weights). If apd's decimal +// arithmetic (esp. Exp) diverges from CPython's libmpdec at prec=10, this fails +// and tells us exactly where. +func TestPipelineWeightsMatch(t *testing.T) { + cases := loadPipelineCases(t) + for _, c := range cases { + c := c + t.Run(c.Name, func(t *testing.T) { + got, err := LogprobsToWeights(c.Logprobs, c.Temperature, c.TopP, c.TopK, c.MinP) + if err != nil { + t.Fatalf("LogprobsToWeights: %v", err) + } + if len(got) != len(c.ExpectedWeights) { + t.Fatalf("weight count %d, want %d (got=%v want=%v)", + len(got), len(c.ExpectedWeights), got, c.ExpectedWeights) + } + var sum int64 + for tid, w := range got { + sum += w + if want, ok := c.ExpectedWeights[tid]; !ok || w != want { + t.Errorf("weight[%s] = %d, want %d", tid, w, c.ExpectedWeights[tid]) + } + } + if sum != weightScale { + t.Errorf("weights sum %d, want %d", sum, weightScale) + } + }) + } +} + +// TestPipelineTokenMatch runs the full pipeline + sampling and asserts the +// sampled token matches the reference. This is the end-to-end §4+§5+§6 gate. +func TestPipelineTokenMatch(t *testing.T) { + cases := loadPipelineCases(t) + for _, c := range cases { + c := c + t.Run(c.Name, func(t *testing.T) { + rng := NewFromSeedString(c.SeedStr) + got, err := DecimalSampleFromLogprobs( + c.Logprobs, rng, c.Temperature, c.TopP, c.TopK, c.MinP) + if err != nil { + t.Fatalf("DecimalSampleFromLogprobs: %v", err) + } + if got != c.ExpectedToken { + t.Errorf("token = %q, want %q", got, c.ExpectedToken) + } + }) + } +} + +// TestPipelineDeterministic: same inputs twice -> identical weights. +func TestPipelineDeterministic(t *testing.T) { + cases := loadPipelineCases(t) + c := cases[len(cases)-1] // the ten-token case + a, err := LogprobsToWeights(c.Logprobs, c.Temperature, c.TopP, c.TopK, c.MinP) + if err != nil { + t.Fatal(err) + } + b, err := LogprobsToWeights(c.Logprobs, c.Temperature, c.TopP, c.TopK, c.MinP) + if err != nil { + t.Fatal(err) + } + keys := make([]string, 0, len(a)) + for k := range a { + keys = append(keys, k) + } + sort.Strings(keys) + for _, k := range keys { + if a[k] != b[k] { + t.Errorf("nondeterministic weight[%s]: %d vs %d", k, a[k], b[k]) + } + } +} + +func loadPipelineCases(t *testing.T) []pipelineCase { + t.Helper() + // Reuse the same fixture loader as rng_test via a fresh decode with the + // richer case shape. + var v struct { + Cases []pipelineCase `json:"cases"` + } + decodeVectors(t, &v) + if len(v.Cases) == 0 { + t.Fatal("no cases") + } + return v.Cases +} diff --git a/decentralized-api/internal/validation/detsample/reject_test.go b/decentralized-api/internal/validation/detsample/reject_test.go new file mode 100644 index 0000000000..33acb7711b --- /dev/null +++ b/decentralized-api/internal/validation/detsample/reject_test.go @@ -0,0 +1,83 @@ +package detsample + +import "testing" + +// TestRejectionLimit pins the boundary computation. The pipeline only ever uses +// n = 2^16 (a power of two -> sentinel 0), so these non-power-of-2 values are +// the only coverage of the reject path's limit math. +func TestRejectionLimit(t *testing.T) { + // n divides 2^64 -> sentinel 0 (accept all). + for _, n := range []uint64{1, 2, 4, 256, 65536, 1 << 40} { + if got := rejectionLimit(n); got != 0 { + t.Errorf("rejectionLimit(%d) = %d, want 0 (n divides 2^64)", n, got) + } + } + // n=3: 2^64 mod 3 = 1 -> limit = 2^64 - 1 = MaxUint64. + if got, want := rejectionLimit(3), ^uint64(0); got != want { + t.Errorf("rejectionLimit(3) = %d, want %d", got, want) + } + // n=10: 2^64 mod 10 = 6 -> limit = 2^64 - 6 = MaxUint64 - 5. + if got, want := rejectionLimit(10), ^uint64(0)-5; got != want { + t.Errorf("rejectionLimit(10) = %d, want %d", got, want) + } +} + +// TestReduceUnbiasedRejects forces the reject-and-retry branch with an injected +// draw source (impossible to hit deterministically via the real RNG), proving a +// draw >= limit is rejected and the next accepted. +func TestReduceUnbiasedRejects(t *testing.T) { + limit := rejectionLimit(10) // 2^64 - 6 + draws := []uint64{limit, limit + 1, 43} + i := 0 + got := reduceUnbiased(10, func() uint64 { v := draws[i]; i++; return v }) + if got != 43%10 { + t.Errorf("reduceUnbiased = %d, want %d", got, 43%10) + } + if i != 3 { + t.Errorf("consumed %d draws, want 3 (first two rejected)", i) + } +} + +// TestUint64BelowParity replays the non-power-of-2 uint64_below vectors from the +// Python reference — cross-language parity of the limit + modulo path. +func TestUint64BelowParity(t *testing.T) { + var v struct { + Uint64Below struct { + Seed string `json:"seed"` + Cases []struct { + N uint64 `json:"n"` + Draws []uint64 `json:"draws"` + } `json:"cases"` + } `json:"uint64_below"` + } + decodeVectors(t, &v) + if len(v.Uint64Below.Cases) == 0 { + t.Fatal("no uint64_below cases") + } + for _, c := range v.Uint64Below.Cases { + rng := NewFromSeedString(v.Uint64Below.Seed) + for i, want := range c.Draws { + got, err := Uint64Below(rng, c.N) + if err != nil { + t.Fatal(err) + } + if got != want { + t.Errorf("n=%d draw %d: got %d, want %d", c.N, i, got, want) + } + } + } +} + +// TestSampleCategoricalWeightsErrors covers the guard paths (no silent fallback). +func TestSampleCategoricalWeightsErrors(t *testing.T) { + rng := NewFromSeedString("x") + if _, err := SampleCategoricalWeights([]int64{1, -2, 3}, rng); err == nil { + t.Error("negative weight should error") + } + if _, err := SampleCategoricalWeights([]int64{0, 0}, rng); err == nil { + t.Error("zero total should error") + } + if _, err := SampleCategoricalWeights(nil, rng); err == nil { + t.Error("empty weights should error") + } +} diff --git a/decentralized-api/internal/validation/detsample/rng.go b/decentralized-api/internal/validation/detsample/rng.go new file mode 100644 index 0000000000..1190dbfb16 --- /dev/null +++ b/decentralized-api/internal/validation/detsample/rng.go @@ -0,0 +1,123 @@ +// Package detsample is the Go side of the deterministic-sampling contract +// (see ../DETERMINISTIC_SAMPLING_CONTRACT.md). It reproduces the vLLM reference +// pipeline bit-for-bit so the chain validator can replay an executor's sampling. +// +// This file implements the portable, dependency-free half: the SHA256 +// counter-mode RNG and the integer categorical sampler (contract §5, §6). Both +// are exact integer/crypto operations and require no decimal library. The +// decimal pipeline (logprobs -> integer weights, contract §4) lands separately. +package detsample + +import ( + "crypto/sha256" + "encoding/binary" + "fmt" +) + +// Sha256CounterRNG is the portable RNG from contract §5: +// +// u64 = big-endian uint64 of first 8 bytes of SHA256(seedBytes || be_u64(counter)) +// +// counter starts at 0 and advances by one per draw. Identical sequence to the +// Python Sha256CounterRNG for the same seed string. +type Sha256CounterRNG struct { + seed []byte + counter uint64 +} + +// NewFromSeedString creates an RNG seeded with the UTF-8 bytes of s. +func NewFromSeedString(s string) *Sha256CounterRNG { + return &Sha256CounterRNG{seed: []byte(s), counter: 0} +} + +// NextU64 returns the next pseudorandom uint64 and advances the counter. +func (r *Sha256CounterRNG) NextU64() uint64 { + buf := make([]byte, len(r.seed)+8) + copy(buf, r.seed) + binary.BigEndian.PutUint64(buf[len(r.seed):], r.counter) + sum := sha256.Sum256(buf) + r.counter++ + return binary.BigEndian.Uint64(sum[:8]) +} + +// IterU64 returns count draws from a fresh RNG seeded with seed. Mirrors the +// Python iter_u64 helper used for the reference vector. +func IterU64(seed string, count int) []uint64 { + r := NewFromSeedString(seed) + out := make([]uint64, count) + for i := 0; i < count; i++ { + out[i] = r.NextU64() + } + return out +} + +// rejectionLimit returns the largest multiple-of-n boundary below 2^64: a draw +// >= limit must be rejected for an unbiased result (contract §6). A return of 0 +// is the sentinel "n divides 2^64, accept every draw" (e.g. the 2^16 weight +// scale, where the reject path is never taken). +func rejectionLimit(n uint64) uint64 { + // 2^64 mod n, computed in uint64 (2^64 == MaxUint64 + 1). + twoTo64ModN := (^uint64(0)%n + 1) % n + if twoTo64ModN == 0 { + return 0 + } + return -twoTo64ModN // wraps to 2^64 - twoTo64ModN +} + +// reduceUnbiased draws from next() until a value falls below the rejection limit, +// then reduces it mod n (contract §6). Split out from Uint64Below so the +// reject-and-retry branch is unit-testable with an injected draw source (the +// SHA256 RNG almost never produces a rejectable value naturally). +func reduceUnbiased(n uint64, next func() uint64) uint64 { + limit := rejectionLimit(n) + for { + x := next() + if limit == 0 || x < limit { + return x % n + } + } +} + +// Uint64Below returns an unbiased draw in [0, n) via rejection sampling +// (contract §6). n must be > 0. +func Uint64Below(r *Sha256CounterRNG, n uint64) (uint64, error) { + if n == 0 { + return 0, fmt.Errorf("detsample: n must be > 0") + } + return reduceUnbiased(n, r.NextU64), nil +} + +// SampleCategoricalWeights draws an index in [0, len(weights)) with probability +// proportional to the non-negative integer weights (contract §6). The RNG state +// advances. Returns an error on a negative weight or a non-positive total (no +// silent fallback — a zero total in zero-tolerance validation is an upstream +// bug). +func SampleCategoricalWeights(weights []int64, r *Sha256CounterRNG) (int, error) { + if len(weights) == 0 { + return 0, fmt.Errorf("detsample: weights is empty") + } + var total uint64 + for i, w := range weights { + if w < 0 { + return 0, fmt.Errorf("detsample: negative weight at index %d: %d", i, w) + } + total += uint64(w) + } + if total == 0 { + return 0, fmt.Errorf("detsample: weights sum to zero") + } + + rv, err := Uint64Below(r, total) + if err != nil { + return 0, err + } + var cum uint64 + for i, w := range weights { + cum += uint64(w) + if rv < cum { + return i, nil + } + } + // Unreachable when total == sum(weights); kept for completeness. + return len(weights) - 1, nil +} diff --git a/decentralized-api/internal/validation/detsample/rng_test.go b/decentralized-api/internal/validation/detsample/rng_test.go new file mode 100644 index 0000000000..14ddec5779 --- /dev/null +++ b/decentralized-api/internal/validation/detsample/rng_test.go @@ -0,0 +1,106 @@ +package detsample + +import ( + "encoding/json" + "os" + "path/filepath" + "sort" + "testing" +) + +// The vectors live in the parent package's testdata; this sub-package reads them +// directly so its tests stay dependency-free (importable/testable offline). +type vectorCase struct { + Name string `json:"name"` + SeedStr string `json:"seed_str"` + ExpectedWeights map[string]int64 `json:"expected_weights"` + ExpectedToken string `json:"expected_token"` +} + +type vectors struct { + ContractVersion string `json:"contract_version"` + WeightScale int64 `json:"weight_scale"` + RNGReference struct { + Seed string `json:"seed"` + FirstU64 []uint64 `json:"first_u64"` + } `json:"rng_reference"` + Cases []vectorCase `json:"cases"` +} + +// decodeVectors reads the shared fixture and unmarshals it into target, so both +// rng_test and pipeline_test can decode the shape they need. +func decodeVectors(t *testing.T, target interface{}) { + t.Helper() + path := filepath.Join("..", "testdata", "conformance_vectors.json") + raw, err := os.ReadFile(path) + if err != nil { + t.Fatalf("read %s: %v", path, err) + } + if err := json.Unmarshal(raw, target); err != nil { + t.Fatalf("unmarshal: %v", err) + } +} + +func loadVectors(t *testing.T) vectors { + t.Helper() + var v vectors + decodeVectors(t, &v) + return v +} + +// TestRNGReference: the SHA256 counter RNG matches the pinned reference stream. +func TestRNGReference(t *testing.T) { + v := loadVectors(t) + got := IterU64(v.RNGReference.Seed, len(v.RNGReference.FirstU64)) + for i, want := range v.RNGReference.FirstU64 { + if got[i] != want { + t.Errorf("IterU64(%q)[%d] = %d, want %d", + v.RNGReference.Seed, i, got[i], want) + } + } + const wantFirst = uint64(4286832458236889005) + if got[0] != wantFirst { + t.Errorf("first u64 = %d, want %d", got[0], wantFirst) + } +} + +// TestCategoricalReplayFromExpectedWeights cross-validates the RNG + integer +// categorical sampler against the reference implementation *without* the decimal +// pipeline: it feeds each vector's committed expected_weights into the Go +// sampler and asserts the sampled token matches expected_token. This proves the +// §5/§6 half of cross-language parity is bit-identical today. The decimal half +// (§4, logprobs -> weights) is validated once that pipeline lands. +func TestCategoricalReplayFromExpectedWeights(t *testing.T) { + v := loadVectors(t) + for _, c := range v.Cases { + c := c + t.Run(c.Name, func(t *testing.T) { + // Build the weight list in lexicographic token-ID-string order + // (contract §3), matching Python's sorted(weights.keys()). + tids := make([]string, 0, len(c.ExpectedWeights)) + for tid := range c.ExpectedWeights { + tids = append(tids, tid) + } + sort.Strings(tids) + + weights := make([]int64, len(tids)) + var sum int64 + for i, tid := range tids { + weights[i] = c.ExpectedWeights[tid] + sum += weights[i] + } + if sum != v.WeightScale { + t.Fatalf("%s: weights sum %d != scale %d", c.Name, sum, v.WeightScale) + } + + rng := NewFromSeedString(c.SeedStr) + idx, err := SampleCategoricalWeights(weights, rng) + if err != nil { + t.Fatalf("%s: sample error: %v", c.Name, err) + } + if got := tids[idx]; got != c.ExpectedToken { + t.Errorf("%s: sampled %q, want %q", c.Name, got, c.ExpectedToken) + } + }) + } +} diff --git a/decentralized-api/internal/validation/detsample/seed.go b/decentralized-api/internal/validation/detsample/seed.go new file mode 100644 index 0000000000..bccbf2caac --- /dev/null +++ b/decentralized-api/internal/validation/detsample/seed.go @@ -0,0 +1,65 @@ +package detsample + +// Chain-bound seed derivation — Go side of gonka-ai/vllm#56. Mirrors the Python +// derive_chain_bound_seed byte-for-byte so the executor and the chain validator +// derive the same RNG seed. The seed binds Stage-1 replay to chain provenance +// (inference_id), which request-controlled prompt material cannot provide. + +import ( + "crypto/sha256" + "encoding/hex" + "fmt" + "strconv" + "unicode/utf8" +) + +const seedDomainTag = "gonka-deterministic-sampling-v1" + +const ( + idMinOrd = 0x21 + idMaxOrd = 0x7E + maxInferenceIDLen = 256 +) + +// DeriveChainBoundSeed returns the lowercase SHA256 hex digest to seed a +// Sha256CounterRNG. userSeed is int64 (vLLM's seed is int-only; the int64 type +// enforces the pinned signed-64-bit range). inferenceID must be printable ASCII +// (0x21..0x7E), non-empty, and at most maxInferenceIDLen characters. It fails +// closed on any other input rather than falling back to request-controlled +// material. +func DeriveChainBoundSeed(userSeed int64, inferenceID string) (string, error) { + if inferenceID == "" { + return "", fmt.Errorf("detsample: inference_id must be non-empty") + } + if utf8.RuneCountInString(inferenceID) > maxInferenceIDLen { + return "", fmt.Errorf( + "detsample: inference_id too long (>%d characters)", maxInferenceIDLen) + } + // Language-invariant charset (contract): printable ASCII only, so the + // accept/reject boundary is identical to the Python executor. + for _, ch := range inferenceID { + if ch < idMinOrd || ch > idMaxOrd { + return "", fmt.Errorf( + "detsample: inference_id must be printable ASCII (0x21..0x7E); "+ + "found U+%04X", ch) + } + } + + seedBytes := []byte(strconv.FormatInt(userSeed, 10)) + idBytes := []byte(inferenceID) + + // Byte-length-prefixed, domain-separated framing (prefixes count bytes). + material := make([]byte, 0, len(seedDomainTag)+len(seedBytes)+len(idBytes)+64) + material = append(material, seedDomainTag...) + material = append(material, "\nuser_seed_len="...) + material = append(material, strconv.Itoa(len(seedBytes))...) + material = append(material, '\n') + material = append(material, seedBytes...) + material = append(material, "\ninference_id_len="...) + material = append(material, strconv.Itoa(len(idBytes))...) + material = append(material, '\n') + material = append(material, idBytes...) + + sum := sha256.Sum256(material) + return hex.EncodeToString(sum[:]), nil +} diff --git a/decentralized-api/internal/validation/detsample/seed_test.go b/decentralized-api/internal/validation/detsample/seed_test.go new file mode 100644 index 0000000000..76dc75bd6b --- /dev/null +++ b/decentralized-api/internal/validation/detsample/seed_test.go @@ -0,0 +1,84 @@ +package detsample + +import "testing" + +type seedAccept struct { + UserSeed int64 `json:"user_seed"` + InferenceID string `json:"inference_id"` + ExpectedSeed string `json:"expected_seed"` +} + +type seedReject struct { + InferenceID string `json:"inference_id"` + Reason string `json:"reason"` +} + +type seedVectors struct { + SeedDerivation struct { + DomainTag string `json:"domain_tag"` + Accept []seedAccept `json:"accept"` + RejectInferenceID []seedReject `json:"reject_inference_id"` + } `json:"seed_derivation"` +} + +func loadSeedVectors(t *testing.T) seedVectors { + t.Helper() + var v seedVectors + decodeVectors(t, &v) + if len(v.SeedDerivation.Accept) == 0 { + t.Fatal("no seed_derivation accept cases") + } + return v +} + +// TestChainBoundSeedAccept: the Go derivation reproduces the Python digests +// bit-for-bit (cross-language parity for the seed domain). +func TestChainBoundSeedAccept(t *testing.T) { + v := loadSeedVectors(t) + if v.SeedDerivation.DomainTag != seedDomainTag { + t.Errorf("domain_tag = %q, want %q", v.SeedDerivation.DomainTag, seedDomainTag) + } + for _, c := range v.SeedDerivation.Accept { + c := c + t.Run(c.InferenceID, func(t *testing.T) { + got, err := DeriveChainBoundSeed(c.UserSeed, c.InferenceID) + if err != nil { + t.Fatalf("DeriveChainBoundSeed(%d, %q): %v", c.UserSeed, c.InferenceID, err) + } + if got != c.ExpectedSeed { + t.Errorf("seed(%d, %q) = %s, want %s", + c.UserSeed, c.InferenceID, got, c.ExpectedSeed) + } + }) + } +} + +// TestChainBoundSeedReject: every invalid inference id the reference rejects is +// also rejected here (identical fail-closed boundary). +func TestChainBoundSeedReject(t *testing.T) { + v := loadSeedVectors(t) + for _, c := range v.SeedDerivation.RejectInferenceID { + c := c + t.Run(c.Reason, func(t *testing.T) { + if _, err := DeriveChainBoundSeed(7, c.InferenceID); err == nil { + t.Errorf("expected rejection for %q (%s), got none", c.InferenceID, c.Reason) + } + }) + } +} + +// TestChainBoundSeedDomainSeparation: same user_seed, different inference_id -> +// different seed (the whole point of chain binding). +func TestChainBoundSeedDomainSeparation(t *testing.T) { + a, err := DeriveChainBoundSeed(7, "chain-abc") + if err != nil { + t.Fatal(err) + } + b, err := DeriveChainBoundSeed(7, "chain-xyz") + if err != nil { + t.Fatal(err) + } + if a == b { + t.Error("different inference_id must yield different seed") + } +} diff --git a/decentralized-api/internal/validation/detsample/sequence.go b/decentralized-api/internal/validation/detsample/sequence.go new file mode 100644 index 0000000000..30d73775a1 --- /dev/null +++ b/decentralized-api/internal/validation/detsample/sequence.go @@ -0,0 +1,143 @@ +package detsample + +// Sequence-level Stage-1 replay. VerifyPosition classifies one position; this +// layer replays a whole response and aggregates a three-valued verdict, mirroring +// the Python serving-side orchestrator (validation_sampling.verify_sequence) so +// the chain (Go) and vLLM (Python) reach the same verdict from the same artifact. +// +// RNG semantics: one seed per position (Decision B = per-position). The sequence +// composes each position's seed as fmt.Sprintf("%s|%d", BaseSeed, i); a fresh RNG +// per position keeps the variable draw-count of rejection sampling at one position +// from desyncing the replay of the rest of the sequence. This matches the Python +// side byte-for-byte. + +import "fmt" + +// SequencePosition is one position's replay data within a sequence. Unlike +// Position it carries no SeedStr — the sequence derives it from BaseSeed. A nil +// Logprobs marks a position with no replay data (Inconclusive, not Fraud). +type SequencePosition struct { + Logprobs map[string]string + ReportedToken string +} + +// SequenceRequest holds the request-level parameters shared by every position. +type SequenceRequest struct { + ContractVersion string + SeedDomain string + BaseSeed string + Temperature string + TopP *string + TopK *int + MinP *string + Greedy bool // temperature == 0 for the whole request (contract §7) +} + +// SequenceResult aggregates a whole-response replay. +type SequenceResult struct { + Verdict Verdict + FraudPosition int // first fraud position; -1 if none + NHonest int + NInconclusive int + Reason string +} + +// VerifySequence replays a response position-by-position and aggregates: +// +// any position Fraud -> Fraud (reports the first such position) +// else >= 1 Honest -> Honest (Inconclusive positions defer to Stage-2) +// else all Inconclusive -> Inconclusive (Stage-1 carries no signal) +// +// Version/seed-domain/greedy gating and the unbounded-support gate run before any +// per-position replay, so a request the validator structurally cannot cover never +// yields Fraud. +func VerifySequence(req SequenceRequest, positions []SequencePosition) SequenceResult { + if req.ContractVersion != SupportedContractVersion { + return seqInconclusive("unsupported contract version %q (validator supports %q)", + req.ContractVersion, SupportedContractVersion) + } + if req.SeedDomain != SupportedSeedDomain { + return seqInconclusive("unsupported seed domain %q (validator supports %q)", + req.SeedDomain, SupportedSeedDomain) + } + if req.Greedy { + return seqInconclusive("greedy (temperature 0): sequence check not applicable") + } + if !supportIsBounded(req.TopK, req.MinP) { + // top_p alone (especially high temperature) and pure-temperature sampling + // have unbounded support: the nucleus can exceed the signed top-K, so the + // reported set cannot faithfully reproduce the filter. Defer to Stage-2. + return seqInconclusive("unbounded support (no top_k/min_p): deferred to distance check") + } + + nHonest, nInconclusive := 0, 0 + for i, pos := range positions { + if pos.Logprobs == nil { + nInconclusive++ + continue + } + r := VerifyPosition(Position{ + ContractVersion: req.ContractVersion, + SeedDomain: req.SeedDomain, + Logprobs: pos.Logprobs, + Temperature: req.Temperature, + TopP: req.TopP, + TopK: req.TopK, + MinP: req.MinP, + SeedStr: fmt.Sprintf("%s|%d", req.BaseSeed, i), + ReportedToken: pos.ReportedToken, + }) + switch r.Verdict { + case VerdictFraud: + return SequenceResult{ + Verdict: VerdictFraud, + FraudPosition: i, + NHonest: nHonest, + NInconclusive: nInconclusive, + Reason: r.Reason, + } + case VerdictHonest: + nHonest++ + default: + nInconclusive++ + } + } + + if nHonest > 0 { + return SequenceResult{ + Verdict: VerdictHonest, + FraudPosition: -1, + NHonest: nHonest, + NInconclusive: nInconclusive, + } + } + return SequenceResult{ + Verdict: VerdictInconclusive, + FraudPosition: -1, + NInconclusive: nInconclusive, + Reason: "no replayable position (all inconclusive)", + } +} + +// supportIsBounded reports whether the request's filter pins the support to a +// finite, exactly-reproducible set: true iff top_k or min_p is active. Mirrors +// validation_sampling._support_is_bounded (Python). +func supportIsBounded(topK *int, minP *string) bool { + if topK != nil && *topK > 0 { + return true + } + if minP != nil { + if v, err := parseDec(*minP); err == nil && v.Cmp(zeroDecimal) > 0 { + return true + } + } + return false +} + +func seqInconclusive(format string, a ...any) SequenceResult { + return SequenceResult{ + Verdict: VerdictInconclusive, + FraudPosition: -1, + Reason: fmt.Sprintf(format, a...), + } +} diff --git a/decentralized-api/internal/validation/detsample/sequence_conformance_test.go b/decentralized-api/internal/validation/detsample/sequence_conformance_test.go new file mode 100644 index 0000000000..8e6be0f339 --- /dev/null +++ b/decentralized-api/internal/validation/detsample/sequence_conformance_test.go @@ -0,0 +1,107 @@ +package detsample + +import "testing" + +// Cross-language conformance for the sequence layer: the Go VerifySequence / +// MAEDistance must reproduce the same aggregation and distance the Python +// reference committed to testdata/conformance_vectors.json (generated by +// scripts/gen_conformance_vectors.py). This extends the primitive-level parity +// (RNG/pipeline/seed) to the sequence level. + +type confSeqPosition struct { + Logprobs map[string]string `json:"logprobs"` // JSON null -> nil (missing data) + ReportedToken string `json:"reported_token"` +} + +type confSeqExpected struct { + Verdict string `json:"verdict"` + FraudPosition int `json:"fraud_position"` + NHonest int `json:"n_honest"` + NInconclusive int `json:"n_inconclusive"` +} + +type confSequenceCase struct { + Name string `json:"name"` + BaseSeed string `json:"base_seed"` + ContractVersion string `json:"contract_version"` + SeedDomain string `json:"seed_domain"` + Temperature string `json:"temperature"` + TopP *string `json:"top_p"` + TopK *int `json:"top_k"` + MinP *string `json:"min_p"` + Greedy bool `json:"greedy"` + Positions []confSeqPosition `json:"positions"` + Expected confSeqExpected `json:"expected"` +} + +type confDistanceCase struct { + Name string `json:"name"` + Executor []map[string]float64 `json:"executor"` + Validator []map[string]float64 `json:"validator"` + ExpectedMAE float64 `json:"expected_mae"` +} + +func TestSequenceConformance(t *testing.T) { + var v struct { + Cases []confSequenceCase `json:"sequence_cases"` + } + decodeVectors(t, &v) + if len(v.Cases) == 0 { + t.Fatal("no sequence_cases in vectors") + } + for _, c := range v.Cases { + c := c + t.Run(c.Name, func(t *testing.T) { + positions := make([]SequencePosition, len(c.Positions)) + for i, p := range c.Positions { + positions[i] = SequencePosition{ + Logprobs: p.Logprobs, + ReportedToken: p.ReportedToken, + } + } + r := VerifySequence(SequenceRequest{ + ContractVersion: c.ContractVersion, + SeedDomain: c.SeedDomain, + BaseSeed: c.BaseSeed, + Temperature: c.Temperature, + TopP: c.TopP, + TopK: c.TopK, + MinP: c.MinP, + Greedy: c.Greedy, + }, positions) + + if string(r.Verdict) != c.Expected.Verdict { + t.Errorf("verdict = %s (%s), want %s", r.Verdict, r.Reason, c.Expected.Verdict) + } + if r.FraudPosition != c.Expected.FraudPosition { + t.Errorf("fraudPosition = %d, want %d", r.FraudPosition, c.Expected.FraudPosition) + } + if r.NHonest != c.Expected.NHonest { + t.Errorf("nHonest = %d, want %d", r.NHonest, c.Expected.NHonest) + } + if r.NInconclusive != c.Expected.NInconclusive { + t.Errorf("nInconclusive = %d, want %d", r.NInconclusive, c.Expected.NInconclusive) + } + }) + } +} + +func TestDistanceConformance(t *testing.T) { + var v struct { + Cases []confDistanceCase `json:"distance_cases"` + } + decodeVectors(t, &v) + if len(v.Cases) == 0 { + t.Fatal("no distance_cases in vectors") + } + for _, c := range v.Cases { + c := c + t.Run(c.Name, func(t *testing.T) { + // Exact equality: sorted-key summation makes the float64 accumulation + // order identical to the Python reference (byte-exact distance). + if got := MAEDistance(c.Executor, c.Validator); got != c.ExpectedMAE { + t.Errorf("MAEDistance = %v, want %v", got, c.ExpectedMAE) + } + }) + } +} diff --git a/decentralized-api/internal/validation/detsample/sequence_test.go b/decentralized-api/internal/validation/detsample/sequence_test.go new file mode 100644 index 0000000000..f123ad8657 --- /dev/null +++ b/decentralized-api/internal/validation/detsample/sequence_test.go @@ -0,0 +1,134 @@ +package detsample + +import ( + "fmt" + "testing" +) + +const ( + seqBaseSeed = "42|[1,2,3]" + seqTemp = "1.0" +) + +func seqTopK() *int { k := 20; return &k } // bounded support + +// seqLogprobs are four positions of canonical decimal logprob strings. +var seqLogprobs = []map[string]string{ + {"5": "-0.5", "10": "-1.2", "2": "-2.5", "100": "-3.9", "7": "-4.1"}, + {"3": "-0.3", "42": "-1.1", "9": "-2.2", "11": "-3.0", "1": "-4.5"}, + {"8": "-0.7", "6": "-1.5", "4": "-2.0", "20": "-2.9", "15": "-3.3"}, + {"12": "-0.9", "13": "-1.8", "14": "-2.4", "16": "-3.1", "17": "-3.7"}, +} + +// honestToken is the token an honest executor sharing the decimal path reports at +// position pos: the same pipeline + per-position seed VerifySequence replays. +func honestToken(t *testing.T, logprobs map[string]string, pos int) string { + t.Helper() + tok, err := DecimalSampleFromLogprobs( + logprobs, + NewFromSeedString(fmt.Sprintf("%s|%d", seqBaseSeed, pos)), + seqTemp, nil, seqTopK(), nil) + if err != nil { + t.Fatalf("sample position %d: %v", pos, err) + } + return tok +} + +func honestSequence(t *testing.T) []SequencePosition { + t.Helper() + positions := make([]SequencePosition, len(seqLogprobs)) + for i, lp := range seqLogprobs { + positions[i] = SequencePosition{Logprobs: lp, ReportedToken: honestToken(t, lp, i)} + } + return positions +} + +func boundedReq() SequenceRequest { + return SequenceRequest{ + ContractVersion: SupportedContractVersion, + SeedDomain: SupportedSeedDomain, + BaseSeed: seqBaseSeed, + Temperature: seqTemp, + TopK: seqTopK(), + } +} + +func TestVerifySequenceHonest(t *testing.T) { + r := VerifySequence(boundedReq(), honestSequence(t)) + if r.Verdict != VerdictHonest { + t.Fatalf("verdict = %s (%s), want honest", r.Verdict, r.Reason) + } + if r.NHonest != len(seqLogprobs) || r.FraudPosition != -1 { + t.Errorf("nHonest=%d fraudPos=%d, want %d and -1", r.NHonest, r.FraudPosition, len(seqLogprobs)) + } +} + +func TestVerifySequenceFlagsFirstTamperedPosition(t *testing.T) { + positions := honestSequence(t) + positions[2].ReportedToken = "999999" // a token the RNG would not have drawn + r := VerifySequence(boundedReq(), positions) + if r.Verdict != VerdictFraud || r.FraudPosition != 2 { + t.Fatalf("verdict=%s fraudPos=%d, want fraud at 2", r.Verdict, r.FraudPosition) + } +} + +func TestVerifySequenceUnboundedSupportInconclusive(t *testing.T) { + req := boundedReq() + req.TopK = nil // top_p-only / pure temperature: unbounded support + p := "0.9" + req.TopP = &p + r := VerifySequence(req, honestSequence(t)) + if r.Verdict != VerdictInconclusive { + t.Fatalf("verdict = %s, want inconclusive", r.Verdict) + } +} + +func TestVerifySequenceGreedyInconclusive(t *testing.T) { + req := boundedReq() + req.Greedy = true + if r := VerifySequence(req, honestSequence(t)); r.Verdict != VerdictInconclusive { + t.Fatalf("verdict = %s, want inconclusive", r.Verdict) + } +} + +func TestVerifySequenceVersionGating(t *testing.T) { + req := boundedReq() + req.ContractVersion = "9.9.9" + if r := VerifySequence(req, honestSequence(t)); r.Verdict != VerdictInconclusive { + t.Fatalf("verdict = %s, want inconclusive", r.Verdict) + } +} + +func TestVerifySequenceMissingLogprobsIsInconclusiveNotFraud(t *testing.T) { + positions := honestSequence(t) + positions[1].Logprobs = nil // no replay data at this position + r := VerifySequence(boundedReq(), positions) + if r.Verdict != VerdictHonest { + t.Fatalf("verdict = %s, want honest (missing != fraud)", r.Verdict) + } + if r.NInconclusive != 1 || r.NHonest != len(seqLogprobs)-1 { + t.Errorf("nInconclusive=%d nHonest=%d, want 1 and %d", r.NInconclusive, r.NHonest, len(seqLogprobs)-1) + } +} + +func TestSupportIsBounded(t *testing.T) { + k := 20 + zero := 0 + mp := "0.02" + mpZero := "0" + cases := []struct { + topK *int + minP *string + want bool + }{ + {&k, nil, true}, + {nil, &mp, true}, + {nil, nil, false}, // pure temperature + {&zero, &mpZero, false}, // disabled sentinels + } + for i, c := range cases { + if got := supportIsBounded(c.topK, c.minP); got != c.want { + t.Errorf("case %d: supportIsBounded = %v, want %v", i, got, c.want) + } + } +} diff --git a/decentralized-api/internal/validation/detsample/verify.go b/decentralized-api/internal/validation/detsample/verify.go new file mode 100644 index 0000000000..70d50f8556 --- /dev/null +++ b/decentralized-api/internal/validation/detsample/verify.go @@ -0,0 +1,107 @@ +package detsample + +// Validator-facing replay API (Stage-1 "Check 2"). This is the layer the chain +// validator calls: given one artifact position, replay the sampling and classify +// the outcome. It sits on top of the bit-verified primitives (LogprobsToWeights, +// Sha256CounterRNG, SampleCategoricalWeights). +// +// The verdict is deliberately three-valued. A replay mismatch is only *fraud* +// when the validator could faithfully reproduce the executor's computation; a +// version it does not support, or its own replay error, is *inconclusive*, never +// fraud. Collapsing these into a bool would let a validator-side or +// version-skew problem punish an honest executor (a false-fraud), which in a +// slashing system is far more costly than a missed detection. + +import "fmt" + +// What this validator can faithfully replay. An artifact declaring anything else +// is Inconclusive (version-unsupported), not Fraud. +const ( + SupportedContractVersion = "1.0.0" + SupportedSeedDomain = seedDomainTag +) + +// Verdict is the classified outcome of a replay. +type Verdict string + +const ( + // VerdictHonest: the replay reproduced the reported token exactly. + VerdictHonest Verdict = "honest" + // VerdictFraud: the validator faithfully replayed and got a different token. + VerdictFraud Verdict = "fraud" + // VerdictInconclusive: the validator could not faithfully replay (unsupported + // version, greedy position, or a replay error). Must not be treated as fraud. + VerdictInconclusive Verdict = "inconclusive" +) + +// Result is a verdict plus a human-readable reason (empty for Honest). +type Result struct { + Verdict Verdict + Reason string +} + +// Position is one artifact position to replay. Logprobs are canonical decimal +// strings (contract §1); SeedStr is the already-composed RNG seed (contract §8 +// output, e.g. from DeriveChainBoundSeed). ContractVersion / SeedDomain are the +// executor's declared versions, checked before any fraud verdict. +type Position struct { + ContractVersion string + SeedDomain string + Logprobs map[string]string + Temperature string + TopP *string + TopK *int + MinP *string + SeedStr string + ReportedToken string + Greedy bool // temperature == 0 (argmax; contract §7) +} + +// VerifyPosition replays one position with zero tolerance and classifies it. +// Version gating and the greedy exemption run before any fraud verdict. +func VerifyPosition(p Position) Result { + if p.ContractVersion != SupportedContractVersion { + return inconclusive( + "unsupported contract version %q (validator supports %q)", + p.ContractVersion, SupportedContractVersion) + } + if p.SeedDomain != SupportedSeedDomain { + return inconclusive( + "unsupported seed domain %q (validator supports %q)", + p.SeedDomain, SupportedSeedDomain) + } + if p.Greedy { + // Contract §7: temperature 0 is argmax and never consults the RNG, so + // the sequence check provides no signal here; defer to the distance check. + return inconclusive("greedy position (temperature 0): sequence check not applicable") + } + // A non-greedy position must have temperature > 0 (contract §4). Guard + // explicitly rather than relying on a downstream divide-by-zero: a + // non-positive or unparseable temperature is an inconsistent artifact, not + // executor fraud. + if T, err := parseDec(p.Temperature); err != nil || T.Cmp(zeroDecimal) <= 0 { + return inconclusive( + "non-positive or unparseable temperature %q with greedy flag unset", + p.Temperature) + } + + rng := NewFromSeedString(p.SeedStr) + replayed, err := DecimalSampleFromLogprobs( + p.Logprobs, rng, p.Temperature, p.TopP, p.TopK, p.MinP) + if err != nil { + // A replay error is a validator-side inability to reproduce, not proof + // of fraud. + return inconclusive("replay error: %v", err) + } + if replayed != p.ReportedToken { + return Result{ + Verdict: VerdictFraud, + Reason: fmt.Sprintf("replayed %q but executor reported %q", replayed, p.ReportedToken), + } + } + return Result{Verdict: VerdictHonest} +} + +func inconclusive(format string, a ...any) Result { + return Result{Verdict: VerdictInconclusive, Reason: fmt.Sprintf(format, a...)} +} diff --git a/decentralized-api/internal/validation/detsample/verify_test.go b/decentralized-api/internal/validation/detsample/verify_test.go new file mode 100644 index 0000000000..2cff375805 --- /dev/null +++ b/decentralized-api/internal/validation/detsample/verify_test.go @@ -0,0 +1,134 @@ +package detsample + +import "testing" + +// positionFromVector builds an honest Position from a conformance case. An +// honest executor declares the versions this validator supports. +func positionFromVector(c vectorCaseFull) Position { + return Position{ + ContractVersion: SupportedContractVersion, + SeedDomain: SupportedSeedDomain, + Logprobs: c.Logprobs, + Temperature: c.Temperature, + TopP: c.TopP, + TopK: c.TopK, + MinP: c.MinP, + SeedStr: c.SeedStr, + ReportedToken: c.ExpectedToken, + } +} + +// vectorCaseFull carries the fields VerifyPosition needs from a case. +type vectorCaseFull struct { + Name string `json:"name"` + Logprobs map[string]string `json:"logprobs"` + Temperature string `json:"temperature"` + TopP *string `json:"top_p"` + TopK *int `json:"top_k"` + MinP *string `json:"min_p"` + SeedStr string `json:"seed_str"` + ExpectedToken string `json:"expected_token"` +} + +func loadFullVectors(t *testing.T) []vectorCaseFull { + t.Helper() + var full struct { + Cases []vectorCaseFull `json:"cases"` + } + decodeVectors(t, &full) + if len(full.Cases) == 0 { + t.Fatal("no cases") + } + return full.Cases +} + +// TestVerifyPositionHonest: an untampered position replays to Honest. +func TestVerifyPositionHonest(t *testing.T) { + cases := loadFullVectors(t) + for _, c := range cases { + c := c + t.Run(c.Name, func(t *testing.T) { + r := VerifyPosition(positionFromVector(c)) + if r.Verdict != VerdictHonest { + t.Errorf("verdict = %s (%s), want honest", r.Verdict, r.Reason) + } + }) + } +} + +// TestVerifyPositionFraud: a tampered reported token replays to Fraud. +func TestVerifyPositionFraud(t *testing.T) { + cases := loadFullVectors(t) + for _, c := range cases { + c := c + // Pick any token id different from the honest one. + var tampered string + for tid := range c.Logprobs { + if tid != c.ExpectedToken { + tampered = tid + break + } + } + if tampered == "" { + continue // single-token case: nothing to tamper to + } + t.Run(c.Name, func(t *testing.T) { + p := positionFromVector(c) + p.ReportedToken = tampered + r := VerifyPosition(p) + if r.Verdict != VerdictFraud { + t.Errorf("verdict = %s (%s), want fraud", r.Verdict, r.Reason) + } + }) + } +} + +// TestVerifyPositionVersionGating: unknown versions are Inconclusive, not Fraud. +func TestVerifyPositionVersionGating(t *testing.T) { + cases := loadFullVectors(t) + base := positionFromVector(cases[0]) + + badContract := base + badContract.ContractVersion = "9.9.9" + if r := VerifyPosition(badContract); r.Verdict != VerdictInconclusive { + t.Errorf("bad contract version: verdict = %s, want inconclusive", r.Verdict) + } + + badSeed := base + badSeed.SeedDomain = "some-other-domain" + if r := VerifyPosition(badSeed); r.Verdict != VerdictInconclusive { + t.Errorf("bad seed domain: verdict = %s, want inconclusive", r.Verdict) + } + + // A tampered token under an unsupported version is still Inconclusive, never + // Fraud — version gating must win. + badContract.ReportedToken = "definitely-wrong" + if r := VerifyPosition(badContract); r.Verdict != VerdictInconclusive { + t.Errorf("tamper under bad version: verdict = %s, want inconclusive", r.Verdict) + } +} + +// TestVerifyPositionGreedy: temperature 0 is Inconclusive (no Stage-1 signal). +func TestVerifyPositionGreedy(t *testing.T) { + cases := loadFullVectors(t) + p := positionFromVector(cases[0]) + p.Greedy = true + p.ReportedToken = "anything" // must not matter + if r := VerifyPosition(p); r.Verdict != VerdictInconclusive { + t.Errorf("greedy: verdict = %s, want inconclusive", r.Verdict) + } +} + +// TestVerifyPositionNonPositiveTemperature: a non-greedy position with +// temperature <= 0 (or unparseable) is Inconclusive, not a crash or Fraud. +func TestVerifyPositionNonPositiveTemperature(t *testing.T) { + cases := loadFullVectors(t) + for _, temp := range []string{"0", "0.0", "-0.5", "not-a-number"} { + p := positionFromVector(cases[0]) + p.Greedy = false + p.Temperature = temp + if r := VerifyPosition(p); r.Verdict != VerdictInconclusive { + t.Errorf("temperature %q: verdict = %s, want inconclusive", temp, r.Verdict) + } + } +} diff --git a/decentralized-api/internal/validation/testdata/conformance_vectors.json b/decentralized-api/internal/validation/testdata/conformance_vectors.json new file mode 100644 index 0000000000..233d4b3908 --- /dev/null +++ b/decentralized-api/internal/validation/testdata/conformance_vectors.json @@ -0,0 +1,800 @@ +{ + "contract_version": "1.0.0", + "decimal": { + "prec": 10, + "rounding": "ROUND_HALF_EVEN" + }, + "weight_scale": 65536, + "rng_reference": { + "seed": "reference_seed_v1", + "first_u64": [ + 4286832458236889005, + 12281003819428572724, + 12352776571910749143, + 12178488218135958089, + 6205195570139478562 + ] + }, + "float_to_string": [ + { + "description": "plain -0.05 as float64", + "repr": "-0.05" + }, + { + "description": "-0.05 as float32 widened to float64", + "repr": "-0.05000000074505806" + }, + { + "description": "0.1 as float32 widened to float64", + "repr": "0.10000000149011612" + } + ], + "cases": [ + { + "name": "single_token", + "logprobs": { + "791": "-0.05" + }, + "temperature": "1.0", + "top_p": null, + "top_k": null, + "min_p": null, + "seed_str": "42|[1,2,3]", + "expected_weights": { + "791": 65536 + }, + "expected_weight_sum": 65536, + "expected_token": "791" + }, + { + "name": "three_token_plain", + "logprobs": { + "791": "-0.05", + "1": "-3.2", + "5": "-2.1" + }, + "temperature": "1.0", + "top_p": null, + "top_k": null, + "min_p": null, + "seed_str": "42|[1,2,3]", + "expected_weights": { + "1": 2397, + "5": 7201, + "791": 55938 + }, + "expected_weight_sum": 65536, + "expected_token": "791" + }, + { + "name": "two_token_near_tie", + "logprobs": { + "5": "-0.6931471805599453", + "7": "-0.6931471805599453" + }, + "temperature": "1.0", + "top_p": null, + "top_k": null, + "min_p": null, + "seed_str": "42|[1,2,3]", + "expected_weights": { + "5": 32768, + "7": 32768 + }, + "expected_weight_sum": 65536, + "expected_token": "7" + }, + { + "name": "top_k_truncation", + "logprobs": { + "791": "-0.05", + "1": "-3.2", + "5": "-2.1", + "9": "-1.0" + }, + "temperature": "1.0", + "top_p": null, + "top_k": 2, + "min_p": null, + "seed_str": "42|[1,2,3]", + "expected_weights": { + "791": 47259, + "9": 18277 + }, + "expected_weight_sum": 65536, + "expected_token": "791" + }, + { + "name": "top_p_nucleus", + "logprobs": { + "791": "-0.05", + "1": "-3.2", + "5": "-2.1", + "9": "-1.0" + }, + "temperature": "1.0", + "top_p": "0.9", + "top_k": null, + "min_p": null, + "seed_str": "42|[1,2,3]", + "expected_weights": { + "5": 5567, + "791": 43245, + "9": 16724 + }, + "expected_weight_sum": 65536, + "expected_token": "791" + }, + { + "name": "min_p_filter", + "logprobs": { + "791": "-0.05", + "1": "-3.2", + "5": "-2.1", + "9": "-1.0" + }, + "temperature": "1.0", + "top_p": null, + "top_k": null, + "min_p": "0.05", + "seed_str": "42|[1,2,3]", + "expected_weights": { + "5": 5567, + "791": 43245, + "9": 16724 + }, + "expected_weight_sum": 65536, + "expected_token": "791" + }, + { + "name": "temperature_small", + "logprobs": { + "791": "-0.05", + "1": "-3.2", + "5": "-2.1" + }, + "temperature": "0.1", + "top_p": null, + "top_k": null, + "min_p": null, + "seed_str": "42|[1,2,3]", + "expected_weights": { + "1": 0, + "5": 0, + "791": 65536 + }, + "expected_weight_sum": 65536, + "expected_token": "791" + }, + { + "name": "temperature_large", + "logprobs": { + "791": "-0.05", + "1": "-3.2", + "5": "-2.1" + }, + "temperature": "2.0", + "top_p": null, + "top_k": null, + "min_p": null, + "seed_str": "42|[1,2,3]", + "expected_weights": { + "1": 8664, + "5": 15017, + "791": 41855 + }, + "expected_weight_sum": 65536, + "expected_token": "791" + }, + { + "name": "float32_widened_logprob", + "logprobs": { + "791": "-0.05000000074505806", + "1": "-3.0" + }, + "temperature": "0.7", + "top_p": null, + "top_k": null, + "min_p": null, + "seed_str": "42|[1,2,3]", + "expected_weights": { + "1": 955, + "791": 64581 + }, + "expected_weight_sum": 65536, + "expected_token": "791" + }, + { + "name": "ten_tokens", + "logprobs": { + "3": "-0.1", + "10": "-0.2", + "42": "-0.30000000000000004", + "100": "-0.4", + "256": "-0.5", + "512": "-0.6000000000000001", + "777": "-0.7000000000000001", + "1024": "-0.8", + "2048": "-0.9", + "4096": "-1.0" + }, + "temperature": "0.8", + "top_p": "0.95", + "top_k": null, + "min_p": null, + "seed_str": "42|[1,2,3]", + "expected_weights": { + "10": 9525, + "100": 7418, + "1024": 4499, + "2048": 3970, + "256": 6546, + "3": 10793, + "4096": 3504, + "42": 8406, + "512": 5777, + "777": 5098 + }, + "expected_weight_sum": 65536, + "expected_token": "3" + } + ], + "uint64_below": { + "seed": "reference_seed_v1", + "count": 8, + "cases": [ + { + "n": 3, + "draws": [ + 1, + 1, + 0, + 2, + 1, + 0, + 0, + 2 + ] + }, + { + "n": 10, + "draws": [ + 5, + 4, + 3, + 9, + 2, + 9, + 1, + 0 + ] + }, + { + "n": 1000, + "draws": [ + 5, + 724, + 143, + 89, + 562, + 449, + 921, + 110 + ] + }, + { + "n": 65537, + "draws": [ + 12997, + 43373, + 2752, + 34948, + 5254, + 28917, + 42537, + 24722 + ] + }, + { + "n": 999983, + "draws": [ + 594664, + 773438, + 484737, + 309324, + 502931, + 719637, + 764984, + 890163 + ] + } + ] + }, + "seed_derivation": { + "domain_tag": "gonka-deterministic-sampling-v1", + "accept": [ + { + "user_seed": 7, + "inference_id": "chain-abc", + "expected_seed": "910b688db5b2061e66385acf0ee665682d5e01bab5d1d8d2cdde9a2612a6e6c2" + }, + { + "user_seed": 7, + "inference_id": "chain-xyz", + "expected_seed": "10668913c3b68c1da4d32a882523ee44cd37af73d4d4361dd0dd8d71f1fbee4b" + }, + { + "user_seed": 42, + "inference_id": "devshard-escrow1-100", + "expected_seed": "c12c2cbf4165c5b1a4240bc69c4c527ada1b8c65c51b35de017d15be22c7b3d9" + }, + { + "user_seed": -1, + "inference_id": "chain-abc", + "expected_seed": "3d06091b4635031cf457f333dd2d3c64de7eaf56dfa4ae4e6bcc37d81c90c515" + }, + { + "user_seed": 9223372036854775807, + "inference_id": "x", + "expected_seed": "934d232ad3fca38d5cde7e9c8dd8132155ab91e8674977973ea64e431abd72c6" + } + ], + "reject_inference_id": [ + { + "inference_id": "", + "reason": "empty" + }, + { + "inference_id": " chain-abc ", + "reason": "whitespace (space is 0x20, below 0x21)" + }, + { + "inference_id": "chain\tabc", + "reason": "control char (tab)" + }, + { + "inference_id": "chain-é", + "reason": "non-ASCII" + }, + { + "inference_id": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", + "reason": "too long (>256)" + } + ] + }, + "sequence_cases": [ + { + "name": "honest_top_k", + "base_seed": "42|[1,2,3]", + "contract_version": "1.0.0", + "seed_domain": "gonka-deterministic-sampling-v1", + "temperature": "1.0", + "top_p": null, + "top_k": 20, + "min_p": null, + "greedy": false, + "positions": [ + { + "logprobs": { + "5": "-0.5", + "10": "-1.2", + "2": "-2.5", + "100": "-3.9", + "7": "-4.1" + }, + "reported_token": "5" + }, + { + "logprobs": { + "3": "-0.3", + "42": "-1.1", + "9": "-2.2", + "11": "-3.0", + "1": "-4.5" + }, + "reported_token": "1" + }, + { + "logprobs": { + "8": "-0.7", + "6": "-1.5", + "4": "-2.0", + "20": "-2.9", + "15": "-3.3" + }, + "reported_token": "6" + } + ], + "expected": { + "verdict": "honest", + "fraud_position": -1, + "n_honest": 3, + "n_inconclusive": 0 + } + }, + { + "name": "honest_min_p", + "base_seed": "42|[1,2,3]", + "contract_version": "1.0.0", + "seed_domain": "gonka-deterministic-sampling-v1", + "temperature": "1.0", + "top_p": null, + "top_k": null, + "min_p": "0.02", + "greedy": false, + "positions": [ + { + "logprobs": { + "5": "-0.5", + "10": "-1.2", + "2": "-2.5", + "100": "-3.9", + "7": "-4.1" + }, + "reported_token": "5" + }, + { + "logprobs": { + "3": "-0.3", + "42": "-1.1", + "9": "-2.2", + "11": "-3.0", + "1": "-4.5" + }, + "reported_token": "11" + }, + { + "logprobs": { + "8": "-0.7", + "6": "-1.5", + "4": "-2.0", + "20": "-2.9", + "15": "-3.3" + }, + "reported_token": "6" + } + ], + "expected": { + "verdict": "honest", + "fraud_position": -1, + "n_honest": 3, + "n_inconclusive": 0 + } + }, + { + "name": "fraud_at_pos_1", + "base_seed": "42|[1,2,3]", + "contract_version": "1.0.0", + "seed_domain": "gonka-deterministic-sampling-v1", + "temperature": "1.0", + "top_p": null, + "top_k": 20, + "min_p": null, + "greedy": false, + "positions": [ + { + "logprobs": { + "5": "-0.5", + "10": "-1.2", + "2": "-2.5", + "100": "-3.9", + "7": "-4.1" + }, + "reported_token": "5" + }, + { + "logprobs": { + "3": "-0.3", + "42": "-1.1", + "9": "-2.2", + "11": "-3.0", + "1": "-4.5" + }, + "reported_token": "999999" + }, + { + "logprobs": { + "8": "-0.7", + "6": "-1.5", + "4": "-2.0", + "20": "-2.9", + "15": "-3.3" + }, + "reported_token": "6" + } + ], + "expected": { + "verdict": "fraud", + "fraud_position": 1, + "n_honest": 1, + "n_inconclusive": 0 + } + }, + { + "name": "honest_with_missing_position", + "base_seed": "42|[1,2,3]", + "contract_version": "1.0.0", + "seed_domain": "gonka-deterministic-sampling-v1", + "temperature": "1.0", + "top_p": null, + "top_k": 20, + "min_p": null, + "greedy": false, + "positions": [ + { + "logprobs": { + "5": "-0.5", + "10": "-1.2", + "2": "-2.5", + "100": "-3.9", + "7": "-4.1" + }, + "reported_token": "5" + }, + { + "logprobs": null, + "reported_token": "" + }, + { + "logprobs": { + "8": "-0.7", + "6": "-1.5", + "4": "-2.0", + "20": "-2.9", + "15": "-3.3" + }, + "reported_token": "6" + } + ], + "expected": { + "verdict": "honest", + "fraud_position": -1, + "n_honest": 2, + "n_inconclusive": 1 + } + }, + { + "name": "inconclusive_unbounded_top_p", + "base_seed": "42|[1,2,3]", + "contract_version": "1.0.0", + "seed_domain": "gonka-deterministic-sampling-v1", + "temperature": "1.0", + "top_p": "0.9", + "top_k": null, + "min_p": null, + "greedy": false, + "positions": [ + { + "logprobs": { + "5": "-0.5", + "10": "-1.2", + "2": "-2.5", + "100": "-3.9", + "7": "-4.1" + }, + "reported_token": "5" + }, + { + "logprobs": { + "3": "-0.3", + "42": "-1.1", + "9": "-2.2", + "11": "-3.0", + "1": "-4.5" + }, + "reported_token": "3" + }, + { + "logprobs": { + "8": "-0.7", + "6": "-1.5", + "4": "-2.0", + "20": "-2.9", + "15": "-3.3" + }, + "reported_token": "8" + } + ], + "expected": { + "verdict": "inconclusive", + "fraud_position": -1, + "n_honest": 0, + "n_inconclusive": 0 + } + }, + { + "name": "inconclusive_greedy", + "base_seed": "42|[1,2,3]", + "contract_version": "1.0.0", + "seed_domain": "gonka-deterministic-sampling-v1", + "temperature": "0", + "top_p": null, + "top_k": 20, + "min_p": null, + "greedy": true, + "positions": [ + { + "logprobs": { + "5": "-0.5", + "10": "-1.2", + "2": "-2.5", + "100": "-3.9", + "7": "-4.1" + }, + "reported_token": "5" + }, + { + "logprobs": { + "3": "-0.3", + "42": "-1.1", + "9": "-2.2", + "11": "-3.0", + "1": "-4.5" + }, + "reported_token": "3" + }, + { + "logprobs": { + "8": "-0.7", + "6": "-1.5", + "4": "-2.0", + "20": "-2.9", + "15": "-3.3" + }, + "reported_token": "8" + } + ], + "expected": { + "verdict": "inconclusive", + "fraud_position": -1, + "n_honest": 0, + "n_inconclusive": 0 + } + }, + { + "name": "inconclusive_bad_version", + "base_seed": "42|[1,2,3]", + "contract_version": "9.9.9", + "seed_domain": "gonka-deterministic-sampling-v1", + "temperature": "1.0", + "top_p": null, + "top_k": 20, + "min_p": null, + "greedy": false, + "positions": [ + { + "logprobs": { + "5": "-0.5", + "10": "-1.2", + "2": "-2.5", + "100": "-3.9", + "7": "-4.1" + }, + "reported_token": "5" + }, + { + "logprobs": { + "3": "-0.3", + "42": "-1.1", + "9": "-2.2", + "11": "-3.0", + "1": "-4.5" + }, + "reported_token": "3" + }, + { + "logprobs": { + "8": "-0.7", + "6": "-1.5", + "4": "-2.0", + "20": "-2.9", + "15": "-3.3" + }, + "reported_token": "8" + } + ], + "expected": { + "verdict": "inconclusive", + "fraud_position": -1, + "n_honest": 0, + "n_inconclusive": 0 + } + } + ], + "distance_cases": [ + { + "name": "identical", + "executor": [ + { + "5": -0.5, + "10": -1.2, + "2": -2.5, + "100": -3.9, + "7": -4.1 + } + ], + "validator": [ + { + "5": -0.5, + "10": -1.2, + "2": -2.5, + "100": -3.9, + "7": -4.1 + } + ], + "expected_mae": 0.0 + }, + { + "name": "uniform_shift", + "executor": [ + { + "5": -0.5, + "10": -1.2, + "2": -2.5, + "100": -3.9, + "7": -4.1 + } + ], + "validator": [ + { + "5": -1.0, + "10": -1.7, + "2": -3.0, + "100": -4.4, + "7": -4.6 + } + ], + "expected_mae": 0.5000000000000001 + }, + { + "name": "length_mismatch", + "executor": [ + { + "5": -0.5, + "10": -1.2, + "2": -2.5, + "100": -3.9, + "7": -4.1 + }, + { + "3": -0.3, + "42": -1.1, + "9": -2.2, + "11": -3.0, + "1": -4.5 + } + ], + "validator": [ + { + "5": -0.5, + "10": -1.2, + "2": -2.5, + "100": -3.9, + "7": -4.1 + } + ], + "expected_mae": 10.0 + }, + { + "name": "missing_token", + "executor": [ + { + "5": -0.5, + "10": -1.2, + "2": -2.5, + "100": -3.9, + "7": -4.1 + } + ], + "validator": [ + { + "5": -0.5, + "10": -1.2, + "2": -2.5, + "100": -3.9 + } + ], + "expected_mae": 2.0 + } + ] +}