fix(warmup): gate implausible prices and cap confidence at ingestion

scripts/warmup-prices.ts

@@ -1,23 +1,24 @@
-import { config as loadEnv } from 'dotenv'
-loadEnv();
+import { config as loadEnv } from 'dotenv';
 import { chainIdToName, normalizeTokenAddress } from "../src/chains";
-import { createPool } from '../src/db'
-import { DefiLlamaClient } from '../src/defillama'
+import { priceCurveLpUsd } from '../src/curve';
+import { createPool } from '../src/db';
+import { DefiLlamaClient } from '../src/defillama';
+import { capConfidence, CURVE_CONFIDENCE, isPlausiblePrice } from '../src/format';
 import {
   getBatchHistoricalPrices,
   getExistingExactTimestamps,
   insertTokenPrices,
 } from "../src/queries";
-import { estimateBlockByTimestamp, getChainClient, readVaultSharePrice } from '../src/rpc'
-import { priceCurveLpUsd } from '../src/curve'
+import { estimateBlockByTimestamp, getChainClient, readVaultSharePrice } from '../src/rpc';
 import {
   isTodayNormalized,
   normalizedDaysInRange,
   normalizeToEndOfDay,
   nowUnix,
   parseCliDate,
 } from "../src/time";
-import type { HistoricalRequestTuple, KongVaultListItem, TokenPriceWrite } from '../src/types'
+import type { HistoricalRequestTuple, KongVaultListItem, TokenPriceWrite } from '../src/types';
+loadEnv();
 
 const databaseUrl = process.env.DATABASE_URL
 if (!databaseUrl) {
@@ -33,6 +34,8 @@ const stats: WarmupStats = {
   insertedDirect: 0,
   insertedDerived: 0,
   insertedCurve: 0,
+  guardedDirect: 0,
+  guardedCurve: 0,
 }
 const defiLlama = new DefiLlamaClient(undefined, () => {
   stats.retries += 1
@@ -48,6 +51,7 @@ interface NormalizedVault {
   chainId: number
   vaultToken: `0x${string}`
   underlyingToken: `0x${string}`
+  underlyingSymbol: string | null
   symbol: string | null
   apiVersion: string | null
   decimals: number
@@ -61,13 +65,15 @@ interface WarmupStats {
   insertedDirect: number
   insertedDerived: number
   insertedCurve: number
+  guardedDirect: number
+  guardedCurve: number
 }
 
 function sleep(milliseconds: number): Promise<void> {
   return new Promise(resolve => setTimeout(resolve, milliseconds))
 }
 
-function parseArgs(argv: string[]): { start: number; end: number } {
+function parseArgs(argv: string[]): { start: number; end: number; chain?: string; token?: `0x${string}` } {
   const options = new Map<string, string>()
   for (let index = 0; index < argv.length; index += 1) {
     const current = argv[index]
@@ -87,7 +93,10 @@ function parseArgs(argv: string[]): { start: number; end: number } {
     throw new Error('--start must be <= --end')
   }
 
-  return { start, end }
+  const chain = options.get('--chain')?.toLowerCase()
+  const token = options.has('--token') ? normalizeTokenAddress(options.get('--token')!) : undefined
+
+  return { start, end, chain, token }
 }
 
 async function fetchYearnVaults(): Promise<NormalizedVault[]> {
@@ -111,6 +120,7 @@ async function fetchYearnVaults(): Promise<NormalizedVault[]> {
         chainId: item.chainId,
         vaultToken: normalizeTokenAddress(item.address),
         underlyingToken: normalizeTokenAddress(item.asset.address),
+        underlyingSymbol: item.asset.symbol ?? null,
         symbol: item.symbol,
         apiVersion: item.apiVersion,
         decimals: item.decimals,
@@ -123,6 +133,22 @@ async function fetchYearnVaults(): Promise<NormalizedVault[]> {
   return vaults
 }
 
+function filterVaults(vaults: NormalizedVault[], chain?: string, token?: `0x${string}`): NormalizedVault[] {
+  if (!chain && !token) {
+    return vaults
+  }
+
+  return vaults.filter(vault => {
+    if (chain && vault.chain !== chain) {
+      return false
+    }
+    if (token && vault.vaultToken !== token && vault.underlyingToken !== token) {
+      return false
+    }
+    return true
+  })
+}
+
 function buildDailyTimestamps(start: number, end: number): number[] {
   return normalizedDaysInRange(start, end)
 }
@@ -229,14 +255,22 @@ async function warmDirectPrices(
         if (responseCoin) {
           for (const price of responseCoin.prices) {
             returnedTimestamps.add(normalizeToEndOfDay(price.timestamp))
+            // Price guard: drop implausible DefiLlama prices (0 for legacy Curve
+            // LPs, ~1e10 from the stale Curve-LP bug) so they never enter the
+            // table and outrank the Curve fallback.
+            if (!isPlausiblePrice(price.price)) {
+              stats.guardedDirect += 1
+              console.warn(`guard:price ${tokenKey} ${price.timestamp} ${price.price}`)
+              continue
+            }
             const [chain, token] = tokenKey.split(':')
             writes.push({
               chain,
               token,
               timestamp: normalizeToEndOfDay(price.timestamp),
               price: price.price,
               symbol: responseCoin.symbol ?? null,
-              confidence: price.confidence ?? null,
+              confidence: capConfidence(price.confidence),
               source: 'defillama',
             })
           }
@@ -266,12 +300,13 @@ async function warmCurveFallbackPrices(
   // Underlying tokens DefiLlama can't price (e.g. old Curve LP tokens) leave a
   // gap that cascades into derived vault prices. Fill those from the Curve
   // pool's on-chain virtual price.
-  const underlyings = new Map<string, { chain: string; chainId: number; token: `0x${string}` }>()
+  const underlyings = new Map<string, { chain: string; chainId: number; token: `0x${string}`; symbol: string | null }>()
   for (const vault of vaults) {
     underlyings.set(`${vault.chain}:${vault.underlyingToken}`, {
       chain: vault.chain,
       chainId: vault.chainId,
       token: vault.underlyingToken,
+      symbol: vault.underlyingSymbol,
     })
   }
 
@@ -282,13 +317,13 @@ async function warmCurveFallbackPrices(
     }
   }
 
-  const [existingDefillama, existingCurve] = await Promise.all([
-    getExistingExactTimestamps(pool, requests, 'defillama'),
-    getExistingExactTimestamps(pool, requests, 'curve'),
-  ])
+  const existingCurve = await getExistingExactTimestamps(pool, requests, 'curve')
   const missing = requests.filter(request => {
     const key = `${request.chain}:${request.token}:${request.timestamp}`
-    return isTodayNormalized(request.timestamp) || (!existingDefillama.has(key) && !existingCurve.has(key))
+    if (isTodayNormalized(request.timestamp)) {
+      return true
+    }
+    return !existingCurve.has(key)
   })
 
   await runInGroups(missing, async request => {
@@ -315,13 +350,19 @@ async function warmCurveFallbackPrices(
         return
       }
 
+      if (!isPlausiblePrice(price)) {
+        stats.guardedCurve += 1
+        console.warn(`guard:curve ${request.chain}:${request.token} ${request.timestamp} ${price}`)
+        return
+      }
+
       await insertTokenPrices(pool, [{
         chain: request.chain,
         token: request.token,
         timestamp: request.timestamp,
         price,
-        symbol: null,
-        confidence: null,
+        symbol: underlying.symbol,
+        confidence: CURVE_CONFIDENCE,
         source: 'curve',
       }])
       stats.insertedCurve += 1
@@ -409,9 +450,13 @@ async function warmDerivedVaultPrices(
 }
 
 try {
-  const { start, end } = parseArgs(process.argv.slice(2))
+  const { start, end, chain, token } = parseArgs(process.argv.slice(2))
   const timestamps = buildDailyTimestamps(start, end)
-  const vaults = await fetchYearnVaults()
+  const vaults = filterVaults(await fetchYearnVaults(), chain, token)
+
+  if ((chain || token) && vaults.length === 0) {
+    throw new Error(`No vaults matched filter: ${chain ?? 'any chain'} / ${token ?? 'any token'}`)
+  }
 
   console.info(`Warmup start: ${timestamps.length} days, ${vaults.length} vaults`)
   await warmDirectPrices(vaults, timestamps, stats)

src/format.ts

@@ -23,3 +23,27 @@ export function optionalResponseNumber(value: string | number | null): number |
 
   return toResponseNumber(value)
 }
+
+// DeFiLlama's confidence is a 0-1 quality score, but it occasionally comes back
+// marginally above 1 (e.g. 1.01). Clamp to [0, 1] so stored values stay in range.
+export function capConfidence(value: number | null | undefined): number | null {
+  return value == null ? null : Math.max(0, Math.min(value, 1))
+}
+
+// Sanity ceiling on a single token's USD price. No real token/vault share trades
+// above this; DefiLlama returns 0 for some legacy Curve LPs and ~1e10 for its
+// stale Curve-LP bug — both garbage. Guard at ingestion so they never enter the
+// table and outrank the Curve fallback.
+// ponytail: absolute cap (tune if a legit token ever exceeds it); upgrade path is
+// a per-token expected-range check.
+export const MAX_PLAUSIBLE_PRICE = 1_000_000
+
+// Curve LP price = on-chain virtual_price × coin0's USD price (from DefiLlama),
+// exact only for stableswap pools. Shipped at full confidence so it ranks as a
+// trusted fallback; the residual uncertainty is coin0's price, not the LP math.
+export const CURVE_CONFIDENCE = 1
+
+// A served USD price is only valid in (0, MAX]: anything else is provider garbage.
+export function isPlausiblePrice(price: number): boolean {
+  return Number.isFinite(price) && price > 0 && price <= MAX_PLAUSIBLE_PRICE
+}

src/queries.ts

@@ -1,12 +1,17 @@
 import type { Pool } from '@neondatabase/serverless'
 import { SOURCE_PRIORITY, type DbPriceRow, type ExactPriceRecord, type HistoricalRequestTuple, type PriceSource, type RangeRequest, type TokenPriceWrite } from './types'
-import { optionalResponseNumber, toResponseNumber } from './format'
+import { MAX_PLAUSIBLE_PRICE, optionalResponseNumber, toResponseNumber } from './format'
 import { pgTimestampToUnix, unixToIsoTimestamp, isTodayNormalized } from './time'
 
 function buildSourceCaseExpression(column = 'tp.source'): string {
   return `CASE ${column} ${SOURCE_PRIORITY.map((source, index) => `WHEN '${source}' THEN ${index + 1}`).join(' ')} ELSE 999 END`
 }
 
+// Stored prices outside the plausible range — provider garbage like 0, negatives,
+// or the ~1e10 stale-Curve-LP bug — must never be served or outrank a good
+// fallback. The ingestion guard stops new garbage; this filters any already stored.
+const PLAUSIBLE_PRICE_SQL = `tp.price > 0 AND tp.price <= ${MAX_PLAUSIBLE_PRICE}`
+
 export async function getExactHistoricalPrice(
   pool: Pool,
   request: HistoricalRequestTuple,
@@ -49,7 +54,7 @@ export async function getBatchHistoricalPrices(
         ON tp.chain = r.chain
        AND tp.token = r.token
        AND tp.timestamp = r.timestamp
-      WHERE tp.source = $${sourceIndex}
+      WHERE tp.source = $${sourceIndex} AND ${PLAUSIBLE_PRICE_SQL}
       ORDER BY tp.chain, tp.token, tp.timestamp
     `
   } else {
@@ -61,6 +66,7 @@ export async function getBatchHistoricalPrices(
         ON tp.chain = r.chain
        AND tp.token = r.token
        AND tp.timestamp = r.timestamp
+      WHERE ${PLAUSIBLE_PRICE_SQL}
       ORDER BY tp.chain, tp.token, tp.timestamp, ${buildSourceCaseExpression()}
     `
   }
@@ -107,7 +113,7 @@ export async function getRangeHistoricalPrices(
         ON tp.chain = r.chain
        AND tp.token = r.token
        AND tp.timestamp BETWEEN r.start_timestamp AND r.end_timestamp
-      WHERE tp.source = $${sourceIndex}
+      WHERE tp.source = $${sourceIndex} AND ${PLAUSIBLE_PRICE_SQL}
       ORDER BY tp.chain, tp.token, tp.timestamp
     `
   } else {
@@ -119,6 +125,7 @@ export async function getRangeHistoricalPrices(
         ON tp.chain = r.chain
        AND tp.token = r.token
        AND tp.timestamp BETWEEN r.start_timestamp AND r.end_timestamp
+      WHERE ${PLAUSIBLE_PRICE_SQL}
       ORDER BY tp.chain, tp.token, tp.timestamp, ${buildSourceCaseExpression()}
     `
   }

test/format.test.ts

@@ -0,0 +1,32 @@
+import { describe, expect, it } from 'vitest'
+import { capConfidence, isPlausiblePrice, MAX_PLAUSIBLE_PRICE } from '../src/format'
+
+describe('isPlausiblePrice', () => {
+  it('accepts prices in (0, MAX]', () => {
+    expect(isPlausiblePrice(1)).toBe(true)
+    expect(isPlausiblePrice(0.0001)).toBe(true)
+    expect(isPlausiblePrice(MAX_PLAUSIBLE_PRICE)).toBe(true)
+  })
+
+  it('rejects provider garbage', () => {
+    expect(isPlausiblePrice(0)).toBe(false)
+    expect(isPlausiblePrice(-1)).toBe(false)
+    expect(isPlausiblePrice(MAX_PLAUSIBLE_PRICE + 1)).toBe(false)
+    expect(isPlausiblePrice(1e10)).toBe(false)
+    expect(isPlausiblePrice(Number.NaN)).toBe(false)
+    expect(isPlausiblePrice(Number.POSITIVE_INFINITY)).toBe(false)
+  })
+})
+
+describe('capConfidence', () => {
+  it('clamps to [0, 1]', () => {
+    expect(capConfidence(1.01)).toBe(1)
+    expect(capConfidence(-0.5)).toBe(0)
+    expect(capConfidence(0.7)).toBe(0.7)
+  })
+
+  it('passes through null/undefined as null', () => {
+    expect(capConfidence(null)).toBeNull()
+    expect(capConfidence(undefined)).toBeNull()
+  })
+})