fix(prices): denylist legacy Curve LPs with bad DefiLlama prices

scripts/warmup-prices.ts

@@ -1,23 +1,26 @@
-import { config as loadEnv } from 'dotenv'
-loadEnv();
+import { config as loadEnv } from 'dotenv';
 import { chainIdToName, normalizeTokenAddress } from "../src/chains";
-import { createPool } from '../src/db'
-import { DefiLlamaClient } from '../src/defillama'
+import { priceCurveLpUsd } from '../src/curve';
+import { createPool } from '../src/db';
+import { DefiLlamaClient } from '../src/defillama';
+import { isBadDefiLlamaTokenKey } from '../src/bad-defillama-tokens';
+import { shouldAttemptCurveFallback } from '../src/curve-fallback';
+import { capConfidence } from '../src/format';
 import {
   getBatchHistoricalPrices,
   getExistingExactTimestamps,
   insertTokenPrices,
 } from "../src/queries";
-import { estimateBlockByTimestamp, getChainClient, readVaultSharePrice } from '../src/rpc'
-import { priceCurveLpUsd } from '../src/curve'
+import { estimateBlockByTimestamp, getChainClient, readVaultSharePrice } from '../src/rpc';
 import {
   isTodayNormalized,
   normalizedDaysInRange,
   normalizeToEndOfDay,
   nowUnix,
   parseCliDate,
 } from "../src/time";
-import type { HistoricalRequestTuple, KongVaultListItem, TokenPriceWrite } from '../src/types'
+import type { HistoricalRequestTuple, KongVaultListItem, TokenPriceWrite } from '../src/types';
+loadEnv();
 
 const databaseUrl = process.env.DATABASE_URL
 if (!databaseUrl) {
@@ -33,6 +36,7 @@ const stats: WarmupStats = {
   insertedDirect: 0,
   insertedDerived: 0,
   insertedCurve: 0,
+  skippedBadDefiLlama: 0,
 }
 const defiLlama = new DefiLlamaClient(undefined, () => {
   stats.retries += 1
@@ -48,6 +52,7 @@ interface NormalizedVault {
   chainId: number
   vaultToken: `0x${string}`
   underlyingToken: `0x${string}`
+  underlyingSymbol: string | null
   symbol: string | null
   apiVersion: string | null
   decimals: number
@@ -61,13 +66,14 @@ interface WarmupStats {
   insertedDirect: number
   insertedDerived: number
   insertedCurve: number
+  skippedBadDefiLlama: number
 }
 
 function sleep(milliseconds: number): Promise<void> {
   return new Promise(resolve => setTimeout(resolve, milliseconds))
 }
 
-function parseArgs(argv: string[]): { start: number; end: number } {
+function parseArgs(argv: string[]): { start: number; end: number; chain?: string; token?: `0x${string}` } {
   const options = new Map<string, string>()
   for (let index = 0; index < argv.length; index += 1) {
     const current = argv[index]
@@ -87,7 +93,10 @@ function parseArgs(argv: string[]): { start: number; end: number } {
     throw new Error('--start must be <= --end')
   }
 
-  return { start, end }
+  const chain = options.get('--chain')?.toLowerCase()
+  const token = options.has('--token') ? normalizeTokenAddress(options.get('--token')!) : undefined
+
+  return { start, end, chain, token }
 }
 
 async function fetchYearnVaults(): Promise<NormalizedVault[]> {
@@ -111,6 +120,7 @@ async function fetchYearnVaults(): Promise<NormalizedVault[]> {
         chainId: item.chainId,
         vaultToken: normalizeTokenAddress(item.address),
         underlyingToken: normalizeTokenAddress(item.asset.address),
+        underlyingSymbol: item.asset.symbol ?? null,
         symbol: item.symbol,
         apiVersion: item.apiVersion,
         decimals: item.decimals,
@@ -123,6 +133,22 @@ async function fetchYearnVaults(): Promise<NormalizedVault[]> {
   return vaults
 }
 
+function filterVaults(vaults: NormalizedVault[], chain?: string, token?: `0x${string}`): NormalizedVault[] {
+  if (!chain && !token) {
+    return vaults
+  }
+
+  return vaults.filter(vault => {
+    if (chain && vault.chain !== chain) {
+      return false
+    }
+    if (token && vault.vaultToken !== token && vault.underlyingToken !== token) {
+      return false
+    }
+    return true
+  })
+}
+
 function buildDailyTimestamps(start: number, end: number): number[] {
   return normalizedDaysInRange(start, end)
 }
@@ -214,6 +240,16 @@ async function warmDirectPrices(
   }).length
 
   const groupedMissing = groupMissingRequests(requests, existing)
+  // Hardcoded denylist: legacy Curve LPs where DefiLlama returns 0 / multi-million
+  // garbage that would outrank the Curve virtual-price fallback. Skip the API
+  // call so Curve can fill instead.
+  for (const tokenKey of Object.keys(groupedMissing)) {
+    if (isBadDefiLlamaTokenKey(tokenKey)) {
+      stats.skippedBadDefiLlama += groupedMissing[tokenKey].length
+      console.warn(`skip:bad-defillama-curve ${tokenKey} (${groupedMissing[tokenKey].length} timestamps)`)
+      delete groupedMissing[tokenKey]
+    }
+  }
   const payloads = buildDefiLlamaPayloads(groupedMissing)
 
   await runInGroups(payloads, async payload => {
@@ -236,7 +272,7 @@ async function warmDirectPrices(
               timestamp: normalizeToEndOfDay(price.timestamp),
               price: price.price,
               symbol: responseCoin.symbol ?? null,
-              confidence: price.confidence ?? null,
+              confidence: capConfidence(price.confidence),
               source: 'defillama',
             })
           }
@@ -266,12 +302,13 @@ async function warmCurveFallbackPrices(
   // Underlying tokens DefiLlama can't price (e.g. old Curve LP tokens) leave a
   // gap that cascades into derived vault prices. Fill those from the Curve
   // pool's on-chain virtual price.
-  const underlyings = new Map<string, { chain: string; chainId: number; token: `0x${string}` }>()
+  const underlyings = new Map<string, { chain: string; chainId: number; token: `0x${string}`; symbol: string | null }>()
   for (const vault of vaults) {
     underlyings.set(`${vault.chain}:${vault.underlyingToken}`, {
       chain: vault.chain,
       chainId: vault.chainId,
       token: vault.underlyingToken,
+      symbol: vault.underlyingSymbol,
     })
   }
 
@@ -282,14 +319,18 @@ async function warmCurveFallbackPrices(
     }
   }
 
+  // Gate historical Curve probes on both sources: a valid DefiLlama row fills the
+  // gap (SOURCE_PRIORITY), and an existing Curve row must not be recomputed.
+  // getExistingExactTimestamps('defillama') already excludes denylisted LP rows, so
+  // those four tokens still appear missing and remain eligible for Curve fallback.
+  // Today always refreshes regardless of what is already stored.
   const [existingDefillama, existingCurve] = await Promise.all([
     getExistingExactTimestamps(pool, requests, 'defillama'),
     getExistingExactTimestamps(pool, requests, 'curve'),
   ])
-  const missing = requests.filter(request => {
-    const key = `${request.chain}:${request.token}:${request.timestamp}`
-    return isTodayNormalized(request.timestamp) || (!existingDefillama.has(key) && !existingCurve.has(key))
-  })
+  const missing = requests.filter(request =>
+    shouldAttemptCurveFallback(request, existingDefillama, existingCurve),
+  )
 
   await runInGroups(missing, async request => {
     try {
@@ -303,6 +344,8 @@ async function warmCurveFallbackPrices(
 
       const blockNumber = await estimateBlockByTimestamp(client, underlying.chainId, request.timestamp)
 
+      // Approximate: virtual_price × DefiLlama(coin0). Residual uncertainty includes
+      // coin0 (interest-bearing yTokens on legacy Y/yBUSD/PAX pools).
       const price = await priceCurveLpUsd(client, underlying.chainId, underlying.token, blockNumber, async coin => {
         const coinKey = `${request.chain}:${coin}`
         stats.apiCalls += 1
@@ -320,7 +363,7 @@ async function warmCurveFallbackPrices(
         token: request.token,
         timestamp: request.timestamp,
         price,
-        symbol: null,
+        symbol: underlying.symbol,
         confidence: null,
         source: 'curve',
       }])
@@ -409,9 +452,13 @@ async function warmDerivedVaultPrices(
 }
 
 try {
-  const { start, end } = parseArgs(process.argv.slice(2))
+  const { start, end, chain, token } = parseArgs(process.argv.slice(2))
   const timestamps = buildDailyTimestamps(start, end)
-  const vaults = await fetchYearnVaults()
+  const vaults = filterVaults(await fetchYearnVaults(), chain, token)
+
+  if ((chain || token) && vaults.length === 0) {
+    throw new Error(`No vaults matched filter: ${chain ?? 'any chain'} / ${token ?? 'any token'}`)
+  }
 
   console.info(`Warmup start: ${timestamps.length} days, ${vaults.length} vaults`)
   await warmDirectPrices(vaults, timestamps, stats)

src/bad-defillama-tokens.ts

@@ -0,0 +1,41 @@
+import { normalizeTokenKey } from './chains'
+
+// Legacy Curve LP tokens where DefiLlama stores garbage (price = 0 and/or
+// multi-million spikes) that outranks the Curve virtual-price fallback.
+// Enumerated from production token_prices: source=defillama AND
+// (price<=0 OR price>1e6) AND a curve-source row exists for the same token.
+//
+// Curve is a best-effort fallback via priceCurveLpUsd (virtual_price × coin0):
+// for Y / yBUSD / PAX the goal is suppressing invalid DefiLlama spikes on drained
+// legacy lending pools (coin0 is an interest-bearing yToken, so the price is
+// approximate). sUSD plain3 is the straightforward stablecoin case.
+const BAD_DEFILLAMA_CURVE_LPS = [
+  // Curve Y pool — all-zero DefiLlama history in DB (~901 rows); drained lending pool
+  ['ethereum', '0xdF5e0e81Dff6FAF3A7e52BA697820c5e32D806A8'], // yDAI+yUSDC+yUSDT+yTUSD
+  // Curve yBUSD pool — zeros + ~7e7 spikes (~920 rows); drained lending pool
+  ['ethereum', '0x3B3Ac5386837Dc563660FB6a0937DFAa5924333B'], // yDAI+yUSDC+yUSDT+yBUSD
+  // Curve PAX pool — zeros + ~2e6 spikes (~916 rows); drained lending pool
+  ['ethereum', '0xD905e2eaeBe188fc92179b6350807D8bd91Db0D8'], // ypaxCrv
+  // Curve sUSD plain3 — all-zero DefiLlama (~329 rows); plain stablecoin LP
+  ['ethereum', '0xC25a3A3b969415c80451098fa907EC722572917F'], // crvPlain3andSUSD
+] as const
+
+export const BAD_DEFILLAMA_TOKEN_KEYS: ReadonlySet<string> = new Set(
+  BAD_DEFILLAMA_CURVE_LPS.map(([chain, token]) => normalizeTokenKey(chain, token)),
+)
+
+export function isBadDefiLlamaToken(chain: string, token: string): boolean {
+  try {
+    return BAD_DEFILLAMA_TOKEN_KEYS.has(normalizeTokenKey(chain, token))
+  } catch {
+    return false
+  }
+}
+
+export function isBadDefiLlamaTokenKey(tokenKey: string): boolean {
+  const separatorIndex = tokenKey.indexOf(':')
+  if (separatorIndex <= 0 || separatorIndex === tokenKey.length - 1) {
+    return false
+  }
+  return isBadDefiLlamaToken(tokenKey.slice(0, separatorIndex), tokenKey.slice(separatorIndex + 1))
+}

src/curve-fallback.ts

@@ -0,0 +1,20 @@
+import { isTodayNormalized } from './time'
+import type { HistoricalRequestTuple } from './types'
+
+// Decide whether warmCurveFallbackPrices should probe Curve for this request.
+// Historical: only when neither a usable DefiLlama row nor a Curve row exists.
+// Today: always refresh. Denylisted DefiLlama rows are already excluded by
+// getExistingExactTimestamps(..., 'defillama'), so they appear missing here.
+export function shouldAttemptCurveFallback(
+  request: HistoricalRequestTuple,
+  existingDefillama: ReadonlySet<string>,
+  existingCurve: ReadonlySet<string>,
+  now?: number,
+): boolean {
+  if (isTodayNormalized(request.timestamp, now)) {
+    return true
+  }
+
+  const key = `${request.chain}:${request.token}:${request.timestamp}`
+  return !existingDefillama.has(key) && !existingCurve.has(key)
+}

src/format.ts

@@ -23,3 +23,9 @@ export function optionalResponseNumber(value: string | number | null): number |
 
   return toResponseNumber(value)
 }
+
+// DeFiLlama's confidence is a 0-1 quality score, but it occasionally comes back
+// marginally above 1 (e.g. 1.01). Clamp to [0, 1] so stored values stay in range.
+export function capConfidence(value: number | null | undefined): number | null {
+  return value == null ? null : Math.max(0, Math.min(value, 1))
+}

src/queries.ts

@@ -1,4 +1,5 @@
 import type { Pool } from '@neondatabase/serverless'
+import { BAD_DEFILLAMA_TOKEN_KEYS } from './bad-defillama-tokens'
 import { SOURCE_PRIORITY, type DbPriceRow, type ExactPriceRecord, type HistoricalRequestTuple, type PriceSource, type RangeRequest, type TokenPriceWrite } from './types'
 import { optionalResponseNumber, toResponseNumber } from './format'
 import { pgTimestampToUnix, unixToIsoTimestamp, isTodayNormalized } from './time'
@@ -7,6 +8,24 @@ function buildSourceCaseExpression(column = 'tp.source'): string {
   return `CASE ${column} ${SOURCE_PRIORITY.map((source, index) => `WHEN '${source}' THEN ${index + 1}`).join(' ')} ELSE 999 END`
 }
 
+// Drop DefiLlama rows for legacy Curve LPs known to return garbage
+// (see bad-defillama-tokens). Those rows would otherwise outrank the Curve
+// virtual-price fallback via SOURCE_PRIORITY.
+function buildBadDefiLlamaFilter(columnPrefix = 'tp'): string {
+  if (BAD_DEFILLAMA_TOKEN_KEYS.size === 0) {
+    return 'TRUE'
+  }
+  const pairs = [...BAD_DEFILLAMA_TOKEN_KEYS].map(key => {
+    const separatorIndex = key.indexOf(':')
+    const chain = key.slice(0, separatorIndex).replace(/'/g, "''")
+    const token = key.slice(separatorIndex + 1).replace(/'/g, "''")
+    return `('${chain}', '${token}')`
+  })
+  return `NOT (${columnPrefix}.source = 'defillama' AND (${columnPrefix}.chain, ${columnPrefix}.token) IN (${pairs.join(', ')}))`
+}
+
+const BAD_DEFILLAMA_SQL = buildBadDefiLlamaFilter()
+
 export async function getExactHistoricalPrice(
   pool: Pool,
   request: HistoricalRequestTuple,
@@ -49,7 +68,7 @@ export async function getBatchHistoricalPrices(
         ON tp.chain = r.chain
        AND tp.token = r.token
        AND tp.timestamp = r.timestamp
-      WHERE tp.source = $${sourceIndex}
+      WHERE tp.source = $${sourceIndex} AND ${BAD_DEFILLAMA_SQL}
       ORDER BY tp.chain, tp.token, tp.timestamp
     `
   } else {
@@ -61,6 +80,7 @@ export async function getBatchHistoricalPrices(
         ON tp.chain = r.chain
        AND tp.token = r.token
        AND tp.timestamp = r.timestamp
+      WHERE ${BAD_DEFILLAMA_SQL}
       ORDER BY tp.chain, tp.token, tp.timestamp, ${buildSourceCaseExpression()}
     `
   }
@@ -107,7 +127,7 @@ export async function getRangeHistoricalPrices(
         ON tp.chain = r.chain
        AND tp.token = r.token
        AND tp.timestamp BETWEEN r.start_timestamp AND r.end_timestamp
-      WHERE tp.source = $${sourceIndex}
+      WHERE tp.source = $${sourceIndex} AND ${BAD_DEFILLAMA_SQL}
       ORDER BY tp.chain, tp.token, tp.timestamp
     `
   } else {
@@ -119,6 +139,7 @@ export async function getRangeHistoricalPrices(
         ON tp.chain = r.chain
        AND tp.token = r.token
        AND tp.timestamp BETWEEN r.start_timestamp AND r.end_timestamp
+      WHERE ${BAD_DEFILLAMA_SQL}
       ORDER BY tp.chain, tp.token, tp.timestamp, ${buildSourceCaseExpression()}
     `
   }