external-enrichment: Umgebungsfelder nicht mehr bei Refetch-Fehlschlag löschen (#429)

CAMS-Luftqualität, EEA-Lärm und Klimanormalen wurden bei jedem nächtlichen
Lauf komplett neu in den locationContext geschrieben; schlug ein Provider
transient fehl, fiel das Feld ersatzlos weg statt auf den Vorlauf
zurückzufallen — das erzeugte die schwankende Datenabdeckung in den
Settings. mergeLocationContextWithPrevious behält jetzt den alten Wert pro
Feld bei Fehlschlag. Coverage zeigt zusätzlich pro Quelle das Datum der
letzten erfolgreichen Aktualisierung, damit ein dauerhaft toter Provider
sichtbar bleibt statt sich hinter alten Daten zu verstecken.

Author: haexhub

components/settings/SettingsExternalDataCard.vue

@@ -54,6 +54,7 @@ interface ExternalDataSourceCoverage {
   total: number
   covered: number
   byCountry: ExternalDataCoverageCountryRow[]
+  lastSuccessAt: string | null
 }
 
 const { t, te } = useI18n()
@@ -307,6 +308,9 @@ onMounted(async () => {
             </span>
           </div>
           <Progress :model-value="percentOf(coverageFor(source.id)!.covered, coverageFor(source.id)!.total)" />
+          <p v-if="coverageFor(source.id)!.lastSuccessAt" class="text-xs text-muted-foreground">
+            {{ $t('settings.externalData.coverage.lastSuccessAt', { at: formatBatchDate(coverageFor(source.id)!.lastSuccessAt) }) }}
+          </p>
           <p v-if="coverageFor(source.id)!.byCountry.length === 0" class="text-xs text-muted-foreground">
             {{ $t('settings.externalData.coverage.empty') }}
           </p>

i18n/locales/de.json

@@ -410,6 +410,7 @@
       "coverage": {
         "title": "Datenabdeckung",
         "summary": "{covered} von {total} geocodierten Auktionen ({percent} %)",
+        "lastSuccessAt": "Zuletzt erfolgreich aktualisiert: {at}",
         "empty": "Noch keine geocodierten Auktionen für diese Quelle.",
         "loadError": "Datenabdeckung konnte nicht geladen werden."
       },

i18n/locales/en.json

@@ -410,6 +410,7 @@
       "coverage": {
         "title": "Data coverage",
         "summary": "{covered} of {total} geocoded auctions ({percent}%)",
+        "lastSuccessAt": "Last successfully updated: {at}",
         "empty": "No geocoded auctions for this source yet.",
         "loadError": "Could not load data coverage."
       },

server/api/settings/external-data/coverage.get.test.ts

@@ -88,4 +88,57 @@ describe('GET /api/settings/external-data/coverage', () => {
       { country: 'fr', total: 50, covered: 45 },
     ])
   })
+
+  it('takes the most recent lastSuccessAt across countries, only for the three sources it tracks', async () => {
+    vi.stubGlobal('defineEventHandler', (handler: unknown) => handler)
+
+    const { getPool } = await import('~/server/utils/db')
+    vi.mocked(getPool).mockReturnValue({
+      query: async () => ({
+        rows: [
+          {
+            country: 'de',
+            geocoded_total: '10',
+            cams_air_quality: '5',
+            cams_air_quality_last_success_at: '2026-08-01T00:00:00.000Z',
+            open_meteo_climate_normals: '5',
+            open_meteo_climate_normals_last_success_at: '2026-07-15T00:00:00.000Z',
+            eea_environmental_noise_directive: '5',
+            eea_environmental_noise_directive_last_success_at: null,
+            eu_flood_risk_areas: '5',
+            copernicus_effis: '5',
+            fr_dvf_geolocated: '5',
+          },
+          {
+            country: 'fr',
+            geocoded_total: '10',
+            cams_air_quality: '5',
+            cams_air_quality_last_success_at: '2026-08-10T00:00:00.000Z',
+            open_meteo_climate_normals: '5',
+            open_meteo_climate_normals_last_success_at: '2026-06-01T00:00:00.000Z',
+            eea_environmental_noise_directive: '5',
+            eea_environmental_noise_directive_last_success_at: '2026-08-05T00:00:00.000Z',
+            eu_flood_risk_areas: '5',
+            copernicus_effis: '5',
+            fr_dvf_geolocated: '5',
+          },
+        ],
+      }),
+    } as never)
+
+    const handler = (await import('./coverage.get')).default as unknown as () => Promise<{
+      sources: Array<{ id: string; lastSuccessAt: string | null }>
+    }>
+    const { sources } = await handler()
+    const bySourceId = Object.fromEntries(sources.map((source) => [source.id, source.lastSuccessAt]))
+
+    expect(bySourceId['cams-air-quality']).toBe('2026-08-10T00:00:00.000Z')
+    expect(bySourceId['open-meteo-climate-normals']).toBe('2026-07-15T00:00:00.000Z')
+    expect(bySourceId['eea-environmental-noise-directive']).toBe('2026-08-05T00:00:00.000Z')
+    // Hazards/market already fall back to their previous value on failure, so
+    // their freshness isn't tracked here.
+    expect(bySourceId['eu-flood-risk-areas']).toBeNull()
+    expect(bySourceId['copernicus-effis']).toBeNull()
+    expect(bySourceId['fr-dvf-geolocated']).toBeNull()
+  })
 })

server/tasks/external-enrichment.test.ts

@@ -530,7 +530,7 @@ describe('runExternalEnrichment', () => {
       platform: 'se-kronofogden',
       country: 'se',
       externalId: '1',
-    }))
+    }), null)
     expect(writeLocationEnrichmentCache).toHaveBeenCalledWith({
       'se-kronofogden:1': expect.objectContaining({
         locationContext,
@@ -697,4 +697,87 @@ describe('withLocationContextEnhancers', () => {
     expect(summary.errors).toHaveLength(1)
     expect(summary.errors[0]).toContain('second')
   })
+
+  it('restores air quality, climate normals and noise from the previous run when this run cannot refresh them', async () => {
+    vi.stubGlobal('defineTask', (def: unknown) => def)
+    const { withLocationContextEnhancers } = await import('./external-enrichment')
+    const summary = {
+      processed: 0,
+      written: 0,
+      skippedMissingCoordinates: 0,
+      marketComparisons: 0,
+      landValueBaselines: 0,
+      hazards: 0,
+      locationContexts: 0,
+      staleResults: 0,
+      providerFailures: 0,
+      errors: [] as string[],
+      durationMs: 0,
+    }
+    const previousContext: LocationContext = {
+      ...locationContext,
+      environment: {
+        ...locationContext.environment,
+        airQuality: {
+          index: 30,
+          level: 'fair',
+          particulateMatter10: 12,
+          particulateMatter25: 8,
+          nitrogenDioxide: 15,
+          ozone: 40,
+          observedAt: '2026-08-01T00:00:00.000Z',
+          sourceLabel: 'CAMS',
+          sourceUrl: 'https://example.test/cams',
+          checkedAt: '2026-08-01T00:00:00.000Z',
+        },
+        climateNormals: {
+          periodStartYear: 1991,
+          periodEndYear: 2020,
+          months: [],
+          sourceLabel: 'Open-Meteo',
+          sourceUrl: 'https://example.test/climate',
+          checkedAt: '2026-07-01T00:00:00.000Z',
+        },
+        reportedNoise: [{
+          source: 'road',
+          indicator: 'lden',
+          level: 'medium',
+          bandLabel: '60-64 dB Lden',
+          minDb: 60,
+          maxDb: 64,
+          value: 2,
+          sourceLayerName: null,
+          sourceLabel: 'EEA',
+          sourceUrl: 'https://example.test/eea',
+          checkedAt: '2026-06-01T00:00:00.000Z',
+        }],
+      },
+    }
+    // This run's OSM rebuild has none of the previous run's enhancer fields yet.
+    const freshContext: LocationContext = {
+      ...locationContext,
+      environment: { ...locationContext.environment, airQuality: null, climateNormals: null, reportedNoise: undefined },
+    }
+    const baseAdapter = {
+      id: 'base',
+      sourceVersion: 'v1',
+      supports: () => true,
+      context: vi.fn(async () => freshContext),
+    }
+    const failingEnhancer = {
+      id: 'cams-air-quality',
+      sourceVersion: 'v1',
+      supports: () => true,
+      enhance: vi.fn(async () => {
+        throw new Error('rate limited')
+      }),
+    }
+
+    const adapter = withLocationContextEnhancers(baseAdapter, [failingEnhancer], summary)
+    const result = await adapter.context(auction(), previousContext)
+
+    expect(result?.environment.airQuality).toEqual(previousContext.environment.airQuality)
+    expect(result?.environment.climateNormals).toEqual(previousContext.environment.climateNormals)
+    expect(result?.environment.reportedNoise).toEqual(previousContext.environment.reportedNoise)
+  })
 })

server/tasks/external-enrichment.ts

@@ -14,6 +14,7 @@ import { createEeaEnvironmentalNoiseEnhancer } from '~/server/utils/external-dat
 import { createCamsAirQualityEnhancer } from '~/server/utils/external-data/cams-air-quality'
 import { createOpenMeteoClimateNormalsEnhancer } from '~/server/utils/external-data/open-meteo-climate'
 import { createLocalOsmLocationContextAdapter } from '~/server/utils/external-data/osm-location-context'
+import { mergeLocationContextWithPrevious } from '~/server/utils/external-data/location-context-merge'
 import {
   getStoredExternalDataSourceConfig,
   getConfigurableExternalDataSource,
@@ -49,7 +50,7 @@ export interface LocationContextAdapter {
   id: string
   sourceVersion: string
   supports(auction: Auction): boolean
-  context(auction: Auction): Promise<LocationContext | null>
+  context(auction: Auction, previous?: LocationContext | null): Promise<LocationContext | null>
 }
 
 export interface LocationContextEnhancer {
@@ -175,7 +176,7 @@ export async function runExternalEnrichment(
       throwIfTaskAborted(signal)
       const hazards = await allHazards(auction, hazardAdapters, summary)
       throwIfTaskAborted(signal)
-      const locationContext = await firstLocationContext(auction, locationContextAdapters, summary)
+      const locationContext = await firstLocationContext(auction, locationContextAdapters, summary, previous?.locationContext ?? null)
       throwIfTaskAborted(signal)
 
       if (marketComparison) summary.marketComparisons++
@@ -315,11 +316,12 @@ async function firstLocationContext(
   auction: Auction,
   adapters: LocationContextAdapter[],
   summary: ExternalEnrichmentSummary,
+  previous: LocationContext | null,
 ): Promise<LocationContext | null> {
   for (const adapter of adapters) {
     if (!adapter.supports(auction)) continue
     try {
-      const result = await adapter.context(auction)
+      const result = await adapter.context(auction, previous)
       if (result) return result
     } catch (err) {
       recordProviderFailure(summary, adapter.id, 'location context', auction, err)
@@ -477,8 +479,8 @@ export function withLocationContextEnhancers(
     id: [adapter.id, ...enhancers.map((enhancer) => enhancer.id)].join('+'),
     sourceVersion: [adapter.sourceVersion, ...enhancers.map((enhancer) => enhancer.sourceVersion)].join(','),
     supports: (auction) => adapter.supports(auction),
-    async context(auction) {
-      let context = await adapter.context(auction)
+    async context(auction, previous) {
+      let context = await adapter.context(auction, previous)
       if (!context) return null
       for (const enhancer of enhancers) {
         if (!enhancer.supports(auction, context)) continue
@@ -492,7 +494,7 @@ export function withLocationContextEnhancers(
           recordProviderFailure(summary, enhancer.id, 'location context enhancer', auction, err)
         }
       }
-      return context
+      return previous ? mergeLocationContextWithPrevious(context, previous) : context
     },
   }
 }

server/utils/external-data/coverage.ts

@@ -27,12 +27,22 @@ export interface ExternalDataSourceCoverage {
   total: number
   covered: number
   byCountry: ExternalDataCoverageCountryRow[]
+  /** Most recent `checkedAt` this source actually succeeded at, across every
+   *  auction — null when never tracked or never succeeded. A transient
+   *  fetch failure keeps the previous run's value (external-enrichment.ts's
+   *  mergeLocationContextWithPrevious), so unlike `covered` this only stalls
+   *  when the source has stopped succeeding for every auction, surfacing a
+   *  dead provider that a stable coverage percentage would otherwise hide. */
+  lastSuccessAt: string | null
 }
 
 // One row per country, geocoded_total plus one covered-count column per
 // source in COVERAGE_SOURCE_IDS (column name = source id with '-' -> '_').
 // A single query grouped by country avoids six-way N+1 scans over
-// location_enrichment.
+// location_enrichment. lastSuccessAt columns are only meaningful for the
+// three sources subject to that transient-failure merge above — hazards and
+// market comparison already fall back to their previous whole value on
+// failure (external-enrichment.ts), so they're never tracked here.
 const COVERAGE_QUERY = `
   SELECT
     a.country,
@@ -41,14 +51,20 @@ const COVERAGE_QUERY = `
       WHERE a.lat IS NOT NULL AND a.lng IS NOT NULL
         AND nullif(le.enrichment->'locationContext'->'environment'->'airQuality', 'null'::jsonb) IS NOT NULL
     ) AS cams_air_quality,
+    max(le.enrichment->'locationContext'->'environment'->'airQuality'->>'checkedAt') AS cams_air_quality_last_success_at,
     count(*) FILTER (
       WHERE a.lat IS NOT NULL AND a.lng IS NOT NULL
         AND le.enrichment->'locationContext'->'environment'->'climateNormals' IS NOT NULL
     ) AS open_meteo_climate_normals,
+    max(le.enrichment->'locationContext'->'environment'->'climateNormals'->>'checkedAt') AS open_meteo_climate_normals_last_success_at,
     count(*) FILTER (
       WHERE a.lat IS NOT NULL AND a.lng IS NOT NULL
         AND jsonb_array_length(coalesce(le.enrichment->'locationContext'->'environment'->'reportedNoise', '[]'::jsonb)) > 0
     ) AS eea_environmental_noise_directive,
+    max((
+      SELECT max(n->>'checkedAt')
+      FROM jsonb_array_elements(coalesce(le.enrichment->'locationContext'->'environment'->'reportedNoise', '[]'::jsonb)) n
+    )) AS eea_environmental_noise_directive_last_success_at,
     count(*) FILTER (
       WHERE a.lat IS NOT NULL AND a.lng IS NOT NULL
         AND EXISTS (
@@ -84,13 +100,19 @@ const COVERAGE_COLUMN_BY_SOURCE_ID: Record<CoverageSourceId, string> = {
   'fr-dvf-geolocated': 'fr_dvf_geolocated',
 }
 
-type CoverageRow = { country: string; geocoded_total: string } & Record<string, string>
+const LAST_SUCCESS_COLUMN_BY_SOURCE_ID: Partial<Record<CoverageSourceId, string>> = {
+  'cams-air-quality': 'cams_air_quality_last_success_at',
+  'open-meteo-climate-normals': 'open_meteo_climate_normals_last_success_at',
+  'eea-environmental-noise-directive': 'eea_environmental_noise_directive_last_success_at',
+}
+
+type CoverageRow = { country: string; geocoded_total: string } & Record<string, string | null>
 
 export async function computeExternalDataCoverage(db: Pool): Promise<ExternalDataSourceCoverage[]> {
   const { rows } = await db.query<CoverageRow>(COVERAGE_QUERY)
 
   const bySource = new Map<CoverageSourceId, ExternalDataSourceCoverage>(
-    COVERAGE_SOURCE_IDS.map((id) => [id, { id, total: 0, covered: 0, byCountry: [] }]),
+    COVERAGE_SOURCE_IDS.map((id) => [id, { id, total: 0, covered: 0, byCountry: [], lastSuccessAt: null }]),
   )
 
   for (const row of rows) {
@@ -103,6 +125,11 @@ export async function computeExternalDataCoverage(db: Pool): Promise<ExternalDat
       entry.total += total
       entry.covered += covered
       entry.byCountry.push({ country: row.country, total, covered })
+      const lastSuccessColumn = LAST_SUCCESS_COLUMN_BY_SOURCE_ID[id]
+      const rowLastSuccessAt = lastSuccessColumn ? row[lastSuccessColumn] : null
+      if (rowLastSuccessAt && (!entry.lastSuccessAt || rowLastSuccessAt > entry.lastSuccessAt)) {
+        entry.lastSuccessAt = rowLastSuccessAt
+      }
     }
   }