Skip to content

Commit 5c5d357

Browse files
haexhubclaude
haexhub
and
claude
authored
feat(crawler): add Italy via Aste Giudiziarie Inlinea (#27)
* feat(crawler): add Italy via Aste Giudiziarie Inlinea (astegiudiziarie.it) Adds a new `agi` platform crawler covering all 21 Italian regions. The official PVP portal requires SPID authentication for its search API, so we use astegiudiziarie.it (Ministry-authorised, public HTML + JSON API). Two-phase fetch per region: 1. POST search/map → all lot IDs + basic data for the region 2. POST search/Data → full details in batches of 50 Session cookie (ASP.NET Core) is obtained by a single GET /results before each crawl. enrichOne() fetches the detail page and scrapes PDF allegato links (perizia → gutachten, avviso → bekanntmachung). Address assembly handles both pre-geocoded Italian addresses (CAP present) and street-only entries (appends comune + provincia for Nominatim). Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com> * fix(crawler): address code review for PR #27 (AGI + shared cleanup) - agi/constants: derive PORTAL_REGION_NAMES from IT_REGION_NAMES instead of duplicating - agi/list: remove unused MapEntry fields (latitudine/longitudine/prezzoBase/dataPubblicazione), add AbortSignal.timeout() to all fetches, map esito.Sigla → aufgehoben, set detailUrlUpstream to null when urlSchedaDettagliata is missing, catch per-batch errors in fetchAllDetails - agi/detail: add timeout to fetchDetailInfo, prefer gutachten PDFs over first-in-DOM, prefer data-src over src for lazy-loaded images, extract applyDetailInfo helper, skip enrichment when detailUrlUpstream is null - types/auction: detailUrlUpstream is now string | null (no specific page for some lots) - at/detail, zvbawu/detail: null guard for detailUrlUpstream (TypeScript narrowing) - refactor: extract shared classifyAttachment utility, remove duplicate classify() fns from biddit/detail and zvg-portal/detail Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com> --------- Co-authored-by: Claude Sonnet 4.6 <noreply@anthropic.com>
1 parent 6fcb896 commit 5c5d357

12 files changed

Lines changed: 531 additions & 48 deletions

File tree

‎server/crawlers/agi/constants.ts‎

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,42 @@
1+
import type { RegionInfo } from '../types'
2+
3+
export const AGI_BASE = 'https://www.astegiudiziarie.it'
4+
export const AGI_API_BASE = 'https://webapi.astegiudiziarie.it/api'
5+
export const COUNTRY = 'it'
6+
export const UA = 'Mozilla/5.0 (X11; Linux x86_64; rv:130.0) Gecko/20100101 Firefox/130.0'
7+
8+
/** Batch size for the search/Data API call. */
9+
export const DETAIL_BATCH_SIZE = 50
10+
11+
/** idGenere=1 means Immobili (real estate) on the portal. */
12+
export const ID_GENERE_IMMOBILI = 1
13+
14+
export const IT_REGIONS: readonly RegionInfo[] = [
15+
{ code: 'abruzzo', name: 'Abruzzo' },
16+
{ code: 'basilicata', name: 'Basilicata' },
17+
{ code: 'calabria', name: 'Calabria' },
18+
{ code: 'campania', name: 'Campania' },
19+
{ code: 'emilia-romagna', name: 'Emilia-Romagna' },
20+
{ code: 'friuli-venezia-giulia', name: 'Friuli-Venezia Giulia' },
21+
{ code: 'lazio', name: 'Lazio' },
22+
{ code: 'liguria', name: 'Liguria' },
23+
{ code: 'lombardia', name: 'Lombardia' },
24+
{ code: 'marche', name: 'Marche' },
25+
{ code: 'molise', name: 'Molise' },
26+
{ code: 'piemonte', name: 'Piemonte' },
27+
{ code: 'puglia', name: 'Puglia' },
28+
{ code: 'sardegna', name: 'Sardegna' },
29+
{ code: 'sicilia', name: 'Sicilia' },
30+
{ code: 'toscana', name: 'Toscana' },
31+
{ code: 'trentino-alto-adige', name: 'Trentino-Alto Adige' },
32+
{ code: 'umbria', name: 'Umbria' },
33+
{ code: 'valle-daosta', name: "Valle d'Aosta" },
34+
{ code: 'veneto', name: 'Veneto' },
35+
] as const
36+
37+
export const IT_REGION_NAMES: Record<string, string> = Object.fromEntries(
38+
IT_REGIONS.map((r) => [r.code, r.name]),
39+
)
40+
41+
/** Maps the canonical region code to the Italian region name used by the portal API. */
42+
export const PORTAL_REGION_NAMES: Record<string, string> = IT_REGION_NAMES

‎server/crawlers/agi/detail.ts‎

Lines changed: 133 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,133 @@
1+
import * as cheerio from 'cheerio'
2+
import type { Attachment, Auction } from '~/types/auction'
3+
import { AGI_BASE, UA } from './constants'
4+
import { allegatoKind } from './text'
5+
6+
interface DetailInfo {
7+
attachments: Attachment[]
8+
pdfUrl: string | null
9+
pdfUrlUpstream: string | null
10+
fotoCount: number
11+
thumbnailUrl: string | null
12+
}
13+
14+
/** Fetch the detail page HTML and extract attachments and photo count. */
15+
async function fetchDetailInfo(detailUpstream: string): Promise<DetailInfo> {
16+
const res = await fetch(detailUpstream, {
17+
headers: { 'User-Agent': UA, Accept: 'text/html,*/*' },
18+
signal: AbortSignal.timeout(15_000),
19+
})
20+
if (!res.ok) throw new Error(`[agi] detail page HTTP ${res.status}: ${detailUpstream}`)
21+
const html = await res.text()
22+
const $ = cheerio.load(html)
23+
24+
const attachments: Attachment[] = []
25+
let pdfUpstream: string | null = null
26+
const seenPaths = new Set<string>()
27+
let fotoCount = 0
28+
let thumbnailUrl: string | null = null
29+
30+
// Collect all /allegato/ hrefs (PDF and image files)
31+
$('a[href^="/allegato/"]').each((_, el) => {
32+
const path = $(el).attr('href') ?? ''
33+
if (seenPaths.has(path)) return
34+
seenPaths.add(path)
35+
36+
const filename = path.split('/').find((seg) => seg.includes('.')) ?? path
37+
const lowerFile = filename.toLowerCase()
38+
39+
if (lowerFile.endsWith('.pdf')) {
40+
const kind = allegatoKind(filename)
41+
const upstreamUrl = `${AGI_BASE}${path}`
42+
attachments.push({
43+
kind,
44+
label: $(el).text().trim() || filename,
45+
filename,
46+
sizeBytes: null,
47+
fileId: path,
48+
proxyUrl: upstreamUrl,
49+
})
50+
// Prefer gutachten as primary PDF; fall back to the first PDF found
51+
if (kind === 'gutachten') {
52+
pdfUpstream = upstreamUrl
53+
} else if (!pdfUpstream) {
54+
pdfUpstream = upstreamUrl
55+
}
56+
}
57+
})
58+
59+
// Count photo attachments from /allegato/foto-* img tags; prefer data-src for lazy-loaded images
60+
$('img[src^="/allegato/foto-"], img[data-src^="/allegato/foto-"]').each((_, el) => {
61+
const src = $(el).attr('data-src') ?? $(el).attr('src') ?? ''
62+
if (!src.toLowerCase().includes('/allegato/foto-')) return
63+
if (!seenPaths.has(src)) {
64+
seenPaths.add(src)
65+
fotoCount++
66+
if (!thumbnailUrl) thumbnailUrl = `${AGI_BASE}${src}`
67+
}
68+
})
69+
70+
return {
71+
attachments,
72+
pdfUrl: pdfUpstream,
73+
pdfUrlUpstream: pdfUpstream,
74+
fotoCount,
75+
thumbnailUrl,
76+
}
77+
}
78+
79+
function applyDetailInfo(auction: Auction, info: DetailInfo): void {
80+
if (info.attachments.length > 0) auction.attachments = info.attachments
81+
if (info.pdfUrl) {
82+
auction.pdfUrl = info.pdfUrl
83+
auction.pdfUrlUpstream = info.pdfUrlUpstream
84+
}
85+
if (info.fotoCount > 0) {
86+
auction.fotoCount = info.fotoCount
87+
if (info.thumbnailUrl && !auction.thumbnailUrl) {
88+
auction.thumbnailUrl = info.thumbnailUrl
89+
}
90+
}
91+
}
92+
93+
export interface EnrichResult {
94+
enriched: number
95+
errors: number
96+
}
97+
98+
/** Enrich a batch of auctions with detail-page data (attachments, PDFs). */
99+
export async function enrichInBatches(
100+
auctions: Auction[],
101+
concurrency = 5,
102+
): Promise<EnrichResult> {
103+
let enriched = 0
104+
let errors = 0
105+
let cursor = 0
106+
107+
async function worker() {
108+
while (cursor < auctions.length) {
109+
const idx = cursor++
110+
const auction = auctions[idx]
111+
if (!auction) continue
112+
if (!auction.detailUrlUpstream) { enriched++; continue }
113+
try {
114+
const info = await fetchDetailInfo(auction.detailUrlUpstream)
115+
applyDetailInfo(auction, info)
116+
enriched++
117+
} catch (err) {
118+
errors++
119+
console.warn(`[agi] enrichOne failed for ${auction.zvgId}: ${(err as Error).message}`)
120+
}
121+
}
122+
}
123+
124+
await Promise.all(Array.from({ length: concurrency }, worker))
125+
return { enriched, errors }
126+
}
127+
128+
/** Enrich a single auction in place. Used by the enrich task. */
129+
export async function enrichSingle(auction: Auction): Promise<void> {
130+
if (!auction.detailUrlUpstream) return
131+
const info = await fetchDetailInfo(auction.detailUrlUpstream)
132+
applyDetailInfo(auction, info)
133+
}

‎server/crawlers/agi/index.ts‎

Lines changed: 75 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,75 @@
1+
import type { Auction, CrawlResult } from '~/types/auction'
2+
import type { CrawlOptions, PlatformCrawler } from '../types'
3+
import {
4+
AGI_BASE,
5+
COUNTRY,
6+
IT_REGIONS,
7+
IT_REGION_NAMES,
8+
PORTAL_REGION_NAMES,
9+
} from './constants'
10+
import { fetchSession, fetchMapData, fetchAllDetails, buildAuctions } from './list'
11+
import { enrichInBatches, enrichSingle } from './detail'
12+
13+
const PLATFORM_ID = 'agi'
14+
15+
async function crawl(opts: CrawlOptions): Promise<CrawlResult> {
16+
const regionCode = opts.region.toLowerCase()
17+
const regionName = IT_REGION_NAMES[regionCode]
18+
const portalRegion = PORTAL_REGION_NAMES[regionCode]
19+
if (!regionName || !portalRegion) {
20+
throw new Error(`[agi] Unbekannte Region: ${opts.region}`)
21+
}
22+
const enrichDetails = opts.enrichDetails ?? true
23+
24+
const cookies = await fetchSession()
25+
const mapEntries = await fetchMapData(portalRegion, cookies)
26+
27+
if (mapEntries.length === 0) {
28+
return {
29+
platform: PLATFORM_ID,
30+
source: AGI_BASE,
31+
countries: [COUNTRY],
32+
regions: [regionName],
33+
fetchedAt: new Date().toISOString(),
34+
totalReported: 0,
35+
auctions: [],
36+
}
37+
}
38+
39+
const ids = mapEntries.map((e) => e.idLotto)
40+
const details = await fetchAllDetails(ids, cookies)
41+
const auctions = buildAuctions(mapEntries, details, regionName, PLATFORM_ID)
42+
43+
if (enrichDetails && auctions.length > 0) {
44+
const result = await enrichInBatches(auctions)
45+
if (result.errors > 0) {
46+
console.warn(
47+
`[agi] ${regionCode}: enriched ${result.enriched}/${auctions.length}, ${result.errors} detail fetches failed`,
48+
)
49+
}
50+
}
51+
52+
return {
53+
platform: PLATFORM_ID,
54+
source: AGI_BASE,
55+
countries: [COUNTRY],
56+
regions: [regionName],
57+
fetchedAt: new Date().toISOString(),
58+
totalReported: auctions.length,
59+
auctions,
60+
}
61+
}
62+
63+
async function enrichOne(auction: Auction): Promise<void> {
64+
await enrichSingle(auction)
65+
}
66+
67+
export const agiCrawler: PlatformCrawler = {
68+
id: PLATFORM_ID,
69+
name: 'Aste Giudiziarie Inlinea (Italia)',
70+
baseUrl: AGI_BASE,
71+
country: COUNTRY,
72+
regions: IT_REGIONS,
73+
crawl,
74+
enrichOne,
75+
}

0 commit comments

Comments
 (0)