dcaribou · dcaribou · Jun 24, 2026 · May 30, 2026 · Jun 23, 2026 · Jun 23, 2026
diff --git a/.gitignore b/.gitignore
@@ -66,3 +66,9 @@ storage
 
 # generated zip archive
 *.zip
+
+# DuckDB database files
+*.db
+
+# Plan documents
+docs/plans/
diff --git a/CLAUDE.md b/CLAUDE.md
@@ -9,6 +9,8 @@ Football (soccer) dataset built from transfermarkt.com data. Raw data is acquire
 - `data/raw/transfermarkt-scraper/` - raw JSON data from the scraper
 - `data/raw/transfermarkt-api/` - raw JSON data from the API
 - `data/prep/` - prepared dataset CSV files (gzipped)
+- `data/competitions.json` - generated list of competitions (overwritten on every `acquire_local --asset competitions` run)
+- `data/supplemental_competitions.json` - hand-crafted competition records merged into `data/competitions.json` on each run; use this for competitions that don't appear in Transfermarkt's confederation hierarchy (e.g. play-offs); see [docs/developer-guide.md](docs/developer-guide.md) for details
 - `dbt/` - dbt project root (models, profiles, packages)
 - `dbt/models/base/` - base models that parse raw JSON into tables (one per source)
 - `dbt/models/curated/` - curated models that join/transform base models into final datasets

diff --git a/config.yml b/config.yml
@@ -1,6 +1,9 @@
 defintions:
   seasons: [2012, 2013, 2014, 2015, 2016, 2017, 2018, 2019, 2020, 2021, 2022, 2023, 2024, 2025]
   competition_ids:
+    # Allowlist for acquisition. Competitions not in Transfermarkt's confederation hierarchy
+    # (e.g. play-offs) won't be scraped by tfmkt competitions even if listed here —
+    # add those to data/supplemental_competitions.json instead.
     # European first-tier leagues
     - L1    # Germany - Bundesliga
     - ES1   # Spain - LaLiga
@@ -10,6 +13,9 @@ defintions:
     - NL1   # Netherlands - Eredivisie
     - PO1   # Portugal - Liga Portugal
     - BE1   # Belgium - Jupiler Pro League
+    - EJPL  # Belgium - Jupiler Pro League Champions' Play-Offs
+    - POBE  # Belgium - Jupiler Pro League Europe Play-Offs
+    - BPO4  # Belgium - Jupiler Pro League Relegation Play-offs
     - TR1   # Turkey - Super Lig
     - GR1   # Greece - Super League 1
     - SC1   # Scotland - Scottish Premiership

diff --git a/data/competitions.json b/data/competitions.json
diff --git a/data/countries.json b/data/countries.json
diff --git a/data/raw/transfermarkt-scraper.dvc b/data/raw/transfermarkt-scraper.dvc
@@ -1,6 +1,6 @@
 outs:
-- md5: f08257faab036ec0e93b1a1e923d0373.dir
-  size: 510528071
+- md5: 6829abea768831f7632fc20be9b8cc7c.dir
+  size: 509943406
   nfiles: 74
   hash: md5
   path: transfermarkt-scraper
diff --git a/data/supplemental_competitions.json b/data/supplemental_competitions.json
@@ -0,0 +1,3 @@
+{"type": "competition", "parent": {"type": "confederation", "href": "/wettbewerbe/europa"}, "country_id": "19", "country_name": "Belgium", "country_code": "BE1", "competition_type": "play_off", "competition_name": "Jupiler Pro League Champions' Play-Offs", "href": "/jupiler-pro-league-playoff-i/startseite/wettbewerb/EJPL"}
+{"type": "competition", "parent": {"type": "confederation", "href": "/wettbewerbe/europa"}, "country_id": "19", "country_name": "Belgium", "country_code": "BE1", "competition_type": "play_off", "competition_name": "Jupiler Pro League Europe Play-Offs", "href": "/jupiler-pro-league-europe-play-offs/startseite/wettbewerb/POBE"}
+{"type": "competition", "parent": {"type": "confederation", "href": "/wettbewerbe/europa"}, "country_id": "19", "country_name": "Belgium", "country_code": "BE1", "competition_type": "play_off", "competition_name": "Jupiler Pro League Relegation Play-offs", "href": "/jupiler-pro-league-relegation-play-offs/startseite/wettbewerb/BPO4"}
diff --git a/data/tournament_editions.json b/data/tournament_editions.json
@@ -1,3 +1,4 @@
+{"type": "tournament_edition", "href": "/weltmeisterschaft/startseite/pokalwettbewerb/FIWC/saison_id/2025", "year": "2026", "season": "2025"}
 {"type": "tournament_edition", "href": "/weltmeisterschaft/startseite/pokalwettbewerb/FIWC/saison_id/2021", "year": "2022", "season": "2021"}
 {"type": "tournament_edition", "href": "/weltmeisterschaft/startseite/pokalwettbewerb/FIWC/saison_id/2017", "year": "2018", "season": "2017"}
 {"type": "tournament_edition", "href": "/weltmeisterschaft/startseite/pokalwettbewerb/FIWC/saison_id/2013", "year": "2014", "season": "2013"}

diff --git a/dbt/dbt_project.yml b/dbt/dbt_project.yml
@@ -43,7 +43,7 @@ models:
 vars:
   competition_codes: [
       'ES1', 'GB1', 'L1', 'IT1', 'FR1', 'GR1', 'PO1', 
-      'BE1', 'UKR1', 'BE1', 'RU1', 'DK1', 'SC1', 'TR1',
+      'BE1', 'EJPL', 'POBE', 'BPO4', 'UKR1', 'RU1', 'DK1', 'SC1', 'TR1',
       'NL1', 'NLP', 'NLSC', 'BESC', 'DFB', 'POSU', 'POCP',
       'CGB', 'UKRP', 'RUP', 'GBCS', 'SUC', 'DFL', 'CDR',
       'RUSS', 'GRP', 'SFA', 'CIT', 'DKP', 'FRCH', 'UKRS',

diff --git a/dbt/models/base/transfermarkt_scraper/base_national_teams.sql b/dbt/models/base/transfermarkt_scraper/base_national_teams.sql
@@ -14,7 +14,10 @@ with
 
 select
     national_team_id,
-    json_extract_string(json_row, '$.name') as name,
+    coalesce(
+        nullif(json_extract_string(json_row, '$.name'), ''),
+        json_extract_string(json_row, '$.parent.country_name')
+    ) as name,
     json_extract_string(json_row, '$.code') as team_code,
     json_extract_string(json_row, '$.parent.country_id') as country_id,
     case

diff --git a/dbt/models/curated/models.yml b/dbt/models/curated/models.yml
@@ -188,7 +188,7 @@ models:
             - player_club_domestic_competition_id
       - dbt_expectations.expect_table_row_count_to_be_between:
           min_value: 430000
-          max_value: 650000
+          max_value: 850000
     columns:
       - name: player_id
         tests:

diff --git a/dbt/tests/expected_number_of_first_tier_leagues.sql b/dbt/tests/expected_number_of_first_tier_leagues.sql
@@ -1,9 +1,10 @@
 {#
-    Verify the total number of first-tier leagues matches the expected count.
-    Should be updated when new leagues are added to competitions.json.
+    Verify the total number of first-tier leagues is within the expected range.
+    COL1 (Colombia Liga BetPlay - Finalización) is seasonally inactive in Jun-Aug,
+    so the count fluctuates between 31 (off-season) and 32 (full season).
 #}
 
 select count(*) as league_count
 from {{ ref('competitions') }}
 where sub_type = 'first_tier'
-having count(*) != 32
+having count(*) not between 31 and 32
diff --git a/docs/developer-guide.md b/docs/developer-guide.md
@@ -73,6 +73,22 @@ This populates `data/raw/transfermarkt-api` with the collected data. You can als
 cd scripts/acquiring && python transfermarkt-api.py --season 2024
 ```
 
+### Adding or extending competitions
+
+Which competitions are scraped is controlled by the `competition_ids` list in [`config.yml`](../config.yml). To add a competition, append its Transfermarkt competition ID (the last path segment of its URL, e.g. `GB1`) to that list.
+
+**Competitions not in Transfermarkt's confederation hierarchy** (such as domestic play-offs) will never appear in the output of `tfmkt competitions`, even if their ID is in `config.yml`. For these, add a hand-crafted JSONL record to [`data/supplemental_competitions.json`](../data/supplemental_competitions.json). Records in that file are merged into `data/competitions.json` on every `acquire_local --asset competitions` run, so they survive re-acquisition. `data/competitions.json` itself is a generated file — do not edit it directly.
+
+When crafting a supplemental record, match the shape of existing entries in `data/competitions.json` and set `competition_type` to a value that maps to the correct URL scheme in `tfmkt/common.py`:
+
+| `competition_type`   | URL generated by the scraper                              |
+|----------------------|-----------------------------------------------------------|
+| `first_tier`         | `.../startseite/wettbewerb/{ID}/plus/0?saison_id={year}` |
+| `domestic_cup` / `domestic_super_cup` | `.../startseite/pokalwettbewerb/{ID}?saison_id={year}` |
+| anything else        | `.../startseite/wettbewerb/{ID}?saison_id={year}`        |
+
+Play-offs and other non-standard competitions should use `play_off` (or any value not in the table above) to get the plain `wettbewerb` URL.
+
 ## Data preparation
 
 "Preparing" is the process of transforming raw data into a high-quality dataset. This is done in SQL using [dbt](https://docs.getdbt.com/) and [DuckDB](https://duckdb.org/).

diff --git a/docs/plans/314-not-null-transfers-player-name.md b/docs/plans/314-not-null-transfers-player-name.md
diff --git a/docs/plans/321-missing-clubs-competitions.md b/docs/plans/321-missing-clubs-competitions.md
diff --git a/scripts/acquiring/transfermarkt-scraper.py b/scripts/acquiring/transfermarkt-scraper.py
@@ -98,7 +98,7 @@ def all(cls):
 # Explicit casts prevent type-mismatch errors in UNION ALL BY NAME when
 # an existing file has all-null values and a new scrape has actual strings.
 VARCHAR_CASTS = {
-  'clubs': ['coach_name', 'total_market_value', 'league_position'],
+  'clubs': ['coach_name', 'coach_href', 'total_market_value', 'league_position'],
   'games': ['home_club_position', 'away_club_position'],
   'players': ['day_of_last_contract_extension', 'current_market_value', 'highest_market_value'],
   'national_team_players': ['day_of_last_contract_extension', 'current_market_value', 'highest_market_value'],
@@ -350,6 +350,23 @@ def acquire_competitions():
 
     logging.info(f"Scraped {len(lines)} competitions, kept {len(filtered)} matching existing config")
 
+    # Merge supplemental competitions (play-offs etc. not in Transfermarkt's confederation hierarchy)
+    supp_file = pathlib.Path("data/supplemental_competitions.json")
+    if supp_file.exists():
+      supp_ids_seen = set()
+      existing_ids = {_competition_id_from_href(json.loads(l).get('href', '')) for l in filtered}
+      with open(str(supp_file)) as sf:
+        for line in sf:
+          line = line.strip()
+          if not line:
+            continue
+          record = json.loads(line)
+          comp_id = _competition_id_from_href(record.get('href', ''))
+          if comp_id in allowed_ids and comp_id not in existing_ids and comp_id not in supp_ids_seen:
+            filtered.append(line)
+            supp_ids_seen.add(comp_id)
+      logging.info(f"Merged {len(supp_ids_seen)} supplemental competitions")
+
     with open(str(output_file), 'w') as f:
       f.write('\n'.join(filtered))
     logging.info(f"Wrote {len(filtered)} competitions to {output_file}")