Merge pull request #54 from open-energy-transition/feat/fes-workbook-download-and-extract

Add FES workbook extractor

Author: brynpickering

.github/workflows/update-gb-model-pinned-env.yaml

@@ -44,7 +44,7 @@ jobs:
       uses: actions/upload-artifact@v4
       with:
         # all are uploaded to the same artifact as they will have different names for each platform so won't clash.
-        name: lockfiles
+        name: lockfiles-${{ matrix.os }}
         path: envs/**/*.pin.txt
 
   create-pull-request:
@@ -55,8 +55,9 @@ jobs:
     - name: Download all artifacts
       uses: actions/download-artifact@v5
       with:
-        name: lockfiles
+        name: lockfiles-*
         path: envs/
+        merge-multiple: true
 
     - name: Create Pull Request
       uses: peter-evans/create-pull-request@v7

config/gb-model/config.common.yaml

@@ -2,10 +2,14 @@
 #
 # SPDX-License-Identifier: MIT
 
+# docs in https://pypsa-eur.readthedocs.io/en/latest/configuration.html#run
+run:
+  name: "GB"
 
 urls:
   gb-etys-boundaries: https://api.neso.energy/dataset/997f4820-1ad4-499b-b1fe-4b8d3d7fbc72/resource/e914fcec-1dc9-4f1f-97e7-59c0d9521bea/download/etys-boundary-gis-data-mar25.zip
   transmission-availability: https://www.neso.energy/document/211021/download
+  fes-workbook: https://www.neso.energy/document/199971/download # FES 2021 workbook
 target_crs: "EPSG:27700"
 area_loss_tolerance_percent: 0.01 # percentage of area loss tolerated when splitting regions
 min_region_area: 1000000 # minimum area of a region in square meters (1 km²) for keeping it after splitting
@@ -48,3 +52,21 @@ region_operations:
   - [39, 99]
   - [74, 75, 76]
   - [4, "5w"]
+
+fes-sheet-config:
+  SV.34:
+    usecols: "I:AN"
+    skiprows: 5
+    index_col: 0
+    header: 0
+    rename:
+      index: scenario
+      columns: year
+
+  BB1:
+    usecols: "A:C,E,H:AK" # Ignore columns: "Baseline (2020)", "Share of GSP" (empty), "DNO License Area"
+    skiprows: 0
+    index_col: [0, 1, 2, 3]
+    header: 0
+    rename:
+      columns: year

doc/gb-model/data_sources.rst

@@ -13,6 +13,14 @@ The following table provides an overview of the data sources used exclusively in
 For data sources used in PyPSA-Eur, see `this page <../data_sources.html>`_.
 Different licenses apply to the data sources.
 
-For data source
-.. toctree::
-   :maxdepth: 1
+---------------------------------
+The Future Energy Scenarios (FES)
+---------------------------------
+
+`The FES <https://www.neso.energy/publications/future-energy-scenarios-fes>`_ is the primary data source for defining the model, both for GB and other European countries.
+Here, we use the 2021 FES data workbook.
+Tables from the workbook we use are:
+
+- BB1: Building Block Data
+- SV.34: Installed BECCS generation capacity (GW)
+

doc/gb-model/release_notes.rst

@@ -12,6 +12,7 @@ Release Notes
 Upcoming Release
 ================
 
+- Add FES workbook data download and sheet extraction rule (#50).
 - Restructured documentation.
 - Added modelling methodology documentation (#20).
 - Added GB custom geographic boundary rule and script (#13).

envs/gb-model/workflow.yaml

@@ -22,5 +22,7 @@ dependencies:
 - pandas>=2.1
 - geopandas>=1
 - xarray>=2024.03.0,<2025.07.0
+- openpyxl
+
 - pip:
   - pdfplumber==0.9.0 # technically noarch but imagemagick dependency not available on Windows

rules/gb-model.smk

@@ -13,13 +13,15 @@ configfile: "config/gb-model/config.common.yaml"
 
 
 # Rule to download and extract ETYS boundary data
-rule retrieve_etys_boundary_data:
+rule download_data:
+    message:
+        "Download {wildcards.gb_data} GB model data."
     output:
-        boundary_shp="data/gb-model/etys-boundary-gis-data.zip",
+        downloaded="data/gb-model/downloaded/{gb_data}",
     params:
-        url=config["urls"]["gb-etys-boundaries"],
+        url=lambda wildcards: config["urls"][Path(wildcards.gb_data).stem],
     log:
-        logs("retrieve_etys_boundary_data.log"),
+        logs("download_{gb_data}.log"),
     localrule: True
     conda:
         "../envs/gb-model/workflow.yaml"
@@ -31,7 +33,7 @@ rule retrieve_etys_boundary_data:
 rule create_region_shapes:
     input:
         country_shapes=resources("country_shapes.geojson"),
-        etys_boundary_lines=rules.retrieve_etys_boundary_data.output.boundary_shp,
+        etys_boundary_lines="data/gb-model/downloaded/gb-etys-boundaries.zip",
     output:
         raw_region_shapes=resources("raw_region_shapes.geojson"),
     log:
@@ -60,23 +62,9 @@ rule manual_region_merger:
         "../scripts/gb-model/manual_region_merger.py"
 
 
-rule download_transmission_availability_pdf:
-    output:
-        pdf_report="data/gb-model/transmission-availability.pdf",
-    params:
-        url=config["urls"]["transmission-availability"],
-    log:
-        logs("transmission_availability.log"),
-    localrule: True
-    conda:
-        "../envs/gb-model/workflow.yaml"
-    shell:
-        "curl -sSLvo {output} {params.url}"
-
-
 rule extract_transmission_availability:
     input:
-        pdf_report=rules.download_transmission_availability_pdf.output.pdf_report,
+        pdf_report="data/gb-model/downloaded/transmission-availability.pdf",
     output:
         csv=resources("transmission_availability.csv"),
     log:
@@ -85,3 +73,20 @@ rule extract_transmission_availability:
         "../envs/gb-model/workflow.yaml"
     script:
         "../scripts/gb-model/extract_transmission_availability.py"
+
+
+rule extract_fes_workbook_sheet:
+    message:
+        "Extract FES workbook sheet {wildcards.fes_sheet} and process into machine-readable, 'tidy' dataframe format according to defined configuration."
+    input:
+        workbook="data/gb-model/downloaded/fes-workbook.xlsx",
+    output:
+        csv=resources("fes/{fes_sheet}.csv"),
+    params:
+        sheet_extract_config=lambda wildcards: config["fes-sheet-config"][
+            wildcards.fes_sheet
+        ],
+    log:
+        logs("extract_fes_{fes_sheet}.log"),
+    script:
+        "../scripts/gb-model/extract_fes_sheet.py"

scripts/gb-model/extract_fes_sheet.py

@@ -0,0 +1,65 @@
+# SPDX-FileCopyrightText:  gb-open-market-model contributors
+#
+# SPDX-License-Identifier: MIT
+
+
+"""
+FES worksheet extractor.
+
+This is a generalised script to extract tables based on their manually configured positions in non-machine-readable Excel sheets.
+"""
+
+import logging
+
+import pandas as pd
+
+from scripts._helpers import configure_logging, set_scenario_config
+
+logger = logging.getLogger(__name__)
+
+
+def extract_fes_worksheet(
+    excel_path: str, sheet_name: str, sheet_config: dict
+) -> pd.DataFrame:
+    """
+    Extract FES worksheet data from an Excel file.
+
+    Args:
+        excel_path (str): Path to the Excel file.
+        sheet_name (str): Name of the sheet to extract.
+        sheet_config (dict): Configuration for the sheet extraction.
+
+    Returns:
+        pd.DataFrame: A DataFrame containing the extracted FES worksheet data.
+    """
+    renamers = sheet_config.pop("rename", {})
+    fes_data = pd.read_excel(excel_path, sheet_name=sheet_name, **sheet_config)
+
+    fes_data = fes_data.rename_axis(**renamers)
+
+    if pd.notnull(sheet_config["header"]):
+        fes_data = fes_data.stack(sheet_config["header"])
+
+    if not (unnamed_data := fes_data.filter(regex="Unnamed")).empty:
+        logger.error(
+            "The extracted DataFrame contains unnamed columns/rows, please check the configuration."
+            f"First rows of unnamed data:\n{unnamed_data.head()}"
+        )
+        raise
+    return fes_data.to_frame("data")
+
+
+if __name__ == "__main__":
+    if "snakemake" not in globals():
+        from scripts._helpers import mock_snakemake
+
+        snakemake = mock_snakemake("extract_fes_sheet")
+    configure_logging(snakemake)
+    set_scenario_config(snakemake)
+
+    availability_df = extract_fes_worksheet(
+        snakemake.input.workbook,
+        snakemake.wildcards.fes_sheet,
+        snakemake.params.sheet_extract_config,
+    )
+    availability_df.to_csv(snakemake.output.csv)