fix(outputs): broadcast scalar existing mixed with period-Map siblings

A unit with a constant `existing` capacity sharing an entity class with
a period-Map sibling surfaces from SpineDbReader as a single frame
`[name, x, value]` where the constant unit's row carries a NULL index
(`x = null`; the Map index is named `x` by Spine's silent default).

`_per_entity_param_lf` classified `is_scalar` per-FRAME (does an index
column exist?) instead of per-ROW, so the null-index scalar became an
explicit `(e, d=null)` row that never joined the period grid in
`_resolve_per_period_lf` and was silently filled with 0. This zeroed
the output-facing `p_entity_all_existing` for such units, surfacing as
negative VRE curtailment (`potential - flow` with `potential` derived
from a 0 capacity), plus 0 in the capacity / capacity-factor / pre-
existing fixed-cost reports.

Output-only: the LP flow bound uses `p_process_existing_count` (scalar-
correct), and `p_entity_all_existing` enters the objective only as an
opt-in constant term (off by default) — dispatch, investments, flows,
and the optimal objective are unchanged.

Fix: per-row scalar detection `is_scalar = period_col.is_null()` in both
mirror implementations (`_derived_npv`, `_derived_existing`). Adds a
regression test exercising the mixed scalar+Map shape via InMemoryReader.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

Author: jkiviluo

flextool/engine_polars/_derived_existing.py

@@ -242,11 +242,19 @@ def _per_entity_param_lf(source: "InputSource",
         extra = [c for c in cols if c not in ("name", "value")]
         period_col = "period" if "period" in extra else (extra[0] if extra else None)
         if period_col is not None:
+            # Per-ROW scalar detection — see the matching helper in
+            # ``_derived_npv._per_entity_param_lf`` for the full rationale.
+            # A constant ``existing`` (etc.) stored in an entity class
+            # that ALSO carries period-Map values for sibling entities
+            # surfaces with a NULL index (``x = null``).  That row must
+            # broadcast across the period universe (``is_scalar=True``),
+            # not be emitted as an explicit ``(e, d=null)`` row that
+            # never joins the period grid and silently zeroes out.
             parts.append(df.lazy().select(
                 alias_to_axis("name", "e"),
                 alias_to_axis(period_col, "d"),
                 pl.col("value").cast(pl.Float64, strict=False),
-                pl.lit(False).alias("is_scalar"),
+                pl.col(period_col).is_null().alias("is_scalar"),
             ))
         else:
             parts.append(df.lazy().select(

flextool/engine_polars/_derived_npv.py

@@ -201,12 +201,23 @@ def _per_entity_param_lf(source: "InputSource",
         extra = [c for c in cols if c not in ("name", "value")]
         period_col = "period" if "period" in extra else (extra[0] if extra else None)
         if period_col is not None:
+            # Per-ROW scalar detection.  A scalar value stored alongside
+            # Map values in the same entity class surfaces with a NULL
+            # index ((e.g. ``x = null``) — Spine emits one ``x`` column
+            # for the whole class, so a unit with a constant ``existing``
+            # while a sibling unit carries a period-Map shows up as a
+            # null-period row.  Such rows must broadcast across the
+            # period universe (``is_scalar=True``), NOT be treated as an
+            # explicit ``(e, d=null)`` row that fails to join the period
+            # grid and silently fills 0.  Classifying by ``period_col``'s
+            # null-ness per row (rather than per frame) covers the mixed
+            # scalar+Map class.
             parts.append(df.lazy().select(
                 alias_to_axis("name", "e"),
                 alias_to_axis(pl.col(period_col).cast(pl.Utf8, strict=False),
                               "d"),
                 pl.col("value").cast(pl.Float64, strict=False),
-                pl.lit(False).alias("is_scalar"),
+                pl.col(period_col).is_null().alias("is_scalar"),
             ))
         else:
             parts.append(df.lazy().select(

tests/engine_polars/test_per_entity_scalar_map_mix.py

@@ -0,0 +1,108 @@
+"""Regression — per-entity scalar value mixed with period-Map values in
+the same entity class.
+
+When one unit carries a *constant* ``existing`` capacity while a sibling
+unit in the same class carries a *period-Map* ``existing``, the Spine
+reader returns a single frame with one shared index column (named ``x``
+by Spine's silent default) where the constant unit's row has a NULL
+index::
+
+    name         x      value
+    wind_scalar  null   31.5     <- scalar, broadcast across periods
+    wind_map     p2024  10.8     <- period-Map row
+    wind_map     p2025  22.8
+
+``_per_entity_param_lf`` must classify the null-index row as a SCALAR
+(``is_scalar=True``) so ``_resolve_per_period_lf`` broadcasts it across
+the period universe.  The historical bug classified the whole frame by
+the *presence* of the index column, so the null-index scalar became an
+explicit ``(e, d=null)`` row that never joined the period grid and was
+silently filled with 0 — surfacing downstream as a zeroed-out existing
+capacity (e.g. negative VRE curtailment = ``potential - flow`` with
+``potential`` computed from a 0 capacity).
+
+Both mirror implementations of the helper are covered.
+"""
+from __future__ import annotations
+
+import polars as pl
+import pytest
+
+from flextool.engine_polars import _derived_existing as de
+from flextool.engine_polars import _derived_npv as dn
+
+
+def _mixed_existing_source():
+    """InMemoryReader whose ``unit.existing`` mixes a scalar (null index)
+    with a period-Map, mirroring ``SpineDbReader.parameter`` output."""
+    from flextool.engine_polars._inmemory_reader import InMemoryReader
+
+    frame = pl.DataFrame(
+        {
+            "name": ["wind_scalar", "wind_map", "wind_map", "wind_map"],
+            "x": [None, "p2024", "p2025", "p2026"],
+            "value": [31.5, 10.8, 22.8, 22.8],
+        }
+    )
+    entities = {"unit": pl.DataFrame({"name": ["wind_scalar", "wind_map"]})}
+    return InMemoryReader(entities, {("unit", "existing"): frame})
+
+
+@pytest.mark.parametrize("module", [de, dn], ids=["derived_existing", "derived_npv"])
+def test_null_index_row_is_scalar(module):
+    """A null-index row must be flagged ``is_scalar=True``; the Map rows
+    must stay ``is_scalar=False`` with their explicit period."""
+    src = _mixed_existing_source()
+    out = module._per_entity_param_lf(src, "existing").collect()
+
+    scalar_row = out.filter(pl.col("e") == "wind_scalar")
+    assert scalar_row.height == 1
+    assert scalar_row["is_scalar"][0] is True
+    assert scalar_row["d"][0] is None
+    assert scalar_row["value"][0] == pytest.approx(31.5)
+
+    map_rows = out.filter(pl.col("e") == "wind_map")
+    assert map_rows.height == 3
+    assert not any(map_rows["is_scalar"].to_list())
+    assert set(map_rows["d"].to_list()) == {"p2024", "p2025", "p2026"}
+
+
+@pytest.mark.parametrize("module", [de, dn], ids=["derived_existing", "derived_npv"])
+def test_scalar_broadcasts_across_period_grid(module):
+    """The scalar must resolve to its constant value at EVERY period in
+    the grid, not be zero-filled.  This is the load-bearing assertion:
+    the bug zeroed ``wind_scalar`` at every period."""
+    src = _mixed_existing_source()
+    per = module._per_entity_param_lf(src, "existing")
+
+    grid = pl.LazyFrame(
+        {
+            "e": ["wind_scalar", "wind_scalar", "wind_map", "wind_map"],
+            "d": ["p2025", "p2026", "p2025", "p2026"],
+        }
+    )
+    resolved = module._resolve_per_period_lf(per, grid, fill=0.0).collect()
+
+    scalar_res = resolved.filter(pl.col("e") == "wind_scalar").sort("d")
+    assert scalar_res["value"].to_list() == pytest.approx([31.5, 31.5])
+
+    map_res = resolved.filter(pl.col("e") == "wind_map").sort("d")
+    assert map_res["value"].to_list() == pytest.approx([22.8, 22.8])
+
+
+def test_pre_existing_broadcasts_scalar_to_all_periods():
+    """End-to-end through ``p_entity_pre_existing_lf``: the scalar unit's
+    pre-existing capacity must be present (non-zero) at the active
+    period, matching the period-Map unit's treatment."""
+    src = _mixed_existing_source()
+    pre = de.p_entity_pre_existing_lf(
+        src, active_solve="y2025", period_in_use=["p2025"]
+    ).collect()
+
+    scalar_res = pre.filter(pl.col("e") == "wind_scalar")
+    assert scalar_res.height == 1
+    assert scalar_res["value"][0] == pytest.approx(31.5)
+
+    map_res = pre.filter(pl.col("e") == "wind_map")
+    assert map_res.height == 1
+    assert map_res["value"][0] == pytest.approx(22.8)