From ca5fde74d249179f17d35669193623c61af527c6 Mon Sep 17 00:00:00 2001 From: FBumann <117816358+FBumann@users.noreply.github.com> Date: Mon, 1 Jun 2026 14:47:39 +0200 Subject: [PATCH 01/10] refactor(common): split DataArray conversion into a 3-rung strictness ladder Replace the as_dataarray + _as_dataarray_lax pair (and the enforce_level_coverage flag) with three public entry points, each including the previous one: - as_dataarray: convert only (the former _as_dataarray_lax). Used by __matmul__, where dims missing from the constant must not be broadcast in (they would be contracted away as common dims). - broadcast_to_coords: convert + broadcast against coords (the former broadcasting as_dataarray). Used by expression arithmetic. - align_to_coords: convert + broadcast + enforce the coords contract. Used by add_variables / add_constraints (unchanged signature). The broadcasting mechanics live in one shared private core (_broadcast_core) that reports MultiIndex-level projections instead of applying policy. The entry points decide what a partial projection or coverage gap means: broadcast_to_coords warns (arithmetic convention), align_to_coords raises (coords contract). This removes the enforce_level_coverage flag and keeps validation concerns out of the broadcasting layer. No behavior changes; all call sites keep their semantics. New tests pin the ladder contrasts and the matmul dim-contraction rules. Co-Authored-By: Claude Opus 4.8 (1M context) --- doc/release_notes.rst | 2 +- linopy/common.py | 235 +++++++++++++++------------------ linopy/expressions.py | 12 +- linopy/variables.py | 5 +- test/test_common.py | 111 +++++++++++++--- test/test_linear_expression.py | 41 ++++++ 6 files changed, 254 insertions(+), 152 deletions(-) diff --git a/doc/release_notes.rst b/doc/release_notes.rst index f775b8cd..8dbf1788 100644 --- a/doc/release_notes.rst +++ b/doc/release_notes.rst @@ -71,7 +71,7 @@ Most users should keep calling ``model.solve(...)``. If you want more control, y **Internal** -* ``linopy.common.as_dataarray`` is the single broadcasting primitive; strict subset-dim / coord-value checks live in ``validate_alignment`` (via ``align_to_coords`` in ``add_variables`` / ``add_constraints``). When ``coords`` is a mapping, extra keys beyond the positional ``dims`` are broadcast in rather than dropped. +* ``linopy.common`` exposes a three-rung strictness ladder for DataArray conversion: ``as_dataarray`` (convert only; used by ``__matmul__`` so dims missing from the constant are not broadcast in and contracted away), ``broadcast_to_coords`` (convert and broadcast against ``coords``; used by expression arithmetic), and ``align_to_coords`` (convert, broadcast, and enforce the coords contract via ``validate_alignment``; used by ``add_variables`` / ``add_constraints``). When ``coords`` is a mapping, extra keys beyond the positional ``dims`` are broadcast in rather than dropped (broadcast rung and up). * Each ``Solver`` subclass now overrides at most three hooks: ``_build_direct`` (build the native model), ``_run_direct`` (run it), and ``_run_file`` (run the solver on an LP/MPS file). File-only solvers (CBC, GLPK, CPLEX, SCIP, Knitro, COPT, MindOpt) only override ``_run_file``. * New ``ConstraintLabelIndex`` cached on ``Model.constraints`` (mirrors the existing ``Variables.label_index``); ``ConstraintBase`` gains ``active_labels()`` and a ``range`` property; ``CSRConstraint`` exposes ``coords``. * ``linopy.common`` gains ``values_to_lookup_array``; the legacy pandas-based helpers ``series_to_lookup_array`` and ``lookup_vals`` are removed. diff --git a/linopy/common.py b/linopy/common.py index 235b17c7..a19d3c75 100644 --- a/linopy/common.py +++ b/linopy/common.py @@ -12,7 +12,7 @@ from collections.abc import Callable, Generator, Hashable, Iterable, Mapping, Sequence from functools import cached_property, partial, reduce, wraps from pathlib import Path -from typing import TYPE_CHECKING, Any, Generic, TypeVar, overload +from typing import TYPE_CHECKING, Any, Generic, NamedTuple, TypeVar, overload from warnings import warn import numpy as np @@ -216,20 +216,21 @@ def numpy_to_dataarray( return DataArray(arr, coords=coords, dims=dims, **kwargs) -def _as_dataarray_lax( +def as_dataarray( arr: Any, coords: CoordsLike | None = None, dims: DimsLike | None = None, **kwargs: Any, ) -> DataArray: """ - Type-dispatched DataArray conversion without any coords validation. + Convert ``arr`` to a DataArray. - This is the conversion primitive used by ``as_dataarray``: it picks the - right constructor for each supported input type but does not check the - result against ``coords``. Callers that need ``coords`` to govern the - output (dim order, shared-dim values, missing-dim expansion) should use - ``as_dataarray`` instead. + Picks the right constructor for each supported input type (pandas, + polars, numpy, scalar, DataArray) and labels positional axes with + ``dims`` / ``coords``. The result is not reshaped against ``coords``: + dims are neither expanded, reordered, nor projected onto MultiIndex + dims. Use :func:`broadcast_to_coords` or :func:`align_to_coords` when + ``coords`` should govern the result's shape. """ if isinstance(arr, pd.Series | pd.DataFrame): arr = pandas_to_dataarray(arr, coords=coords, dims=dims, **kwargs) @@ -285,34 +286,31 @@ def _as_multiindex(coord_values: Any) -> pd.MultiIndex | None: return None +class _LevelProjection(NamedTuple): + """Record of one MultiIndex-level projection performed by ``_broadcast_core``.""" + + dim: Hashable + levels: list[Hashable] + is_partial: bool # input carried only a subset of the MI's levels + has_gap: bool # projection left entries of the MI dim uncovered (NaN) + + def _project_onto_multiindex_levels( arr: DataArray, expected: dict[Hashable, Any], - *, - enforce_coverage: bool, -) -> DataArray: +) -> tuple[DataArray, list[_LevelProjection]]: """ - Map ``arr`` dims that are levels of a stacked-MultiIndex coords dim onto it. + Map ``arr`` dims that name levels of a stacked-MultiIndex coords dim onto it. - A dim of ``arr`` that is not itself a coords dim but names a level of a - stacked-MultiIndex coords dim ``D`` is projected onto ``D`` by selecting, - for every entry of ``D``, the ``arr`` value at that entry's level values. - A subset of ``D``'s levels broadcasts across the remaining ones; the full - set aligns element-wise. ``arr`` is returned unchanged when it carries no - such level dims. + For every entry of the MultiIndex dim, select the ``arr`` value at that + entry's level values. A subset of levels broadcasts across the remaining + ones; the full set aligns element-wise. ``arr`` is returned unchanged + when it carries no level dims. - Raises ``ValueError`` if a level name belongs to more than one MI dim - (ambiguous) or if a referenced level value is missing from ``arr``. When - ``enforce_coverage`` is set, also raises if the projection leaves entries - of ``D`` uncovered (the input did not span the full MultiIndex). - - On the non-enforcing (arithmetic) path, projections that the v1 - arithmetic convention will require the caller to make explicit emit an - :class:`~linopy.EvolvingAPIWarning`: aligning a *subset* of ``D``'s - levels (an implicit broadcast — future §9/§10) and aligning the full - level set when it leaves gaps (an implicit NaN-fill — future §5/§8). - Aligning the full level set with full coverage is convention-clean and - stays silent. + Raises ``ValueError`` only on structural errors: a level name owned by + two MI dims, or a level value missing from ``arr``. Partial projections + and coverage gaps are recorded in the returned ``_LevelProjection`` list; + the caller decides how to treat them. """ level_owner: dict[Hashable, Hashable] = {} owner_mi: dict[Hashable, pd.MultiIndex] = {} @@ -340,6 +338,7 @@ def _project_onto_multiindex_levels( if owner is not None: groups.setdefault(owner, []).append(d) + projections: list[_LevelProjection] = [] for dim, levels in groups.items(): mi = owner_mi[dim] selectors = { @@ -354,92 +353,34 @@ def _project_onto_multiindex_levels( f"{dim!r}: value {err} is missing." ) from err arr = arr.assign_coords(Coordinates.from_pandas_multiindex(mi, dim)) - is_partial = len(levels) < sum(name is not None for name in mi.names) - has_gap = bool(arr.isnull().any()) - if enforce_coverage: - if has_gap: - raise ValueError( - f"Input does not cover every entry of MultiIndex dimension " - f"{dim!r} (aligned from level(s) {levels})." - ) - elif is_partial or has_gap: - kind = ( - f"broadcasting level subset {levels}" - if is_partial - else f"filling uncovered entries with NaN (from level(s) {levels})" - ) - warn( - f"multiindex-projection: implicitly {kind} onto MultiIndex " - f"dimension {dim!r}. The v1 arithmetic convention will require " - f"this to be explicit; reindex onto the dimension or use a " - f"named method with `join=` to keep current behavior.", - EvolvingAPIWarning, - stacklevel=2, + projections.append( + _LevelProjection( + dim=dim, + levels=levels, + is_partial=len(levels) < sum(name is not None for name in mi.names), + has_gap=bool(arr.isnull().any()), ) + ) - return arr + return arr, projections -def as_dataarray( +def _broadcast_core( arr: Any, coords: CoordsLike | None = None, dims: DimsLike | None = None, - *, - enforce_level_coverage: bool = False, **kwargs: Any, -) -> DataArray: +) -> tuple[DataArray, list[_LevelProjection]]: """ - Convert ``arr`` to a DataArray and broadcast it against ``coords``. - - When ``coords`` carries named dimensions, the result is aligned with - those coords: - - - positional inputs (numpy, polars, unnamed pandas, scalar) are labeled - with the coord dim names by position; - - for every dim shared between ``arr`` and ``coords``, same-values- - different-order coordinates are reindexed to ``coords`` order; - - dims present in ``coords`` but not in ``arr`` are expanded to the - ``coords`` shape; - - dims of ``arr`` that name levels of a stacked-MultiIndex ``coords`` - dim are projected onto that dim (a subset of levels broadcasts, the - full set aligns element-wise); - - the result is transposed to ``coords`` order. - - Dimensions present in ``arr`` but not in ``coords`` are preserved so - standard xarray broadcasting keeps working. Disagreeing coord values - on a shared dim (i.e. value sets that are not equal as sets) are - passed through unchanged: downstream xarray alignment decides how to - combine them. To enforce that ``arr.dims`` ⊆ ``coords.dims`` and that - shared coord values match, use ``validate_alignment`` (called - automatically for ``lower``, ``upper``, and ``mask`` in - :meth:`~linopy.model.Model.add_variables` and for ``mask`` in - :meth:`~linopy.model.Model.add_constraints`). + Convert ``arr`` and broadcast it against ``coords`` (shared mechanics). - Parameters - ---------- - arr - Input scalar / list / numpy / polars / pandas / DataArray. - coords - Mapping of dim name → coord values, or a sequence of ``pd.Index`` - / unnamed sequences. ``None`` falls back to xarray's default - labeling (no broadcasting). - dims - Optional dim-names hint, used for positional inputs and to bias - pandas-axis interpretation. - enforce_level_coverage - When projecting onto a stacked-MultiIndex dim, raise if the input - leaves entries of that dim uncovered. Set by the strict callers - (``add_variables`` / ``add_constraints`` via ``align_to_coords``). - **kwargs - Forwarded to the underlying DataArray construction. - - Returns - ------- - DataArray - Broadcast against ``coords`` (extra dims preserved). + Returns the broadcast DataArray together with the MultiIndex-level + projections performed along the way, so the public entry points can + apply their own policy (warn or raise) to partial projections and + coverage gaps. """ if coords is None: - return _as_dataarray_lax(arr, coords, dims, **kwargs) + return as_dataarray(arr, coords, dims, **kwargs), [] if isinstance(coords, list | tuple) and any(isinstance(c, tuple) for c in coords): # xarray reads bare `(a, b)` as `(dim_name, values)`; normalize so a @@ -448,7 +389,7 @@ def as_dataarray( expected = _coords_to_dict(coords, dims=dims) if not expected: - return _as_dataarray_lax(arr, coords, dims, **kwargs) + return as_dataarray(arr, coords, dims, **kwargs), [] if isinstance(arr, pd.Series | pd.DataFrame): converted = _named_pandas_to_dataarray(arr) @@ -458,7 +399,7 @@ def as_dataarray( if not isinstance(arr, DataArray): # numpy/polars/unnamed-pandas inputs are positional — their only # meaningful information is the values; any axis labels are - # auto-generated. Default dims to coords' keys so the lax conversion + # auto-generated. Default dims to coords' keys so the conversion # labels axes correctly (instead of dim_0/dim_1), then re-assign # coords from expected so positional inputs align to coords by # position. A shape mismatch surfaces here as a clear xarray @@ -466,9 +407,9 @@ def as_dataarray( # "coordinates do not match" further down. if dims is None: dims = list(expected) - arr = _as_dataarray_lax(arr, coords, dims=dims, **kwargs) + arr = as_dataarray(arr, coords, dims=dims, **kwargs) # Skip MultiIndex dims — re-assigning a PandasMultiIndex coord emits - # a FutureWarning and isn't needed (the lax pass already used it). + # a FutureWarning and isn't needed (the conversion already used it). arr = arr.assign_coords( { d: expected[d] @@ -477,9 +418,7 @@ def as_dataarray( } ) - arr = _project_onto_multiindex_levels( - arr, expected, enforce_coverage=enforce_level_coverage - ) + arr, projections = _project_onto_multiindex_levels(arr, expected) for dim, coord_values in expected.items(): if dim not in arr.dims: @@ -491,10 +430,7 @@ def as_dataarray( if actual_idx.equals(expected_idx): continue # Same values, different order → reindex to match expected order. - # Different value sets are left alone: downstream xarray alignment - # (e.g. xr.align in arithmetic) handles them. Callers needing strict - # value matching (add_variables / add_constraints) should use - # ``validate_alignment`` after this call. + # Different value sets are left alone for downstream xarray alignment. if len(actual_idx) == len(expected_idx) and set(actual_idx) == set( expected_idx ): @@ -543,7 +479,51 @@ def as_dataarray( name=arr.name, ) - return arr + return arr, projections + + +def broadcast_to_coords( + arr: Any, + coords: CoordsLike | None = None, + dims: DimsLike | None = None, + **kwargs: Any, +) -> DataArray: + """ + Convert ``arr`` to a DataArray and broadcast it against ``coords``. + + When ``coords`` carries named dimensions, the result is aligned with + them: positional inputs are labeled by position, shared dims with equal + values in a different order are reindexed, dims missing from ``arr`` + are expanded, dims naming levels of a stacked-MultiIndex coords dim are + projected onto it, and the result is transposed to ``coords`` order. + + Dims of ``arr`` not present in ``coords``, and shared dims with + disagreeing value sets, pass through unchanged so downstream xarray + alignment can handle them. Use :func:`align_to_coords` to enforce that + ``arr`` stays within ``coords``. + + Implicit MultiIndex-level projections (a level subset, or one that + leaves entries uncovered) emit an :class:`~linopy.EvolvingAPIWarning`; + the v1 arithmetic convention will require them to be explicit. + """ + da, projections = _broadcast_core(arr, coords, dims, **kwargs) + for p in projections: + if not p.is_partial and not p.has_gap: + continue + kind = ( + f"broadcasting level subset {p.levels}" + if p.is_partial + else f"filling uncovered entries with NaN (from level(s) {p.levels})" + ) + warn( + f"multiindex-projection: implicitly {kind} onto MultiIndex " + f"dimension {p.dim!r}. The v1 arithmetic convention will require " + f"this to be explicit; reindex onto the dimension or use a " + f"named method with `join=` to keep current behavior.", + EvolvingAPIWarning, + stacklevel=2, + ) + return da def validate_alignment( @@ -617,24 +597,29 @@ def align_to_coords( **kwargs: Any, ) -> DataArray: """ - Convert ``value`` with :func:`as_dataarray` and enforce the coords contract. + Convert and broadcast ``value`` against ``coords``, enforcing the coords contract. - Used by :meth:`~linopy.model.Model.add_variables` for ``lower``, ``upper``, - and ``mask``, and by :meth:`~linopy.model.Model.add_constraints` for - ``mask``. Raises :class:`ValueError` with a message that names ``label`` - when ``value`` cannot be aligned to ``coords``. Coords-parsing errors - propagate unchanged. + On top of :func:`broadcast_to_coords` this requires that ``value`` stays + within ``coords``: no extra dims, no disagreeing coord values, and no + MultiIndex coverage gaps. Errors are raised as :class:`ValueError` / + :class:`TypeError` naming ``label``; coords-parsing errors propagate + unchanged. """ if coords is not None: _coords_to_dict(coords, dims=dims) try: - da = as_dataarray( - value, coords, dims=dims, enforce_level_coverage=True, **kwargs - ) + da, projections = _broadcast_core(value, coords, dims=dims, **kwargs) except TypeError as err: raise TypeError(f"{label} could not be aligned to coords: {err}") from err except (ValueError, CoordinateValidationError) as err: raise ValueError(f"{label} could not be aligned to coords: {err}") from err + for p in projections: + if p.has_gap: + raise ValueError( + f"{label} could not be aligned to coords: input does not cover " + f"every entry of MultiIndex dimension {p.dim!r} (aligned from " + f"level(s) {p.levels})." + ) validate_alignment(da, coords, dims=dims, label=label) return da diff --git a/linopy/expressions.py b/linopy/expressions.py index 7342d22a..e0abf5c3 100644 --- a/linopy/expressions.py +++ b/linopy/expressions.py @@ -47,9 +47,9 @@ from linopy.common import ( EmptyDeprecationWrapper, LocIndexer, - _as_dataarray_lax, as_dataarray, assign_multiindex_safe, + broadcast_to_coords, check_common_keys_values, check_has_nulls, check_has_nulls_polars, @@ -583,7 +583,7 @@ def _add_constant( # so that missing data does not silently propagate through arithmetic. if np.isscalar(other) and join is None: return self.assign(const=self.const.fillna(0) + other) - da = as_dataarray(other, coords=self.coords, dims=self.coord_dims) + da = broadcast_to_coords(other, coords=self.coords, dims=self.coord_dims) self_const, da, needs_data_reindex = self._align_constant( da, fill_value=0, join=join ) @@ -612,7 +612,7 @@ def _apply_constant_op( - factor (other) is filled with fill_value (0 for mul, 1 for div) - coeffs and const are filled with 0 (additive identity) """ - factor = as_dataarray(other, coords=self.coords, dims=self.coord_dims) + factor = broadcast_to_coords(other, coords=self.coords, dims=self.coord_dims) self_const, factor, needs_data_reindex = self._align_constant( factor, fill_value=fill_value, join=join ) @@ -1104,7 +1104,7 @@ def to_constraint( ) if isinstance(rhs, CONSTANT_TYPES): - rhs = as_dataarray(rhs, coords=self.coords, dims=self.coord_dims) + rhs = broadcast_to_coords(rhs, coords=self.coords, dims=self.coord_dims) extra_dims = set(rhs.dims) - set(self.coord_dims) if extra_dims: @@ -1687,7 +1687,7 @@ def __matmul__( Matrix multiplication with other, similar to xarray dot. """ if not isinstance(other, LinearExpression | variables.Variable): - other = _as_dataarray_lax(other, coords=self.coords, dims=self.coord_dims) + other = as_dataarray(other, coords=self.coords, dims=self.coord_dims) common_dims = list(set(self.coord_dims).intersection(other.dims)) return (self * other).sum(dim=common_dims) @@ -2173,7 +2173,7 @@ def __matmul__( "Higher order non-linear expressions are not yet supported." ) - other = _as_dataarray_lax(other, coords=self.coords, dims=self.coord_dims) + other = as_dataarray(other, coords=self.coords, dims=self.coord_dims) common_dims = list(set(self.coord_dims).intersection(other.dims)) return (self * other).sum(dim=common_dims) diff --git a/linopy/variables.py b/linopy/variables.py index cbf2fb87..b8397c59 100644 --- a/linopy/variables.py +++ b/linopy/variables.py @@ -37,6 +37,7 @@ VariableLabelIndex, as_dataarray, assign_multiindex_safe, + broadcast_to_coords, check_has_nulls, check_has_nulls_polars, filter_nulls_polars, @@ -327,7 +328,9 @@ def to_linexpr( linopy.LinearExpression Linear expression with the variables and coefficients. """ - coefficient = as_dataarray(coefficient, coords=self.coords, dims=self.dims) + coefficient = broadcast_to_coords( + coefficient, coords=self.coords, dims=self.dims + ) coefficient = coefficient.reindex_like(self.labels, fill_value=0) coefficient = coefficient.fillna(0) ds = Dataset({"coeffs": coefficient, "vars": self.labels}).expand_dims( diff --git a/test/test_common.py b/test/test_common.py index 61ae6f2d..954fc3ce 100644 --- a/test/test_common.py +++ b/test/test_common.py @@ -24,6 +24,7 @@ as_dataarray, assign_multiindex_safe, best_int, + broadcast_to_coords, get_dims_with_index_levels, is_constant, iterate_slices, @@ -349,11 +350,22 @@ def test_as_dataarray_with_ndarray_coords_dict_dims_aligned() -> None: def test_as_dataarray_with_ndarray_coords_dict_set_dims_not_aligned() -> None: - """Coords is source of truth: extra coord entries broadcast into the result.""" + """as_dataarray converts only: dims label the axes, extra coord entries are dropped.""" target_dims = ("dim_0", "dim_1") target_coords = {"dim_0": ["a", "b"], "dim_2": ["A", "B"]} arr = np.array([[1, 2], [3, 4]]) da = as_dataarray(arr, coords=target_coords, dims=target_dims) + assert da.dims == target_dims + assert list(da.coords["dim_0"].values) == ["a", "b"] + assert "dim_2" not in da.coords + + +def test_broadcast_to_coords_with_ndarray_coords_dict_set_dims_not_aligned() -> None: + """Coords is source of truth: extra coord entries broadcast into the result.""" + target_dims = ("dim_0", "dim_1") + target_coords = {"dim_0": ["a", "b"], "dim_2": ["A", "B"]} + arr = np.array([[1, 2], [3, 4]]) + da = broadcast_to_coords(arr, coords=target_coords, dims=target_dims) # dims labels the positional axes; coords adds dim_2 by broadcast. assert set(da.dims) == {"dim_0", "dim_1", "dim_2"} assert list(da.coords["dim_0"].values) == ["a", "b"] @@ -489,7 +501,7 @@ def test_as_dataarray_with_unsupported_type() -> None: as_dataarray(lambda x: 1, dims=["dim1"], coords=[["a"]]) -def test_as_dataarray_preserves_extra_dims_for_broadcasting() -> None: +def test_broadcast_to_coords_preserves_extra_dims() -> None: """Extra dims in the input are not rejected — they broadcast downstream.""" arr = DataArray( [[1, 2], [3, 4], [5, 6]], @@ -497,21 +509,21 @@ def test_as_dataarray_preserves_extra_dims_for_broadcasting() -> None: coords={"a": [0, 1, 2], "t": [10, 20]}, ) coords = {"a": [0, 1, 2]} - da = as_dataarray(arr, coords=coords) + da = broadcast_to_coords(arr, coords=coords) assert set(da.dims) == {"a", "t"} assert list(da.coords["t"].values) == [10, 20] -def test_as_dataarray_keeps_disjoint_shared_dim_values() -> None: +def test_broadcast_to_coords_keeps_disjoint_shared_dim_values() -> None: """Different value sets on a shared dim are passed through (xr.align handles).""" arr = DataArray([1, 2, 3, 4, 5], dims=["a"], coords={"a": [0, 1, 2, 3, 4]}) coords = {"a": [2, 3]} - da = as_dataarray(arr, coords=coords) + da = broadcast_to_coords(arr, coords=coords) # No exception, no reindex; downstream alignment intersects. assert list(da.coords["a"].values) == [0, 1, 2, 3, 4] -def test_as_dataarray_expands_missing_multiindex_dim_keeps_levels() -> None: +def test_broadcast_to_coords_expands_missing_multiindex_dim_keeps_levels() -> None: """ Broadcasting a missing MultiIndex dim must keep its level coords intact. @@ -527,7 +539,7 @@ def test_as_dataarray_expands_missing_multiindex_dim_keeps_levels() -> None: labels = DataArray( [[1], [2], [3], [4]], coords={**sc, "name": ["1"]}, dims=["snapshot", "name"] ) - coeff = as_dataarray( + coeff = broadcast_to_coords( DataArray([1.0], coords={"name": ["1"]}, dims=["name"]), coords=labels.coords, dims=labels.dims, @@ -536,7 +548,7 @@ def test_as_dataarray_expands_missing_multiindex_dim_keeps_levels() -> None: coeff.reindex_like(labels, fill_value=0) -def test_as_dataarray_broadcasts_single_multiindex_level() -> None: +def test_broadcast_to_coords_broadcasts_single_multiindex_level() -> None: """ A constant indexed by one MultiIndex level broadcasts across the MI dim. @@ -550,7 +562,7 @@ def test_as_dataarray_broadcasts_single_multiindex_level() -> None: by_level1 = DataArray([10.0, 20.0], coords={"level1": [1, 2]}, dims=["level1"]) with pytest.warns(EvolvingAPIWarning, match=r"broadcasting level subset"): - da = as_dataarray(by_level1, coords=coords, dims=["dim_3"]) + da = broadcast_to_coords(by_level1, coords=coords, dims=["dim_3"]) assert da.dims == ("dim_3",) assert isinstance(da.indexes["dim_3"], pd.MultiIndex) @@ -560,7 +572,7 @@ def test_as_dataarray_broadcasts_single_multiindex_level() -> None: assert da.sel(dim_3=(2, "b")).item() == 20.0 -def test_as_dataarray_stacks_full_multiindex_levels() -> None: +def test_broadcast_to_coords_stacks_full_multiindex_levels() -> None: """ A constant indexed by all MI level names stacks element-wise into the MI dim. @@ -576,7 +588,7 @@ def test_as_dataarray_stacks_full_multiindex_levels() -> None: weights = pd.Series([10.0, 20.0], index=subset) with pytest.warns(EvolvingAPIWarning, match=r"filling uncovered entries with NaN"): - da = as_dataarray(weights, coords=coords, dims=["dim_3"]) + da = broadcast_to_coords(weights, coords=coords, dims=["dim_3"]) assert da.dims == ("dim_3",) assert isinstance(da.indexes["dim_3"], pd.MultiIndex) @@ -586,7 +598,7 @@ def test_as_dataarray_stacks_full_multiindex_levels() -> None: assert np.isnan(da.sel(dim_3=(2, "a")).item()) -def test_as_dataarray_full_multiindex_full_coverage_is_silent() -> None: +def test_broadcast_to_coords_full_multiindex_full_coverage_is_silent() -> None: """ Full-level, fully-covering alignment is convention-clean → no warning. @@ -601,13 +613,13 @@ def test_as_dataarray_full_multiindex_full_coverage_is_silent() -> None: with warnings.catch_warnings(): warnings.simplefilter("error", EvolvingAPIWarning) - da = as_dataarray(full, coords=coords, dims=["dim_3"]) + da = broadcast_to_coords(full, coords=coords, dims=["dim_3"]) assert da.dims == ("dim_3",) assert da.values.tolist() == [1.0, 2.0, 3.0, 4.0] -def test_as_dataarray_level_projection_ambiguous_raises() -> None: +def test_broadcast_to_coords_level_projection_ambiguous_raises() -> None: """A level name shared by two MI dims cannot be resolved.""" a = pd.MultiIndex.from_product([[1, 2], ["a", "b"]], names=("shared", "x")) b = pd.MultiIndex.from_product([[1, 2], ["c", "d"]], names=("shared", "y")) @@ -618,10 +630,10 @@ def test_as_dataarray_level_projection_ambiguous_raises() -> None: arr = DataArray([1.0, 2.0], coords={"shared": [1, 2]}, dims=["shared"]) with pytest.raises(ValueError, match=r"shared.*shared by MultiIndex"): - as_dataarray(arr, coords=coords) + broadcast_to_coords(arr, coords=coords) -def test_as_dataarray_level_projection_missing_value_raises() -> None: +def test_broadcast_to_coords_level_projection_missing_value_raises() -> None: """A level value absent from the input cannot be broadcast.""" idx = pd.MultiIndex.from_product([[1, 2], ["a", "b"]], names=("level1", "level2")) idx.name = "dim_3" @@ -629,19 +641,80 @@ def test_as_dataarray_level_projection_missing_value_raises() -> None: by_level1 = DataArray([10.0, 20.0], coords={"level1": [1, 9]}, dims=["level1"]) with pytest.raises(ValueError, match=r"Cannot align level.*is missing"): - as_dataarray(by_level1, coords=coords, dims=["dim_3"]) + broadcast_to_coords(by_level1, coords=coords, dims=["dim_3"]) -def test_as_dataarray_unrelated_multiindex_series_still_unstacks() -> None: +def test_broadcast_to_coords_unrelated_multiindex_series_still_unstacks() -> None: """A MI Series whose levels match no coords MI dim keeps unstacking.""" sub = pd.MultiIndex.from_product([["p", "q"], [1, 2]], names=["foo", "bar"]) series = pd.Series([1.0, 2.0, 3.0, 4.0], index=sub) - da = as_dataarray(series, coords={"time": [0, 1, 2]}) + da = broadcast_to_coords(series, coords={"time": [0, 1, 2]}) assert set(da.dims) == {"time", "foo", "bar"} +# --------------------------------------------------------------------------- +# Strictness ladder: as_dataarray ⊂ broadcast_to_coords ⊂ align_to_coords +# --------------------------------------------------------------------------- + + +def test_as_dataarray_does_not_expand_missing_coord_dims() -> None: + """as_dataarray converts; only broadcast_to_coords expands missing dims.""" + coords = {"a": [0, 1], "b": [10, 20]} + arr = np.array([1, 2]) + + converted = as_dataarray(arr, coords=coords, dims=["a"]) + assert converted.dims == ("a",) + + broadcast = broadcast_to_coords(arr, coords=coords, dims=["a"]) + assert broadcast.dims == ("a", "b") + + +def test_broadcast_to_coords_passes_extra_dims_align_to_coords_rejects() -> None: + """Extra dims pass through the broadcast rung but fail the strict rung.""" + arr = DataArray( + [[1, 2], [3, 4]], dims=["a", "t"], coords={"a": [0, 1], "t": [10, 20]} + ) + coords = {"a": [0, 1]} + + da = broadcast_to_coords(arr, coords=coords) + assert set(da.dims) == {"a", "t"} + + with pytest.raises(ValueError, match=r"not declared in coords"): + align_to_coords(arr, coords, label="lower bound") + + +def test_align_to_coords_rejects_multiindex_coverage_gap() -> None: + """A coverage gap warns on the broadcast rung but raises on the strict rung.""" + idx = pd.MultiIndex.from_product([[1, 2], ["a", "b"]], names=("level1", "level2")) + idx.name = "dim_3" + coords = xr.Coordinates.from_pandas_multiindex(idx, "dim_3") + subset = pd.MultiIndex.from_tuples([(1, "a"), (2, "b")], names=["level1", "level2"]) + weights = pd.Series([10.0, 20.0], index=subset) + + with pytest.warns(EvolvingAPIWarning, match=r"filling uncovered entries"): + broadcast_to_coords(weights, coords=coords, dims=["dim_3"]) + + with pytest.raises(ValueError, match=r"does not cover every entry"): + align_to_coords(weights, coords, dims=["dim_3"], label="lower bound") + + +def test_align_to_coords_allows_partial_level_broadcast_silently() -> None: + """Per-level bounds broadcast across the MI dim without the arithmetic warning.""" + idx = pd.MultiIndex.from_product([[1, 2], ["a", "b"]], names=("level1", "level2")) + idx.name = "dim_3" + coords = xr.Coordinates.from_pandas_multiindex(idx, "dim_3") + by_level1 = DataArray([10.0, 20.0], coords={"level1": [1, 2]}, dims=["level1"]) + + with warnings.catch_warnings(): + warnings.simplefilter("error", EvolvingAPIWarning) + da = align_to_coords(by_level1, coords, dims=["dim_3"], label="lower bound") + + assert da.sel(dim_3=(1, "b")).item() == 10.0 + assert da.sel(dim_3=(2, "a")).item() == 20.0 + + def test_validate_alignment_rejects_extra_dims() -> None: arr = DataArray( [[1, 2], [3, 4]], dims=["a", "b"], coords={"a": [0, 1], "b": [0, 1]} diff --git a/test/test_linear_expression.py b/test/test_linear_expression.py index 1ea20b00..82aba70e 100644 --- a/test/test_linear_expression.py +++ b/test/test_linear_expression.py @@ -546,6 +546,47 @@ def test_matmul_expr_and_const(x: Variable, y: Variable) -> None: assert_linequal(expr.dot(const), target) +def test_matmul_contracts_only_shared_dims(z: Variable) -> None: + """ + A @ b contracts the genuinely shared dims and keeps the rest. + + ``z`` has dims (dim_0, dim_1); ``b`` has (dim_1, location). Only dim_1 + is shared, so the result must keep dim_0 and location. A conversion that + broadcast ``b`` to ``z``'s coords would expand dim_0 into ``b`` and + contract it away too — collapsing the result to (location,) only. + """ + expr = 1 * z + b = xr.DataArray( + np.ones((3, 2)), + coords={"dim_1": expr.data.indexes["dim_1"], "location": ["L1", "L2"]}, + dims=["dim_1", "location"], + ) + + res = expr @ b + + assert set(res.coord_dims) == {"dim_0", "location"} + assert_linequal(res, (expr * b).sum("dim_1")) + + +def test_matmul_contracts_all_dims_when_const_covers_them(z: Variable) -> None: + """B covering all of a's dims (and more) contracts a's dims, keeping b's extras.""" + expr = 1 * z # dims (dim_0, dim_1) + b = xr.DataArray( + np.ones((2, 3, 2)), + coords={ + "dim_0": expr.data.indexes["dim_0"], + "dim_1": expr.data.indexes["dim_1"], + "location": ["L1", "L2"], + }, + dims=["dim_0", "dim_1", "location"], + ) + + res = expr @ b + + assert set(res.coord_dims) == {"location"} + assert_linequal(res, (expr * b).sum(["dim_0", "dim_1"])) + + def test_matmul_wrong_input(x: Variable, y: Variable, z: Variable) -> None: expr = 10 * x + y + z with pytest.raises(TypeError): From 18c65c4ce661ae571db84111d7360d6d752e9f16 Mon Sep 17 00:00:00 2001 From: FBumann <117816358+FBumann@users.noreply.github.com> Date: Mon, 1 Jun 2026 15:00:27 +0200 Subject: [PATCH 02/10] docs: shorten release-notes bullet on conversion helpers Co-Authored-By: Claude Opus 4.8 (1M context) --- doc/release_notes.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/release_notes.rst b/doc/release_notes.rst index 8dbf1788..6485b862 100644 --- a/doc/release_notes.rst +++ b/doc/release_notes.rst @@ -71,7 +71,7 @@ Most users should keep calling ``model.solve(...)``. If you want more control, y **Internal** -* ``linopy.common`` exposes a three-rung strictness ladder for DataArray conversion: ``as_dataarray`` (convert only; used by ``__matmul__`` so dims missing from the constant are not broadcast in and contracted away), ``broadcast_to_coords`` (convert and broadcast against ``coords``; used by expression arithmetic), and ``align_to_coords`` (convert, broadcast, and enforce the coords contract via ``validate_alignment``; used by ``add_variables`` / ``add_constraints``). When ``coords`` is a mapping, extra keys beyond the positional ``dims`` are broadcast in rather than dropped (broadcast rung and up). +* ``linopy.common`` provides three DataArray conversion helpers of increasing strictness: ``as_dataarray`` (convert only), ``broadcast_to_coords`` (convert and broadcast against ``coords``), and ``align_to_coords`` (convert, broadcast, and enforce the coords contract). * Each ``Solver`` subclass now overrides at most three hooks: ``_build_direct`` (build the native model), ``_run_direct`` (run it), and ``_run_file`` (run the solver on an LP/MPS file). File-only solvers (CBC, GLPK, CPLEX, SCIP, Knitro, COPT, MindOpt) only override ``_run_file``. * New ``ConstraintLabelIndex`` cached on ``Model.constraints`` (mirrors the existing ``Variables.label_index``); ``ConstraintBase`` gains ``active_labels()`` and a ``range`` property; ``CSRConstraint`` exposes ``coords``. * ``linopy.common`` gains ``values_to_lookup_array``; the legacy pandas-based helpers ``series_to_lookup_array`` and ``lookup_vals`` are removed. From b5f2cdb453f334c2fbae68a0a7d602bf6362abe5 Mon Sep 17 00:00:00 2001 From: FBumann <117816358+FBumann@users.noreply.github.com> Date: Mon, 1 Jun 2026 15:56:45 +0200 Subject: [PATCH 03/10] refactor(common): rename _broadcast_core to _broadcast_to_coords Private-twin convention: _broadcast_to_coords is the raw implementation of broadcast_to_coords (returns projection events instead of applying policy), shared with align_to_coords. Co-Authored-By: Claude Opus 4.8 (1M context) --- linopy/common.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/linopy/common.py b/linopy/common.py index a19d3c75..baf175a8 100644 --- a/linopy/common.py +++ b/linopy/common.py @@ -287,7 +287,7 @@ def _as_multiindex(coord_values: Any) -> pd.MultiIndex | None: class _LevelProjection(NamedTuple): - """Record of one MultiIndex-level projection performed by ``_broadcast_core``.""" + """Record of one MultiIndex-level projection performed by ``_broadcast_to_coords``.""" dim: Hashable levels: list[Hashable] @@ -365,7 +365,7 @@ def _project_onto_multiindex_levels( return arr, projections -def _broadcast_core( +def _broadcast_to_coords( arr: Any, coords: CoordsLike | None = None, dims: DimsLike | None = None, @@ -506,7 +506,7 @@ def broadcast_to_coords( leaves entries uncovered) emit an :class:`~linopy.EvolvingAPIWarning`; the v1 arithmetic convention will require them to be explicit. """ - da, projections = _broadcast_core(arr, coords, dims, **kwargs) + da, projections = _broadcast_to_coords(arr, coords, dims, **kwargs) for p in projections: if not p.is_partial and not p.has_gap: continue @@ -608,7 +608,7 @@ def align_to_coords( if coords is not None: _coords_to_dict(coords, dims=dims) try: - da, projections = _broadcast_core(value, coords, dims=dims, **kwargs) + da, projections = _broadcast_to_coords(value, coords, dims=dims, **kwargs) except TypeError as err: raise TypeError(f"{label} could not be aligned to coords: {err}") from err except (ValueError, CoordinateValidationError) as err: From 633d5b9ff0ffa18a3dc8a23e08ec123b0d6806ed Mon Sep 17 00:00:00 2001 From: FBumann <117816358+FBumann@users.noreply.github.com> Date: Mon, 1 Jun 2026 16:16:41 +0200 Subject: [PATCH 04/10] fix(expressions): as_expression converts constants with the broadcast rung The constraint lhs/rhs setters call as_expression(value, model, coords=self.coords, dims=self.coord_dims); forwarding those kwargs to the convert-only as_dataarray dropped the broadcasting these setters relied on (e.g. a MultiIndex-level-indexed rhs failed with an xarray AlignmentError instead of being projected onto the stacked dim). Use broadcast_to_coords instead. The other as_expression callers pass only dims (no coords), for which both rungs behave identically. Adds regression tests for the rhs setter: missing-dim broadcast and MultiIndex-level projection. Co-Authored-By: Claude Opus 4.8 (1M context) --- linopy/expressions.py | 4 ++-- test/test_constraint.py | 39 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 41 insertions(+), 2 deletions(-) diff --git a/linopy/expressions.py b/linopy/expressions.py index e0abf5c3..27918ce7 100644 --- a/linopy/expressions.py +++ b/linopy/expressions.py @@ -2291,7 +2291,7 @@ def as_expression( model : linopy.Model, optional Assigned model, by default None **kwargs : - Keyword arguments passed to `linopy.as_dataarray`. + Keyword arguments passed to `linopy.common.broadcast_to_coords`. Returns ------- @@ -2308,7 +2308,7 @@ def as_expression( return obj.to_linexpr() else: try: - obj = as_dataarray(obj, **kwargs) + obj = broadcast_to_coords(obj, **kwargs) except ValueError as e: raise ValueError("Cannot convert to LinearExpression") from e return LinearExpression(obj, model) diff --git a/test/test_constraint.py b/test/test_constraint.py index a1b33d66..d3581de9 100644 --- a/test/test_constraint.py +++ b/test/test_constraint.py @@ -453,6 +453,45 @@ def test_constraint_rhs_setter_with_expression_and_constant( assert mc.lhs.nterm == 2 +def test_constraint_rhs_setter_broadcasts_missing_dim() -> None: + """Rhs assignment broadcasts against the constraint coords: missing dims expand.""" + m = Model() + x = m.add_variables( + coords=[pd.RangeIndex(2, name="i"), pd.RangeIndex(3, name="j")], name="x" + ) + con = m.add_constraints(1 * x >= 0, name="con") + + con.rhs = xr.DataArray([1.0, 2.0], dims=["i"], coords={"i": [0, 1]}) # type: ignore + + assert dict(con.rhs.sizes) == {"i": 2, "j": 3} + assert (con.rhs.sel(i=1) == 2.0).all() + + +def test_constraint_rhs_setter_projects_multiindex_level() -> None: + """ + Rhs indexed by one MultiIndex level is projected onto the stacked dim. + + Regression: as_expression must convert constants with the broadcast rung + (broadcast_to_coords), not plain conversion — otherwise the level dim + collides with the MI level coord downstream (xarray AlignmentError). + """ + idx = pd.MultiIndex.from_product([[1, 2], ["a", "b"]], names=("level1", "level2")) + idx.name = "dim_3" + coords = xr.Coordinates.from_pandas_multiindex(idx, "dim_3") + m = Model() + x = m.add_variables(coords=coords, name="x") + con = m.add_constraints(1 * x >= 0, name="con") + + rhs_by_level = xr.DataArray( + [10.0, 20.0], coords={"level1": [1, 2]}, dims=["level1"] + ) + with pytest.warns(linopy.EvolvingAPIWarning, match="broadcasting level subset"): + con.rhs = rhs_by_level # type: ignore + + assert con.rhs.sel(dim_3=(1, "b")).item() == 10.0 + assert con.rhs.sel(dim_3=(2, "a")).item() == 20.0 + + def test_constraint_labels_setter_invalid(c: linopy.constraints.CSRConstraint) -> None: # Test that assigning labels raises AttributeError (Constraint is frozen) with pytest.raises(AttributeError): From b14df0ee444338ba89ef6780e28185407ba30ae8 Mon Sep 17 00:00:00 2001 From: FBumann <117816358+FBumann@users.noreply.github.com> Date: Mon, 1 Jun 2026 20:21:43 +0200 Subject: [PATCH 05/10] refactor(common): rename align_to_coords to strict_broadcast_to_coords MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The function never aligns anything — it broadcasts and raises on any mismatch it cannot resolve by broadcasting alone. "Align" is also the word that invites join= proposals (aligns take joins, broadcasts do not), so the name now states what it is: the same broadcast as broadcast_to_coords with a strict failure mode (zip(strict=True) semantics). Error messages keep the "could not be aligned to coords" wording so tests in the base branch (#732) stay untouched. Co-Authored-By: Claude Opus 4.8 (1M context) --- doc/release_notes.rst | 2 +- linopy/common.py | 14 +++++++------- linopy/model.py | 16 +++++++++++----- test/test_common.py | 34 ++++++++++++++++++++-------------- 4 files changed, 39 insertions(+), 27 deletions(-) diff --git a/doc/release_notes.rst b/doc/release_notes.rst index 6485b862..f878689d 100644 --- a/doc/release_notes.rst +++ b/doc/release_notes.rst @@ -71,7 +71,7 @@ Most users should keep calling ``model.solve(...)``. If you want more control, y **Internal** -* ``linopy.common`` provides three DataArray conversion helpers of increasing strictness: ``as_dataarray`` (convert only), ``broadcast_to_coords`` (convert and broadcast against ``coords``), and ``align_to_coords`` (convert, broadcast, and enforce the coords contract). +* ``linopy.common`` provides three DataArray conversion helpers of increasing strictness: ``as_dataarray`` (convert only), ``broadcast_to_coords`` (convert and broadcast against ``coords``), and ``strict_broadcast_to_coords`` (the same broadcast, but any mismatch raises). * Each ``Solver`` subclass now overrides at most three hooks: ``_build_direct`` (build the native model), ``_run_direct`` (run it), and ``_run_file`` (run the solver on an LP/MPS file). File-only solvers (CBC, GLPK, CPLEX, SCIP, Knitro, COPT, MindOpt) only override ``_run_file``. * New ``ConstraintLabelIndex`` cached on ``Model.constraints`` (mirrors the existing ``Variables.label_index``); ``ConstraintBase`` gains ``active_labels()`` and a ``range`` property; ``CSRConstraint`` exposes ``coords``. * ``linopy.common`` gains ``values_to_lookup_array``; the legacy pandas-based helpers ``series_to_lookup_array`` and ``lookup_vals`` are removed. diff --git a/linopy/common.py b/linopy/common.py index baf175a8..5edfaf51 100644 --- a/linopy/common.py +++ b/linopy/common.py @@ -229,7 +229,7 @@ def as_dataarray( polars, numpy, scalar, DataArray) and labels positional axes with ``dims`` / ``coords``. The result is not reshaped against ``coords``: dims are neither expanded, reordered, nor projected onto MultiIndex - dims. Use :func:`broadcast_to_coords` or :func:`align_to_coords` when + dims. Use :func:`broadcast_to_coords` or :func:`strict_broadcast_to_coords` when ``coords`` should govern the result's shape. """ if isinstance(arr, pd.Series | pd.DataFrame): @@ -499,7 +499,7 @@ def broadcast_to_coords( Dims of ``arr`` not present in ``coords``, and shared dims with disagreeing value sets, pass through unchanged so downstream xarray - alignment can handle them. Use :func:`align_to_coords` to enforce that + alignment can handle them. Use :func:`strict_broadcast_to_coords` to enforce that ``arr`` stays within ``coords``. Implicit MultiIndex-level projections (a level subset, or one that @@ -588,7 +588,7 @@ def validate_alignment( ) -def align_to_coords( +def strict_broadcast_to_coords( value: Any, coords: CoordsLike | None, *, @@ -597,11 +597,11 @@ def align_to_coords( **kwargs: Any, ) -> DataArray: """ - Convert and broadcast ``value`` against ``coords``, enforcing the coords contract. + :func:`broadcast_to_coords` with a strict failure mode. - On top of :func:`broadcast_to_coords` this requires that ``value`` stays - within ``coords``: no extra dims, no disagreeing coord values, and no - MultiIndex coverage gaps. Errors are raised as :class:`ValueError` / + The same broadcast, but anything it cannot resolve by broadcasting alone + raises instead of passing through: extra dims, disagreeing coord values, + and MultiIndex coverage gaps. Errors are raised as :class:`ValueError` / :class:`TypeError` naming ``label``; coords-parsing errors propagate unchanged. """ diff --git a/linopy/model.py b/linopy/model.py index e374c101..6a2abcb0 100644 --- a/linopy/model.py +++ b/linopy/model.py @@ -28,12 +28,12 @@ from linopy import solvers from linopy.common import ( - align_to_coords, as_dataarray, assign_multiindex_safe, best_int, maybe_replace_signs, replace_by_map, + strict_broadcast_to_coords, to_path, ) from linopy.constants import ( @@ -774,8 +774,12 @@ def add_variables( "Semi-continuous variables require a positive scalar lower bound." ) - lower_da = align_to_coords(lower, coords, label="lower bound", **kwargs) - upper_da = align_to_coords(upper, coords, label="upper bound", **kwargs) + lower_da = strict_broadcast_to_coords( + lower, coords, label="lower bound", **kwargs + ) + upper_da = strict_broadcast_to_coords( + upper, coords, label="upper bound", **kwargs + ) data = Dataset( { "lower": lower_da, @@ -788,7 +792,7 @@ def add_variables( self._check_valid_dim_names(data) if mask is not None: - mask = align_to_coords( + mask = strict_broadcast_to_coords( mask, coords if coords is not None else data.coords, label="mask", @@ -1057,7 +1061,9 @@ def add_constraints( (data,) = xr.broadcast(data, exclude=[TERM_DIM]) if mask is not None: - mask = align_to_coords(mask, data.coords, label="mask").astype(bool) + mask = strict_broadcast_to_coords(mask, data.coords, label="mask").astype( + bool + ) # Auto-mask based on null expressions or NaN RHS (use numpy for speed) if self.auto_mask: diff --git a/test/test_common.py b/test/test_common.py index 954fc3ce..8d0b1933 100644 --- a/test/test_common.py +++ b/test/test_common.py @@ -20,7 +20,6 @@ from linopy import EvolvingAPIWarning, LinearExpression, Model, Variable from linopy.common import ( align, - align_to_coords, as_dataarray, assign_multiindex_safe, best_int, @@ -29,6 +28,7 @@ is_constant, iterate_slices, maybe_group_terms_polars, + strict_broadcast_to_coords, validate_alignment, ) from linopy.testing import assert_linequal, assert_varequal @@ -655,7 +655,7 @@ def test_broadcast_to_coords_unrelated_multiindex_series_still_unstacks() -> Non # --------------------------------------------------------------------------- -# Strictness ladder: as_dataarray ⊂ broadcast_to_coords ⊂ align_to_coords +# Strictness ladder: as_dataarray ⊂ broadcast_to_coords ⊂ strict_broadcast_to_coords # --------------------------------------------------------------------------- @@ -671,7 +671,9 @@ def test_as_dataarray_does_not_expand_missing_coord_dims() -> None: assert broadcast.dims == ("a", "b") -def test_broadcast_to_coords_passes_extra_dims_align_to_coords_rejects() -> None: +def test_broadcast_to_coords_passes_extra_dims_strict_broadcast_to_coords_rejects() -> ( + None +): """Extra dims pass through the broadcast rung but fail the strict rung.""" arr = DataArray( [[1, 2], [3, 4]], dims=["a", "t"], coords={"a": [0, 1], "t": [10, 20]} @@ -682,10 +684,10 @@ def test_broadcast_to_coords_passes_extra_dims_align_to_coords_rejects() -> None assert set(da.dims) == {"a", "t"} with pytest.raises(ValueError, match=r"not declared in coords"): - align_to_coords(arr, coords, label="lower bound") + strict_broadcast_to_coords(arr, coords, label="lower bound") -def test_align_to_coords_rejects_multiindex_coverage_gap() -> None: +def test_strict_broadcast_to_coords_rejects_multiindex_coverage_gap() -> None: """A coverage gap warns on the broadcast rung but raises on the strict rung.""" idx = pd.MultiIndex.from_product([[1, 2], ["a", "b"]], names=("level1", "level2")) idx.name = "dim_3" @@ -697,10 +699,10 @@ def test_align_to_coords_rejects_multiindex_coverage_gap() -> None: broadcast_to_coords(weights, coords=coords, dims=["dim_3"]) with pytest.raises(ValueError, match=r"does not cover every entry"): - align_to_coords(weights, coords, dims=["dim_3"], label="lower bound") + strict_broadcast_to_coords(weights, coords, dims=["dim_3"], label="lower bound") -def test_align_to_coords_allows_partial_level_broadcast_silently() -> None: +def test_strict_broadcast_to_coords_allows_partial_level_broadcast_silently() -> None: """Per-level bounds broadcast across the MI dim without the arithmetic warning.""" idx = pd.MultiIndex.from_product([[1, 2], ["a", "b"]], names=("level1", "level2")) idx.name = "dim_3" @@ -709,7 +711,9 @@ def test_align_to_coords_allows_partial_level_broadcast_silently() -> None: with warnings.catch_warnings(): warnings.simplefilter("error", EvolvingAPIWarning) - da = align_to_coords(by_level1, coords, dims=["dim_3"], label="lower bound") + da = strict_broadcast_to_coords( + by_level1, coords, dims=["dim_3"], label="lower bound" + ) assert da.sel(dim_3=(1, "b")).item() == 10.0 assert da.sel(dim_3=(2, "a")).item() == 20.0 @@ -755,22 +759,24 @@ def test_validate_alignment_label_in_error() -> None: validate_alignment(arr, {"a": [0, 1]}, label="lower bound") -def test_align_to_coords_wraps_conversion_errors() -> None: +def test_strict_broadcast_to_coords_wraps_conversion_errors() -> None: with pytest.raises(ValueError, match=r"lower bound could not be aligned"): - align_to_coords(np.array([1, 2]), {"x": [0, 1, 2]}, label="lower bound") + strict_broadcast_to_coords( + np.array([1, 2]), {"x": [0, 1, 2]}, label="lower bound" + ) -def test_align_to_coords_preserves_type_errors() -> None: +def test_strict_broadcast_to_coords_preserves_type_errors() -> None: """Unsupported input types stay TypeError (don't become ValueError).""" with pytest.raises(TypeError, match=r"lower bound could not be aligned"): - align_to_coords(lambda x: x, {"x": [0, 1, 2]}, label="lower bound") + strict_broadcast_to_coords(lambda x: x, {"x": [0, 1, 2]}, label="lower bound") -def test_align_to_coords_does_not_relabel_coords_errors() -> None: +def test_strict_broadcast_to_coords_does_not_relabel_coords_errors() -> None: """Coords-side TypeError carries its own message, not the value label.""" mi = pd.MultiIndex.from_product([[0, 1], ["a", "b"]], names=["i", "j"]) with pytest.raises(TypeError, match=r"MultiIndex.*must have \.name set"): - align_to_coords(np.array([1, 2, 3, 4]), [mi], label="lower bound") + strict_broadcast_to_coords(np.array([1, 2, 3, 4]), [mi], label="lower bound") class TestCoordsToDictRules: From 54ed91a9b4959f0d30d7774b239fac737605b9f0 Mon Sep 17 00:00:00 2001 From: FBumann <117816358+FBumann@users.noreply.github.com> Date: Mon, 1 Jun 2026 21:29:25 +0200 Subject: [PATCH 06/10] refactor(common): apply review polish to the strictness ladder - Document the one non-obvious policy in strict_broadcast_to_coords: partial-level broadcasts are silent (bounds-broadcast feature), unlike the warning on the broadcast rung. - Unify the first parameter name across the ladder (value -> arr). - Un-invert the warning-policy loop in broadcast_to_coords. - Rename the test whose name forced an awkward signature wrap to a behavior-oriented name (test_extra_dims_pass_broadcast_rung_fail_strict_rung). Co-Authored-By: Claude Opus 4.8 (1M context) --- linopy/common.py | 38 +++++++++++++++++++++----------------- test/test_common.py | 4 +--- 2 files changed, 22 insertions(+), 20 deletions(-) diff --git a/linopy/common.py b/linopy/common.py index 5edfaf51..44eab4b4 100644 --- a/linopy/common.py +++ b/linopy/common.py @@ -508,21 +508,20 @@ def broadcast_to_coords( """ da, projections = _broadcast_to_coords(arr, coords, dims, **kwargs) for p in projections: - if not p.is_partial and not p.has_gap: - continue - kind = ( - f"broadcasting level subset {p.levels}" - if p.is_partial - else f"filling uncovered entries with NaN (from level(s) {p.levels})" - ) - warn( - f"multiindex-projection: implicitly {kind} onto MultiIndex " - f"dimension {p.dim!r}. The v1 arithmetic convention will require " - f"this to be explicit; reindex onto the dimension or use a " - f"named method with `join=` to keep current behavior.", - EvolvingAPIWarning, - stacklevel=2, - ) + if p.is_partial or p.has_gap: + kind = ( + f"broadcasting level subset {p.levels}" + if p.is_partial + else f"filling uncovered entries with NaN (from level(s) {p.levels})" + ) + warn( + f"multiindex-projection: implicitly {kind} onto MultiIndex " + f"dimension {p.dim!r}. The v1 arithmetic convention will require " + f"this to be explicit; reindex onto the dimension or use a " + f"named method with `join=` to keep current behavior.", + EvolvingAPIWarning, + stacklevel=2, + ) return da @@ -589,7 +588,7 @@ def validate_alignment( def strict_broadcast_to_coords( - value: Any, + arr: Any, coords: CoordsLike | None, *, label: str, @@ -604,11 +603,16 @@ def strict_broadcast_to_coords( and MultiIndex coverage gaps. Errors are raised as :class:`ValueError` / :class:`TypeError` naming ``label``; coords-parsing errors propagate unchanged. + + Partial-level broadcasts (input indexed by a subset of a MultiIndex's + levels) are silent here — they are the documented bounds-broadcast + feature, not the arithmetic-convention concern that makes + :func:`broadcast_to_coords` warn. """ if coords is not None: _coords_to_dict(coords, dims=dims) try: - da, projections = _broadcast_to_coords(value, coords, dims=dims, **kwargs) + da, projections = _broadcast_to_coords(arr, coords, dims=dims, **kwargs) except TypeError as err: raise TypeError(f"{label} could not be aligned to coords: {err}") from err except (ValueError, CoordinateValidationError) as err: diff --git a/test/test_common.py b/test/test_common.py index 8d0b1933..030630bf 100644 --- a/test/test_common.py +++ b/test/test_common.py @@ -671,9 +671,7 @@ def test_as_dataarray_does_not_expand_missing_coord_dims() -> None: assert broadcast.dims == ("a", "b") -def test_broadcast_to_coords_passes_extra_dims_strict_broadcast_to_coords_rejects() -> ( - None -): +def test_extra_dims_pass_broadcast_rung_fail_strict_rung() -> None: """Extra dims pass through the broadcast rung but fail the strict rung.""" arr = DataArray( [[1, 2], [3, 4]], dims=["a", "t"], coords={"a": [0, 1], "t": [10, 20]} From 9d118114aa44849a15892e8662c1c797c3762ee0 Mon Sep 17 00:00:00 2001 From: FBumann <117816358+FBumann@users.noreply.github.com> Date: Mon, 1 Jun 2026 21:38:21 +0200 Subject: [PATCH 07/10] docs(common): add numpydoc Parameters/Returns to the three public rungs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Parameter entries carry descriptions only — types live in the function signatures. Co-Authored-By: Claude Opus 4.8 (1M context) --- linopy/common.py | 52 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 52 insertions(+) diff --git a/linopy/common.py b/linopy/common.py index 44eab4b4..2b473342 100644 --- a/linopy/common.py +++ b/linopy/common.py @@ -231,6 +231,22 @@ def as_dataarray( dims are neither expanded, reordered, nor projected onto MultiIndex dims. Use :func:`broadcast_to_coords` or :func:`strict_broadcast_to_coords` when ``coords`` should govern the result's shape. + + Parameters + ---------- + arr + The input to convert. + coords + Coordinate values used to label positional axes. + dims + Dimension names used to label positional axes. + **kwargs + Forwarded to the underlying DataArray construction. + + Returns + ------- + DataArray + The converted input, dims and entries as ``arr`` provides them. """ if isinstance(arr, pd.Series | pd.DataFrame): arr = pandas_to_dataarray(arr, coords=coords, dims=dims, **kwargs) @@ -505,6 +521,24 @@ def broadcast_to_coords( Implicit MultiIndex-level projections (a level subset, or one that leaves entries uncovered) emit an :class:`~linopy.EvolvingAPIWarning`; the v1 arithmetic convention will require them to be explicit. + + Parameters + ---------- + arr + The input to convert and broadcast. + coords + Coordinate values the result is broadcast against. ``None`` falls + back to plain conversion. + dims + Dimension names used to label positional axes. + **kwargs + Forwarded to the underlying DataArray construction. + + Returns + ------- + DataArray + Broadcast against ``coords``; extra dims and disagreeing entries + pass through. """ da, projections = _broadcast_to_coords(arr, coords, dims, **kwargs) for p in projections: @@ -608,6 +642,24 @@ def strict_broadcast_to_coords( levels) are silent here — they are the documented bounds-broadcast feature, not the arithmetic-convention concern that makes :func:`broadcast_to_coords` warn. + + Parameters + ---------- + arr + The input to convert and broadcast. + coords + Coordinate values the result must stay within. + label + Name of the argument in error messages (e.g. ``"lower bound"``). + dims + Dimension names used to label positional axes. + **kwargs + Forwarded to the underlying DataArray construction. + + Returns + ------- + DataArray + Broadcast against ``coords`` and verified to stay within it. """ if coords is not None: _coords_to_dict(coords, dims=dims) From a26b92e7933e988669e90546dafbc8861161475b Mon Sep 17 00:00:00 2001 From: FBumann <117816358+FBumann@users.noreply.github.com> Date: Tue, 2 Jun 2026 11:00:18 +0200 Subject: [PATCH 08/10] refactor(common): unify the broadcast rungs into broadcast_to_coords(strict=...) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Per review discussion: one public function instead of two, with strict as a keyword flag. - strict=True (default): any mismatch with coords raises, naming label in the error — the former strict_broadcast_to_coords. - strict=False: mismatches pass through for downstream xarray alignment — the former loose broadcast_to_coords, used by arithmetic. Strict is the default so that forgetting the flag adds safety rather than silently dropping validation. MI handling preserved exactly per mode (strict: silent partial / raise on gap; non-strict: EvolvingAPIWarning) — the scenario-B deprecation warnings land separately in #732. Call sites: model.py bounds/mask drop the long name (strict is default); arithmetic and as_expression pass strict=False explicitly. Co-Authored-By: Claude Opus 4.8 (1M context) --- doc/release_notes.rst | 2 +- linopy/common.py | 141 +++++++++++++++++------------------------- linopy/expressions.py | 14 +++-- linopy/model.py | 16 ++--- linopy/variables.py | 2 +- test/test_common.py | 54 ++++++++-------- 6 files changed, 99 insertions(+), 130 deletions(-) diff --git a/doc/release_notes.rst b/doc/release_notes.rst index f878689d..c0dfa445 100644 --- a/doc/release_notes.rst +++ b/doc/release_notes.rst @@ -71,7 +71,7 @@ Most users should keep calling ``model.solve(...)``. If you want more control, y **Internal** -* ``linopy.common`` provides three DataArray conversion helpers of increasing strictness: ``as_dataarray`` (convert only), ``broadcast_to_coords`` (convert and broadcast against ``coords``), and ``strict_broadcast_to_coords`` (the same broadcast, but any mismatch raises). +* ``linopy.common`` provides two DataArray conversion helpers: ``as_dataarray`` (convert only) and ``broadcast_to_coords`` (convert and broadcast against ``coords``). The latter takes ``strict`` (default ``True``): any mismatch with ``coords`` raises, naming ``label`` in the error; ``strict=False`` passes mismatches through for downstream xarray alignment. * Each ``Solver`` subclass now overrides at most three hooks: ``_build_direct`` (build the native model), ``_run_direct`` (run it), and ``_run_file`` (run the solver on an LP/MPS file). File-only solvers (CBC, GLPK, CPLEX, SCIP, Knitro, COPT, MindOpt) only override ``_run_file``. * New ``ConstraintLabelIndex`` cached on ``Model.constraints`` (mirrors the existing ``Variables.label_index``); ``ConstraintBase`` gains ``active_labels()`` and a ``range`` property; ``CSRConstraint`` exposes ``coords``. * ``linopy.common`` gains ``values_to_lookup_array``; the legacy pandas-based helpers ``series_to_lookup_array`` and ``lookup_vals`` are removed. diff --git a/linopy/common.py b/linopy/common.py index 2b473342..71e05c0b 100644 --- a/linopy/common.py +++ b/linopy/common.py @@ -229,7 +229,7 @@ def as_dataarray( polars, numpy, scalar, DataArray) and labels positional axes with ``dims`` / ``coords``. The result is not reshaped against ``coords``: dims are neither expanded, reordered, nor projected onto MultiIndex - dims. Use :func:`broadcast_to_coords` or :func:`strict_broadcast_to_coords` when + dims. Use :func:`broadcast_to_coords` when ``coords`` should govern the result's shape. Parameters @@ -502,6 +502,9 @@ def broadcast_to_coords( arr: Any, coords: CoordsLike | None = None, dims: DimsLike | None = None, + *, + strict: bool = True, + label: str | None = None, **kwargs: Any, ) -> DataArray: """ @@ -513,14 +516,18 @@ def broadcast_to_coords( are expanded, dims naming levels of a stacked-MultiIndex coords dim are projected onto it, and the result is transposed to ``coords`` order. - Dims of ``arr`` not present in ``coords``, and shared dims with - disagreeing value sets, pass through unchanged so downstream xarray - alignment can handle them. Use :func:`strict_broadcast_to_coords` to enforce that - ``arr`` stays within ``coords``. + ``strict`` decides what happens to anything broadcasting alone cannot + resolve — extra dims, disagreeing coord values, and MultiIndex coverage + gaps: - Implicit MultiIndex-level projections (a level subset, or one that - leaves entries uncovered) emit an :class:`~linopy.EvolvingAPIWarning`; - the v1 arithmetic convention will require them to be explicit. + - ``strict=True`` (default): raise, naming ``label`` in the error. + Partial-level broadcasts stay silent (the documented bounds-broadcast + feature). + - ``strict=False``: pass through unchanged so downstream xarray + alignment can handle them. Implicit MultiIndex-level projections + (a level subset, or one that leaves entries uncovered) emit an + :class:`~linopy.EvolvingAPIWarning`; the v1 arithmetic convention + will require them to be explicit. Parameters ---------- @@ -531,31 +538,56 @@ def broadcast_to_coords( back to plain conversion. dims Dimension names used to label positional axes. + strict + Check that the result stays within ``coords`` (raise on violation) + instead of passing violations through. + label + Name of the argument in error messages (e.g. ``"lower bound"``); + only used when ``strict=True``. **kwargs Forwarded to the underlying DataArray construction. Returns ------- DataArray - Broadcast against ``coords``; extra dims and disagreeing entries - pass through. - """ - da, projections = _broadcast_to_coords(arr, coords, dims, **kwargs) + Broadcast against ``coords``. + """ + if not strict: + da, projections = _broadcast_to_coords(arr, coords, dims, **kwargs) + for p in projections: + if p.is_partial or p.has_gap: + kind = ( + f"broadcasting level subset {p.levels}" + if p.is_partial + else f"filling uncovered entries with NaN (from level(s) {p.levels})" + ) + warn( + f"multiindex-projection: implicitly {kind} onto MultiIndex " + f"dimension {p.dim!r}. The v1 arithmetic convention will require " + f"this to be explicit; reindex onto the dimension or use a " + f"named method with `join=` to keep current behavior.", + EvolvingAPIWarning, + stacklevel=2, + ) + return da + + subject = label or "Value" + if coords is not None: + _coords_to_dict(coords, dims=dims) + try: + da, projections = _broadcast_to_coords(arr, coords, dims=dims, **kwargs) + except TypeError as err: + raise TypeError(f"{subject} could not be aligned to coords: {err}") from err + except (ValueError, CoordinateValidationError) as err: + raise ValueError(f"{subject} could not be aligned to coords: {err}") from err for p in projections: - if p.is_partial or p.has_gap: - kind = ( - f"broadcasting level subset {p.levels}" - if p.is_partial - else f"filling uncovered entries with NaN (from level(s) {p.levels})" - ) - warn( - f"multiindex-projection: implicitly {kind} onto MultiIndex " - f"dimension {p.dim!r}. The v1 arithmetic convention will require " - f"this to be explicit; reindex onto the dimension or use a " - f"named method with `join=` to keep current behavior.", - EvolvingAPIWarning, - stacklevel=2, + if p.has_gap: + raise ValueError( + f"{subject} could not be aligned to coords: input does not cover " + f"every entry of MultiIndex dimension {p.dim!r} (aligned from " + f"level(s) {p.levels})." ) + validate_alignment(da, coords, dims=dims, label=label) return da @@ -621,65 +653,6 @@ def validate_alignment( ) -def strict_broadcast_to_coords( - arr: Any, - coords: CoordsLike | None, - *, - label: str, - dims: DimsLike | None = None, - **kwargs: Any, -) -> DataArray: - """ - :func:`broadcast_to_coords` with a strict failure mode. - - The same broadcast, but anything it cannot resolve by broadcasting alone - raises instead of passing through: extra dims, disagreeing coord values, - and MultiIndex coverage gaps. Errors are raised as :class:`ValueError` / - :class:`TypeError` naming ``label``; coords-parsing errors propagate - unchanged. - - Partial-level broadcasts (input indexed by a subset of a MultiIndex's - levels) are silent here — they are the documented bounds-broadcast - feature, not the arithmetic-convention concern that makes - :func:`broadcast_to_coords` warn. - - Parameters - ---------- - arr - The input to convert and broadcast. - coords - Coordinate values the result must stay within. - label - Name of the argument in error messages (e.g. ``"lower bound"``). - dims - Dimension names used to label positional axes. - **kwargs - Forwarded to the underlying DataArray construction. - - Returns - ------- - DataArray - Broadcast against ``coords`` and verified to stay within it. - """ - if coords is not None: - _coords_to_dict(coords, dims=dims) - try: - da, projections = _broadcast_to_coords(arr, coords, dims=dims, **kwargs) - except TypeError as err: - raise TypeError(f"{label} could not be aligned to coords: {err}") from err - except (ValueError, CoordinateValidationError) as err: - raise ValueError(f"{label} could not be aligned to coords: {err}") from err - for p in projections: - if p.has_gap: - raise ValueError( - f"{label} could not be aligned to coords: input does not cover " - f"every entry of MultiIndex dimension {p.dim!r} (aligned from " - f"level(s) {p.levels})." - ) - validate_alignment(da, coords, dims=dims, label=label) - return da - - def _coords_to_dict( coords: Sequence[Sequence | pd.Index] | Mapping, dims: DimsLike | None = None, diff --git a/linopy/expressions.py b/linopy/expressions.py index 27918ce7..673eaba9 100644 --- a/linopy/expressions.py +++ b/linopy/expressions.py @@ -583,7 +583,9 @@ def _add_constant( # so that missing data does not silently propagate through arithmetic. if np.isscalar(other) and join is None: return self.assign(const=self.const.fillna(0) + other) - da = broadcast_to_coords(other, coords=self.coords, dims=self.coord_dims) + da = broadcast_to_coords( + other, coords=self.coords, dims=self.coord_dims, strict=False + ) self_const, da, needs_data_reindex = self._align_constant( da, fill_value=0, join=join ) @@ -612,7 +614,9 @@ def _apply_constant_op( - factor (other) is filled with fill_value (0 for mul, 1 for div) - coeffs and const are filled with 0 (additive identity) """ - factor = broadcast_to_coords(other, coords=self.coords, dims=self.coord_dims) + factor = broadcast_to_coords( + other, coords=self.coords, dims=self.coord_dims, strict=False + ) self_const, factor, needs_data_reindex = self._align_constant( factor, fill_value=fill_value, join=join ) @@ -1104,7 +1108,9 @@ def to_constraint( ) if isinstance(rhs, CONSTANT_TYPES): - rhs = broadcast_to_coords(rhs, coords=self.coords, dims=self.coord_dims) + rhs = broadcast_to_coords( + rhs, coords=self.coords, dims=self.coord_dims, strict=False + ) extra_dims = set(rhs.dims) - set(self.coord_dims) if extra_dims: @@ -2308,7 +2314,7 @@ def as_expression( return obj.to_linexpr() else: try: - obj = broadcast_to_coords(obj, **kwargs) + obj = broadcast_to_coords(obj, strict=False, **kwargs) except ValueError as e: raise ValueError("Cannot convert to LinearExpression") from e return LinearExpression(obj, model) diff --git a/linopy/model.py b/linopy/model.py index 6a2abcb0..aa0e5d29 100644 --- a/linopy/model.py +++ b/linopy/model.py @@ -31,9 +31,9 @@ as_dataarray, assign_multiindex_safe, best_int, + broadcast_to_coords, maybe_replace_signs, replace_by_map, - strict_broadcast_to_coords, to_path, ) from linopy.constants import ( @@ -774,12 +774,8 @@ def add_variables( "Semi-continuous variables require a positive scalar lower bound." ) - lower_da = strict_broadcast_to_coords( - lower, coords, label="lower bound", **kwargs - ) - upper_da = strict_broadcast_to_coords( - upper, coords, label="upper bound", **kwargs - ) + lower_da = broadcast_to_coords(lower, coords, label="lower bound", **kwargs) + upper_da = broadcast_to_coords(upper, coords, label="upper bound", **kwargs) data = Dataset( { "lower": lower_da, @@ -792,7 +788,7 @@ def add_variables( self._check_valid_dim_names(data) if mask is not None: - mask = strict_broadcast_to_coords( + mask = broadcast_to_coords( mask, coords if coords is not None else data.coords, label="mask", @@ -1061,9 +1057,7 @@ def add_constraints( (data,) = xr.broadcast(data, exclude=[TERM_DIM]) if mask is not None: - mask = strict_broadcast_to_coords(mask, data.coords, label="mask").astype( - bool - ) + mask = broadcast_to_coords(mask, data.coords, label="mask").astype(bool) # Auto-mask based on null expressions or NaN RHS (use numpy for speed) if self.auto_mask: diff --git a/linopy/variables.py b/linopy/variables.py index b8397c59..755a3afc 100644 --- a/linopy/variables.py +++ b/linopy/variables.py @@ -329,7 +329,7 @@ def to_linexpr( Linear expression with the variables and coefficients. """ coefficient = broadcast_to_coords( - coefficient, coords=self.coords, dims=self.dims + coefficient, coords=self.coords, dims=self.dims, strict=False ) coefficient = coefficient.reindex_like(self.labels, fill_value=0) coefficient = coefficient.fillna(0) diff --git a/test/test_common.py b/test/test_common.py index 030630bf..845ac2e1 100644 --- a/test/test_common.py +++ b/test/test_common.py @@ -28,7 +28,6 @@ is_constant, iterate_slices, maybe_group_terms_polars, - strict_broadcast_to_coords, validate_alignment, ) from linopy.testing import assert_linequal, assert_varequal @@ -365,7 +364,7 @@ def test_broadcast_to_coords_with_ndarray_coords_dict_set_dims_not_aligned() -> target_dims = ("dim_0", "dim_1") target_coords = {"dim_0": ["a", "b"], "dim_2": ["A", "B"]} arr = np.array([[1, 2], [3, 4]]) - da = broadcast_to_coords(arr, coords=target_coords, dims=target_dims) + da = broadcast_to_coords(arr, coords=target_coords, dims=target_dims, strict=False) # dims labels the positional axes; coords adds dim_2 by broadcast. assert set(da.dims) == {"dim_0", "dim_1", "dim_2"} assert list(da.coords["dim_0"].values) == ["a", "b"] @@ -509,7 +508,7 @@ def test_broadcast_to_coords_preserves_extra_dims() -> None: coords={"a": [0, 1, 2], "t": [10, 20]}, ) coords = {"a": [0, 1, 2]} - da = broadcast_to_coords(arr, coords=coords) + da = broadcast_to_coords(arr, coords=coords, strict=False) assert set(da.dims) == {"a", "t"} assert list(da.coords["t"].values) == [10, 20] @@ -518,7 +517,7 @@ def test_broadcast_to_coords_keeps_disjoint_shared_dim_values() -> None: """Different value sets on a shared dim are passed through (xr.align handles).""" arr = DataArray([1, 2, 3, 4, 5], dims=["a"], coords={"a": [0, 1, 2, 3, 4]}) coords = {"a": [2, 3]} - da = broadcast_to_coords(arr, coords=coords) + da = broadcast_to_coords(arr, coords=coords, strict=False) # No exception, no reindex; downstream alignment intersects. assert list(da.coords["a"].values) == [0, 1, 2, 3, 4] @@ -543,6 +542,7 @@ def test_broadcast_to_coords_expands_missing_multiindex_dim_keeps_levels() -> No DataArray([1.0], coords={"name": ["1"]}, dims=["name"]), coords=labels.coords, dims=labels.dims, + strict=False, ) assert set(coeff.xindexes) == {"snapshot", "period", "timestep", "name"} coeff.reindex_like(labels, fill_value=0) @@ -562,7 +562,7 @@ def test_broadcast_to_coords_broadcasts_single_multiindex_level() -> None: by_level1 = DataArray([10.0, 20.0], coords={"level1": [1, 2]}, dims=["level1"]) with pytest.warns(EvolvingAPIWarning, match=r"broadcasting level subset"): - da = broadcast_to_coords(by_level1, coords=coords, dims=["dim_3"]) + da = broadcast_to_coords(by_level1, coords=coords, dims=["dim_3"], strict=False) assert da.dims == ("dim_3",) assert isinstance(da.indexes["dim_3"], pd.MultiIndex) @@ -588,7 +588,7 @@ def test_broadcast_to_coords_stacks_full_multiindex_levels() -> None: weights = pd.Series([10.0, 20.0], index=subset) with pytest.warns(EvolvingAPIWarning, match=r"filling uncovered entries with NaN"): - da = broadcast_to_coords(weights, coords=coords, dims=["dim_3"]) + da = broadcast_to_coords(weights, coords=coords, dims=["dim_3"], strict=False) assert da.dims == ("dim_3",) assert isinstance(da.indexes["dim_3"], pd.MultiIndex) @@ -613,7 +613,7 @@ def test_broadcast_to_coords_full_multiindex_full_coverage_is_silent() -> None: with warnings.catch_warnings(): warnings.simplefilter("error", EvolvingAPIWarning) - da = broadcast_to_coords(full, coords=coords, dims=["dim_3"]) + da = broadcast_to_coords(full, coords=coords, dims=["dim_3"], strict=False) assert da.dims == ("dim_3",) assert da.values.tolist() == [1.0, 2.0, 3.0, 4.0] @@ -630,7 +630,7 @@ def test_broadcast_to_coords_level_projection_ambiguous_raises() -> None: arr = DataArray([1.0, 2.0], coords={"shared": [1, 2]}, dims=["shared"]) with pytest.raises(ValueError, match=r"shared.*shared by MultiIndex"): - broadcast_to_coords(arr, coords=coords) + broadcast_to_coords(arr, coords=coords, strict=False) def test_broadcast_to_coords_level_projection_missing_value_raises() -> None: @@ -641,7 +641,7 @@ def test_broadcast_to_coords_level_projection_missing_value_raises() -> None: by_level1 = DataArray([10.0, 20.0], coords={"level1": [1, 9]}, dims=["level1"]) with pytest.raises(ValueError, match=r"Cannot align level.*is missing"): - broadcast_to_coords(by_level1, coords=coords, dims=["dim_3"]) + broadcast_to_coords(by_level1, coords=coords, dims=["dim_3"], strict=False) def test_broadcast_to_coords_unrelated_multiindex_series_still_unstacks() -> None: @@ -649,13 +649,13 @@ def test_broadcast_to_coords_unrelated_multiindex_series_still_unstacks() -> Non sub = pd.MultiIndex.from_product([["p", "q"], [1, 2]], names=["foo", "bar"]) series = pd.Series([1.0, 2.0, 3.0, 4.0], index=sub) - da = broadcast_to_coords(series, coords={"time": [0, 1, 2]}) + da = broadcast_to_coords(series, coords={"time": [0, 1, 2]}, strict=False) assert set(da.dims) == {"time", "foo", "bar"} # --------------------------------------------------------------------------- -# Strictness ladder: as_dataarray ⊂ broadcast_to_coords ⊂ strict_broadcast_to_coords +# Strictness: as_dataarray (convert) ⊂ broadcast_to_coords(strict=False) ⊂ broadcast_to_coords(strict=True) # --------------------------------------------------------------------------- @@ -667,7 +667,7 @@ def test_as_dataarray_does_not_expand_missing_coord_dims() -> None: converted = as_dataarray(arr, coords=coords, dims=["a"]) assert converted.dims == ("a",) - broadcast = broadcast_to_coords(arr, coords=coords, dims=["a"]) + broadcast = broadcast_to_coords(arr, coords=coords, dims=["a"], strict=False) assert broadcast.dims == ("a", "b") @@ -678,14 +678,14 @@ def test_extra_dims_pass_broadcast_rung_fail_strict_rung() -> None: ) coords = {"a": [0, 1]} - da = broadcast_to_coords(arr, coords=coords) + da = broadcast_to_coords(arr, coords=coords, strict=False) assert set(da.dims) == {"a", "t"} with pytest.raises(ValueError, match=r"not declared in coords"): - strict_broadcast_to_coords(arr, coords, label="lower bound") + broadcast_to_coords(arr, coords, label="lower bound") -def test_strict_broadcast_to_coords_rejects_multiindex_coverage_gap() -> None: +def test_broadcast_to_coords_rejects_multiindex_coverage_gap() -> None: """A coverage gap warns on the broadcast rung but raises on the strict rung.""" idx = pd.MultiIndex.from_product([[1, 2], ["a", "b"]], names=("level1", "level2")) idx.name = "dim_3" @@ -694,13 +694,13 @@ def test_strict_broadcast_to_coords_rejects_multiindex_coverage_gap() -> None: weights = pd.Series([10.0, 20.0], index=subset) with pytest.warns(EvolvingAPIWarning, match=r"filling uncovered entries"): - broadcast_to_coords(weights, coords=coords, dims=["dim_3"]) + broadcast_to_coords(weights, coords=coords, dims=["dim_3"], strict=False) with pytest.raises(ValueError, match=r"does not cover every entry"): - strict_broadcast_to_coords(weights, coords, dims=["dim_3"], label="lower bound") + broadcast_to_coords(weights, coords, dims=["dim_3"], label="lower bound") -def test_strict_broadcast_to_coords_allows_partial_level_broadcast_silently() -> None: +def test_broadcast_to_coords_allows_partial_level_broadcast_silently() -> None: """Per-level bounds broadcast across the MI dim without the arithmetic warning.""" idx = pd.MultiIndex.from_product([[1, 2], ["a", "b"]], names=("level1", "level2")) idx.name = "dim_3" @@ -709,9 +709,7 @@ def test_strict_broadcast_to_coords_allows_partial_level_broadcast_silently() -> with warnings.catch_warnings(): warnings.simplefilter("error", EvolvingAPIWarning) - da = strict_broadcast_to_coords( - by_level1, coords, dims=["dim_3"], label="lower bound" - ) + da = broadcast_to_coords(by_level1, coords, dims=["dim_3"], label="lower bound") assert da.sel(dim_3=(1, "b")).item() == 10.0 assert da.sel(dim_3=(2, "a")).item() == 20.0 @@ -757,24 +755,22 @@ def test_validate_alignment_label_in_error() -> None: validate_alignment(arr, {"a": [0, 1]}, label="lower bound") -def test_strict_broadcast_to_coords_wraps_conversion_errors() -> None: +def test_broadcast_to_coords_wraps_conversion_errors() -> None: with pytest.raises(ValueError, match=r"lower bound could not be aligned"): - strict_broadcast_to_coords( - np.array([1, 2]), {"x": [0, 1, 2]}, label="lower bound" - ) + broadcast_to_coords(np.array([1, 2]), {"x": [0, 1, 2]}, label="lower bound") -def test_strict_broadcast_to_coords_preserves_type_errors() -> None: +def test_broadcast_to_coords_preserves_type_errors() -> None: """Unsupported input types stay TypeError (don't become ValueError).""" with pytest.raises(TypeError, match=r"lower bound could not be aligned"): - strict_broadcast_to_coords(lambda x: x, {"x": [0, 1, 2]}, label="lower bound") + broadcast_to_coords(lambda x: x, {"x": [0, 1, 2]}, label="lower bound") -def test_strict_broadcast_to_coords_does_not_relabel_coords_errors() -> None: +def test_broadcast_to_coords_does_not_relabel_coords_errors() -> None: """Coords-side TypeError carries its own message, not the value label.""" mi = pd.MultiIndex.from_product([[0, 1], ["a", "b"]], names=["i", "j"]) with pytest.raises(TypeError, match=r"MultiIndex.*must have \.name set"): - strict_broadcast_to_coords(np.array([1, 2, 3, 4]), [mi], label="lower bound") + broadcast_to_coords(np.array([1, 2, 3, 4]), [mi], label="lower bound") class TestCoordsToDictRules: From a6984d85221b457c182a14d421fd21eff07e55c8 Mon Sep 17 00:00:00 2001 From: FBumann <117816358+FBumann@users.noreply.github.com> Date: Tue, 2 Jun 2026 11:26:59 +0200 Subject: [PATCH 09/10] feat(common): require label when broadcast_to_coords is strict Restores the contract align_to_coords always had: strict-mode errors must name their subject ("lower bound could not be aligned..." rather than "Value could not be aligned..."). Enforced both statically (overloads: strict=True requires label: str, strict=False forbids it) and at runtime (TypeError). Co-Authored-By: Claude Opus 4.8 (1M context) --- linopy/common.py | 37 +++++++++++++++++++++++++++++++++---- test/test_common.py | 6 ++++++ 2 files changed, 39 insertions(+), 4 deletions(-) diff --git a/linopy/common.py b/linopy/common.py index 71e05c0b..e82f1f6d 100644 --- a/linopy/common.py +++ b/linopy/common.py @@ -12,7 +12,7 @@ from collections.abc import Callable, Generator, Hashable, Iterable, Mapping, Sequence from functools import cached_property, partial, reduce, wraps from pathlib import Path -from typing import TYPE_CHECKING, Any, Generic, NamedTuple, TypeVar, overload +from typing import TYPE_CHECKING, Any, Generic, Literal, NamedTuple, TypeVar, overload from warnings import warn import numpy as np @@ -498,6 +498,30 @@ def _broadcast_to_coords( return arr, projections +@overload +def broadcast_to_coords( + arr: Any, + coords: CoordsLike | None = ..., + dims: DimsLike | None = ..., + *, + strict: Literal[True] = ..., + label: str, + **kwargs: Any, +) -> DataArray: ... + + +@overload +def broadcast_to_coords( + arr: Any, + coords: CoordsLike | None = ..., + dims: DimsLike | None = ..., + *, + strict: Literal[False], + label: None = ..., + **kwargs: Any, +) -> DataArray: ... + + def broadcast_to_coords( arr: Any, coords: CoordsLike | None = None, @@ -542,8 +566,8 @@ def broadcast_to_coords( Check that the result stays within ``coords`` (raise on violation) instead of passing violations through. label - Name of the argument in error messages (e.g. ``"lower bound"``); - only used when ``strict=True``. + Name of the input in error messages (e.g. ``"lower bound"``). + Required when ``strict=True``, not accepted otherwise. **kwargs Forwarded to the underlying DataArray construction. @@ -571,7 +595,12 @@ def broadcast_to_coords( ) return da - subject = label or "Value" + if label is None: + raise TypeError( + "broadcast_to_coords(strict=True) requires `label` to name the " + "input in error messages, e.g. label='lower bound'." + ) + subject = label if coords is not None: _coords_to_dict(coords, dims=dims) try: diff --git a/test/test_common.py b/test/test_common.py index 845ac2e1..d7dfe283 100644 --- a/test/test_common.py +++ b/test/test_common.py @@ -755,6 +755,12 @@ def test_validate_alignment_label_in_error() -> None: validate_alignment(arr, {"a": [0, 1]}, label="lower bound") +def test_broadcast_to_coords_strict_requires_label() -> None: + """strict=True without label raises: errors must name their subject.""" + with pytest.raises(TypeError, match=r"requires `label`"): + broadcast_to_coords(np.array([1, 2]), {"x": [0, 1]}) # type: ignore[call-overload] + + def test_broadcast_to_coords_wraps_conversion_errors() -> None: with pytest.raises(ValueError, match=r"lower bound could not be aligned"): broadcast_to_coords(np.array([1, 2]), {"x": [0, 1, 2]}, label="lower bound") From 00990e8ae30f8e7fc85c87783349e04f13303afa Mon Sep 17 00:00:00 2001 From: FBumann <117816358+FBumann@users.noreply.github.com> Date: Tue, 2 Jun 2026 11:53:34 +0200 Subject: [PATCH 10/10] feat(common): deprecate implicit MI-level projection everywhere (scenario B) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Per the #737 review discussion and Fabian's decision: implicit level projection is deprecated and will raise under the v1 convention, so the EvolvingAPIWarning now fires in both modes of broadcast_to_coords — the MI check is the same for every use case: - input missing a whole level: warn (strict and non-strict) - coverage gap (level combinations without a value): warn (non-strict) / raise (strict — no downstream layer to defer the NaN to) Warning emission lives in one helper, _warn_implicit_projections, with a TODO(#738) to migrate to LinopySemanticsWarning once #717 lands. Also clarifies the MultiIndex terminology everywhere: an MI dim has *levels* and *level combinations* (one tuple per position). Docstrings carry the glossary, the coverage-gap error names the missing combinations explicitly, and "entry" is gone from messages. User-facing: add_variables / add_constraints with per-period-style bounds now emit the deprecation warning (PyPSA multi-investment). Co-Authored-By: Claude Opus 4.8 (1M context) --- doc/release_notes.rst | 2 +- linopy/common.py | 108 ++++++++++++++++++++++++++++++------------ test/test_common.py | 25 ++++++---- test/test_variable.py | 9 +++- 4 files changed, 102 insertions(+), 42 deletions(-) diff --git a/doc/release_notes.rst b/doc/release_notes.rst index c0dfa445..7738d1aa 100644 --- a/doc/release_notes.rst +++ b/doc/release_notes.rst @@ -55,7 +55,7 @@ Most users should keep calling ``model.solve(...)``. If you want more control, y **Bug Fixes** * ``add_variables`` / ``add_constraints``: extends 0.7.0's coords-as-truth rule to ``lower``, ``upper`` and ``mask`` for every bound type and dim order. Pandas ``Series`` / ``DataFrame`` bounds or masks missing a dimension are broadcast to ``coords`` instead of being silently dropped (`#709 `__); the variable's dimension order always follows ``coords`` (`#706 `__); bare-tuple coord entries (``coords=[(0, 1, 2)]``) now behave like lists. Mismatched values or extra dims raise ``ValueError`` with a labelled message; sparse-coord masks (formerly a v0.6.3 ``FutureWarning``, #580) raise ``ValueError``, and masks with dims not in the data raise ``ValueError`` instead of ``AssertionError``. -* Pandas inputs whose index names *levels* of a stacked-``MultiIndex`` ``coords`` dimension are now projected onto that dimension: a level subset broadcasts across the others, the full set aligns element-wise. This fixes PyPSA multi-investment arithmetic (e.g. an expression over a ``(period, timestep)`` ``snapshot`` MultiIndex times a ``period``-indexed weighting). In ``add_variables`` / ``add_constraints`` the input must cover every entry of the MultiIndex or a ``ValueError`` is raised. On the arithmetic path the same projections still work but now emit an ``EvolvingAPIWarning`` when they rely on an *implicit* broadcast (level subset) or NaN-fill (uncovered entries) — the upcoming v1 arithmetic convention will require these to be made explicit (e.g. ``.reindex`` onto the dimension, or a named ``.mul(..., join=...)``). Aligning the full level set with full coverage stays silent. +* Pandas inputs whose index names *levels* of a stacked-``MultiIndex`` ``coords`` dimension are now projected onto that dimension: a level subset broadcasts across the others, the full set aligns element-wise. This fixes PyPSA multi-investment arithmetic (e.g. an expression over a ``(period, timestep)`` ``snapshot`` MultiIndex times a ``period``-indexed weighting). In ``add_variables`` / ``add_constraints`` the input must provide a value for every level combination of the MultiIndex or a ``ValueError`` is raised (the error lists the missing combinations). **Implicit level projections are deprecated**: they emit an ``EvolvingAPIWarning`` everywhere — in arithmetic *and* in ``add_variables`` / ``add_constraints`` — and will raise under the upcoming v1 convention. Project the input onto the dimension explicitly (select with the dimension's level values) to keep current behavior. Aligning the full level set with full coverage stays silent. * ``add_piecewise_formulation`` now produces a reproducible dimension order in the broadcast breakpoint array. The previous set-based expansion gave a hash-randomized order that varied between processes. * SOS constraints on masked variables no longer cause solver-specific failures (Gurobi ``IndexError``, Xpress ``?404 Invalid column number``, LP parse errors, silent set corruption). ``Model.solve()`` and ``Model.to_file()`` now raise a clear ``NotImplementedError`` referring users to `#688 `__; pass ``reformulate_sos=True`` as a workaround. * ``Model.solve(..., reformulate_sos=True)`` now actually reformulates SOS constraints even when the solver supports them natively. Previously it was silently ignored with a warning. diff --git a/linopy/common.py b/linopy/common.py index e82f1f6d..2ed979a3 100644 --- a/linopy/common.py +++ b/linopy/common.py @@ -303,12 +303,19 @@ def _as_multiindex(coord_values: Any) -> pd.MultiIndex | None: class _LevelProjection(NamedTuple): - """Record of one MultiIndex-level projection performed by ``_broadcast_to_coords``.""" + """ + Record of one MultiIndex-level projection performed by ``_broadcast_to_coords``. + + Terminology: a stacked MultiIndex dim has *levels* (its component index + names, e.g. ``period`` / ``timestep``) and *level combinations* (its + elements — one tuple per position, e.g. ``(2030, 't1')``). + """ dim: Hashable levels: list[Hashable] is_partial: bool # input carried only a subset of the MI's levels - has_gap: bool # projection left entries of the MI dim uncovered (NaN) + has_gap: bool # some level combinations of the MI dim got no value (NaN) + missing: list[Any] # the level combinations that got no value def _project_onto_multiindex_levels( @@ -318,10 +325,10 @@ def _project_onto_multiindex_levels( """ Map ``arr`` dims that name levels of a stacked-MultiIndex coords dim onto it. - For every entry of the MultiIndex dim, select the ``arr`` value at that - entry's level values. A subset of levels broadcasts across the remaining - ones; the full set aligns element-wise. ``arr`` is returned unchanged - when it carries no level dims. + For every level combination of the MultiIndex dim, select the ``arr`` + value at that combination's level values. A subset of levels broadcasts + across the remaining ones; the full set aligns element-wise. ``arr`` is + returned unchanged when it carries no level dims. Raises ``ValueError`` only on structural errors: a level name owned by two MI dims, or a level value missing from ``arr``. Partial projections @@ -369,12 +376,21 @@ def _project_onto_multiindex_levels( f"{dim!r}: value {err} is missing." ) from err arr = arr.assign_coords(Coordinates.from_pandas_multiindex(mi, dim)) + # A level combination is "missing" when the projection gave it no + # value at any position of the other dims. + null_mask = arr.isnull() + other_dims = [d for d in arr.dims if d != dim] + if other_dims: + null_mask = null_mask.any(other_dims) + has_gap = bool(null_mask.any()) + missing = list(arr.indexes[dim][null_mask.values]) if has_gap else [] projections.append( _LevelProjection( dim=dim, levels=levels, is_partial=len(levels) < sum(name is not None for name in mi.names), - has_gap=bool(arr.isnull().any()), + has_gap=has_gap, + missing=missing, ) ) @@ -545,13 +561,21 @@ def broadcast_to_coords( gaps: - ``strict=True`` (default): raise, naming ``label`` in the error. - Partial-level broadcasts stay silent (the documented bounds-broadcast - feature). - ``strict=False``: pass through unchanged so downstream xarray - alignment can handle them. Implicit MultiIndex-level projections - (a level subset, or one that leaves entries uncovered) emit an - :class:`~linopy.EvolvingAPIWarning`; the v1 arithmetic convention - will require them to be explicit. + alignment can handle them. + + A stacked-MultiIndex dim of ``coords`` has *levels* (its component + index names, e.g. ``period`` / ``timestep``) and *level combinations* + (its elements — one tuple per position, e.g. ``(2030, 't1')``). Inputs + indexed by levels instead of the dim itself are implicitly projected + onto the dim's level combinations. These projections are deprecated in + both modes and emit an :class:`~linopy.EvolvingAPIWarning`; the v1 + convention will require them to be explicit. Two cases: + + - input misses a whole level → broadcasts across it; warns in both modes. + - input gives some level combinations no value (a *coverage gap*) → + warns under ``strict=False``, raises under ``strict=True`` (the error + lists the missing combinations). Parameters ---------- @@ -578,21 +602,7 @@ def broadcast_to_coords( """ if not strict: da, projections = _broadcast_to_coords(arr, coords, dims, **kwargs) - for p in projections: - if p.is_partial or p.has_gap: - kind = ( - f"broadcasting level subset {p.levels}" - if p.is_partial - else f"filling uncovered entries with NaN (from level(s) {p.levels})" - ) - warn( - f"multiindex-projection: implicitly {kind} onto MultiIndex " - f"dimension {p.dim!r}. The v1 arithmetic convention will require " - f"this to be explicit; reindex onto the dimension or use a " - f"named method with `join=` to keep current behavior.", - EvolvingAPIWarning, - stacklevel=2, - ) + _warn_implicit_projections(projections) return da if label is None: @@ -611,15 +621,51 @@ def broadcast_to_coords( raise ValueError(f"{subject} could not be aligned to coords: {err}") from err for p in projections: if p.has_gap: + preview = ", ".join(str(c) for c in p.missing[:5]) + if len(p.missing) > 5: + preview += f", … ({len(p.missing)} in total)" raise ValueError( - f"{subject} could not be aligned to coords: input does not cover " - f"every entry of MultiIndex dimension {p.dim!r} (aligned from " - f"level(s) {p.levels})." + f"{subject} could not be aligned to coords: no value for " + f"{len(p.missing)} level combination(s) of MultiIndex dimension " + f"{p.dim!r}: {preview}. The input is indexed by level(s) " + f"{p.levels} and must cover every combination." ) + _warn_implicit_projections(projections) validate_alignment(da, coords, dims=dims, label=label) return da +def _warn_implicit_projections(projections: list[_LevelProjection]) -> None: + """ + Deprecation warnings for implicit MultiIndex-level projections. + + The same check in every mode (scenario B of the #732 / #737 discussion): + implicit projection is deprecated and raises under the v1 convention. The + strict path raises on coverage gaps before reaching here, so only partial + levels warn there; the non-strict path warns for both. + + TODO(#738): migrate to ``warn_legacy()`` / ``LinopySemanticsWarning`` + once the v1 semantics infrastructure (#717) lands. + """ + for p in projections: + if p.is_partial or p.has_gap: + kind = ( + f"broadcasting level subset {p.levels}" + if p.is_partial + else f"filling uncovered level combinations with NaN " + f"(from level(s) {p.levels})" + ) + warn( + f"multiindex-projection: implicitly {kind} onto MultiIndex " + f"dimension {p.dim!r}. This is deprecated and will raise under " + f"the v1 convention; project the input onto the dimension " + f"explicitly (select with the dimension's level values) to " + f"keep current behavior.", + EvolvingAPIWarning, + stacklevel=3, + ) + + def validate_alignment( arr: DataArray, coords: CoordsLike | None, diff --git a/test/test_common.py b/test/test_common.py index d7dfe283..86735547 100644 --- a/test/test_common.py +++ b/test/test_common.py @@ -578,7 +578,7 @@ def test_broadcast_to_coords_stacks_full_multiindex_levels() -> None: PyPSA's storage_weightings is a pandas Series over a (period, timestep) MultiIndex subset (the last snapshot of each period); it must align onto - the matching entries of the 'snapshot' MultiIndex. Entries the subset does + the matching level combinations of the 'snapshot' MultiIndex. Combinations the subset does not cover are left as NaN (broadcast path). """ idx = pd.MultiIndex.from_product([[1, 2], ["a", "b"]], names=("level1", "level2")) @@ -587,7 +587,9 @@ def test_broadcast_to_coords_stacks_full_multiindex_levels() -> None: subset = pd.MultiIndex.from_tuples([(1, "a"), (2, "b")], names=["level1", "level2"]) weights = pd.Series([10.0, 20.0], index=subset) - with pytest.warns(EvolvingAPIWarning, match=r"filling uncovered entries with NaN"): + with pytest.warns( + EvolvingAPIWarning, match=r"filling uncovered level combinations" + ): da = broadcast_to_coords(weights, coords=coords, dims=["dim_3"], strict=False) assert da.dims == ("dim_3",) @@ -693,22 +695,29 @@ def test_broadcast_to_coords_rejects_multiindex_coverage_gap() -> None: subset = pd.MultiIndex.from_tuples([(1, "a"), (2, "b")], names=["level1", "level2"]) weights = pd.Series([10.0, 20.0], index=subset) - with pytest.warns(EvolvingAPIWarning, match=r"filling uncovered entries"): + with pytest.warns( + EvolvingAPIWarning, match=r"filling uncovered level combinations" + ): broadcast_to_coords(weights, coords=coords, dims=["dim_3"], strict=False) - with pytest.raises(ValueError, match=r"does not cover every entry"): + with pytest.raises(ValueError, match=r"no value for .* level combination"): broadcast_to_coords(weights, coords, dims=["dim_3"], label="lower bound") -def test_broadcast_to_coords_allows_partial_level_broadcast_silently() -> None: - """Per-level bounds broadcast across the MI dim without the arithmetic warning.""" +def test_broadcast_to_coords_strict_partial_level_warns() -> None: + """ + Per-level bounds broadcast across the MI dim, with the deprecation warning. + + Scenario B (#732 / #737 discussion): implicit MI-level projection is + deprecated everywhere, including the strict (bounds/mask) path, and will + raise under the v1 convention. + """ idx = pd.MultiIndex.from_product([[1, 2], ["a", "b"]], names=("level1", "level2")) idx.name = "dim_3" coords = xr.Coordinates.from_pandas_multiindex(idx, "dim_3") by_level1 = DataArray([10.0, 20.0], coords={"level1": [1, 2]}, dims=["level1"]) - with warnings.catch_warnings(): - warnings.simplefilter("error", EvolvingAPIWarning) + with pytest.warns(EvolvingAPIWarning, match=r"broadcasting level subset"): da = broadcast_to_coords(by_level1, coords, dims=["dim_3"], label="lower bound") assert da.sel(dim_3=(1, "b")).item() == 10.0 diff --git a/test/test_variable.py b/test/test_variable.py index d3629c0e..c5e315bd 100644 --- a/test/test_variable.py +++ b/test/test_variable.py @@ -866,7 +866,12 @@ def test_single_level_bound_broadcasts( self, model: "Model", midx: pd.MultiIndex ) -> None: bound = DataArray([5, 6], dims=["l1"], coords={"l1": [0, 1]}) - var = model.add_variables(upper=bound, coords=[midx], name="x") + # Implicit level projection is deprecated (scenario B) — warns until + # the v1 convention makes it an error. + with pytest.warns( + linopy.EvolvingAPIWarning, match=r"broadcasting level subset" + ): + var = model.add_variables(upper=bound, coords=[midx], name="x") assert var.dims == ("multi",) assert (var.data.upper == [5, 5, 6, 6]).all() @@ -875,5 +880,5 @@ def test_incomplete_level_bound_raises( ) -> None: subset = pd.MultiIndex.from_tuples([(0, "a"), (1, "b")], names=("l1", "l2")) bound = pd.Series([1, 2], index=subset) - with pytest.raises(ValueError, match="does not cover every entry"): + with pytest.raises(ValueError, match="no value for .* level combination"): model.add_variables(upper=bound, coords=[midx], name="x")