From bbe68ffa0cd299ff5c431c2cafa5b1127d2d8648 Mon Sep 17 00:00:00 2001 From: Zio Gabber <78922322+Gabrymi93@users.noreply.github.com> Date: Fri, 16 Oct 2026 19:38:17 +0100 Subject: [PATCH 1/2] fix(core): register Excel misto numerico+stringa come VARCHAR MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Le colonne object miste (numeri + stringhe, es. footer 'Applied filters' degli export Terna) fallivano al register pandas→DuckDB: DuckDB deduceva il tipo numerico dai valori dominanti e castava male la stringa. Stringifiche le colonne miste al register; i consumer SQL continuano a usare cast_int/cast_double. --- tests/test_clean_duckdb_read.py | 39 +++++++++++++++++++++++++++++++++ toolkit/core/read_excel.py | 38 ++++++++++++++++++++++++++++++++ 2 files changed, 77 insertions(+) diff --git a/tests/test_clean_duckdb_read.py b/tests/test_clean_duckdb_read.py index 9bf90cb..f56cbd0 100644 --- a/tests/test_clean_duckdb_read.py +++ b/tests/test_clean_duckdb_read.py @@ -392,6 +392,45 @@ def test_read_excel_compact_columns_rename_ods(tmp_path: Path): assert rows == [(2022, "ROMA", "1.600"), (2022, "TORINO", "987")] +@pytest.mark.policy +def test_read_excel_mixed_numeric_string_column_registers_as_varchar(tmp_path: Path): + """policy: colonne object miste (numeri + stringhe) si registrano come VARCHAR. + + Regression: l'export xlsx Terna Download Center termina con una riga footer + "Applied filters: ..." nella prima colonna numerica. DuckDB deduceva INT32 + dai valori dominanti e falliva sul cast della stringa al register — anche + quando il clean.sql filtrava gia' la riga con TRY_CAST. + """ + input_file = tmp_path / "footer.xlsx" + pd.DataFrame( + [ + {"Anno": 2022, "Regione": "PIEMONTE", "Valore": 1.5}, + {"Anno": 2022, "Regione": "LAZIO", "Valore": 2.5}, + {"Anno": "Applied filters: Year is 2022;", "Regione": None, "Valore": None}, + ] + ).to_excel(input_file, index=False, engine="openpyxl") + + with safe_connect() as con: + logger = logging.getLogger("tests.clean.duckdb_read.xlsx_mixed_footer") + + info = duckdb_read.read_raw_to_relation( + con, [input_file], {"header": True}, "fallback", logger + ) + + assert info.source == "excel" + rows = con.execute( + 'SELECT "Anno", "Regione", "Valore" FROM raw_input ORDER BY "Valore" NULLS LAST' + ).fetchall() + assert rows == [ + ("2022", "PIEMONTE", 1.5), + ("2022", "LAZIO", 2.5), + ("Applied filters: Year is 2022;", None, None), + ] + types = {r[0]: r[1] for r in con.execute("DESCRIBE raw_input").fetchall()} + assert types["Anno"] == "VARCHAR" + assert types["Valore"] == "DOUBLE" + + @pytest.mark.policy def test_read_excel_source_label_uses_all_batch_suffixes(tmp_path: Path): """policy: source_label considera tutti i suffix del batch, non solo il primo.""" diff --git a/toolkit/core/read_excel.py b/toolkit/core/read_excel.py index ba59e79..ceb1219 100644 --- a/toolkit/core/read_excel.py +++ b/toolkit/core/read_excel.py @@ -34,6 +34,43 @@ def _trim_excel_dataframe(df: pd.DataFrame) -> pd.DataFrame: ) +def _normalize_mixed_excel_columns(df: pd.DataFrame) -> pd.DataFrame: + """Stringify object columns that mix numbers and strings. + + DuckDB's pandas converter infers a numeric type from the dominant values + of an object column and hard-fails on stray strings (e.g. the + "Applied filters" footer row emitted by the Terna Download Center xlsx + export). Stringifying mixed columns keeps registration total; SQL + consumers cast with the toolkit macros (cast_int/cast_double). + """ + import numbers + + import pandas as pd + + for col in df.columns: + series = df[col] + if series.dtype != object: + continue + has_str = False + has_num = False + for value in series: + if value is None or (isinstance(value, float) and pd.isna(value)): + continue + if isinstance(value, str): + has_str = True + elif isinstance(value, numbers.Number): + has_num = True + if has_str and has_num: + df[col] = series.map( + lambda value: ( + None + if value is None or (isinstance(value, float) and pd.isna(value)) + else (value if isinstance(value, str) else str(value)) + ) + ) + return df + + def _load_excel_frame( input_file: Path, read_cfg: dict[str, Any], @@ -122,6 +159,7 @@ def _execute_excel_read( params_used = frame_params combined = pd.concat(frames, ignore_index=True) if len(frames) > 1 else frames[0] + combined = _normalize_mixed_excel_columns(combined) con.register(RAW_INPUT_DF_VIEW, combined) con.execute(f"CREATE OR REPLACE VIEW {RAW_INPUT_VIEW} AS SELECT * FROM {RAW_INPUT_DF_VIEW};") From a8edba2d7bb00927ee14c81199889df4ab784c46 Mon Sep 17 00:00:00 2001 From: Zio Gabber <78922322+Gabrymi93@users.noreply.github.com> Date: Fri, 16 Oct 2026 19:44:44 +0100 Subject: [PATCH 2/2] refactor(core): _normalize_mixed_excel_columns pura (copy-on-write) Review finding: l'helper mutava in-place il DataFrame single-frame. Ora copia solo quando trova una colonna mista da stringificare; input mai mutato. Aggiunto test pure policy. --- tests/test_clean_duckdb_read.py | 20 ++++++++++++++++++++ toolkit/core/read_excel.py | 10 +++++++--- 2 files changed, 27 insertions(+), 3 deletions(-) diff --git a/tests/test_clean_duckdb_read.py b/tests/test_clean_duckdb_read.py index f56cbd0..977a157 100644 --- a/tests/test_clean_duckdb_read.py +++ b/tests/test_clean_duckdb_read.py @@ -431,6 +431,26 @@ def test_read_excel_mixed_numeric_string_column_registers_as_varchar(tmp_path: P assert types["Valore"] == "DOUBLE" +@pytest.mark.policy +def test_normalize_mixed_excel_columns_is_pure(tmp_path: Path): + """policy: _normalize_mixed_excel_columns non muta il DataFrame di input.""" + from toolkit.core.read_excel import _normalize_mixed_excel_columns + + original = pd.DataFrame( + [ + {"Anno": 2022, "Valore": 1.5}, + {"Anno": "Applied filters", "Valore": None}, + ] + ) + snapshot = original.copy(deep=True) + + result = _normalize_mixed_excel_columns(original) + + pd.testing.assert_frame_equal(original, snapshot) + assert result is not original + assert list(result["Anno"]) == ["2022", "Applied filters"] + + @pytest.mark.policy def test_read_excel_source_label_uses_all_batch_suffixes(tmp_path: Path): """policy: source_label considera tutti i suffix del batch, non solo il primo.""" diff --git a/toolkit/core/read_excel.py b/toolkit/core/read_excel.py index ceb1219..9550bda 100644 --- a/toolkit/core/read_excel.py +++ b/toolkit/core/read_excel.py @@ -35,18 +35,20 @@ def _trim_excel_dataframe(df: pd.DataFrame) -> pd.DataFrame: def _normalize_mixed_excel_columns(df: pd.DataFrame) -> pd.DataFrame: - """Stringify object columns that mix numbers and strings. + """Return a copy of ``df`` with mixed number+string object columns stringified. DuckDB's pandas converter infers a numeric type from the dominant values of an object column and hard-fails on stray strings (e.g. the "Applied filters" footer row emitted by the Terna Download Center xlsx export). Stringifying mixed columns keeps registration total; SQL consumers cast with the toolkit macros (cast_int/cast_double). + Pure: the input frame is never mutated (copy-on-write). """ import numbers import pandas as pd + result: pd.DataFrame | None = None for col in df.columns: series = df[col] if series.dtype != object: @@ -61,14 +63,16 @@ def _normalize_mixed_excel_columns(df: pd.DataFrame) -> pd.DataFrame: elif isinstance(value, numbers.Number): has_num = True if has_str and has_num: - df[col] = series.map( + if result is None: + result = df.copy() + result[col] = series.map( lambda value: ( None if value is None or (isinstance(value, float) and pd.isna(value)) else (value if isinstance(value, str) else str(value)) ) ) - return df + return df if result is None else result def _load_excel_frame(