Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
59 changes: 59 additions & 0 deletions tests/test_clean_duckdb_read.py
Original file line number Diff line number Diff line change
Expand Up @@ -392,6 +392,65 @@ def test_read_excel_compact_columns_rename_ods(tmp_path: Path):
assert rows == [(2022, "ROMA", "1.600"), (2022, "TORINO", "987")]


@pytest.mark.policy
def test_read_excel_mixed_numeric_string_column_registers_as_varchar(tmp_path: Path):
"""policy: colonne object miste (numeri + stringhe) si registrano come VARCHAR.
Regression: l'export xlsx Terna Download Center termina con una riga footer
"Applied filters: ..." nella prima colonna numerica. DuckDB deduceva INT32
dai valori dominanti e falliva sul cast della stringa al register — anche
quando il clean.sql filtrava gia' la riga con TRY_CAST.
"""
input_file = tmp_path / "footer.xlsx"
pd.DataFrame(
[
{"Anno": 2022, "Regione": "PIEMONTE", "Valore": 1.5},
{"Anno": 2022, "Regione": "LAZIO", "Valore": 2.5},
{"Anno": "Applied filters: Year is 2022;", "Regione": None, "Valore": None},
]
).to_excel(input_file, index=False, engine="openpyxl")

with safe_connect() as con:
logger = logging.getLogger("tests.clean.duckdb_read.xlsx_mixed_footer")

info = duckdb_read.read_raw_to_relation(
con, [input_file], {"header": True}, "fallback", logger
)

assert info.source == "excel"
rows = con.execute(
'SELECT "Anno", "Regione", "Valore" FROM raw_input ORDER BY "Valore" NULLS LAST'
).fetchall()
assert rows == [
("2022", "PIEMONTE", 1.5),
("2022", "LAZIO", 2.5),
("Applied filters: Year is 2022;", None, None),
]
types = {r[0]: r[1] for r in con.execute("DESCRIBE raw_input").fetchall()}
assert types["Anno"] == "VARCHAR"
assert types["Valore"] == "DOUBLE"


@pytest.mark.policy
def test_normalize_mixed_excel_columns_is_pure(tmp_path: Path):
"""policy: _normalize_mixed_excel_columns non muta il DataFrame di input."""
from toolkit.core.read_excel import _normalize_mixed_excel_columns

original = pd.DataFrame(
[
{"Anno": 2022, "Valore": 1.5},
{"Anno": "Applied filters", "Valore": None},
]
)
snapshot = original.copy(deep=True)

result = _normalize_mixed_excel_columns(original)

pd.testing.assert_frame_equal(original, snapshot)
assert result is not original
assert list(result["Anno"]) == ["2022", "Applied filters"]


@pytest.mark.policy
def test_read_excel_source_label_uses_all_batch_suffixes(tmp_path: Path):
"""policy: source_label considera tutti i suffix del batch, non solo il primo."""
Expand Down
42 changes: 42 additions & 0 deletions toolkit/core/read_excel.py
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,47 @@ def _trim_excel_dataframe(df: pd.DataFrame) -> pd.DataFrame:
)


def _normalize_mixed_excel_columns(df: pd.DataFrame) -> pd.DataFrame:
"""Return a copy of ``df`` with mixed number+string object columns stringified.
DuckDB's pandas converter infers a numeric type from the dominant values
of an object column and hard-fails on stray strings (e.g. the
"Applied filters" footer row emitted by the Terna Download Center xlsx
export). Stringifying mixed columns keeps registration total; SQL
consumers cast with the toolkit macros (cast_int/cast_double).
Pure: the input frame is never mutated (copy-on-write).
"""
import numbers

import pandas as pd

result: pd.DataFrame | None = None
for col in df.columns:
series = df[col]
if series.dtype != object:
continue
has_str = False
has_num = False
for value in series:
if value is None or (isinstance(value, float) and pd.isna(value)):
continue
if isinstance(value, str):
has_str = True
elif isinstance(value, numbers.Number):
has_num = True
if has_str and has_num:
if result is None:
result = df.copy()
result[col] = series.map(
lambda value: (
None
if value is None or (isinstance(value, float) and pd.isna(value))
else (value if isinstance(value, str) else str(value))
)
)
return df if result is None else result


def _load_excel_frame(
input_file: Path,
read_cfg: dict[str, Any],
Expand Down Expand Up @@ -122,6 +163,7 @@ def _execute_excel_read(
params_used = frame_params

combined = pd.concat(frames, ignore_index=True) if len(frames) > 1 else frames[0]
combined = _normalize_mixed_excel_columns(combined)
con.register(RAW_INPUT_DF_VIEW, combined)
con.execute(f"CREATE OR REPLACE VIEW {RAW_INPUT_VIEW} AS SELECT * FROM {RAW_INPUT_DF_VIEW};")

Expand Down
Loading