From bd0a3c7524afea0d32e0a6e58fdb48067de21c44 Mon Sep 17 00:00:00 2001 From: Thomas Pinder Date: Wed, 21 Oct 2026 18:53:19 +0000 Subject: [PATCH] fix: infer get_data column types from all rows pl.DataFrame infers dtypes from the first 100 rows. On long CBS tables that crashed when a series starts late (all null for 100 rows, as in 86103NED and 00372) and silently truncated floats that first appear after 100 integer rows (1.5 -> 1). Co-Authored-By: Claude Opus 5.5 --- src/cbspy/client.py | 4 +++- tests/test_client.py | 30 +++++++++++++++++++++++++++++- 2 files changed, 32 insertions(+), 2 deletions(-) diff --git a/src/cbspy/client.py b/src/cbspy/client.py index 1ddca28..915b71a 100644 --- a/src/cbspy/client.py +++ b/src/cbspy/client.py @@ -144,7 +144,9 @@ def get_data( renamed[new_key] = value renamed_rows.append(renamed) - return pl.DataFrame(renamed_rows) + # Infer from every row: polars' default looks at the first 100 only, which crashes on series + # that start late and silently truncates floats that first appear after 100 integer rows. + return pl.DataFrame(renamed_rows, infer_schema_length=None) @staticmethod def _build_filter( diff --git a/tests/test_client.py b/tests/test_client.py index b4ca533..1ceddac 100644 --- a/tests/test_client.py +++ b/tests/test_client.py @@ -270,8 +270,36 @@ def test_get_data_empty_dataset(self): assert isinstance(df, pl.DataFrame) assert df.shape[0] == 0 + def _mock_long_table(self, values: list[float | int | None]) -> None: + respx.get(f"{BASE}/ODataFeed/OData/00372/DataProperties").mock( + return_value=httpx.Response( + 200, + json={ + "value": [ + {"Key": "Perioden", "Title": "Perioden", "Type": "TimeDimension"}, + {"Key": "Invoer_1", "Title": "Invoer", "Type": "Topic", "Datatype": "Double"}, + ] + }, + ) + ) + rows = [{"ID": i, "Perioden": f"{1900 + i}JJ00", "Invoer_1": v} for i, v in enumerate(values)] + respx.get(f"{BASE}/ODataFeed/OData/00372/TypedDataSet").mock( + return_value=httpx.Response(200, json={"value": rows}) + ) + + @respx.mock + def test_column_null_for_first_hundred_rows(self): + # Long CBS tables (e.g. 00372, from 1946) often have series that start late. + self._mock_long_table([None] * 100 + [100]) + df = Client().get_data("00372") + assert df["Invoer"][-1] == 100 + + @respx.mock + def test_float_after_hundred_ints_is_not_truncated(self): + self._mock_long_table([1] * 100 + [1.5]) + df = Client().get_data("00372") + assert df["Invoer"][-1] == 1.5 -class TestGetDataFilters: """Tests for the generic filters parameter on get_data().""" def _mock_props(self):