Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 3 additions & 1 deletion src/cbspy/client.py
Original file line number Diff line number Diff line change
Expand Up @@ -144,7 +144,9 @@ def get_data(
renamed[new_key] = value
renamed_rows.append(renamed)

return pl.DataFrame(renamed_rows)
# Infer from every row: polars' default looks at the first 100 only, which crashes on series
# that start late and silently truncates floats that first appear after 100 integer rows.
return pl.DataFrame(renamed_rows, infer_schema_length=None)

@staticmethod
def _build_filter(
Expand Down
30 changes: 29 additions & 1 deletion tests/test_client.py
Original file line number Diff line number Diff line change
Expand Up @@ -270,8 +270,36 @@ def test_get_data_empty_dataset(self):
assert isinstance(df, pl.DataFrame)
assert df.shape[0] == 0

def _mock_long_table(self, values: list[float | int | None]) -> None:
respx.get(f"{BASE}/ODataFeed/OData/00372/DataProperties").mock(
return_value=httpx.Response(
200,
json={
"value": [
{"Key": "Perioden", "Title": "Perioden", "Type": "TimeDimension"},
{"Key": "Invoer_1", "Title": "Invoer", "Type": "Topic", "Datatype": "Double"},
]
},
)
)
rows = [{"ID": i, "Perioden": f"{1900 + i}JJ00", "Invoer_1": v} for i, v in enumerate(values)]
respx.get(f"{BASE}/ODataFeed/OData/00372/TypedDataSet").mock(
return_value=httpx.Response(200, json={"value": rows})
)

@respx.mock
def test_column_null_for_first_hundred_rows(self):
# Long CBS tables (e.g. 00372, from 1946) often have series that start late.
self._mock_long_table([None] * 100 + [100])
df = Client().get_data("00372")
assert df["Invoer"][-1] == 100

@respx.mock
def test_float_after_hundred_ints_is_not_truncated(self):
self._mock_long_table([1] * 100 + [1.5])
df = Client().get_data("00372")
assert df["Invoer"][-1] == 1.5

class TestGetDataFilters:
"""Tests for the generic filters parameter on get_data()."""

def _mock_props(self):
Expand Down
Loading