From 3b8b59791b2a3c87b60c8b2b289b58267c2e7735 Mon Sep 17 00:00:00 2001 From: yustme Date: Thu, 16 Jul 2026 13:42:29 +0200 Subject: [PATCH] fix(semantic-layer): classify numeric/temporal field roles by normalized type The `build`/`add dataset --deep-fields` role heuristic matched raw type names against a Snowflake-flavored whitelist (`NUMBER`, `DECIMAL`, `FLOAT`, `INTEGER`, `INT`). Keboola's own basetype for decimals is `NUMERIC`, which was absent from that list, so numeric measure columns (revenue, amount, counts, ...) silently fell through to `dimension` on both Snowflake and BigQuery -- even when the column was correctly typed. Route the type test through the existing `_normalize_field_type` closed vocabulary instead, and add the missing BigQuery native names (`INT64`, `FLOAT64`, `BIGNUMERIC`). Also make a DATE/TIMESTAMP-typed column classify as `timestamp` regardless of its name (the name tokens stay as a fallback for untyped columns), so a bare `date` column is no longer a dimension. --- .../services/_semantic_layer_internals.py | 3 ++ .../services/semantic_layer_service.py | 26 +++++++++++++---- tests/test_semantic_layer_service.py | 28 +++++++++++++++++++ 3 files changed, 52 insertions(+), 5 deletions(-) diff --git a/src/keboola_agent_cli/services/_semantic_layer_internals.py b/src/keboola_agent_cli/services/_semantic_layer_internals.py index cd461046..631179ff 100644 --- a/src/keboola_agent_cli/services/_semantic_layer_internals.py +++ b/src/keboola_agent_cli/services/_semantic_layer_internals.py @@ -955,6 +955,7 @@ def write_snapshot_to_file(snapshot: dict[str, Any], output_path: Path) -> None: "bigint": "integer", "smallint": "integer", "tinyint": "integer", + "int64": "integer", # BigQuery # decimals "decimal": "decimal", "numeric": "decimal", @@ -963,6 +964,8 @@ def write_snapshot_to_file(snapshot: dict[str, Any], output_path: Path) -> None: "double": "decimal", "real": "decimal", "money": "decimal", + "float64": "decimal", # BigQuery + "bignumeric": "decimal", # BigQuery # booleans "boolean": "boolean", "bool": "boolean", diff --git a/src/keboola_agent_cli/services/semantic_layer_service.py b/src/keboola_agent_cli/services/semantic_layer_service.py index 3d9a4878..2cfa0df4 100644 --- a/src/keboola_agent_cli/services/semantic_layer_service.py +++ b/src/keboola_agent_cli/services/semantic_layer_service.py @@ -35,8 +35,8 @@ from ._semantic_layer_crud import find_target_for_remove as _find_target_for_remove from ._semantic_layer_crud import scan_orphan_constraints as _scan_orphan_constraints from ._semantic_layer_crud import validate_constraint_attrs as _validate_constraint_attrs +from ._semantic_layer_internals import _normalize_field_type, collect_side_from_file from ._semantic_layer_internals import build_export_snapshot as _build_export_snapshot -from ._semantic_layer_internals import collect_side_from_file from ._semantic_layer_internals import default_export_path as _default_export_path from ._semantic_layer_internals import diff_one_type as _diff_one_type_helper from ._semantic_layer_internals import fetch_table_schemas as _fetch_table_schemas @@ -94,7 +94,11 @@ "QTY", "QUANTITY", ) -_NUMERIC_TYPES = ("NUMBER", "DECIMAL", "FLOAT", "INTEGER", "INT") +# Normalized field types (see ``_normalize_field_type``) that represent a +# numeric quantity, and thus qualify a measure-named column as a `measure`. +_NUMERIC_NORMALIZED_TYPES = ("integer", "decimal") +# Normalized field types that represent a point in time. +_TEMPORAL_NORMALIZED_TYPES = ("date", "datetime") def _derive_fqn(table_id: str) -> str: @@ -130,13 +134,25 @@ def _derive_fqn(table_id: str) -> str: def _classify_field_role(name: str, basetype: str) -> str: - """Apply the documented role heuristic to (column name, basetype).""" + """Apply the documented role heuristic to (column name, basetype). + + The type test runs through ``_normalize_field_type`` rather than matching + raw type names, so it is warehouse-agnostic: Snowflake ``NUMBER``, + BigQuery ``INT64`` / ``FLOAT64`` / ``NUMERIC`` and the Keboola basetypes + (``INTEGER`` / ``NUMERIC`` / ``FLOAT``) all resolve to the same normalized + families. Before this, ``NUMERIC`` (Keboola's own basetype for BigQuery + and Snowflake decimals) was absent from the numeric whitelist, so numeric + measure columns silently fell through to ``dimension``. + """ upper = name.upper() if any(upper.startswith(prefix) for prefix in _KEY_PREFIXES): return "key" - if any(tok in upper for tok in _TIMESTAMP_NAMES): + normalized = _normalize_field_type(basetype) + # A DATE / TIMESTAMP column is a timestamp regardless of its name; the + # name tokens stay as a fallback for untyped columns (basetype missing). + if normalized in _TEMPORAL_NORMALIZED_TYPES or any(tok in upper for tok in _TIMESTAMP_NAMES): return "timestamp" - if basetype.upper() in _NUMERIC_TYPES and any(tok in upper for tok in _MEASURE_TOKENS): + if normalized in _NUMERIC_NORMALIZED_TYPES and any(tok in upper for tok in _MEASURE_TOKENS): return "measure" return "dimension" diff --git a/tests/test_semantic_layer_service.py b/tests/test_semantic_layer_service.py index e463ff0f..adc2fa87 100644 --- a/tests/test_semantic_layer_service.py +++ b/tests/test_semantic_layer_service.py @@ -165,6 +165,30 @@ def test_string_with_measure_token_is_dimension(self) -> None: def test_plain_dimension_default(self) -> None: assert _classify_field_role("USER_NAME", "STRING") == "dimension" + def test_numeric_basetype_measure_regression(self) -> None: + # Regression: NUMERIC is Keboola's basetype for Snowflake/BigQuery + # decimals. It used to be absent from the numeric whitelist, so every + # NUMERIC measure column was misclassified as a dimension. + assert _classify_field_role("TOTAL_REVENUE", "NUMERIC") == "measure" + + def test_bigquery_native_numeric_types_are_measures(self) -> None: + # BigQuery native type names (surfaced when types are read straight + # from the warehouse) must classify like their Keboola basetypes. + assert _classify_field_role("COUNT_ORDERS", "INT64") == "measure" + assert _classify_field_role("REVENUE", "FLOAT64") == "measure" + assert _classify_field_role("PRICE", "BIGNUMERIC") == "measure" + + def test_date_typed_column_is_timestamp_regardless_of_name(self) -> None: + # A DATE/TIMESTAMP column is temporal even when its name carries none + # of the timestamp tokens (e.g. a bare "date" column). + assert _classify_field_role("date", "DATE") == "timestamp" + assert _classify_field_role("_timestamp", "TIMESTAMP") == "timestamp" + + def test_missing_basetype_stays_dimension(self) -> None: + # With no basetype (untyped column) the numeric test cannot fire, so a + # measure-named column falls back to dimension. + assert _classify_field_role("total_revenue", "") == "dimension" + # --------------------------------------------------------------------------- # Model resolution @@ -2064,6 +2088,10 @@ class TestNormalizeFieldType: ("DATE", "date"), ("TIMESTAMP_NTZ", "datetime"), ("VARIANT", "json"), + # BigQuery native type names. + ("INT64", "integer"), + ("FLOAT64", "decimal"), + ("BIGNUMERIC", "decimal"), # Unknown types fall through to `"string"` — safest default. ("CUSTOM_UDT", "string"), ("geography", "string"),