From 7bfba06368be907833e2ae2caa6d2803610fbde3 Mon Sep 17 00:00:00 2001 From: JeffreyChen Date: Mon, 22 Jun 2026 01:47:11 +0800 Subject: [PATCH] Add cross-dataset referential integrity checks MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit validate_rows is intra-row and single-table — its unique rule only dedupes within one batch. Add dbt-style generic checks: check_foreign_key (parent/child across two tables), check_unique_key (single/composite), check_accepted_values, and check_row_count, operating on rows already loaded by load_rows/query_sqlite. Wired through facade, executor (four AC_check_* commands), MCP, and the Script Builder with a headless test batch and EN/Zh docs. --- README.md | 7 ++ README/README_zh-CN.md | 7 ++ README/README_zh-TW.md | 7 ++ .../Eng/doc/new_features/v86_features_doc.rst | 43 ++++++++++ docs/source/Eng/eng_index.rst | 1 + .../Zh/doc/new_features/v86_features_doc.rst | 38 +++++++++ docs/source/Zh/zh_index.rst | 1 + je_auto_control/__init__.py | 6 ++ .../gui/script_builder/command_schema.py | 42 ++++++++++ .../utils/executor/action_executor.py | 41 ++++++++++ .../utils/mcp_server/tools/_factories.py | 53 +++++++++++- .../utils/mcp_server/tools/_handlers.py | 21 +++++ je_auto_control/utils/referential/__init__.py | 9 +++ .../utils/referential/referential.py | 66 +++++++++++++++ .../headless/test_referential_batch.py | 80 +++++++++++++++++++ 15 files changed, 421 insertions(+), 1 deletion(-) create mode 100644 docs/source/Eng/doc/new_features/v86_features_doc.rst create mode 100644 docs/source/Zh/doc/new_features/v86_features_doc.rst create mode 100644 je_auto_control/utils/referential/__init__.py create mode 100644 je_auto_control/utils/referential/referential.py create mode 100644 test/unit_test/headless/test_referential_batch.py diff --git a/README.md b/README.md index 18631d97..2cecd8fd 100644 --- a/README.md +++ b/README.md @@ -13,6 +13,7 @@ ## Table of Contents +- [What's new (2026-06-22) — Referential Integrity Checks](#whats-new-2026-06-22--referential-integrity-checks) - [What's new (2026-06-22) — URI-Scheme Value References](#whats-new-2026-06-22--uri-scheme-value-references) - [What's new (2026-06-21) — W3C Baggage Propagation](#whats-new-2026-06-21--w3c-baggage-propagation) - [What's new (2026-06-21) — Dataset Diff (Row-Set Change Report)](#whats-new-2026-06-21--dataset-diff-row-set-change-report) @@ -138,6 +139,12 @@ --- +## What's new (2026-06-22) — Referential Integrity Checks + +Foreign-key, unique, accepted-values and row-count checks across tables. Full reference: [`docs/source/Eng/doc/new_features/v86_features_doc.rst`](docs/source/Eng/doc/new_features/v86_features_doc.rst). + +- **`check_foreign_key` / `check_unique_key` / `check_accepted_values` / `check_row_count`** (`AC_check_foreign_key`, `AC_check_unique_key`, `AC_check_accepted_values`, `AC_check_row_count`): `validate_rows` is intra-row, single-table (its `unique` only dedupes within one batch). This adds dbt-style generic checks — parent/child foreign keys across two tables, single/composite key uniqueness, accepted-values, and row-count bounds — over rows from `load_rows`/`query_sqlite`. Pure-stdlib, deterministic. + ## What's new (2026-06-22) — URI-Scheme Value References Store pointers, not secrets, in config. Full reference: [`docs/source/Eng/doc/new_features/v85_features_doc.rst`](docs/source/Eng/doc/new_features/v85_features_doc.rst). diff --git a/README/README_zh-CN.md b/README/README_zh-CN.md index 2729d517..a68eaf0d 100644 --- a/README/README_zh-CN.md +++ b/README/README_zh-CN.md @@ -12,6 +12,7 @@ ## 目录 +- [本次更新 (2026-06-22) — 参照完整性检查](#本次更新-2026-06-22--参照完整性检查) - [本次更新 (2026-06-22) — URI-Scheme 值引用](#本次更新-2026-06-22--uri-scheme-值引用) - [本次更新 (2026-06-21) — W3C Baggage 传播](#本次更新-2026-06-21--w3c-baggage-传播) - [本次更新 (2026-06-21) — 数据集差异(数据行变更报告)](#本次更新-2026-06-21--数据集差异数据行变更报告) @@ -137,6 +138,12 @@ --- +## 本次更新 (2026-06-22) — 参照完整性检查 + +跨数据表的外键、唯一键、accepted-values 与行数检查。完整参考:[`docs/source/Zh/doc/new_features/v86_features_doc.rst`](../docs/source/Zh/doc/new_features/v86_features_doc.rst)。 + +- **`check_foreign_key` / `check_unique_key` / `check_accepted_values` / `check_row_count`**(`AC_check_foreign_key`、`AC_check_unique_key`、`AC_check_accepted_values`、`AC_check_row_count`):`validate_rows` 是单行、单表(其 `unique` 只在单批次内去重)。本功能补上 dbt 风格通用检查 —— 跨两表的父子外键、单一/复合键唯一性、accepted-values、行数范围 —— 作用于 `load_rows`/`query_sqlite` 的数据行。纯标准库、确定。 + ## 本次更新 (2026-06-22) — URI-Scheme 值引用 在配置中存储指针而非机密。完整参考:[`docs/source/Zh/doc/new_features/v85_features_doc.rst`](../docs/source/Zh/doc/new_features/v85_features_doc.rst)。 diff --git a/README/README_zh-TW.md b/README/README_zh-TW.md index 26522a6d..45e1d60f 100644 --- a/README/README_zh-TW.md +++ b/README/README_zh-TW.md @@ -12,6 +12,7 @@ ## 目錄 +- [本次更新 (2026-06-22) — 參照完整性檢查](#本次更新-2026-06-22--參照完整性檢查) - [本次更新 (2026-06-22) — URI-Scheme 值參照](#本次更新-2026-06-22--uri-scheme-值參照) - [本次更新 (2026-06-21) — W3C Baggage 傳播](#本次更新-2026-06-21--w3c-baggage-傳播) - [本次更新 (2026-06-21) — 資料集差異(資料列變更報告)](#本次更新-2026-06-21--資料集差異資料列變更報告) @@ -137,6 +138,12 @@ --- +## 本次更新 (2026-06-22) — 參照完整性檢查 + +跨資料表的外鍵、唯一鍵、accepted-values 與筆數檢查。完整參考:[`docs/source/Zh/doc/new_features/v86_features_doc.rst`](../docs/source/Zh/doc/new_features/v86_features_doc.rst)。 + +- **`check_foreign_key` / `check_unique_key` / `check_accepted_values` / `check_row_count`**(`AC_check_foreign_key`、`AC_check_unique_key`、`AC_check_accepted_values`、`AC_check_row_count`):`validate_rows` 是單列、單表(其 `unique` 只在單批次內去重)。本功能補上 dbt 風格通用檢查 —— 跨兩表的父子外鍵、單一/複合鍵唯一性、accepted-values、筆數範圍 —— 作用於 `load_rows`/`query_sqlite` 的資料列。純標準函式庫、具決定性。 + ## 本次更新 (2026-06-22) — URI-Scheme 值參照 在設定中儲存指標而非機密。完整參考:[`docs/source/Zh/doc/new_features/v85_features_doc.rst`](../docs/source/Zh/doc/new_features/v85_features_doc.rst)。 diff --git a/docs/source/Eng/doc/new_features/v86_features_doc.rst b/docs/source/Eng/doc/new_features/v86_features_doc.rst new file mode 100644 index 00000000..19fa168d --- /dev/null +++ b/docs/source/Eng/doc/new_features/v86_features_doc.rst @@ -0,0 +1,43 @@ +Referential Integrity Checks +============================ + +``data_quality.validate_rows`` is strictly intra-row, single-table — its +``unique`` rule only dedupes within one batch. There was no parent/child +foreign-key check across two loaded tables, no composite-key uniqueness, and no +standalone accepted-values / row-count assertion. This adds those dbt-style +generic checks over rows already loaded by ``load_rows`` / ``query_sqlite``. + +Pure standard library (``collections``); imports no ``PySide6``. Every function +is pure (rows in, report out), so it is fully deterministic in CI. + +Headless API +------------ + +.. code-block:: python + + from je_auto_control import ( + check_foreign_key, check_unique_key, check_accepted_values, + check_row_count, load_rows, + ) + + orders = load_rows("orders.csv") + users = load_rows("users.csv") + + fk = check_foreign_key(orders, "user_id", users, "id") # {ok, violations, missing} + pk = check_unique_key(orders, ["region", "id"]) # {ok, duplicates} + av = check_accepted_values(orders, "status", ["open", "closed"]) + rc = check_row_count(orders, minimum=1) # {ok, count} + +``check_foreign_key`` flags non-null child values absent from the parent column +(dbt ``relationships``). ``check_unique_key`` reports duplicate single or +composite keys. ``check_accepted_values`` lists non-null values outside the +allowed set. ``check_row_count`` verifies the count falls within optional +``minimum`` / ``maximum`` bounds. Each returns an ``ok`` flag plus details. + +Executor commands +----------------- + +``AC_check_foreign_key``, ``AC_check_unique_key``, ``AC_check_accepted_values``, +and ``AC_check_row_count`` each take JSON ``rows`` (and a column / key / allowed +list) and return the report. All are exposed as MCP tools +(``ac_check_*``) and as Script Builder commands under **Data**. diff --git a/docs/source/Eng/eng_index.rst b/docs/source/Eng/eng_index.rst index 32e28014..a9efc78d 100644 --- a/docs/source/Eng/eng_index.rst +++ b/docs/source/Eng/eng_index.rst @@ -108,6 +108,7 @@ Comprehensive guides for all AutoControl features. doc/new_features/v83_features_doc doc/new_features/v84_features_doc doc/new_features/v85_features_doc + doc/new_features/v86_features_doc doc/ocr_backends/ocr_backends_doc doc/observability/observability_doc doc/operations_layer/operations_layer_doc diff --git a/docs/source/Zh/doc/new_features/v86_features_doc.rst b/docs/source/Zh/doc/new_features/v86_features_doc.rst new file mode 100644 index 00000000..502dd22f --- /dev/null +++ b/docs/source/Zh/doc/new_features/v86_features_doc.rst @@ -0,0 +1,38 @@ +參照完整性檢查 +============ + +``data_quality.validate_rows`` 嚴格限於單列、單表 —— 它的 ``unique`` 規則只在單一批次內去重。沒有跨兩個 +已載入資料表的父子外鍵檢查、沒有複合鍵唯一性,也沒有獨立的 accepted-values / row-count 斷言。本功能在 +已由 ``load_rows`` / ``query_sqlite`` 載入的資料列上,補上這些 dbt 風格的通用檢查。 + +純標準函式庫(``collections``);不匯入 ``PySide6``。每個函式皆為純函式(輸入列、輸出報告),因此在 CI +中完全具決定性。 + +無頭 API +-------- + +.. code-block:: python + + from je_auto_control import ( + check_foreign_key, check_unique_key, check_accepted_values, + check_row_count, load_rows, + ) + + orders = load_rows("orders.csv") + users = load_rows("users.csv") + + fk = check_foreign_key(orders, "user_id", users, "id") # {ok, violations, missing} + pk = check_unique_key(orders, ["region", "id"]) # {ok, duplicates} + av = check_accepted_values(orders, "status", ["open", "closed"]) + rc = check_row_count(orders, minimum=1) # {ok, count} + +``check_foreign_key`` 標記父欄位中不存在的非空子值(dbt ``relationships``)。``check_unique_key`` 回報 +重複的單一或複合鍵。``check_accepted_values`` 列出允許集合之外的非空值。``check_row_count`` 驗證筆數 +落在選用的 ``minimum`` / ``maximum`` 範圍內。每個皆回傳 ``ok`` 旗標加上細節。 + +執行器命令 +---------- + +``AC_check_foreign_key``、``AC_check_unique_key``、``AC_check_accepted_values`` 與 ``AC_check_row_count`` +各自接受 JSON ``rows``(以及欄位 / 鍵 / 允許清單)並回傳報告。全部皆以 MCP 工具(``ac_check_*``)以及 +Script Builder 中 **Data** 分類下的命令提供。 diff --git a/docs/source/Zh/zh_index.rst b/docs/source/Zh/zh_index.rst index 3f79c76e..9e509380 100644 --- a/docs/source/Zh/zh_index.rst +++ b/docs/source/Zh/zh_index.rst @@ -108,6 +108,7 @@ AutoControl 所有功能的完整使用指南。 doc/new_features/v83_features_doc doc/new_features/v84_features_doc doc/new_features/v85_features_doc + doc/new_features/v86_features_doc doc/ocr_backends/ocr_backends_doc doc/observability/observability_doc doc/operations_layer/operations_layer_doc diff --git a/je_auto_control/__init__.py b/je_auto_control/__init__.py index f4004a26..264fde4e 100644 --- a/je_auto_control/__init__.py +++ b/je_auto_control/__init__.py @@ -167,6 +167,10 @@ from je_auto_control.utils.dataset_diff import ( cell_changes, diff_rows, summarize_diff, ) +# Cross-dataset referential integrity (foreign-key / unique / accepted values) +from je_auto_control.utils.referential import ( + check_accepted_values, check_foreign_key, check_row_count, check_unique_key, +) # i18n / l10n testing: pseudo-localize, overflow + catalog checks from je_auto_control.utils.i18n_test import ( check_catalog, check_overflow, pseudo_localize, pseudo_localize_catalog, @@ -841,6 +845,8 @@ def start_autocontrol_gui(*args, **kwargs): "infer_schema", "profile_rows", "categorical_drift", "detect_drift", "ks_two_sample", "psi", "cell_changes", "diff_rows", "summarize_diff", + "check_accepted_values", "check_foreign_key", "check_row_count", + "check_unique_key", "check_catalog", "check_overflow", "pseudo_localize", "pseudo_localize_catalog", "Checkpoint", "CheckpointStore", "run_resumable", diff --git a/je_auto_control/gui/script_builder/command_schema.py b/je_auto_control/gui/script_builder/command_schema.py index ff0e4d51..64ded93e 100644 --- a/je_auto_control/gui/script_builder/command_schema.py +++ b/je_auto_control/gui/script_builder/command_schema.py @@ -1775,6 +1775,48 @@ def _add_resilience_specs(specs: List[CommandSpec]) -> None: ), description="Per-cell {key, column, old, new} changes between row-sets.", )) + specs.append(CommandSpec( + "AC_check_foreign_key", "Data", "Referential: Foreign Key", + fields=( + FieldSpec("child_rows", FieldType.STRING, + placeholder='[{"user_id": 1}]'), + FieldSpec("child_col", FieldType.STRING, placeholder="user_id"), + FieldSpec("parent_rows", FieldType.STRING, + placeholder='[{"id": 1}]'), + FieldSpec("parent_col", FieldType.STRING, placeholder="id"), + ), + description="Every child value must exist in the parent column.", + )) + specs.append(CommandSpec( + "AC_check_unique_key", "Data", "Referential: Unique Key", + fields=( + FieldSpec("rows", FieldType.STRING, + placeholder='[{"id": 1}, {"id": 1}]'), + FieldSpec("cols", FieldType.STRING, + placeholder='id (or ["region", "id"])'), + ), + description="A single or composite key must be unique across rows.", + )) + specs.append(CommandSpec( + "AC_check_accepted_values", "Data", "Referential: Accepted Values", + fields=( + FieldSpec("rows", FieldType.STRING, + placeholder='[{"status": "open"}]'), + FieldSpec("col", FieldType.STRING, placeholder="status"), + FieldSpec("allowed", FieldType.STRING, + placeholder='["open", "closed"]'), + ), + description="Every non-null column value must be in the allowed set.", + )) + specs.append(CommandSpec( + "AC_check_row_count", "Data", "Referential: Row Count", + fields=( + FieldSpec("rows", FieldType.STRING, placeholder='[{"id": 1}]'), + FieldSpec("minimum", FieldType.INT, optional=True), + FieldSpec("maximum", FieldType.INT, optional=True), + ), + description="The row count must fall within the given bounds.", + )) specs.append(CommandSpec( "AC_rate_limit", "Flow", "Rate Limit (Token Bucket)", fields=( diff --git a/je_auto_control/utils/executor/action_executor.py b/je_auto_control/utils/executor/action_executor.py index 8fd2b156..02d81517 100644 --- a/je_auto_control/utils/executor/action_executor.py +++ b/je_auto_control/utils/executor/action_executor.py @@ -3129,6 +3129,43 @@ def _categorical_drift(reference: Any, current: Any) -> Dict[str, Any]: return categorical_drift(reference, current) +def _json_rows(rows: Any) -> Any: + import json + return json.loads(rows) if isinstance(rows, str) else rows + + +def _check_foreign_key(child_rows: Any, child_col: str, parent_rows: Any, + parent_col: str) -> Dict[str, Any]: + """Adapter: foreign-key referential check across two row-sets.""" + from je_auto_control.utils.referential import check_foreign_key + return check_foreign_key(_json_rows(child_rows), child_col, + _json_rows(parent_rows), parent_col) + + +def _check_unique_key(rows: Any, cols: Any) -> Dict[str, Any]: + """Adapter: single/composite key uniqueness check.""" + import json + from je_auto_control.utils.referential import check_unique_key + if isinstance(cols, str) and cols.strip().startswith("["): + cols = json.loads(cols) + return check_unique_key(_json_rows(rows), cols) + + +def _check_accepted_values(rows: Any, col: str, allowed: Any) -> Dict[str, Any]: + """Adapter: accepted-values check for a column.""" + from je_auto_control.utils.referential import check_accepted_values + return check_accepted_values(_json_rows(rows), col, _json_rows(allowed)) + + +def _check_row_count(rows: Any, minimum: Any = None, + maximum: Any = None) -> Dict[str, Any]: + """Adapter: row-count bounds check.""" + from je_auto_control.utils.referential import check_row_count + low = int(minimum) if minimum is not None else None + high = int(maximum) if maximum is not None else None + return check_row_count(_json_rows(rows), low, high) + + def _coerce_diff_inputs(old_rows: Any, new_rows: Any, key: Any): import json if isinstance(old_rows, str): @@ -4249,6 +4286,10 @@ def __init__(self): "AC_categorical_drift": _categorical_drift, "AC_diff_rows": _diff_rows, "AC_cell_changes": _cell_changes, + "AC_check_foreign_key": _check_foreign_key, + "AC_check_unique_key": _check_unique_key, + "AC_check_accepted_values": _check_accepted_values, + "AC_check_row_count": _check_row_count, "AC_unified_diff": _unified_diff, "AC_apply_unified": _apply_unified, "AC_three_way_merge": _three_way_merge, diff --git a/je_auto_control/utils/mcp_server/tools/_factories.py b/je_auto_control/utils/mcp_server/tools/_factories.py index 3469da8c..086ad577 100644 --- a/je_auto_control/utils/mcp_server/tools/_factories.py +++ b/je_auto_control/utils/mcp_server/tools/_factories.py @@ -3343,6 +3343,57 @@ def rate_limit_tools() -> List[MCPTool]: ] +def referential_tools() -> List[MCPTool]: + rows = {"type": "array", "items": {"type": "object"}} + return [ + MCPTool( + name="ac_check_foreign_key", + description=("Every non-null 'child_col' value in 'child_rows' must " + "exist in 'parent_col' of 'parent_rows'. Returns {ok, " + "violations, missing}."), + input_schema=schema( + {"child_rows": rows, "child_col": {"type": "string"}, + "parent_rows": rows, "parent_col": {"type": "string"}}, + ["child_rows", "child_col", "parent_rows", "parent_col"]), + handler=h.check_foreign_key, + annotations=READ_ONLY, + ), + MCPTool( + name="ac_check_unique_key", + description=("A single or composite key ('cols': column name or " + "list) must be unique across 'rows'. Returns {ok, " + "duplicates}."), + input_schema=schema( + {"rows": rows, "cols": {"type": ["string", "array"]}}, + ["rows", "cols"]), + handler=h.check_unique_key, + annotations=READ_ONLY, + ), + MCPTool( + name="ac_check_accepted_values", + description=("Every non-null 'col' value in 'rows' must be within " + "'allowed'. Returns {ok, violations, unexpected}."), + input_schema=schema( + {"rows": rows, "col": {"type": "string"}, + "allowed": {"type": "array"}}, + ["rows", "col", "allowed"]), + handler=h.check_accepted_values, + annotations=READ_ONLY, + ), + MCPTool( + name="ac_check_row_count", + description=("The 'rows' count must fall within optional 'minimum' / " + "'maximum'. Returns {ok, count}."), + input_schema=schema( + {"rows": rows, "minimum": {"type": "integer"}, + "maximum": {"type": "integer"}}, + ["rows"]), + handler=h.check_row_count, + annotations=READ_ONLY, + ), + ] + + def dataset_diff_tools() -> List[MCPTool]: rows_schema = {"type": "array", "items": {"type": "object"}} key_schema = {"type": ["string", "array"]} @@ -5115,7 +5166,7 @@ def media_assert_tools() -> List[MCPTool]: trace_context_tools, baggage_tools, secret_ref_tools, data_profile_tools, http_problem_tools, dotenv_tools, sse_client_tools, layered_config_tools, data_drift_tools, - dataset_diff_tools, + dataset_diff_tools, referential_tools, saga_tools, decision_table_tools, locator_repair_tools, pii_text_tools, sarif_tools, screen_record_tools, diff --git a/je_auto_control/utils/mcp_server/tools/_handlers.py b/je_auto_control/utils/mcp_server/tools/_handlers.py index 63a94e2d..b75a4bb5 100644 --- a/je_auto_control/utils/mcp_server/tools/_handlers.py +++ b/je_auto_control/utils/mcp_server/tools/_handlers.py @@ -1806,6 +1806,27 @@ def cell_changes(old_rows, new_rows, key): return _cell_changes(old_rows, new_rows, key) +def check_foreign_key(child_rows, child_col, parent_rows, parent_col): + from je_auto_control.utils.executor.action_executor import _check_foreign_key + return _check_foreign_key(child_rows, child_col, parent_rows, parent_col) + + +def check_unique_key(rows, cols): + from je_auto_control.utils.executor.action_executor import _check_unique_key + return _check_unique_key(rows, cols) + + +def check_accepted_values(rows, col, allowed): + from je_auto_control.utils.executor.action_executor import ( + _check_accepted_values) + return _check_accepted_values(rows, col, allowed) + + +def check_row_count(rows, minimum=None, maximum=None): + from je_auto_control.utils.executor.action_executor import _check_row_count + return _check_row_count(rows, minimum, maximum) + + def build_provenance(paths, builder_id="je_auto_control"): from je_auto_control.utils.provenance import build_provenance, subject_for subjects = [subject_for(path) for path in paths] diff --git a/je_auto_control/utils/referential/__init__.py b/je_auto_control/utils/referential/__init__.py new file mode 100644 index 00000000..4ffeeb7f --- /dev/null +++ b/je_auto_control/utils/referential/__init__.py @@ -0,0 +1,9 @@ +"""Cross-dataset referential-integrity checks for AutoControl.""" +from je_auto_control.utils.referential.referential import ( + check_accepted_values, check_foreign_key, check_row_count, check_unique_key, +) + +__all__ = [ + "check_accepted_values", "check_foreign_key", "check_row_count", + "check_unique_key", +] diff --git a/je_auto_control/utils/referential/referential.py b/je_auto_control/utils/referential/referential.py new file mode 100644 index 00000000..301ac22b --- /dev/null +++ b/je_auto_control/utils/referential/referential.py @@ -0,0 +1,66 @@ +"""Cross-dataset referential-integrity checks (dbt-style generic tests). + +``data_quality.validate_rows`` is strictly intra-row, single-table — its +``unique`` rule only dedupes within one batch. There was no parent/child +foreign-key check across two loaded tables, no composite-key uniqueness, and no +standalone accepted-values / row-count assertion. This adds those checks over +rows already loaded by ``data_source.load_rows`` / ``sql.query_sqlite``. + +Pure standard library (``collections``); imports no ``PySide6``. Every function +is pure (rows in, report out), so it is fully deterministic in CI. +""" +from collections import Counter +from typing import Any, Dict, List, Sequence, Tuple, Union + +Columns = Union[str, Sequence[str]] +_NULLS = (None, "") + + +def _columns(cols: Columns) -> List[str]: + return [cols] if isinstance(cols, str) else list(cols) + + +def _key_view(key_tuple: Tuple[Any, ...]) -> Any: + return key_tuple[0] if len(key_tuple) == 1 else list(key_tuple) + + +def check_foreign_key(child_rows: Sequence[Dict[str, Any]], child_col: str, + parent_rows: Sequence[Dict[str, Any]], + parent_col: str) -> Dict[str, Any]: + """Every non-null ``child_col`` value must exist in ``parent_col`` (dbt rel).""" + parent_values = {row.get(parent_col) for row in parent_rows} + missing = [row.get(child_col) for row in child_rows + if row.get(child_col) not in _NULLS + and row.get(child_col) not in parent_values] + return {"ok": not missing, "violations": len(missing), + "missing": sorted(set(missing), key=str)} + + +def check_unique_key(rows: Sequence[Dict[str, Any]], + cols: Columns) -> Dict[str, Any]: + """A single or composite key must be unique across ``rows``.""" + columns = _columns(cols) + counts = Counter(tuple(row.get(col) for col in columns) for row in rows) + duplicates = [{"key": _key_view(key_tuple), "count": count} + for key_tuple, count in counts.items() if count > 1] + return {"ok": not duplicates, "duplicates": duplicates} + + +def check_accepted_values(rows: Sequence[Dict[str, Any]], col: str, + allowed: Sequence[Any]) -> Dict[str, Any]: + """Every non-null ``col`` value must be within ``allowed``.""" + allowed_set = set(allowed) + unexpected = [row.get(col) for row in rows + if row.get(col) not in _NULLS + and row.get(col) not in allowed_set] + return {"ok": not unexpected, "violations": len(unexpected), + "unexpected": sorted(set(unexpected), key=str)} + + +def check_row_count(rows: Sequence[Dict[str, Any]], minimum: Any = None, + maximum: Any = None) -> Dict[str, Any]: + """The row count must fall within ``[minimum, maximum]`` when given.""" + count = len(rows) + below = minimum is not None and count < minimum + above = maximum is not None and count > maximum + return {"ok": not (below or above), "count": count} diff --git a/test/unit_test/headless/test_referential_batch.py b/test/unit_test/headless/test_referential_batch.py new file mode 100644 index 00000000..c26953b6 --- /dev/null +++ b/test/unit_test/headless/test_referential_batch.py @@ -0,0 +1,80 @@ +"""Headless tests for referential-integrity checks. Pure stdlib, no Qt.""" +import json + +import je_auto_control as ac +from je_auto_control.utils.referential import ( + check_accepted_values, check_foreign_key, check_row_count, check_unique_key, +) + +_PARENTS = [{"id": 1}, {"id": 2}, {"id": 3}] +_CHILDREN = [{"user_id": 1}, {"user_id": 2}, {"user_id": 9}, {"user_id": None}] + + +def test_foreign_key_reports_orphans(): + report = check_foreign_key(_CHILDREN, "user_id", _PARENTS, "id") + assert report["ok"] is False + assert report["violations"] == 1 and report["missing"] == [9] + + +def test_foreign_key_passes_when_all_present(): + children = [{"user_id": 1}, {"user_id": 2}] + assert check_foreign_key(children, "user_id", _PARENTS, "id")["ok"] is True + + +def test_unique_key_single_and_composite(): + dup = check_unique_key([{"id": 1}, {"id": 1}, {"id": 2}], "id") + assert dup["ok"] is False + assert dup["duplicates"] == [{"key": 1, "count": 2}] + rows = [{"r": "us", "id": 1}, {"r": "us", "id": 1}, {"r": "eu", "id": 1}] + comp = check_unique_key(rows, ["r", "id"]) + assert comp["duplicates"] == [{"key": ["us", 1], "count": 2}] + + +def test_accepted_values(): + rows = [{"s": "open"}, {"s": "closed"}, {"s": "weird"}, {"s": None}] + report = check_accepted_values(rows, "s", ["open", "closed"]) + assert report["ok"] is False and report["unexpected"] == ["weird"] + assert check_accepted_values(rows[:2], "s", ["open", "closed"])["ok"] + + +def test_row_count_bounds(): + rows = [{"id": index} for index in range(5)] + assert check_row_count(rows, minimum=1, maximum=10)["ok"] is True + assert check_row_count(rows, minimum=6)["ok"] is False + assert check_row_count(rows, maximum=4)["ok"] is False + assert check_row_count(rows)["count"] == 5 + + +# --- wiring --------------------------------------------------------------- + +def test_executor_round_trip(): + rec = ac.execute_action([[ + "AC_check_foreign_key", + {"child_rows": json.dumps(_CHILDREN), "child_col": "user_id", + "parent_rows": json.dumps(_PARENTS), "parent_col": "id"}]]) + report = next(v for v in rec.values() if isinstance(v, dict)) + assert report["missing"] == [9] + rec2 = ac.execute_action([[ + "AC_check_unique_key", + {"rows": json.dumps([{"id": 1}, {"id": 1}]), "cols": "id"}]]) + assert next(v for v in rec2.values() + if isinstance(v, dict))["ok"] is False + + +def test_wiring(): + cmds = {"AC_check_foreign_key", "AC_check_unique_key", + "AC_check_accepted_values", "AC_check_row_count"} + assert cmds <= set(ac.executor.known_commands()) + from je_auto_control.utils.mcp_server.tools import build_default_tool_registry + names = {t.name for t in build_default_tool_registry()} + assert {"ac_check_foreign_key", "ac_check_unique_key", + "ac_check_accepted_values", "ac_check_row_count"} <= names + from je_auto_control.gui.script_builder.command_schema import _build_specs + specs = {s.command for s in _build_specs()} + assert cmds <= specs + + +def test_facade_exports(): + for attr in ("check_foreign_key", "check_unique_key", + "check_accepted_values", "check_row_count"): + assert hasattr(ac, attr) and attr in ac.__all__