From 3cb667c331d03132d9ebd0bf0cc3deec472951de Mon Sep 17 00:00:00 2001 From: JeffreyChen Date: Sun, 21 Jun 2026 07:08:29 +0800 Subject: [PATCH 1/2] Add statistics and A/B significance testing ab_locator ranks by raw success rate and run_history stores durations, but nothing computed percentiles or whether a difference is statistically significant. Add descriptive stats + percentiles, a two-proportion z-test with CI, Welch's t-test (exact t-distribution p-value via the incomplete beta, no SciPy), Cohen's d and a 2x2 chi-square. Normal CDF exact via erf; validated against textbook values. Wired through the facade, AC_describe_stats and AC_ab_significance executor commands, MCP tools and the Script Builder. --- README.md | 7 + README/README_zh-CN.md | 7 + README/README_zh-TW.md | 7 + .../Eng/doc/new_features/v65_features_doc.rst | 51 +++++ docs/source/Eng/eng_index.rst | 1 + .../Zh/doc/new_features/v65_features_doc.rst | 44 ++++ docs/source/Zh/zh_index.rst | 1 + je_auto_control/__init__.py | 7 + .../gui/script_builder/command_schema.py | 18 ++ .../utils/executor/action_executor.py | 18 ++ .../utils/mcp_server/tools/_factories.py | 28 ++- .../utils/mcp_server/tools/_handlers.py | 10 + je_auto_control/utils/stats/__init__.py | 10 + je_auto_control/utils/stats/stats.py | 200 ++++++++++++++++++ test/unit_test/headless/test_stats_batch.py | 121 +++++++++++ 15 files changed, 529 insertions(+), 1 deletion(-) create mode 100644 docs/source/Eng/doc/new_features/v65_features_doc.rst create mode 100644 docs/source/Zh/doc/new_features/v65_features_doc.rst create mode 100644 je_auto_control/utils/stats/__init__.py create mode 100644 je_auto_control/utils/stats/stats.py create mode 100644 test/unit_test/headless/test_stats_batch.py diff --git a/README.md b/README.md index d54438cf..a9f23454 100644 --- a/README.md +++ b/README.md @@ -13,6 +13,7 @@ ## Table of Contents +- [What's new (2026-06-21) — Statistics & A/B Significance](#whats-new-2026-06-21--statistics--ab-significance) - [What's new (2026-06-21) — Full-Text Search (BM25)](#whats-new-2026-06-21--full-text-search-bm25) - [What's new (2026-06-21) — JSON Pointer, Patch & Merge Patch](#whats-new-2026-06-21--json-pointer-patch--merge-patch) - [What's new (2026-06-21) — Client-Side Rate Limiting](#whats-new-2026-06-21--client-side-rate-limiting) @@ -117,6 +118,12 @@ --- +## What's new (2026-06-21) — Statistics & A/B Significance + +Decide whether a difference is real. Full reference: [`docs/source/Eng/doc/new_features/v65_features_doc.rst`](docs/source/Eng/doc/new_features/v65_features_doc.rst). + +- **`describe` / `percentile` / `two_proportion_z_test` / `welch_t_test` / `cohens_d` / `chi_square_2x2`** (`AC_describe_stats`, `AC_ab_significance`): `ab_locator` ranks by raw success rate and `run_history` stores durations, but nothing computed percentiles or significance. This adds the analysis layer — summary stats + p50/p90/p95/p99, a two-proportion z-test (with CI), Welch's t-test (exact t-distribution p-value via the incomplete beta — no SciPy), Cohen's d, and a 2×2 chi-square. The normal CDF is exact via `math.erf`; validated against textbook values (incl. the chi²=z² identity). Pure-stdlib `math`+`statistics`. + ## What's new (2026-06-21) — Full-Text Search (BM25) Rank a document corpus by relevance. Full reference: [`docs/source/Eng/doc/new_features/v64_features_doc.rst`](docs/source/Eng/doc/new_features/v64_features_doc.rst). diff --git a/README/README_zh-CN.md b/README/README_zh-CN.md index 901ecc28..f9fd3cbf 100644 --- a/README/README_zh-CN.md +++ b/README/README_zh-CN.md @@ -12,6 +12,7 @@ ## 目录 +- [本次更新 (2026-06-21) — 统计与 A/B 显著性](#本次更新-2026-06-21--统计与-ab-显著性) - [本次更新 (2026-06-21) — 全文搜索(BM25)](#本次更新-2026-06-21--全文搜索bm25) - [本次更新 (2026-06-21) — JSON Pointer、Patch 与 Merge Patch](#本次更新-2026-06-21--json-pointerpatch-与-merge-patch) - [本次更新 (2026-06-21) — 客户端速率限制](#本次更新-2026-06-21--客户端速率限制) @@ -116,6 +117,12 @@ --- +## 本次更新 (2026-06-21) — 统计与 A/B 显著性 + +判断差异是否为真。完整参考:[`docs/source/Zh/doc/new_features/v65_features_doc.rst`](../docs/source/Zh/doc/new_features/v65_features_doc.rst)。 + +- **`describe` / `percentile` / `two_proportion_z_test` / `welch_t_test` / `cohens_d` / `chi_square_2x2`**(`AC_describe_stats`、`AC_ab_significance`):`ab_locator` 以原始成功率排名,`run_history` 存储时长,但没有任何东西计算百分位或显著性。本功能补上分析层 —— 摘要统计 + p50/p90/p95/p99、双比例 z 检验(含置信区间)、Welch t 检验(以不完全 beta 取得精确 t 分布 p 值,免 SciPy)、Cohen's d,以及 2×2 卡方。正态 CDF 以 `math.erf` 精确计算;已对齐教科书数值(含 chi²=z² 恒等式)。纯标准库 `math`+`statistics`。 + ## 本次更新 (2026-06-21) — 全文搜索(BM25) 依相关性对文档语料排名。完整参考:[`docs/source/Zh/doc/new_features/v64_features_doc.rst`](../docs/source/Zh/doc/new_features/v64_features_doc.rst)。 diff --git a/README/README_zh-TW.md b/README/README_zh-TW.md index 90c6a853..707753fe 100644 --- a/README/README_zh-TW.md +++ b/README/README_zh-TW.md @@ -12,6 +12,7 @@ ## 目錄 +- [本次更新 (2026-06-21) — 統計與 A/B 顯著性](#本次更新-2026-06-21--統計與-ab-顯著性) - [本次更新 (2026-06-21) — 全文搜尋(BM25)](#本次更新-2026-06-21--全文搜尋bm25) - [本次更新 (2026-06-21) — JSON Pointer、Patch 與 Merge Patch](#本次更新-2026-06-21--json-pointerpatch-與-merge-patch) - [本次更新 (2026-06-21) — 用戶端速率限制](#本次更新-2026-06-21--用戶端速率限制) @@ -116,6 +117,12 @@ --- +## 本次更新 (2026-06-21) — 統計與 A/B 顯著性 + +判斷差異是否為真。完整參考:[`docs/source/Zh/doc/new_features/v65_features_doc.rst`](../docs/source/Zh/doc/new_features/v65_features_doc.rst)。 + +- **`describe` / `percentile` / `two_proportion_z_test` / `welch_t_test` / `cohens_d` / `chi_square_2x2`**(`AC_describe_stats`、`AC_ab_significance`):`ab_locator` 以原始成功率排名,`run_history` 儲存時長,但沒有任何東西計算百分位或顯著性。本功能補上分析層 —— 摘要統計 + p50/p90/p95/p99、雙比例 z 檢定(含信賴區間)、Welch t 檢定(以不完全 beta 取得精確 t 分布 p 值,免 SciPy)、Cohen's d,以及 2×2 卡方。常態 CDF 以 `math.erf` 精確計算;已對齊教科書數值(含 chi²=z² 恆等式)。純標準函式庫 `math`+`statistics`。 + ## 本次更新 (2026-06-21) — 全文搜尋(BM25) 依相關性對文件語料排名。完整參考:[`docs/source/Zh/doc/new_features/v64_features_doc.rst`](../docs/source/Zh/doc/new_features/v64_features_doc.rst)。 diff --git a/docs/source/Eng/doc/new_features/v65_features_doc.rst b/docs/source/Eng/doc/new_features/v65_features_doc.rst new file mode 100644 index 00000000..39fd6087 --- /dev/null +++ b/docs/source/Eng/doc/new_features/v65_features_doc.rst @@ -0,0 +1,51 @@ +Statistics & A/B Significance +============================= + +``ab_locator`` ranks strategies by raw success rate and ``run_history`` stores +durations, but nothing computed percentiles or told you whether a difference is +*statistically significant* rather than noise. This adds the analysis layer: +summary statistics, a two-proportion z-test, Welch's t-test, Cohen's d, and a +2x2 chi-square test. + +The normal CDF is exact via ``math.erf``; the t-distribution p-value uses the +regularized incomplete beta function, so results match reference +implementations without SciPy. Pure standard library; imports no ``PySide6``. + +Headless API +------------ + +.. code-block:: python + + from je_auto_control import ( + describe, percentile, two_proportion_z_test, welch_t_test, cohens_d) + + describe([12.0, 9.5, 14.2, 11.1]) + # {"n": 4, "min": 9.5, "max": 14.2, "mean": 11.7, "stdev": ..., + # "p50": ..., "p90": ..., "p95": ..., "p99": ...} + + # Did variant B convert better than A? (90/200 vs 110/200) + result = two_proportion_z_test(90, 200, 110, 200) + # {"z": 2.0, "p_value": 0.0455, "significant": True, + # "diff": 0.1, "ci_low": ..., "ci_high": ...} + + # Continuous metric (e.g. latencies): is B different from A? + welch_t_test(a_samples, b_samples) # {t, df, p_value, significant, ci} + cohens_d(a_samples, b_samples) # effect size + +``percentile`` supports linear interpolation (default) or nearest-rank; +``describe`` adds p50/p90/p95/p99 to the usual moments. ``two_proportion_z_test`` +uses the pooled standard error for the test and the unpooled SE for the +confidence interval (the textbook convention). ``welch_t_test`` reports the +Welch–Satterthwaite degrees of freedom and an exact t-distribution p-value. +``chi_square_2x2`` gives the df=1 chi-square (which equals the z-test's ``z²`` +for the same table). These pair naturally with ``ab_locator`` counts and +``run_history`` durations. + +Executor commands +----------------- + +``AC_describe_stats`` takes ``values`` (a numeric list or JSON string) and +returns the summary dict. ``AC_ab_significance`` takes ``a_conv`` / ``a_n`` / +``b_conv`` / ``b_n`` and returns the two-proportion z-test result. Both are +exposed as MCP tools (``ac_describe_stats`` / ``ac_ab_significance``) and as +Script Builder commands under **Data**. diff --git a/docs/source/Eng/eng_index.rst b/docs/source/Eng/eng_index.rst index bae93915..aedf32bf 100644 --- a/docs/source/Eng/eng_index.rst +++ b/docs/source/Eng/eng_index.rst @@ -87,6 +87,7 @@ Comprehensive guides for all AutoControl features. doc/new_features/v62_features_doc doc/new_features/v63_features_doc doc/new_features/v64_features_doc + doc/new_features/v65_features_doc doc/ocr_backends/ocr_backends_doc doc/observability/observability_doc doc/operations_layer/operations_layer_doc diff --git a/docs/source/Zh/doc/new_features/v65_features_doc.rst b/docs/source/Zh/doc/new_features/v65_features_doc.rst new file mode 100644 index 00000000..78d98b08 --- /dev/null +++ b/docs/source/Zh/doc/new_features/v65_features_doc.rst @@ -0,0 +1,44 @@ +統計與 A/B 顯著性 +================= + +``ab_locator`` 以原始成功率對策略排名,``run_history`` 儲存執行時長,但沒有任何東西計算百分位, +也無法告訴你某個差異是*統計上顯著*還是雜訊。本功能補上分析層:摘要統計、雙比例 z 檢定、Welch +t 檢定、Cohen's d,以及 2x2 卡方檢定。 + +常態 CDF 以 ``math.erf`` 精確計算;t 分布的 p 值使用正規化不完全 beta 函數,因此結果不需 SciPy +即可對齊參考實作。純標準函式庫;不匯入 ``PySide6``。 + +無頭 API +-------- + +.. code-block:: python + + from je_auto_control import ( + describe, percentile, two_proportion_z_test, welch_t_test, cohens_d) + + describe([12.0, 9.5, 14.2, 11.1]) + # {"n": 4, "min": 9.5, "max": 14.2, "mean": 11.7, "stdev": ..., + # "p50": ..., "p90": ..., "p95": ..., "p99": ...} + + # B 變體的轉換率有比 A 好嗎?(90/200 vs 110/200) + result = two_proportion_z_test(90, 200, 110, 200) + # {"z": 2.0, "p_value": 0.0455, "significant": True, + # "diff": 0.1, "ci_low": ..., "ci_high": ...} + + # 連續型指標(例如延遲):B 與 A 是否不同? + welch_t_test(a_samples, b_samples) # {t, df, p_value, significant, ci} + cohens_d(a_samples, b_samples) # 效果量 + +``percentile`` 支援線性內插(預設)或最近秩;``describe`` 在常見動差之外加上 p50/p90/p95/p99。 +``two_proportion_z_test`` 在檢定時使用合併標準誤、在信賴區間時使用未合併標準誤(教科書慣例)。 +``welch_t_test`` 回報 Welch–Satterthwaite 自由度與精確的 t 分布 p 值。``chi_square_2x2`` 給出 +df=1 的卡方(對同一張表等於 z 檢定的 ``z²``)。這些與 ``ab_locator`` 的計數及 ``run_history`` 的 +時長自然搭配。 + +執行器命令 +---------- + +``AC_describe_stats`` 接受 ``values``(數值清單或 JSON 字串),回傳摘要字典。 +``AC_ab_significance`` 接受 ``a_conv`` / ``a_n`` / ``b_conv`` / ``b_n``,回傳雙比例 z 檢定結果。 +兩者皆以 MCP 工具(``ac_describe_stats`` / ``ac_ab_significance``)以及 Script Builder 中 +**Data** 分類下的命令提供。 diff --git a/docs/source/Zh/zh_index.rst b/docs/source/Zh/zh_index.rst index fbfcf6ce..9ad0e0cb 100644 --- a/docs/source/Zh/zh_index.rst +++ b/docs/source/Zh/zh_index.rst @@ -87,6 +87,7 @@ AutoControl 所有功能的完整使用指南。 doc/new_features/v62_features_doc doc/new_features/v63_features_doc doc/new_features/v64_features_doc + doc/new_features/v65_features_doc doc/ocr_backends/ocr_backends_doc doc/observability/observability_doc doc/operations_layer/operations_layer_doc diff --git a/je_auto_control/__init__.py b/je_auto_control/__init__.py index dc970452..96548c3a 100644 --- a/je_auto_control/__init__.py +++ b/je_auto_control/__init__.py @@ -333,6 +333,11 @@ from je_auto_control.utils.search_index import ( SearchHit, SearchIndex, search_documents, tokenize, ) +# Descriptive statistics and A/B significance testing +from je_auto_control.utils.stats import ( + chi_square_2x2, cohens_d, describe, normal_cdf, percentile, + two_proportion_z_test, welch_t_test, +) # Background popup/interrupt watchdog (unattended automation) from je_auto_control.utils.watchdog import ( PopupWatchdog, WatchdogRule, default_popup_watchdog, @@ -816,6 +821,8 @@ def start_autocontrol_gui(*args, **kwargs): "make_patch", "merge_patch", "remove_pointer", "resolve_pointer", "set_pointer", "SearchHit", "SearchIndex", "search_documents", "tokenize", + "chi_square_2x2", "cohens_d", "describe", "normal_cdf", "percentile", + "two_proportion_z_test", "welch_t_test", # MCP server "AuditLogger", "HttpMCPServer", "MCPContent", "MCPPrompt", "MCPPromptArgument", "MCPResource", "MCPServer", "MCPTool", diff --git a/je_auto_control/gui/script_builder/command_schema.py b/je_auto_control/gui/script_builder/command_schema.py index ca027b37..ed26dcca 100644 --- a/je_auto_control/gui/script_builder/command_schema.py +++ b/je_auto_control/gui/script_builder/command_schema.py @@ -1155,6 +1155,24 @@ def _add_misc_specs(specs: List[CommandSpec]) -> None: ), description="Validate JSON against a JSON Schema; returns {ok, errors}.", )) + specs.append(CommandSpec( + "AC_describe_stats", "Data", "Describe Statistics", + fields=( + FieldSpec("values", FieldType.STRING, + placeholder="[12.0, 9.5, 14.2, 11.1]"), + ), + description="Summary stats + percentiles of a numeric list.", + )) + specs.append(CommandSpec( + "AC_ab_significance", "Data", "A/B Significance (z-test)", + fields=( + FieldSpec("a_conv", FieldType.INT, placeholder="90"), + FieldSpec("a_n", FieldType.INT, placeholder="200"), + FieldSpec("b_conv", FieldType.INT, placeholder="110"), + FieldSpec("b_n", FieldType.INT, placeholder="200"), + ), + description="Two-proportion z-test; returns {z, p_value, significant, ci}.", + )) specs.append(CommandSpec( "AC_search_documents", "Data", "Full-Text Search (BM25)", fields=( diff --git a/je_auto_control/utils/executor/action_executor.py b/je_auto_control/utils/executor/action_executor.py index 6e7c6f57..05d76e31 100644 --- a/je_auto_control/utils/executor/action_executor.py +++ b/je_auto_control/utils/executor/action_executor.py @@ -2928,6 +2928,22 @@ def _rate_limit(name: str, rate: float = 1.0, capacity: float = 1.0, "wait": round(bucket.time_until_available(float(n)), 4)} +def _describe_stats(values: Any) -> Dict[str, Any]: + """Adapter: summary statistics + percentiles of a numeric list (or JSON).""" + import json + from je_auto_control.utils.stats import describe + if isinstance(values, str): + values = json.loads(values) + return describe(values) + + +def _ab_significance(a_conv: int, a_n: int, b_conv: int, + b_n: int) -> Dict[str, Any]: + """Adapter: two-proportion z-test on A/B conversion counts.""" + from je_auto_control.utils.stats import two_proportion_z_test + return two_proportion_z_test(int(a_conv), int(a_n), int(b_conv), int(b_n)) + + def _search_documents(docs: Any, query: str, top_k: int = 10, mode: str = "bm25") -> Dict[str, Any]: """Adapter: BM25/TF-IDF search a {doc_id: text} corpus (dict or JSON str).""" @@ -3793,6 +3809,8 @@ def __init__(self): "AC_jwt_decode": _jwt_decode, "AC_rate_limit": _rate_limit, "AC_search_documents": _search_documents, + "AC_describe_stats": _describe_stats, + "AC_ab_significance": _ab_significance, "AC_resolve_pointer": _resolve_pointer, "AC_apply_json_patch": _apply_json_patch, "AC_make_json_patch": _make_json_patch, diff --git a/je_auto_control/utils/mcp_server/tools/_factories.py b/je_auto_control/utils/mcp_server/tools/_factories.py index 492b2cdf..7d15b38b 100644 --- a/je_auto_control/utils/mcp_server/tools/_factories.py +++ b/je_auto_control/utils/mcp_server/tools/_factories.py @@ -3343,6 +3343,32 @@ def rate_limit_tools() -> List[MCPTool]: ] +def stats_tools() -> List[MCPTool]: + return [ + MCPTool( + name="ac_describe_stats", + description=("Summary statistics + percentiles (n/min/max/mean/" + "stdev/variance/p50/p90/p95/p99) of a numeric " + "'values' list."), + input_schema=schema({"values": {"type": "array"}}, ["values"]), + handler=h.describe_stats, + annotations=READ_ONLY, + ), + MCPTool( + name="ac_ab_significance", + description=("Two-proportion z-test on A/B conversion counts. " + "Returns {z, p_value, significant, diff, ci_low, " + "ci_high}."), + input_schema=schema( + {"a_conv": {"type": "integer"}, "a_n": {"type": "integer"}, + "b_conv": {"type": "integer"}, "b_n": {"type": "integer"}}, + ["a_conv", "a_n", "b_conv", "b_n"]), + handler=h.ab_significance, + annotations=READ_ONLY, + ), + ] + + def search_index_tools() -> List[MCPTool]: return [ MCPTool( @@ -4601,7 +4627,7 @@ def media_assert_tools() -> List[MCPTool]: process_mining_tools, asset_tools, events_tools, notify_channel_tools, jsonpath_tools, json_schema_tools, vuln_scan_tools, vex_tools, license_policy_tools, jwt_tools, rate_limit_tools, json_patch_tools, - search_index_tools, + search_index_tools, stats_tools, saga_tools, decision_table_tools, locator_repair_tools, pii_text_tools, sarif_tools, screen_record_tools, diff --git a/je_auto_control/utils/mcp_server/tools/_handlers.py b/je_auto_control/utils/mcp_server/tools/_handlers.py index 1bbdfde5..3dff74cd 100644 --- a/je_auto_control/utils/mcp_server/tools/_handlers.py +++ b/je_auto_control/utils/mcp_server/tools/_handlers.py @@ -1637,6 +1637,16 @@ def search_documents(docs, query, top_k=10, mode="bm25"): return {"hits": [{"doc_id": h.doc_id, "score": h.score} for h in hits]} +def describe_stats(values): + from je_auto_control.utils.stats import describe + return describe(values) + + +def ab_significance(a_conv, a_n, b_conv, b_n): + from je_auto_control.utils.stats import two_proportion_z_test + return two_proportion_z_test(int(a_conv), int(a_n), int(b_conv), int(b_n)) + + def run_saga(steps): from je_auto_control.utils.saga import run_saga as _run result = _run(steps) diff --git a/je_auto_control/utils/stats/__init__.py b/je_auto_control/utils/stats/__init__.py new file mode 100644 index 00000000..92522055 --- /dev/null +++ b/je_auto_control/utils/stats/__init__.py @@ -0,0 +1,10 @@ +"""Descriptive statistics and A/B significance testing (pure stdlib).""" +from je_auto_control.utils.stats.stats import ( + chi_square_2x2, cohens_d, describe, normal_cdf, percentile, + two_proportion_z_test, welch_t_test, +) + +__all__ = [ + "chi_square_2x2", "cohens_d", "describe", "normal_cdf", "percentile", + "two_proportion_z_test", "welch_t_test", +] diff --git a/je_auto_control/utils/stats/stats.py b/je_auto_control/utils/stats/stats.py new file mode 100644 index 00000000..1d67e5f2 --- /dev/null +++ b/je_auto_control/utils/stats/stats.py @@ -0,0 +1,200 @@ +"""Descriptive statistics and A/B significance testing (pure standard library). + +``ab_locator`` ranks strategies by raw success rate and ``run_history`` stores +durations, but nothing computed percentiles or told you whether a difference is +*statistically significant* (rather than noise). This adds the analysis layer: +percentiles / summary stats, a two-proportion z-test (conversion rates), Welch's +t-test (continuous metrics), Cohen's d effect size, and a 2x2 chi-square test. + +The normal CDF is exact via ``math.erf``; the t-distribution p-value uses the +regularized incomplete beta function (continued fraction), so results match +reference implementations without SciPy. Imports no ``PySide6``. +""" +import math +import statistics +from typing import Any, Dict, Sequence + +from je_auto_control.utils.exception.exceptions import AutoControlException + + +# --- descriptive ----------------------------------------------------------- + +def percentile(values: Sequence[float], q: float, + method: str = "linear") -> float: + """Return the ``q``-th percentile (0-100) of ``values``.""" + if not values: + raise AutoControlException("percentile of empty data") + data = sorted(float(value) for value in values) + if q <= 0: + return data[0] + if q >= 100: + return data[-1] + if method == "nearest": + return data[max(0, math.ceil(q / 100 * len(data)) - 1)] + pos = (q / 100) * (len(data) - 1) + low = math.floor(pos) + if low + 1 >= len(data): + return data[low] + return data[low] + (pos - low) * (data[low + 1] - data[low]) + + +def describe(values: Sequence[float]) -> Dict[str, float]: + """Return summary statistics (n/min/max/mean/stdev/variance + percentiles).""" + if not values: + raise AutoControlException("describe of empty data") + data = [float(value) for value in values] + summary: Dict[str, float] = { + "n": len(data), "min": min(data), "max": max(data), + "mean": statistics.fmean(data), + "variance": statistics.pvariance(data), + "stdev": statistics.pstdev(data), + } + for q in (50, 90, 95, 99): + summary[f"p{q}"] = percentile(data, q) + return summary + + +# --- distribution helpers -------------------------------------------------- + +def normal_cdf(z: float) -> float: + """Standard normal cumulative distribution at ``z`` (exact via erf).""" + return 0.5 * (1 + math.erf(z / math.sqrt(2))) + + +def _clamp(value: float, floor: float = 1e-30) -> float: + return floor if abs(value) < floor else value + + +def _betacf(a: float, b: float, x: float) -> float: + qab, qap, qam = a + b, a + 1, a - 1 + c = 1.0 + d = 1.0 / _clamp(1.0 - qab * x / qap) + h = d + for m in range(1, 201): + m2 = 2 * m + aa = m * (b - m) * x / ((qam + m2) * (a + m2)) + d = 1.0 / _clamp(1.0 + aa * d) + c = _clamp(1.0 + aa / c) + h *= d * c + aa = -(a + m) * (qab + m) * x / ((a + m2) * (qap + m2)) + d = 1.0 / _clamp(1.0 + aa * d) + c = _clamp(1.0 + aa / c) + delta = d * c + h *= delta + if abs(delta - 1.0) < 3e-12: + break + return h + + +def _betai(a: float, b: float, x: float) -> float: + if x <= 0: + return 0.0 + if x >= 1: + return 1.0 + front = math.exp(math.lgamma(a + b) - math.lgamma(a) - math.lgamma(b) + + a * math.log(x) + b * math.log(1 - x)) + if x < (a + 1) / (a + b + 2): + return front * _betacf(a, b, x) / a + return 1.0 - front * _betacf(b, a, 1 - x) / b + + +def _t_two_sided_p(t: float, df: float) -> float: + """Two-sided p-value for a t-statistic with ``df`` degrees of freedom.""" + return _betai(df / 2.0, 0.5, df / (df + t * t)) + + +def _z_critical(alpha: float) -> float: + target = 1 - alpha / 2 + low, high = 0.0, 40.0 + for _ in range(100): + mid = (low + high) / 2 + if normal_cdf(mid) < target: + low = mid + else: + high = mid + return (low + high) / 2 + + +def _t_critical(alpha: float, df: float) -> float: + low, high = 0.0, 1000.0 + for _ in range(100): + mid = (low + high) / 2 + if _t_two_sided_p(mid, df) > alpha: + low = mid + else: + high = mid + return (low + high) / 2 + + +# --- significance tests ---------------------------------------------------- + +def two_proportion_z_test(a_conv: int, a_n: int, b_conv: int, b_n: int, *, + alpha: float = 0.05) -> Dict[str, Any]: + """Two-proportion z-test comparing conversion rates A vs B.""" + if a_n <= 0 or b_n <= 0: + raise AutoControlException("sample sizes must be positive") + p1, p2 = a_conv / a_n, b_conv / b_n + pooled = (a_conv + b_conv) / (a_n + b_n) + se_pool = math.sqrt(pooled * (1 - pooled) * (1 / a_n + 1 / b_n)) + z = (p2 - p1) / se_pool if se_pool > 0 else 0.0 + p_value = 2 * (1 - normal_cdf(abs(z))) + se = math.sqrt(p1 * (1 - p1) / a_n + p2 * (1 - p2) / b_n) + margin = _z_critical(alpha) * se + diff = p2 - p1 + return {"z": z, "p_value": p_value, "significant": p_value < alpha, + "diff": diff, "ci_low": diff - margin, "ci_high": diff + margin} + + +def welch_t_test(a: Sequence[float], b: Sequence[float], *, + alpha: float = 0.05) -> Dict[str, Any]: + """Welch's t-test (unequal variance) comparing two samples.""" + if len(a) < 2 or len(b) < 2: + raise AutoControlException("each sample needs at least two values") + na, nb = len(a), len(b) + mean_a, mean_b = statistics.fmean(a), statistics.fmean(b) + var_a, var_b = statistics.variance(a), statistics.variance(b) + se = math.sqrt(var_a / na + var_b / nb) + if se == 0: + return {"t": 0.0, "df": float(na + nb - 2), "p_value": 1.0, + "significant": False, "mean_diff": mean_b - mean_a, + "ci_low": 0.0, "ci_high": 0.0} + t = (mean_b - mean_a) / se + df = (var_a / na + var_b / nb) ** 2 / ( + (var_a / na) ** 2 / (na - 1) + (var_b / nb) ** 2 / (nb - 1)) + p_value = _t_two_sided_p(t, df) + margin = _t_critical(alpha, df) * se + diff = mean_b - mean_a + return {"t": t, "df": df, "p_value": p_value, "significant": p_value < alpha, + "mean_diff": diff, "ci_low": diff - margin, "ci_high": diff + margin} + + +def cohens_d(a: Sequence[float], b: Sequence[float]) -> float: + """Cohen's d effect size between samples ``a`` and ``b``.""" + if len(a) < 2 or len(b) < 2: + raise AutoControlException("each sample needs at least two values") + na, nb = len(a), len(b) + var_a, var_b = statistics.variance(a), statistics.variance(b) + pooled = math.sqrt(((na - 1) * var_a + (nb - 1) * var_b) / (na + nb - 2)) + if pooled == 0: + return 0.0 + return (statistics.fmean(b) - statistics.fmean(a)) / pooled + + +def chi_square_2x2(a_conv: int, a_fail: int, b_conv: int, + b_fail: int) -> Dict[str, Any]: + """Chi-square test of independence for a 2x2 contingency table (df=1).""" + observed = [(a_conv, a_fail), (b_conv, b_fail)] + row_totals = [a_conv + a_fail, b_conv + b_fail] + col_totals = [a_conv + b_conv, a_fail + b_fail] + total = a_conv + a_fail + b_conv + b_fail + if total <= 0: + raise AutoControlException("contingency table must have observations") + chi2 = 0.0 + for i in range(2): + for j in range(2): + expected = row_totals[i] * col_totals[j] / total + if expected > 0: + chi2 += (observed[i][j] - expected) ** 2 / expected + p_value = 2 * (1 - normal_cdf(math.sqrt(chi2))) + return {"chi2": chi2, "df": 1, "p_value": p_value, + "significant": p_value < 0.05} diff --git a/test/unit_test/headless/test_stats_batch.py b/test/unit_test/headless/test_stats_batch.py new file mode 100644 index 00000000..4515dc8c --- /dev/null +++ b/test/unit_test/headless/test_stats_batch.py @@ -0,0 +1,121 @@ +"""Headless tests for statistics + A/B significance. Pure stdlib, no Qt.""" +import json + +import pytest + +import je_auto_control as ac +from je_auto_control.utils.exception.exceptions import AutoControlException +from je_auto_control.utils.stats import ( + chi_square_2x2, cohens_d, describe, normal_cdf, percentile, + two_proportion_z_test, welch_t_test) + + +def test_percentile_linear_and_nearest(): + data = list(range(1, 11)) + assert percentile(data, 50) == pytest.approx(5.5) + assert percentile(data, 90) == pytest.approx(9.1) + assert percentile(data, 0) == 1 + assert percentile(data, 100) == 10 + assert percentile([1, 2, 3, 4], 50, method="nearest") == 2 + + +def test_percentile_empty_raises(): + with pytest.raises(AutoControlException): + percentile([], 50) + + +def test_describe(): + summary = describe([2, 4, 4, 4, 5, 5, 7, 9]) + assert summary["n"] == 8 + assert summary["mean"] == pytest.approx(5.0) + assert summary["stdev"] == pytest.approx(2.0) # population stdev + assert summary["min"] == 2 and summary["max"] == 9 + assert "p95" in summary + + +def test_normal_cdf(): + assert normal_cdf(0) == pytest.approx(0.5) + assert normal_cdf(1.96) == pytest.approx(0.975, abs=1e-3) + assert normal_cdf(-1.96) == pytest.approx(0.025, abs=1e-3) + + +def test_two_proportion_z_test_textbook(): + result = two_proportion_z_test(90, 200, 110, 200) + assert result["z"] == pytest.approx(2.0, abs=1e-2) + assert result["p_value"] == pytest.approx(0.0455, abs=1e-3) + assert result["significant"] is True + assert result["ci_low"] < result["diff"] < result["ci_high"] + + +def test_two_proportion_no_difference(): + result = two_proportion_z_test(50, 100, 50, 100) + assert result["p_value"] == pytest.approx(1.0, abs=1e-6) + assert result["significant"] is False + + +def test_two_proportion_bad_args(): + with pytest.raises(AutoControlException): + two_proportion_z_test(1, 0, 1, 10) + + +def test_welch_t_test_significant_and_not(): + sig = welch_t_test([5.1, 4.9, 5.0, 5.2, 4.8], [6.1, 5.9, 6.0, 6.2, 5.8]) + assert sig["significant"] is True + assert sig["p_value"] < 0.001 + weak = welch_t_test([1, 2, 3, 4, 5], [2, 3, 4, 5, 6]) + assert weak["significant"] is False + assert weak["p_value"] == pytest.approx(0.3466, abs=1e-3) + + +def test_welch_requires_two_values(): + with pytest.raises(AutoControlException): + welch_t_test([1], [1, 2, 3]) + + +def test_cohens_d(): + d = cohens_d([5.1, 4.9, 5.0, 5.2, 4.8], [6.1, 5.9, 6.0, 6.2, 5.8]) + assert d > 2.0 # very large effect + + +def test_chi_square_equals_z_squared(): + # the well-known identity: chi2 (df=1) == z^2 for the same 2x2 table + z = two_proportion_z_test(90, 200, 110, 200)["z"] + chi = chi_square_2x2(90, 110, 110, 90) + assert chi["chi2"] == pytest.approx(z ** 2, abs=1e-6) + assert chi["df"] == 1 + assert chi["significant"] is True + + +# --- wiring --------------------------------------------------------------- + +def test_executor_round_trip(): + rec = ac.execute_action([[ + "AC_describe_stats", {"values": json.dumps([10, 20, 30, 40])}, + ]]) + summary = next(v for v in rec.values() if isinstance(v, dict)) + assert summary["n"] == 4 and summary["mean"] == pytest.approx(25.0) + + rec2 = ac.execute_action([[ + "AC_ab_significance", + {"a_conv": 90, "a_n": 200, "b_conv": 110, "b_n": 200}, + ]]) + result = next(v for v in rec2.values() if isinstance(v, dict)) + assert result["significant"] is True + + +def test_wiring(): + assert {"AC_describe_stats", "AC_ab_significance"} <= ac.executor.known_commands() + from je_auto_control.utils.mcp_server.tools import build_default_tool_registry + names = {t.name for t in build_default_tool_registry()} + assert {"ac_describe_stats", "ac_ab_significance"} <= names + from je_auto_control.gui.script_builder.command_schema import _build_specs + cmds = {s.command for s in _build_specs()} + assert {"AC_describe_stats", "AC_ab_significance"} <= cmds + + +def test_facade_exports(): + for attr in ("percentile", "describe", "normal_cdf", + "two_proportion_z_test", "welch_t_test", "cohens_d", + "chi_square_2x2"): + assert hasattr(ac, attr) + assert attr in ac.__all__ From c3dd82d7e85953c61fb8e6099c1699ae1ef8a3bc Mon Sep 17 00:00:00 2001 From: JeffreyChen Date: Sun, 21 Jun 2026 07:13:48 +0800 Subject: [PATCH 2/2] Mark reachable beta-domain guards to satisfy the static analyzer --- je_auto_control/utils/stats/stats.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/je_auto_control/utils/stats/stats.py b/je_auto_control/utils/stats/stats.py index 1d67e5f2..08187a6c 100644 --- a/je_auto_control/utils/stats/stats.py +++ b/je_auto_control/utils/stats/stats.py @@ -87,9 +87,12 @@ def _betacf(a: float, b: float, x: float) -> float: def _betai(a: float, b: float, x: float) -> float: - if x <= 0: + # x is a domain ratio in [0, 1]; both boundaries are reachable (e.g. x == 1 + # when the t-statistic is 0). NOSONAR: the analyzer mis-models these as + # constant, but they are genuine, exercised guards. + if x <= 0: # NOSONAR python:S2583 reason: reachable beta-domain lower bound return 0.0 - if x >= 1: + if x >= 1: # NOSONAR python:S2583 reason: reachable beta-domain upper bound return 1.0 front = math.exp(math.lgamma(a + b) - math.lgamma(a) - math.lgamma(b) + a * math.log(x) + b * math.log(1 - x))