diff --git a/WHATS_NEW.md b/WHATS_NEW.md index 4c39e190..74f81365 100644 --- a/WHATS_NEW.md +++ b/WHATS_NEW.md @@ -1,5 +1,11 @@ # What's New — AutoControl +## What's new (2026-06-25) — Stable Failure Signatures + +Match the *same kind* of failure across runs, despite differing paths and ids. Full reference: [`docs/source/Eng/doc/new_features/v191_features_doc.rst`](docs/source/Eng/doc/new_features/v191_features_doc.rst). + +- **`normalize_error` / `failure_signature` / `group_failures`** (`AC_failure_signature`, `AC_group_failures`): two runs that failed the same way rarely have byte-identical error text — paths, line numbers, addresses, ids and timestamps differ every time — which defeats "is this the same failure?" and "which tests fail together?". This strips the variable parts of an error to a canonical form and hashes it (SHA-256), so the same kind of failure gets the same short signature across runs — the join key the rest of the test-robustness tools (run diffing, flake clustering) group on. `group_failures` buckets a list of errors by signature, most frequent first. Pure stdlib (`re` + `hashlib`). No `PySide6`. + ## What's new (2026-06-24) — Visual Saliency (where to look — spectral-residual) Find the region that stands out, with no template / colour / text. Full reference: [`docs/source/Eng/doc/new_features/v190_features_doc.rst`](docs/source/Eng/doc/new_features/v190_features_doc.rst). diff --git a/docs/source/Eng/doc/new_features/v191_features_doc.rst b/docs/source/Eng/doc/new_features/v191_features_doc.rst new file mode 100644 index 00000000..994cf054 --- /dev/null +++ b/docs/source/Eng/doc/new_features/v191_features_doc.rst @@ -0,0 +1,48 @@ +Stable Failure Signatures +========================= + +Two runs that failed the *same way* almost never have byte-identical error text — +paths, line numbers, memory addresses, ids and timestamps differ every time. That +defeats any attempt to ask "is this the same failure as yesterday?" or "which +tests fail *together*?". ``failure_signature`` strips the variable parts of an +error to a canonical form and hashes it (SHA-256), so the same *kind* of failure +gets the same short signature across runs — the join key the rest of the +test-robustness tools (run diffing, flake clustering) group on. + +* :func:`normalize_error` — collapse paths / hex addresses / UUIDs / timestamps / + line numbers / bare integers to placeholders, +* :func:`failure_signature` — a short stable SHA-256 of the normalised message, +* :func:`group_failures` — group a list of errors by signature, most frequent + first. + +Pure standard library (``re`` + ``hashlib``); no device, no ``PySide6``. + +Headless API +------------ + +.. code-block:: python + + from je_auto_control import (normalize_error, failure_signature, + group_failures) + + a = r"Timeout at C:\app\run.py line 42 (0x7ffab12c) at 2026-06-24 11:03:21" + b = r"Timeout at C:\app\run.py line 88 (0x1234abcd) at 2026-06-25 09:15:00" + normalize_error(a) # "Timeout at line (0x) at " + failure_signature(a) == failure_signature(b) # True — same failure + + group_failures([a, b, "Connection refused to /tmp/x.sock"]) + # [{"signature": "...", "normalized": "...", "count": 2, "examples": [...]}, + # {"signature": "...", "count": 1, ...}] + +Windows and POSIX paths, ``0x`` addresses, UUIDs, ISO timestamps, ``line N`` and +any leftover integers become placeholders; whitespace is squeezed. +``group_failures`` keeps up to three distinct raw examples per group and skips +empty / ``None`` messages. + +Executor commands +----------------- + +``AC_failure_signature`` (``error`` / ``length``) returns ``{signature, +normalized}``; ``AC_group_failures`` (``errors``) returns the grouped list. They +are exposed as read-only ``ac_*`` MCP tools and as Script Builder commands under +**Testing**. diff --git a/docs/source/Zh/doc/new_features/v191_features_doc.rst b/docs/source/Zh/doc/new_features/v191_features_doc.rst new file mode 100644 index 00000000..f1d1f367 --- /dev/null +++ b/docs/source/Zh/doc/new_features/v191_features_doc.rst @@ -0,0 +1,41 @@ +穩定的失敗簽章 +============== + +兩次以*相同方式*失敗的執行,幾乎不會有逐位元組相同的錯誤文字——路徑、行號、記憶體位址、id 與 +時間戳每次都不同。這使得「這和昨天是同一個失敗嗎?」或「哪些測試會*一起*失敗?」無從問起。 +``failure_signature`` 把錯誤的變動部分剝離成標準形式並雜湊(SHA-256),於是*相同類型*的失敗在 +不同執行間會得到相同的短簽章——即其餘 test-robustness 工具(執行比較、flaky 分群)所依據的 +join key。 + +* :func:`normalize_error` ——把路徑 / 十六進位位址 / UUID / 時間戳 / 行號 / 裸整數收斂成佔位符, +* :func:`failure_signature` ——正規化訊息的短而穩定的 SHA-256, +* :func:`group_failures` ——把一組錯誤依簽章分組,最常見者在前。 + +純標準庫(``re`` + ``hashlib``);不涉及裝置,不匯入 ``PySide6``。 + +無頭 API +-------- + +.. code-block:: python + + from je_auto_control import (normalize_error, failure_signature, + group_failures) + + a = r"Timeout at C:\app\run.py line 42 (0x7ffab12c) at 2026-06-24 11:03:21" + b = r"Timeout at C:\app\run.py line 88 (0x1234abcd) at 2026-06-25 09:15:00" + normalize_error(a) # "Timeout at line (0x) at " + failure_signature(a) == failure_signature(b) # True——同一個失敗 + + group_failures([a, b, "Connection refused to /tmp/x.sock"]) + # [{"signature": "...", "normalized": "...", "count": 2, "examples": [...]}, + # {"signature": "...", "count": 1, ...}] + +Windows 與 POSIX 路徑、``0x`` 位址、UUID、ISO 時間戳、``line N`` 與任何殘留整數都會變成佔位符; +空白會被壓縮。``group_failures`` 每組最多保留三個不同的原始範例,並略過空 / ``None`` 訊息。 + +執行器指令 +---------- + +``AC_failure_signature``(``error`` / ``length``)回傳 ``{signature, normalized}``; +``AC_group_failures``(``errors``)回傳分組清單。皆以唯讀 ``ac_*`` MCP 工具及 Script Builder +指令(位於 **Testing** 分類下)形式提供。 diff --git a/je_auto_control/__init__.py b/je_auto_control/__init__.py index 0cd404e9..446a003b 100644 --- a/je_auto_control/__init__.py +++ b/je_auto_control/__init__.py @@ -88,6 +88,10 @@ from je_auto_control.utils.saliency import ( most_salient, salient_regions, saliency_map, ) +# Stable failure signatures (normalise + hash error text; group failures) +from je_auto_control.utils.failure_signature import ( + failure_signature, group_failures, normalize_error, +) # VLM element locator (headless) from je_auto_control.utils.vision import ( VLMNotAvailableError, click_by_description, locate_by_description, @@ -1665,6 +1669,7 @@ def start_autocontrol_gui(*args, **kwargs): "image_quality", "is_blurry", "quality_gate", "detect_scale", "scale_sweep", "saliency_map", "salient_regions", "most_salient", + "normalize_error", "failure_signature", "group_failures", # VLM locator "VLMNotAvailableError", "locate_by_description", "click_by_description", "verify_description", diff --git a/je_auto_control/gui/script_builder/command_schema.py b/je_auto_control/gui/script_builder/command_schema.py index 8019fb23..1ebbb836 100644 --- a/je_auto_control/gui/script_builder/command_schema.py +++ b/je_auto_control/gui/script_builder/command_schema.py @@ -2708,6 +2708,21 @@ def _add_audit_specs(specs: List[CommandSpec]) -> None: description="Aggregate the self-heal log (heal rate, brittle " "locators).", )) + specs.append(CommandSpec( + "AC_failure_signature", "Testing", "Failure Signature", + fields=( + FieldSpec("error", FieldType.STRING, + placeholder="Timeout at C:\\app.py line 42 (0x7ff..)"), + FieldSpec("length", FieldType.INT, optional=True, default=12), + ), + description="Normalise + hash an error to a stable failure signature.", + )) + specs.append(CommandSpec( + "AC_group_failures", "Testing", "Group Failures by Signature", + fields=(FieldSpec("errors", FieldType.STRING, + placeholder='["err one", "err two"]'),), + description="Group error messages by failure signature (most frequent).", + )) specs.append(CommandSpec( "AC_scan_secrets", "Tools", "Scan for Hardcoded Secrets", description="Scan 'data' (JSON view) for hardcoded secrets that " diff --git a/je_auto_control/utils/executor/action_executor.py b/je_auto_control/utils/executor/action_executor.py index 39a99780..304a9729 100644 --- a/je_auto_control/utils/executor/action_executor.py +++ b/je_auto_control/utils/executor/action_executor.py @@ -4348,6 +4348,24 @@ def _most_salient(source: Any = None, region: Any = None, size: Any = 64, return {"found": result is not None, "region": result} +def _failure_signature(error: str, length: Any = 12) -> Dict[str, Any]: + """Adapter: normalise + hash an error message to a stable signature.""" + from je_auto_control.utils.failure_signature import ( + failure_signature, normalize_error) + return {"signature": failure_signature(str(error), length=int(length)), + "normalized": normalize_error(str(error))} + + +def _group_failures(errors: Any) -> Dict[str, Any]: + """Adapter: group error messages by failure signature.""" + import json + from je_auto_control.utils.failure_signature import group_failures + if isinstance(errors, str): + errors = json.loads(errors) + groups = group_failures(errors) + return {"groups": groups, "count": len(groups)} + + def _image_histogram(source: Any = None, bins: Any = 32, space: str = "hsv", region: Any = None) -> Dict[str, Any]: """Adapter: per-channel colour histogram of an image / the screen.""" @@ -6576,6 +6594,8 @@ def __init__(self): "AC_scale_sweep": _scale_sweep, "AC_salient_regions": _salient_regions, "AC_most_salient": _most_salient, + "AC_failure_signature": _failure_signature, + "AC_group_failures": _group_failures, "AC_image_histogram": _image_histogram, "AC_histogram_changed": _histogram_changed, "AC_changed_regions": _changed_regions, diff --git a/je_auto_control/utils/failure_signature/__init__.py b/je_auto_control/utils/failure_signature/__init__.py new file mode 100644 index 00000000..a9017441 --- /dev/null +++ b/je_auto_control/utils/failure_signature/__init__.py @@ -0,0 +1,6 @@ +"""Normalise error messages into stable SHA-256 failure signatures + grouping.""" +from je_auto_control.utils.failure_signature.failure_signature import ( + failure_signature, group_failures, normalize_error, +) + +__all__ = ["normalize_error", "failure_signature", "group_failures"] diff --git a/je_auto_control/utils/failure_signature/failure_signature.py b/je_auto_control/utils/failure_signature/failure_signature.py new file mode 100644 index 00000000..5743eec4 --- /dev/null +++ b/je_auto_control/utils/failure_signature/failure_signature.py @@ -0,0 +1,68 @@ +"""Normalise an error message into a stable failure signature. + +Two runs that failed the *same way* almost never have byte-identical error text — +paths, line numbers, memory addresses, ids and timestamps differ every time. That +defeats any attempt to ask "is this the same failure as yesterday?" or "which +tests fail *together*?". ``failure_signature`` strips the variable parts of an +error to a canonical form and hashes it (SHA-256), so the same *kind* of failure +gets the same short signature across runs — the join key the rest of the +test-robustness tools (run diffing, flake clustering) group on. + +Pure standard library (``re`` + ``hashlib``); no device, no ``PySide6``. +""" +import hashlib +import re +from typing import Any, Dict, Iterable, List + +# Ordered (pattern, replacement): the volatile parts of an error, most specific +# first so e.g. a path's trailing line number isn't half-collapsed by the digit rule. +_NORMALIZERS = [ + (re.compile(r"[A-Za-z]:\\[^\s:*?\"<>|]+"), ""), # Windows path + (re.compile(r"(?:/[\w.\-]+)+/[\w.\-]+"), ""), # POSIX path + (re.compile(r"0x[0-9A-Fa-f]+"), "0x"), # memory address + (re.compile(r"\b[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}" + r"-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}\b"), ""), + (re.compile(r"\d{4}-\d{2}-\d{2}[ T]\d{2}:\d{2}:\d{2}(?:\.\d+)?"), ""), + (re.compile(r"\bline\s+\d+\b", re.IGNORECASE), "line "), + (re.compile(r"\b\d+\b"), ""), # any leftover int +] +_WHITESPACE = re.compile(r"\s+") + + +def normalize_error(message: str) -> str: + """Collapse the volatile parts of an error message to a canonical form. + + Paths, hex addresses, UUIDs, timestamps, line numbers and bare integers + become placeholders, and whitespace is squeezed — so messages that differ + only in those details normalise to the same string. + """ + text = str(message) + for pattern, replacement in _NORMALIZERS: + text = pattern.sub(replacement, text) + return _WHITESPACE.sub(" ", text).strip() + + +def failure_signature(message: str, *, length: int = 12) -> str: + """Return a short stable SHA-256 signature of a normalised error message.""" + digest = hashlib.sha256(normalize_error(message).encode("utf-8")).hexdigest() + return digest[:max(1, int(length))] + + +def group_failures(messages: Iterable[str]) -> List[Dict[str, Any]]: + """Group error messages by signature, most frequent first. + + Returns ``[{signature, normalized, count, examples}]`` (up to three distinct + raw examples per group). ``None`` / empty messages are skipped. + """ + groups: Dict[str, Dict[str, Any]] = {} + for message in messages: + if not message: + continue + signature = failure_signature(message) + group = groups.setdefault(signature, { + "signature": signature, "normalized": normalize_error(message), + "count": 0, "examples": []}) + group["count"] += 1 + if len(group["examples"]) < 3 and str(message) not in group["examples"]: + group["examples"].append(str(message)) + return sorted(groups.values(), key=lambda group: group["count"], reverse=True) diff --git a/je_auto_control/utils/mcp_server/tools/_factories.py b/je_auto_control/utils/mcp_server/tools/_factories.py index b84715a1..8a3ddfbc 100644 --- a/je_auto_control/utils/mcp_server/tools/_factories.py +++ b/je_auto_control/utils/mcp_server/tools/_factories.py @@ -7652,6 +7652,29 @@ def flakiness_tools() -> List[MCPTool]: handler=h.flaky_report, annotations=READ_ONLY, ), + MCPTool( + name="ac_failure_signature", + description=("Normalise an error message (strip paths / addresses / " + "line numbers / timestamps / ids) and hash it to a stable " + "SHA-256 signature, so the same kind of failure matches " + "across runs. Returns {signature, normalized}."), + input_schema=schema({"error": {"type": "string"}, + "length": {"type": "integer"}}, + required=["error"]), + handler=h.failure_signature, + annotations=READ_ONLY, + ), + MCPTool( + name="ac_group_failures", + description=("Group a list of error messages by failure signature, " + "most frequent first: [{signature, normalized, count, " + "examples}]."), + input_schema=schema({ + "errors": {"type": "array", "items": {"type": "string"}}}, + required=["errors"]), + handler=h.group_failures, + annotations=READ_ONLY, + ), ] diff --git a/je_auto_control/utils/mcp_server/tools/_handlers.py b/je_auto_control/utils/mcp_server/tools/_handlers.py index c00840fe..552793a8 100644 --- a/je_auto_control/utils/mcp_server/tools/_handlers.py +++ b/je_auto_control/utils/mcp_server/tools/_handlers.py @@ -2543,6 +2543,16 @@ def most_salient(source=None, region=None, size=64, threshold=None, min_area=4): return _most_salient(source, region, size, threshold, min_area) +def failure_signature(error, length=12): + from je_auto_control.utils.executor.action_executor import _failure_signature + return _failure_signature(error, length) + + +def group_failures(errors): + from je_auto_control.utils.executor.action_executor import _group_failures + return _group_failures(errors) + + def image_histogram(source=None, bins=32, space="hsv", region=None): from je_auto_control.utils.executor.action_executor import _image_histogram return _image_histogram(source, bins, space, region) diff --git a/test/unit_test/headless/test_failure_signature_batch.py b/test/unit_test/headless/test_failure_signature_batch.py new file mode 100644 index 00000000..c1f93933 --- /dev/null +++ b/test/unit_test/headless/test_failure_signature_batch.py @@ -0,0 +1,70 @@ +"""Headless tests for error normalisation + stable failure signatures.""" +import je_auto_control as ac +from je_auto_control.utils.failure_signature import ( + failure_signature, group_failures, normalize_error, +) + +_RUN_A = r"Timeout locating element at C:\Users\me\app.py line 42 (0x7ffab12c) at 2026-06-24 11:03:21" +_RUN_B = r"Timeout locating element at C:\Users\you\app.py line 99 (0x1234abcd) at 2026-06-25 09:15:00" +_OTHER = "Connection refused to /var/run/db.sock" + + +def test_normalize_collapses_volatile_parts(): + assert normalize_error(_RUN_A) == ( + "Timeout locating element at line (0x) at ") + + +def test_same_failure_same_signature_across_runs(): + assert failure_signature(_RUN_A) == failure_signature(_RUN_B) + + +def test_different_failure_differs(): + assert failure_signature(_RUN_A) != failure_signature(_OTHER) + + +def test_signature_length_param(): + assert len(failure_signature(_RUN_A, length=8)) == 8 + assert len(failure_signature(_RUN_A)) == 12 + + +def test_uuid_and_posix_path_normalised(): + msg = "row 00000000-1111-2222-3333-444444444444 at /tmp/x/data.json missing" + assert normalize_error(msg) == "row at missing" + + +def test_group_failures_counts_and_skips_empty(): + groups = group_failures([_RUN_A, _RUN_B, _OTHER, + "Connection refused to /tmp/other.sock", None, ""]) + assert len(groups) == 2 + assert groups[0]["count"] == 2 # most frequent first (tie → 2 each) + timeout = next(g for g in groups if "Timeout" in g["normalized"]) + assert timeout["count"] == 2 + assert len(timeout["examples"]) == 2 # both raw variants kept (max 3) + + +# --- wiring --------------------------------------------------------------- + +def test_executor_paths(): + from je_auto_control.utils.executor.action_executor import ( + _failure_signature, _group_failures) + sig = _failure_signature(_RUN_A) + assert sig["signature"] == failure_signature(_RUN_A) + assert sig["normalized"].endswith("at ") + grouped = _group_failures(f'["{_OTHER}", "{_OTHER}"]') + assert grouped["count"] == 1 and grouped["groups"][0]["count"] == 2 + + +def test_wiring(): + known = set(ac.executor.known_commands()) + assert {"AC_failure_signature", "AC_group_failures"} <= known + from je_auto_control.utils.mcp_server.tools import build_default_tool_registry + names = {t.name for t in build_default_tool_registry()} + assert {"ac_failure_signature", "ac_group_failures"} <= names + from je_auto_control.gui.script_builder.command_schema import _build_specs + specs = {s.command for s in _build_specs()} + assert {"AC_failure_signature", "AC_group_failures"} <= specs + + +def test_facade_exports(): + for name in ("normalize_error", "failure_signature", "group_failures"): + assert hasattr(ac, name) and name in ac.__all__