From bb34b5df275807ec1c68a97a4db7ba197a0b4c34 Mon Sep 17 00:00:00 2001 From: JeffreyChen Date: Wed, 24 Jun 2026 02:36:33 +0800 Subject: [PATCH] Add trust-scored template matching (ambiguity / PSR) match_template returns only the top score and clicks it, but a duplicate toolbar button or a near-identical sibling correlates ~0.95 in two places, so a high score is not an unambiguous match. Add a Lowe-style ratio test for pixel templates: inspect the full correlation surface, compare the global peak to the next-best peak outside an exclusion window, compute PSR, and flag strong-but-ambiguous matches. Reuses a new visual_match._score_map. --- README/WHATS_NEW_zh-CN.md | 6 + README/WHATS_NEW_zh-TW.md | 6 + WHATS_NEW.md | 6 + .../doc/new_features/v161_features_doc.rst | 46 +++++++ docs/source/Eng/eng_index.rst | 1 + .../Zh/doc/new_features/v161_features_doc.rst | 41 ++++++ docs/source/Zh/zh_index.rst | 1 + je_auto_control/__init__.py | 7 + .../gui/script_builder/command_schema.py | 15 ++ .../utils/executor/action_executor.py | 17 +++ je_auto_control/utils/match_trust/__init__.py | 6 + .../utils/match_trust/match_trust.py | 130 ++++++++++++++++++ .../utils/mcp_server/tools/_factories.py | 19 +++ .../utils/mcp_server/tools/_handlers.py | 7 + .../utils/visual_match/visual_match.py | 21 +++ .../headless/test_match_trust_batch.py | 81 +++++++++++ 16 files changed, 410 insertions(+) create mode 100644 docs/source/Eng/doc/new_features/v161_features_doc.rst create mode 100644 docs/source/Zh/doc/new_features/v161_features_doc.rst create mode 100644 je_auto_control/utils/match_trust/__init__.py create mode 100644 je_auto_control/utils/match_trust/match_trust.py create mode 100644 test/unit_test/headless/test_match_trust_batch.py diff --git a/README/WHATS_NEW_zh-CN.md b/README/WHATS_NEW_zh-CN.md index 6dc730f3..e738a1e3 100644 --- a/README/WHATS_NEW_zh-CN.md +++ b/README/WHATS_NEW_zh-CN.md @@ -1,5 +1,11 @@ # 本次更新 — AutoControl +## 本次更新 (2026-06-24) — 信任评分模板匹配(歧义 / PSR) + +在点击前就知道某次模板匹配虽强但*有歧义*。完整参考:[`docs/source/Zh/doc/new_features/v161_features_doc.rst`](../docs/source/Zh/doc/new_features/v161_features_doc.rst)。 + +- **`match_with_trust` / `score_peaks`**(`AC_match_with_trust`):`match_template` 只返回最高分并点击——但工具栏中重复的按钮或近乎相同的同类控件可能在两处都相关到 ~0.95,因此高分并非*无歧义*的匹配。本功能为像素模板加入 Lowe 式比值测试(ORB 通过 `feature_match` 已有,`match_template` 从未有):检视整个相关性曲面,比较全局峰值与排除窗口外的次高峰,计算峰值对旁瓣比(PSR),返回带有 `second_score` / `peak_ratio` / `psr` / `is_ambiguous` 的 `TrustedMatch`。重用新增的 `visual_match._score_map`(公开匹配器丢弃的完整 `matchTemplate` 曲面)——不重复任何匹配代码。`haystack` 可注入;不导入 `PySide6`。 + ## 本次更新 (2026-06-23) — 剪贴板文件拖放列表(CF_HDROP) 把一份文件列表放上剪贴板,可直接粘贴进 Explorer。完整参考:[`docs/source/Zh/doc/new_features/v160_features_doc.rst`](../docs/source/Zh/doc/new_features/v160_features_doc.rst)。 diff --git a/README/WHATS_NEW_zh-TW.md b/README/WHATS_NEW_zh-TW.md index 19e5bf8e..c6ca3936 100644 --- a/README/WHATS_NEW_zh-TW.md +++ b/README/WHATS_NEW_zh-TW.md @@ -1,5 +1,11 @@ # 本次更新 — AutoControl +## 本次更新 (2026-06-24) — 信任評分樣板比對(歧義 / PSR) + +在點擊前就知道某次樣板比對雖強但*有歧義*。完整參考:[`docs/source/Zh/doc/new_features/v161_features_doc.rst`](../docs/source/Zh/doc/new_features/v161_features_doc.rst)。 + +- **`match_with_trust` / `score_peaks`**(`AC_match_with_trust`):`match_template` 只回傳最高分並點擊——但工具列中重複的按鈕或近乎相同的同類元件可能在兩處都相關到 ~0.95,因此高分並非*無歧義*的比對。本功能為像素樣板加入 Lowe 式比值測試(ORB 透過 `feature_match` 已有,`match_template` 從未有):檢視整個相關性曲面,比較全域峰值與排除視窗外的次高峰,計算峰值對旁瓣比(PSR),回傳帶有 `second_score` / `peak_ratio` / `psr` / `is_ambiguous` 的 `TrustedMatch`。重用新增的 `visual_match._score_map`(公開比對器丟棄的完整 `matchTemplate` 曲面)——不重複任何比對程式。`haystack` 可注入;不匯入 `PySide6`。 + ## 本次更新 (2026-06-23) — 剪貼簿檔案拖放清單(CF_HDROP) 把一份檔案清單放上剪貼簿,可直接貼進 Explorer。完整參考:[`docs/source/Zh/doc/new_features/v160_features_doc.rst`](../docs/source/Zh/doc/new_features/v160_features_doc.rst)。 diff --git a/WHATS_NEW.md b/WHATS_NEW.md index a90d4ac7..16e37544 100644 --- a/WHATS_NEW.md +++ b/WHATS_NEW.md @@ -1,5 +1,11 @@ # What's New — AutoControl +## What's new (2026-06-24) — Trust-Scored Template Matching (Ambiguity / PSR) + +Know when a template match is strong but *ambiguous* before clicking it. Full reference: [`docs/source/Eng/doc/new_features/v161_features_doc.rst`](docs/source/Eng/doc/new_features/v161_features_doc.rst). + +- **`match_with_trust` / `score_peaks`** (`AC_match_with_trust`): `match_template` returns only the top score and clicks it — but a button repeated in a toolbar or a near-identical sibling correlates ~0.95 in two places, so a high score is not an *unambiguous* match. This adds a Lowe-style ratio test *for pixel templates* (ORB got one via `feature_match`; `match_template` never did): it inspects the whole correlation surface, compares the global peak to the next-best peak outside an exclusion window, computes the peak-to-sidelobe ratio (PSR), and returns a `TrustedMatch` with `second_score` / `peak_ratio` / `psr` / `is_ambiguous`. Reuses a new `visual_match._score_map` (the full `matchTemplate` surface the public matchers discard) — no matching code duplicated. Injectable `haystack`; no `PySide6`. + ## What's new (2026-06-23) — Clipboard File-Drop List (CF_HDROP) Put a list of files on the clipboard, ready to paste into Explorer. Full reference: [`docs/source/Eng/doc/new_features/v160_features_doc.rst`](docs/source/Eng/doc/new_features/v160_features_doc.rst). diff --git a/docs/source/Eng/doc/new_features/v161_features_doc.rst b/docs/source/Eng/doc/new_features/v161_features_doc.rst new file mode 100644 index 00000000..7237337f --- /dev/null +++ b/docs/source/Eng/doc/new_features/v161_features_doc.rst @@ -0,0 +1,46 @@ +Trust-Scored Template Matching (Ambiguity / PSR) +================================================ + +``match_template`` returns the single best score and happily clicks it — but a control +repeated in a toolbar, or a near-identical sibling, correlates ~0.95 in *two* places, so a +high score does **not** mean an *unambiguous* match, and the matcher can confidently click +the wrong one. ``match_with_trust`` adds a Lowe-style ratio test *for pixel templates* +(``feature_match`` already does this for ORB keypoints, but nothing did it for +``match_template``): it inspects the whole correlation surface, compares the global peak to +the next-best peak outside an exclusion window, and computes the peak-to-sidelobe ratio +(PSR), flagging matches that are strong-but-ambiguous. + +It reuses ``visual_match._score_map`` — the full ``matchTemplate`` surface the public matchers +discard — so no matching code is duplicated. The ``haystack`` is injectable (ndarray / path / +PIL); the analysis is unit-testable on synthetic arrays. Imports no ``PySide6``. + +Headless API +------------ + +.. code-block:: python + + from je_auto_control import match_with_trust, score_peaks + + hit = match_with_trust("save_button.png", min_score=0.8) + if hit and not hit.is_ambiguous: + click(*hit.center) + elif hit: + print("ambiguous!", hit.peak_ratio, "second:", hit.second_score) + + # just the metrics, no match object + print(score_peaks("icon.png")) # {best, second, peak_ratio, psr, ambiguous, location} + +``match_with_trust`` returns a ``TrustedMatch`` (``x`` / ``y`` / ``width`` / ``height`` / +``score`` / ``scale`` / ``second_score`` / ``peak_ratio`` / ``psr`` / ``is_ambiguous`` + +``center``) or ``None``. ``is_ambiguous`` is set when the next-best peak scores at least +``ambiguous_ratio`` (default 0.9) times the best. ``psr`` is the peak-to-sidelobe ratio +(``None`` when the sidelobe is perfectly flat). ``score_peaks`` returns just the metric dict +at scale 1.0. + +Executor command +---------------- + +``AC_match_with_trust`` (``template`` / ``min_score`` / ``scales`` / ``ambiguous_ratio`` / +``region`` / ``method`` → ``{found, match}``) is exposed as the MCP tool +``ac_match_with_trust`` (read-only) and as the Script Builder command **Match Template +(trust-scored)** under **Image**. diff --git a/docs/source/Eng/eng_index.rst b/docs/source/Eng/eng_index.rst index 57929869..ec331a15 100644 --- a/docs/source/Eng/eng_index.rst +++ b/docs/source/Eng/eng_index.rst @@ -183,6 +183,7 @@ Comprehensive guides for all AutoControl features. doc/new_features/v158_features_doc doc/new_features/v159_features_doc doc/new_features/v160_features_doc + doc/new_features/v161_features_doc doc/ocr_backends/ocr_backends_doc doc/observability/observability_doc doc/operations_layer/operations_layer_doc diff --git a/docs/source/Zh/doc/new_features/v161_features_doc.rst b/docs/source/Zh/doc/new_features/v161_features_doc.rst new file mode 100644 index 00000000..f24596f2 --- /dev/null +++ b/docs/source/Zh/doc/new_features/v161_features_doc.rst @@ -0,0 +1,41 @@ +信任評分樣板比對(歧義 / PSR) +============================== + +``match_template`` 只回傳最佳分數並直接點擊——但工具列中重複出現的控制項,或近乎相同的 +同類元件,可能在*兩處*都相關到 ~0.95,因此高分**不代表**比對*無歧義*,比對器可能自信地 +點錯目標。``match_with_trust`` 為像素樣板加入 Lowe 式比值測試(``feature_match`` 已對 ORB +關鍵點這麼做,但 ``match_template`` 從未如此):它檢視整個相關性曲面,比較全域峰值與排除 +視窗外的次高峰,並計算峰值對旁瓣比(PSR),標記出強但有歧義的比對。 + +本功能重用 ``visual_match._score_map``——即公開比對器丟棄的完整 ``matchTemplate`` 曲面 +——因此不重複任何比對程式。``haystack`` 可注入(ndarray / 路徑 / PIL);分析可在合成陣列上 +單元測試。不匯入 ``PySide6``。 + +無頭 API +-------- + +.. code-block:: python + + from je_auto_control import match_with_trust, score_peaks + + hit = match_with_trust("save_button.png", min_score=0.8) + if hit and not hit.is_ambiguous: + click(*hit.center) + elif hit: + print("有歧義!", hit.peak_ratio, "次高:", hit.second_score) + + # 只要指標,不要 match 物件 + print(score_peaks("icon.png")) # {best, second, peak_ratio, psr, ambiguous, location} + +``match_with_trust`` 回傳 ``TrustedMatch``(``x`` / ``y`` / ``width`` / ``height`` / +``score`` / ``scale`` / ``second_score`` / ``peak_ratio`` / ``psr`` / ``is_ambiguous`` + +``center``)或 ``None``。當次高峰至少達到最佳值的 ``ambiguous_ratio`` 倍(預設 0.9)時, +``is_ambiguous`` 為真。``psr`` 為峰值對旁瓣比(旁瓣完全平坦時為 ``None``)。``score_peaks`` +僅回傳縮放 1.0 時的指標字典。 + +執行器指令 +---------- + +``AC_match_with_trust``(``template`` / ``min_score`` / ``scales`` / ``ambiguous_ratio`` / +``region`` / ``method`` → ``{found, match}``)以 MCP 工具 ``ac_match_with_trust``(唯讀)及 +Script Builder 指令 **Match Template (trust-scored)**(位於 **Image** 分類下)形式提供。 diff --git a/docs/source/Zh/zh_index.rst b/docs/source/Zh/zh_index.rst index 7193a73f..747a2332 100644 --- a/docs/source/Zh/zh_index.rst +++ b/docs/source/Zh/zh_index.rst @@ -183,6 +183,7 @@ AutoControl 所有功能的完整使用指南。 doc/new_features/v158_features_doc doc/new_features/v159_features_doc doc/new_features/v160_features_doc + doc/new_features/v161_features_doc doc/ocr_backends/ocr_backends_doc doc/observability/observability_doc doc/operations_layer/operations_layer_doc diff --git a/je_auto_control/__init__.py b/je_auto_control/__init__.py index e92b7b31..cfd6b97c 100644 --- a/je_auto_control/__init__.py +++ b/je_auto_control/__init__.py @@ -283,6 +283,10 @@ from je_auto_control.utils.rotated_match import ( RotatedMatch, match_rotated, match_rotated_all, scale_space, ) +# Template-match trustworthiness (second-peak ratio + peak-to-sidelobe) +from je_auto_control.utils.match_trust import ( + TrustedMatch, match_with_trust, score_peaks, +) # Coarse labelled cell grid for VLM grounding (point <-> cell mapping) from je_auto_control.utils.screen_grid import ( GridCell, cell_for_point, grid_cells, point_for_cell, @@ -1198,6 +1202,9 @@ def start_autocontrol_gui(*args, **kwargs): "match_rotated", "match_rotated_all", "scale_space", + "TrustedMatch", + "match_with_trust", + "score_peaks", "GridCell", "grid_cells", "cell_for_point", diff --git a/je_auto_control/gui/script_builder/command_schema.py b/je_auto_control/gui/script_builder/command_schema.py index 8498a99f..ae36dee1 100644 --- a/je_auto_control/gui/script_builder/command_schema.py +++ b/je_auto_control/gui/script_builder/command_schema.py @@ -335,6 +335,21 @@ def _add_image_specs(specs: List[CommandSpec]) -> None: ), description="Find every rotation/scale-tolerant match (NMS-deduped).", )) + specs.append(CommandSpec( + "AC_match_with_trust", "Image", "Match Template (trust-scored)", + fields=( + FieldSpec("template", FieldType.FILE_PATH), + FieldSpec("min_score", FieldType.FLOAT, optional=True, default=0.0, + min_value=0.0, max_value=1.0), + FieldSpec("ambiguous_ratio", FieldType.FLOAT, optional=True, + default=0.9, min_value=0.0, max_value=1.0), + FieldSpec("scales", FieldType.STRING, optional=True, + placeholder="[0.9, 1.0, 1.1]"), + FieldSpec("region", FieldType.STRING, optional=True, + placeholder=_REGION_PLACEHOLDER), + ), + description="Match a template and flag if it is ambiguous (duplicate peak).", + )) specs.append(CommandSpec( "AC_grid_cells", "Image", "Grid Cells (coarse grounding)", fields=( diff --git a/je_auto_control/utils/executor/action_executor.py b/je_auto_control/utils/executor/action_executor.py index e67035c9..00a04851 100644 --- a/je_auto_control/utils/executor/action_executor.py +++ b/je_auto_control/utils/executor/action_executor.py @@ -3323,6 +3323,22 @@ def _match_rotated_all(template: str, min_score: Any = 0.8, scales: Any = None, return {"count": len(matches), "matches": [m.to_dict() for m in matches]} +def _match_with_trust(template: str, min_score: Any = 0.0, scales: Any = None, + ambiguous_ratio: Any = 0.9, region: Any = None, + method: str = "ccoeff_normed") -> Dict[str, Any]: + """Adapter: best template match with trust metrics (ambiguity / PSR).""" + import json + from je_auto_control.utils.match_trust import match_with_trust + if isinstance(region, str): + region = json.loads(region) if region.strip() else None + match = match_with_trust(template, region=region, + scales=_seq_arg(scales, (1.0,)), + method=method, min_score=float(min_score), + ambiguous_ratio=float(ambiguous_ratio)) + return {"found": match is not None, + "match": match.to_dict() if match else None} + + def _region_arg(value: Any) -> Optional[List[int]]: """Coerce a JSON-string / list region arg into a list of ints, or None.""" import json @@ -5779,6 +5795,7 @@ def __init__(self): "AC_match_masked_all": _match_masked_all, "AC_match_rotated": _match_rotated, "AC_match_rotated_all": _match_rotated_all, + "AC_match_with_trust": _match_with_trust, "AC_grid_cells": _grid_cells, "AC_cell_for_point": _cell_for_point, "AC_point_for_cell": _point_for_cell, diff --git a/je_auto_control/utils/match_trust/__init__.py b/je_auto_control/utils/match_trust/__init__.py new file mode 100644 index 00000000..413a5802 --- /dev/null +++ b/je_auto_control/utils/match_trust/__init__.py @@ -0,0 +1,6 @@ +"""Template-match trustworthiness scoring (second-peak ratio + peak-to-sidelobe).""" +from je_auto_control.utils.match_trust.match_trust import ( + TrustedMatch, match_with_trust, score_peaks, +) + +__all__ = ["TrustedMatch", "match_with_trust", "score_peaks"] diff --git a/je_auto_control/utils/match_trust/match_trust.py b/je_auto_control/utils/match_trust/match_trust.py new file mode 100644 index 00000000..8991288e --- /dev/null +++ b/je_auto_control/utils/match_trust/match_trust.py @@ -0,0 +1,130 @@ +"""Trustworthiness scoring for template matches (second-peak ratio + PSR). + +``visual_match.match_template`` returns the single best score and happily clicks it — +but a control repeated in a toolbar, or a near-identical sibling, correlates ~0.95 in +*two* places, so a high score does not mean an *unambiguous* match. This adds a Lowe-style +ratio test *for pixel templates* (``feature_match`` already does it for ORB keypoints, but +nothing did it for ``match_template``): it inspects the whole correlation surface, compares +the global peak against the next-best peak outside an exclusion window, and computes the +peak-to-sidelobe ratio (PSR), flagging matches that are strong-but-ambiguous. + +It reuses ``visual_match._score_map`` (the full ``matchTemplate`` surface the public matchers +discard) plus the shared gray loaders, so no matching code is duplicated. The ``haystack`` is +injectable (ndarray / path / PIL); the search is unit-testable on synthetic arrays. OpenCV + +NumPy are imported lazily. Imports no ``PySide6``. +""" +from dataclasses import asdict, dataclass +from typing import Any, Dict, List, Optional, Sequence + +from je_auto_control.utils.visual_match.visual_match import _score_map + +ImageSource = Any + + +@dataclass(frozen=True) +class TrustedMatch: + """A match plus its trust metrics: second-best score, peak ratio, PSR, ambiguity.""" + + x: int + y: int + width: int + height: int + score: float + scale: float + second_score: float + peak_ratio: float + psr: Optional[float] + is_ambiguous: bool + + @property + def center(self) -> List[int]: + """The match's centre point ``[x, y]`` (ready to click).""" + return [self.x + self.width // 2, self.y + self.height // 2] + + def to_dict(self) -> Dict[str, Any]: + """Return the match as a plain dict including the centre point.""" + data = asdict(self) + data["center"] = self.center + return data + + +def _safe_psr(value: float) -> Optional[float]: + """Round the PSR, or ``None`` when the sidelobe has no variance (a perfect peak).""" + import math + return round(value, 4) if math.isfinite(value) else None + + +def _peak_stats(score_map, exclude_radius: int): + """Return ``(loc, best, second, peak_ratio, psr)`` for one correlation surface.""" + import numpy as np + height, width = score_map.shape + best_y, best_x = divmod(int(np.argmax(score_map)), width) + best = float(score_map[best_y, best_x]) + mask = np.ones(score_map.shape, dtype=bool) + y0, y1 = max(0, best_y - exclude_radius), min(height, best_y + exclude_radius + 1) + x0, x1 = max(0, best_x - exclude_radius), min(width, best_x + exclude_radius + 1) + mask[y0:y1, x0:x1] = False + sidelobe = score_map[mask] + if sidelobe.size: + second, mean, std = (float(sidelobe.max()), float(sidelobe.mean()), + float(sidelobe.std())) + else: + second, mean, std = 0.0, 0.0, 0.0 + peak_ratio = second / best if abs(best) > 1e-9 else 1.0 + psr = (best - mean) / std if std > 1e-9 else float("inf") + return (best_x, best_y), best, second, peak_ratio, psr + + +def _default_radius(template_shape, exclude_radius: Optional[int]) -> int: + """Pick the peak-exclusion radius: caller value, or a quarter of the smaller side.""" + if exclude_radius: + return int(exclude_radius) + return max(3, min(template_shape[:2]) // 4) + + +def score_peaks(template: ImageSource, *, haystack: Optional[ImageSource] = None, + region: Optional[Sequence[int]] = None, + exclude_radius: Optional[int] = None, method: str = "ccoeff_normed", + ambiguous_ratio: float = 0.9) -> Optional[Dict[str, Any]]: + """Return peak/sidelobe trust metrics for ``template`` at scale 1.0, or ``None``. + + ``{best, second, peak_ratio, psr, ambiguous, location}`` — ``peak_ratio`` near 1 + means a second place scored almost as high (ambiguous); ``psr`` is the + peak-to-sidelobe ratio (``None`` when the sidelobe is flat). + """ + score_map, tmpl = _score_map(template, haystack, region=region, method=method) + if score_map is None: + return None + radius = _default_radius(tmpl.shape, exclude_radius) + (peak_x, peak_y), best, second, ratio, psr = _peak_stats(score_map, radius) + return {"best": round(best, 4), "second": round(second, 4), + "peak_ratio": round(ratio, 4), "psr": _safe_psr(psr), + "ambiguous": ratio >= ambiguous_ratio, "location": [peak_x, peak_y]} + + +def match_with_trust(template: ImageSource, *, + haystack: Optional[ImageSource] = None, + region: Optional[Sequence[int]] = None, + scales: Sequence[float] = (1.0,), method: str = "ccoeff_normed", + min_score: float = 0.0, ambiguous_ratio: float = 0.9, + exclude_radius: Optional[int] = None) -> Optional[TrustedMatch]: + """Return the best match (over ``scales``) with trust metrics attached, or ``None``. + + ``is_ambiguous`` is set when the next-best peak scores at least ``ambiguous_ratio`` + times the best — a strong but untrustworthy match the caller should not blindly click. + """ + best_match: Optional[TrustedMatch] = None + for scale in scales: + score_map, tmpl = _score_map(template, haystack, region=region, + method=method, scale=float(scale)) + if score_map is None: + continue + radius = _default_radius(tmpl.shape, exclude_radius) + (peak_x, peak_y), best, second, ratio, psr = _peak_stats(score_map, radius) + if best < min_score or (best_match is not None and best <= best_match.score): + continue + best_match = TrustedMatch(int(peak_x), int(peak_y), tmpl.shape[1], + tmpl.shape[0], round(best, 4), float(scale), + round(second, 4), round(ratio, 4), _safe_psr(psr), + ratio >= ambiguous_ratio) + return best_match diff --git a/je_auto_control/utils/mcp_server/tools/_factories.py b/je_auto_control/utils/mcp_server/tools/_factories.py index 315ae3e9..b6289368 100644 --- a/je_auto_control/utils/mcp_server/tools/_factories.py +++ b/je_auto_control/utils/mcp_server/tools/_factories.py @@ -3600,6 +3600,25 @@ def rotated_match_tools() -> List[MCPTool]: handler=h.match_rotated_all, annotations=READ_ONLY, ), + MCPTool( + name="ac_match_with_trust", + description=("Find 'template' AND judge whether the match is trustworthy " + "vs ambiguous: returns {found, match:{...,score,second_score," + "peak_ratio,psr,is_ambiguous,center}}. is_ambiguous=true means " + "a second place scored ~as high (e.g. a duplicate toolbar " + "button) - do NOT blindly click. 'ambiguous_ratio' (default " + "0.9), 'min_score', 'scales', 'region', 'method'."), + input_schema=schema({ + "template": {"type": "string"}, + "min_score": {"type": "number"}, + "scales": {"type": "array", "items": {"type": "number"}}, + "ambiguous_ratio": {"type": "number"}, + "region": {"type": "array", "items": {"type": "integer"}}, + "method": {"type": "string"}}, + required=["template"]), + handler=h.match_with_trust, + annotations=READ_ONLY, + ), ] diff --git a/je_auto_control/utils/mcp_server/tools/_handlers.py b/je_auto_control/utils/mcp_server/tools/_handlers.py index 6242ff03..018d4eab 100644 --- a/je_auto_control/utils/mcp_server/tools/_handlers.py +++ b/je_auto_control/utils/mcp_server/tools/_handlers.py @@ -2108,6 +2108,13 @@ def match_rotated_all(template, min_score=0.8, scales=None, angles=None, nms_iou, region) +def match_with_trust(template, min_score=0.0, scales=None, ambiguous_ratio=0.9, + region=None, method="ccoeff_normed"): + from je_auto_control.utils.executor.action_executor import _match_with_trust + return _match_with_trust(template, min_score, scales, ambiguous_ratio, + region, method) + + def grid_cells(rows, cols, region=None): from je_auto_control.utils.executor.action_executor import _grid_cells return _grid_cells(rows, cols, region) diff --git a/je_auto_control/utils/visual_match/visual_match.py b/je_auto_control/utils/visual_match/visual_match.py index a4919c75..c58bfada 100644 --- a/je_auto_control/utils/visual_match/visual_match.py +++ b/je_auto_control/utils/visual_match/visual_match.py @@ -92,6 +92,27 @@ def _resize(template, scale: float): return cv2.resize(template, new_size) +def _score_map(template: ImageSource, haystack: Optional[ImageSource] = None, *, + region: Optional[Sequence[int]] = None, + method: str = "ccoeff_normed", scale: float = 1.0): + """Return ``(full correlation score map, scaled gray template)``. + + The map is oriented so higher = better for every method (``sqdiff_normed`` + is inverted). Returns ``(None, template)`` when the template is larger than + the haystack at this scale. This exposes the whole ``matchTemplate`` surface + that the public matchers discard, for trust / threshold / sub-pixel analysis. + """ + import cv2 + tmpl = _resize(_to_gray(template), float(scale)) + hay = _haystack_gray(haystack, region) + if tmpl.shape[0] > hay.shape[0] or tmpl.shape[1] > hay.shape[1]: + return None, tmpl + result = cv2.matchTemplate(hay, tmpl, _method(method)) + if method == "sqdiff_normed": + result = 1.0 - result + return result, tmpl + + def match_template(template: ImageSource, *, haystack: Optional[ImageSource] = None, region: Optional[Sequence[int]] = None, scales: Sequence[float] = (1.0,), min_score: float = 0.8, diff --git a/test/unit_test/headless/test_match_trust_batch.py b/test/unit_test/headless/test_match_trust_batch.py new file mode 100644 index 00000000..9e10a767 --- /dev/null +++ b/test/unit_test/headless/test_match_trust_batch.py @@ -0,0 +1,81 @@ +"""Headless tests for template-match trust scoring on synthetic arrays.""" +import pytest + +import je_auto_control as ac + +np = pytest.importorskip("numpy") +pytest.importorskip("cv2") + +from je_auto_control.utils.match_trust import ( # noqa: E402 + match_with_trust, score_peaks, +) + + +def _template(): + tmpl = np.zeros((24, 24), dtype=np.uint8) + tmpl[:, :12] = 200 # asymmetric so correlation has variance + return tmpl + + +def _haystack(*tops_lefts): + hay = np.zeros((160, 200), dtype=np.uint8) + tmpl = _template() + for top, left in tops_lefts: + hay[top:top + 24, left:left + 24] = tmpl + return hay + + +def test_single_occurrence_is_unambiguous(): + result = match_with_trust(_template(), haystack=_haystack((20, 30)), + min_score=0.8) + assert result is not None + assert result.is_ambiguous is False + assert result.peak_ratio < 0.9 + assert abs(result.x - 30) <= 1 and abs(result.y - 20) <= 1 + + +def test_duplicate_occurrence_is_ambiguous(): + # the same template twice → a near-equal second peak → flagged ambiguous + result = match_with_trust(_template(), haystack=_haystack((20, 30), (20, 140)), + min_score=0.8) + assert result is not None + assert result.is_ambiguous is True + assert result.peak_ratio >= 0.9 + assert result.second_score >= 0.9 + + +def test_score_peaks_reports_metrics(): + peaks = score_peaks(_template(), haystack=_haystack((20, 30))) + assert peaks is not None + assert set(peaks) == {"best", "second", "peak_ratio", "psr", "ambiguous", + "location"} + assert peaks["best"] >= 0.99 + assert peaks["ambiguous"] is False + + +def test_min_score_filters_out(): + assert match_with_trust(_template(), haystack=np.zeros((160, 200), np.uint8), + min_score=0.95) is None + + +def test_psr_present_for_real_peak(): + result = match_with_trust(_template(), haystack=_haystack((20, 30)), + min_score=0.8) + assert result.psr is None or result.psr > 0 + + +# --- wiring --------------------------------------------------------------- + +def test_wiring(): + assert "AC_match_with_trust" in set(ac.executor.known_commands()) + from je_auto_control.utils.mcp_server.tools import build_default_tool_registry + names = {t.name for t in build_default_tool_registry()} + assert "ac_match_with_trust" in names + from je_auto_control.gui.script_builder.command_schema import _build_specs + specs = {s.command for s in _build_specs()} + assert "AC_match_with_trust" in specs + + +def test_facade_exports(): + for name in ("match_with_trust", "score_peaks", "TrustedMatch"): + assert hasattr(ac, name) and name in ac.__all__