From 3e8aeaacaf2c244e679f6e37341b2bc83bc03693 Mon Sep 17 00:00:00 2001 From: JeffreyChen Date: Tue, 23 Jun 2026 21:44:07 +0800 Subject: [PATCH] Add weighted candidate scoring (role + name + proximity) --- README/WHATS_NEW_zh-CN.md | 6 ++ README/WHATS_NEW_zh-TW.md | 6 ++ WHATS_NEW.md | 6 ++ .../doc/new_features/v156_features_doc.rst | 44 ++++++++++ docs/source/Eng/eng_index.rst | 1 + .../Zh/doc/new_features/v156_features_doc.rst | 37 ++++++++ docs/source/Zh/zh_index.rst | 1 + je_auto_control/__init__.py | 7 ++ .../gui/script_builder/command_schema.py | 24 +++++ .../utils/element_scoring/__init__.py | 6 ++ .../utils/element_scoring/element_scoring.py | 88 +++++++++++++++++++ .../utils/executor/action_executor.py | 31 +++++++ .../utils/mcp_server/tools/_factories.py | 35 +++++++- .../utils/mcp_server/tools/_handlers.py | 10 +++ .../headless/test_element_scoring_batch.py | 80 +++++++++++++++++ 15 files changed, 381 insertions(+), 1 deletion(-) create mode 100644 docs/source/Eng/doc/new_features/v156_features_doc.rst create mode 100644 docs/source/Zh/doc/new_features/v156_features_doc.rst create mode 100644 je_auto_control/utils/element_scoring/__init__.py create mode 100644 je_auto_control/utils/element_scoring/element_scoring.py create mode 100644 test/unit_test/headless/test_element_scoring_batch.py diff --git a/README/WHATS_NEW_zh-CN.md b/README/WHATS_NEW_zh-CN.md index 1dd36dd9..0de9b107 100644 --- a/README/WHATS_NEW_zh-CN.md +++ b/README/WHATS_NEW_zh-CN.md @@ -1,5 +1,11 @@ # 本次更新 — AutoControl +## 本次更新 (2026-06-23) — 加权候选评分 + +以信心分数排序模棱两可的元素候选。完整参考:[`docs/source/Zh/doc/new_features/v156_features_doc.rst`](../docs/source/Zh/doc/new_features/v156_features_doc.rst)。 + +- **`score_candidates` / `best_candidate`**(`AC_score_candidates`、`AC_best_candidate`):`anchor_locator` 是单一关系 + 距离排序、`ab_locator` 依耗时竞赛整个策略——两者都不以*加权*混合(角色匹配 + 模糊名称相似度 + 锚点邻近 + 启用状态)排序模棱候选。本功能返回最佳优先的 `ScoredCandidate` 并含 `matched_on` 明细;名称相似度可注入(默认 `fuzzy_ratio`,重用——不新增字符串距离代码)。纯标准库,作用于元素字典;在多个框都可能是目标时驱动自我修复 / grounding。可无头测试。 + ## 本次更新 (2026-06-23) — 几何感知的元素差异与稳定 ID 以重叠跨帧追踪元素,并给予稳定 ID。完整参考:[`docs/source/Zh/doc/new_features/v155_features_doc.rst`](../docs/source/Zh/doc/new_features/v155_features_doc.rst)。 diff --git a/README/WHATS_NEW_zh-TW.md b/README/WHATS_NEW_zh-TW.md index 8ca4301c..3e420563 100644 --- a/README/WHATS_NEW_zh-TW.md +++ b/README/WHATS_NEW_zh-TW.md @@ -1,5 +1,11 @@ # 本次更新 — AutoControl +## 本次更新 (2026-06-23) — 加權候選評分 + +以信心分數排序模稜兩可的元素候選。完整參考:[`docs/source/Zh/doc/new_features/v156_features_doc.rst`](../docs/source/Zh/doc/new_features/v156_features_doc.rst)。 + +- **`score_candidates` / `best_candidate`**(`AC_score_candidates`、`AC_best_candidate`):`anchor_locator` 是單一關係 + 距離排序、`ab_locator` 依耗時競賽整個策略——兩者都不以*加權*混合(角色匹配 + 模糊名稱相似度 + 錨點鄰近 + 啟用狀態)排序模稜候選。本功能回傳最佳優先的 `ScoredCandidate` 並含 `matched_on` 明細;名稱相似度可注入(預設 `fuzzy_ratio`,重用——不新增字串距離程式)。純標準函式庫,作用於元素字典;在多個框都可能是目標時驅動自我修復 / grounding。可無頭測試。 + ## 本次更新 (2026-06-23) — 幾何感知的元素差異與穩定 ID 以重疊跨影格追蹤元素,並給予穩定 ID。完整參考:[`docs/source/Zh/doc/new_features/v155_features_doc.rst`](../docs/source/Zh/doc/new_features/v155_features_doc.rst)。 diff --git a/WHATS_NEW.md b/WHATS_NEW.md index bdffb3f7..f2916e42 100644 --- a/WHATS_NEW.md +++ b/WHATS_NEW.md @@ -1,5 +1,11 @@ # What's New — AutoControl +## What's new (2026-06-23) — Weighted Candidate Scoring + +Rank ambiguous element candidates by a confidence score. Full reference: [`docs/source/Eng/doc/new_features/v156_features_doc.rst`](docs/source/Eng/doc/new_features/v156_features_doc.rst). + +- **`score_candidates` / `best_candidate`** (`AC_score_candidates`, `AC_best_candidate`): `anchor_locator` is a single relation + distance sort and `ab_locator` races whole strategies by elapsed time — neither ranks ambiguous candidates by a *weighted* mix of role match + fuzzy name similarity + anchor proximity + enabled-state. This returns `ScoredCandidate`s best-first with a `matched_on` breakdown; the name similarity is injectable (default `fuzzy_ratio`, reused — no new string-distance code). Pure-stdlib over element dicts; powers self-heal / grounding when several boxes could be the target. Headless-testable. + ## What's new (2026-06-23) — Geometry-Aware Element Diff & Stable IDs Track elements across frames by overlap, with stable IDs. Full reference: [`docs/source/Eng/doc/new_features/v155_features_doc.rst`](docs/source/Eng/doc/new_features/v155_features_doc.rst). diff --git a/docs/source/Eng/doc/new_features/v156_features_doc.rst b/docs/source/Eng/doc/new_features/v156_features_doc.rst new file mode 100644 index 00000000..0a14fac7 --- /dev/null +++ b/docs/source/Eng/doc/new_features/v156_features_doc.rst @@ -0,0 +1,44 @@ +Weighted Candidate Scoring +========================== + +``anchor_locator`` filters by a single spatial relation and sorts by distance, and +``ab_locator`` races *whole strategies* and picks by elapsed time — neither is a +*weighted multi-signal scorer* that ranks ambiguous candidates by combining a role +match, a fuzzy name similarity, proximity to an anchor and enabled state into one +confidence. That is exactly what self-healing / grounding needs when several boxes +could be the target. The name similarity is injectable (defaulting to the project's +``fuzzy_ratio``), so no new string-distance code is added. + +Pure-stdlib over plain element dicts (``role`` / ``name`` / ``x`` / ``y`` / ``width`` / +``height`` / optional ``enabled``), fully unit-testable. Imports no ``PySide6``. + +Headless API +------------ + +.. code-block:: python + + from je_auto_control import score_candidates, best_candidate + + ranked = score_candidates(candidates, want_role="button", want_name="Save", + anchor=(960, 540)) + for c in ranked: + print(round(c.score, 3), c.element["name"], c.matched_on) + + pick = best_candidate(candidates, want_role="button", want_name="Save") + if pick: + click(*[pick.element["x"], pick.element["y"]]) + +``score_candidates`` returns a list of ``ScoredCandidate`` (``element`` / ``score`` / +``matched_on`` breakdown), best-first; each active signal contributes 0..1 and the +score is their mean. ``want_role`` scores 1 on an exact role match, ``want_name`` runs +``name_similarity`` (default ``fuzzy_ratio``), ``anchor`` adds a proximity term, and +``prefer_enabled`` rewards enabled elements. ``best_candidate`` returns the top one (or +``None``). + +Executor commands +----------------- + +``AC_score_candidates`` (``candidates`` / ``want_role`` / ``want_name`` / ``anchor`` → +``{count, scored}``) and ``AC_best_candidate`` (same inputs → ``{found, best}``). They +are exposed as the MCP tools ``ac_score_candidates`` / ``ac_best_candidate`` and as +Script Builder commands under **Native UI**. diff --git a/docs/source/Eng/eng_index.rst b/docs/source/Eng/eng_index.rst index 33d1b05b..93c28570 100644 --- a/docs/source/Eng/eng_index.rst +++ b/docs/source/Eng/eng_index.rst @@ -178,6 +178,7 @@ Comprehensive guides for all AutoControl features. doc/new_features/v153_features_doc doc/new_features/v154_features_doc doc/new_features/v155_features_doc + doc/new_features/v156_features_doc doc/ocr_backends/ocr_backends_doc doc/observability/observability_doc doc/operations_layer/operations_layer_doc diff --git a/docs/source/Zh/doc/new_features/v156_features_doc.rst b/docs/source/Zh/doc/new_features/v156_features_doc.rst new file mode 100644 index 00000000..bbbe570b --- /dev/null +++ b/docs/source/Zh/doc/new_features/v156_features_doc.rst @@ -0,0 +1,37 @@ +加權候選評分 +============ + +``anchor_locator`` 以單一空間關係過濾、依距離排序,``ab_locator`` 競賽*整個策略*並依耗時挑選——兩者都不是把角色 +匹配、模糊名稱相似度、對錨點的鄰近度與啟用狀態合成單一信心的*加權多訊號評分器*。當多個框都可能是目標時, +自我修復 / grounding 正需要這個。名稱相似度可注入(預設為專案的 ``fuzzy_ratio``),因此不新增字串距離程式。 + +純標準函式庫,作用於純元素字典(``role`` / ``name`` / ``x`` / ``y`` / ``width`` / ``height`` / 選用 ``enabled``), +完全可單元測試。不匯入 ``PySide6``。 + +無頭 API +-------- + +.. code-block:: python + + from je_auto_control import score_candidates, best_candidate + + ranked = score_candidates(candidates, want_role="button", want_name="Save", + anchor=(960, 540)) + for c in ranked: + print(round(c.score, 3), c.element["name"], c.matched_on) + + pick = best_candidate(candidates, want_role="button", want_name="Save") + if pick: + click(*[pick.element["x"], pick.element["y"]]) + +``score_candidates`` 回傳 ``ScoredCandidate`` 清單(``element`` / ``score`` / ``matched_on`` 明細),最佳優先;每個 +啟用的訊號貢獻 0..1,分數為其平均。``want_role`` 在角色精確匹配時得 1、``want_name`` 執行 ``name_similarity`` +(預設 ``fuzzy_ratio``)、``anchor`` 加入鄰近項、``prefer_enabled`` 獎勵啟用元素。``best_candidate`` 回傳最佳者 +(或 ``None``)。 + +執行器命令 +---------- + +``AC_score_candidates``(``candidates`` / ``want_role`` / ``want_name`` / ``anchor`` → ``{count, scored}``)與 +``AC_best_candidate``(相同輸入 → ``{found, best}``)。它們以 MCP 工具 ``ac_score_candidates`` / ``ac_best_candidate`` +以及 Script Builder 中 **Native UI** 分類下的命令提供。 diff --git a/docs/source/Zh/zh_index.rst b/docs/source/Zh/zh_index.rst index cc89957f..d0ebb495 100644 --- a/docs/source/Zh/zh_index.rst +++ b/docs/source/Zh/zh_index.rst @@ -178,6 +178,7 @@ AutoControl 所有功能的完整使用指南。 doc/new_features/v153_features_doc doc/new_features/v154_features_doc doc/new_features/v155_features_doc + doc/new_features/v156_features_doc doc/ocr_backends/ocr_backends_doc doc/observability/observability_doc doc/operations_layer/operations_layer_doc diff --git a/je_auto_control/__init__.py b/je_auto_control/__init__.py index 8c547dee..a0019dec 100644 --- a/je_auto_control/__init__.py +++ b/je_auto_control/__init__.py @@ -381,6 +381,10 @@ from je_auto_control.utils.element_diff import ( assign_stable_ids, match_elements, ) +# Weighted candidate scoring (role + name similarity + proximity + enabled) +from je_auto_control.utils.element_scoring import ( + ScoredCandidate, best_candidate, score_candidates, +) # CI workflow annotations (GitHub Actions) from je_auto_control.utils.ci_annotations import ( emit_annotations, format_annotation, @@ -1271,6 +1275,9 @@ def start_autocontrol_gui(*args, **kwargs): "replay_trace", "match_elements", "assign_stable_ids", + "score_candidates", + "best_candidate", + "ScoredCandidate", "emit_annotations", "format_annotation", "ClipboardHistory", "default_clipboard_history", "analyze_heal_log", "heal_stats", "scan_secrets", diff --git a/je_auto_control/gui/script_builder/command_schema.py b/je_auto_control/gui/script_builder/command_schema.py index 4ea62d7d..c09d0823 100644 --- a/je_auto_control/gui/script_builder/command_schema.py +++ b/je_auto_control/gui/script_builder/command_schema.py @@ -2987,6 +2987,30 @@ def _add_set_of_marks_specs(specs: List[CommandSpec]) -> None: ), description="Tag elements with IDs carried across frames by overlap.", )) + specs.append(CommandSpec( + "AC_score_candidates", "Native UI", "Score Candidates", + fields=( + FieldSpec("candidates", FieldType.STRING, + placeholder='[{"role":"button","name":"OK","x":..,"y":..}]'), + FieldSpec("want_role", FieldType.STRING, optional=True), + FieldSpec("want_name", FieldType.STRING, optional=True), + FieldSpec("anchor", FieldType.STRING, optional=True, + placeholder="[x, y]"), + ), + description="Rank candidate elements by role / name / proximity confidence.", + )) + specs.append(CommandSpec( + "AC_best_candidate", "Native UI", "Best Candidate", + fields=( + FieldSpec("candidates", FieldType.STRING, + placeholder='[{"role":"button","name":"OK","x":..,"y":..}]'), + FieldSpec("want_role", FieldType.STRING, optional=True), + FieldSpec("want_name", FieldType.STRING, optional=True), + FieldSpec("anchor", FieldType.STRING, optional=True, + placeholder="[x, y]"), + ), + description="The single highest-scoring candidate element.", + )) specs.append(CommandSpec( "AC_mark_screen", "Native UI", "Set-of-Marks: Number Elements", fields=( diff --git a/je_auto_control/utils/element_scoring/__init__.py b/je_auto_control/utils/element_scoring/__init__.py new file mode 100644 index 00000000..42d66b46 --- /dev/null +++ b/je_auto_control/utils/element_scoring/__init__.py @@ -0,0 +1,6 @@ +"""Weighted candidate scoring (role + name similarity + proximity + enabled).""" +from je_auto_control.utils.element_scoring.element_scoring import ( + ScoredCandidate, best_candidate, score_candidates, +) + +__all__ = ["ScoredCandidate", "best_candidate", "score_candidates"] diff --git a/je_auto_control/utils/element_scoring/element_scoring.py b/je_auto_control/utils/element_scoring/element_scoring.py new file mode 100644 index 00000000..4c84ddf2 --- /dev/null +++ b/je_auto_control/utils/element_scoring/element_scoring.py @@ -0,0 +1,88 @@ +"""Weighted candidate scoring — rank ambiguous elements by role + name + proximity. + +``anchor_locator`` filters by a single spatial relation and sorts by distance, and +``ab_locator`` races *whole strategies* and picks by elapsed time — neither is a +*weighted multi-signal scorer* that ranks ambiguous candidates by combining a role +match, a fuzzy name similarity, proximity to an anchor and enabled-state into one +confidence. That is what self-healing / grounding needs when several boxes could be the +target. The name similarity is injectable (defaulting to the project's ``fuzzy_ratio``), +so no new string-distance code is added. + +Pure-stdlib over plain element dicts (``role`` / ``name`` / ``x`` / ``y`` / ``width`` / +``height`` / optional ``enabled``), fully unit-testable. Imports no ``PySide6``. +""" +import math +from dataclasses import asdict, dataclass +from typing import Any, Callable, Dict, List, Optional, Sequence + +from je_auto_control.utils.fuzzy import fuzzy_ratio + +Element = Dict[str, Any] + + +@dataclass(frozen=True) +class ScoredCandidate: + """One ranked candidate: the element, its 0..1 ``score`` and the per-signal breakdown.""" + + element: Element + score: float + matched_on: Dict[str, float] + + def to_dict(self) -> Dict[str, Any]: + """Return the scored candidate as a plain dict.""" + return asdict(self) + + +def _proximity(element: Element, anchor: Sequence[int]) -> float: + cx = int(element.get("x", 0)) + int(element.get("width", 0)) // 2 + cy = int(element.get("y", 0)) + int(element.get("height", 0)) // 2 + distance = math.hypot(cx - int(anchor[0]), cy - int(anchor[1])) + return 1.0 / (1.0 + distance / 100.0) + + +def score_candidates(candidates: Sequence[Element], *, + want_role: Optional[str] = None, + want_name: Optional[str] = None, + name_similarity: Optional[Callable[[str, str], float]] = None, + prefer_enabled: bool = True, + anchor: Optional[Sequence[int]] = None + ) -> List[ScoredCandidate]: + """Score and rank ``candidates`` best-first by the supplied signals. + + Each active signal contributes 0..1 and the score is their mean: ``want_role`` (1 + on an exact role match), ``want_name`` (via ``name_similarity``, default + ``fuzzy_ratio``), ``anchor`` proximity, and ``prefer_enabled``. ``matched_on`` holds + the per-signal breakdown. + """ + similarity = name_similarity or fuzzy_ratio + scored: List[ScoredCandidate] = [] + for element in candidates: + parts: Dict[str, float] = {} + if want_role is not None: + parts["role"] = (1.0 if str(element.get("role", "")).lower() + == str(want_role).lower() else 0.0) + if want_name is not None: + parts["name"] = float(similarity(want_name, + str(element.get("name", "")))) + if anchor is not None: + parts["proximity"] = _proximity(element, anchor) + if prefer_enabled: + parts["enabled"] = 1.0 if element.get("enabled", True) else 0.0 + score = sum(parts.values()) / len(parts) if parts else 0.0 + scored.append(ScoredCandidate(element, round(score, 4), parts)) + scored.sort(key=lambda candidate: candidate.score, reverse=True) + return scored + + +def best_candidate(candidates: Sequence[Element], *, + want_role: Optional[str] = None, + want_name: Optional[str] = None, + name_similarity: Optional[Callable[[str, str], float]] = None, + prefer_enabled: bool = True, + anchor: Optional[Sequence[int]] = None + ) -> Optional[ScoredCandidate]: + """Return the single highest-scoring candidate (or ``None`` if there are none).""" + scored = score_candidates(candidates, want_role=want_role, want_name=want_name, + name_similarity=name_similarity, + prefer_enabled=prefer_enabled, anchor=anchor) + return scored[0] if scored else None diff --git a/je_auto_control/utils/executor/action_executor.py b/je_auto_control/utils/executor/action_executor.py index 516b42d9..fe457219 100644 --- a/je_auto_control/utils/executor/action_executor.py +++ b/je_auto_control/utils/executor/action_executor.py @@ -3924,6 +3924,35 @@ def _assign_stable_ids(elements: Any, prior: Any = None, return {"count": len(tagged), "elements": tagged} +def _score_candidates(candidates: Any, want_role: Any = None, want_name: Any = None, + anchor: Any = None) -> Dict[str, Any]: + """Adapter: rank candidate element boxes by role / name / proximity.""" + import json + from je_auto_control.utils.element_scoring import score_candidates + if isinstance(candidates, str): + candidates = json.loads(candidates) + if isinstance(anchor, str): + anchor = json.loads(anchor) if anchor.strip() else None + ranked = score_candidates(list(candidates), want_role=want_role, + want_name=want_name, anchor=anchor) + return {"count": len(ranked), "scored": [c.to_dict() for c in ranked]} + + +def _best_candidate(candidates: Any, want_role: Any = None, want_name: Any = None, + anchor: Any = None) -> Dict[str, Any]: + """Adapter: the single highest-scoring candidate element.""" + import json + from je_auto_control.utils.element_scoring import best_candidate + if isinstance(candidates, str): + candidates = json.loads(candidates) + if isinstance(anchor, str): + anchor = json.loads(anchor) if anchor.strip() else None + best = best_candidate(list(candidates), want_role=want_role, + want_name=want_name, anchor=anchor) + return {"found": best is not None, + "best": best.to_dict() if best is not None else None} + + def _with_modifiers(modifiers: Any, actions: Any) -> Dict[str, Any]: """Adapter: run nested actions while modifier keys are held down.""" import json @@ -5685,6 +5714,8 @@ def __init__(self): "AC_replay_trace": _replay_trace, "AC_match_elements": _match_elements, "AC_assign_stable_ids": _assign_stable_ids, + "AC_score_candidates": _score_candidates, + "AC_best_candidate": _best_candidate, "AC_tile_rect": _tile_rect, "AC_grid_rects": _grid_rects, "AC_cascade_rects": _cascade_rects, diff --git a/je_auto_control/utils/mcp_server/tools/_factories.py b/je_auto_control/utils/mcp_server/tools/_factories.py index 75de67ab..a961da75 100644 --- a/je_auto_control/utils/mcp_server/tools/_factories.py +++ b/je_auto_control/utils/mcp_server/tools/_factories.py @@ -3377,6 +3377,39 @@ def element_diff_tools() -> List[MCPTool]: ] +def element_scoring_tools() -> List[MCPTool]: + return [ + MCPTool( + name="ac_score_candidates", + description=("Rank candidate element boxes best-first by a weighted mean " + "of role match ('want_role'), fuzzy name similarity " + "('want_name'), 'anchor' proximity and enabled-state. " + "Returns {count, scored:[{element, score, matched_on}]}."), + input_schema=schema({ + "candidates": {"type": "array", "items": {"type": "object"}}, + "want_role": {"type": "string"}, + "want_name": {"type": "string"}, + "anchor": {"type": "array", "items": {"type": "integer"}}}, + required=["candidates"]), + handler=h.score_candidates, + annotations=READ_ONLY, + ), + MCPTool( + name="ac_best_candidate", + description=("The single highest-scoring candidate element by role / " + "name / proximity. Returns {found, best}."), + input_schema=schema({ + "candidates": {"type": "array", "items": {"type": "object"}}, + "want_role": {"type": "string"}, + "want_name": {"type": "string"}, + "anchor": {"type": "array", "items": {"type": "integer"}}}, + required=["candidates"]), + handler=h.best_candidate, + annotations=READ_ONLY, + ), + ] + + def ssim_tools() -> List[MCPTool]: return [ MCPTool( @@ -6886,7 +6919,7 @@ def media_assert_tools() -> List[MCPTool]: motion_regions_tools, window_zorder_tools, soft_assert_tools, perceptual_diff_tools, window_geometry_tools, cua_action_tools, observation_tools, action_grounding_tools, agent_replay_tools, - element_diff_tools, plugin_sdk_tools, governance_tools, + element_diff_tools, element_scoring_tools, plugin_sdk_tools, governance_tools, credential_lease_tools, egress_tools, approval_testing_tools, trajectory_eval_tools, compliance_tools, agent_trace_tools, video_report_tools, fuzzy_tools, artifact_store_tools, image_dedup_tools, diff --git a/je_auto_control/utils/mcp_server/tools/_handlers.py b/je_auto_control/utils/mcp_server/tools/_handlers.py index 27f257a3..44c27623 100644 --- a/je_auto_control/utils/mcp_server/tools/_handlers.py +++ b/je_auto_control/utils/mcp_server/tools/_handlers.py @@ -2340,6 +2340,16 @@ def assign_stable_ids(elements, prior=None, iou_threshold=0.5): return _assign_stable_ids(elements, prior, iou_threshold) +def score_candidates(candidates, want_role=None, want_name=None, anchor=None): + from je_auto_control.utils.executor.action_executor import _score_candidates + return _score_candidates(candidates, want_role, want_name, anchor) + + +def best_candidate(candidates, want_role=None, want_name=None, anchor=None): + from je_auto_control.utils.executor.action_executor import _best_candidate + return _best_candidate(candidates, want_role, want_name, anchor) + + def detect_drift(reference, current, threshold=0.25, bins=10): from je_auto_control.utils.executor.action_executor import _detect_drift return _detect_drift(reference, current, threshold, bins) diff --git a/test/unit_test/headless/test_element_scoring_batch.py b/test/unit_test/headless/test_element_scoring_batch.py new file mode 100644 index 00000000..540179b0 --- /dev/null +++ b/test/unit_test/headless/test_element_scoring_batch.py @@ -0,0 +1,80 @@ +"""Headless tests for weighted candidate scoring. No Qt.""" +import pytest + +import je_auto_control as ac +from je_auto_control.utils.element_scoring import ( + ScoredCandidate, best_candidate, score_candidates, +) + + +def _candidates(): + return [ + {"role": "button", "name": "Save", "x": 10, "y": 10, "width": 40, + "height": 20, "enabled": True}, + {"role": "button", "name": "Save As", "x": 100, "y": 10, "width": 60, + "height": 20, "enabled": True}, + {"role": "link", "name": "Save", "x": 10, "y": 200, "width": 40, + "height": 20, "enabled": False}, + ] + + +def test_exact_button_near_anchor_ranks_first(): + ranked = score_candidates(_candidates(), want_role="button", want_name="Save", + anchor=(20, 15)) + assert ranked[0].element["name"] == "Save" + assert ranked[0].element["role"] == "button" + assert ranked[0].score > ranked[1].score > ranked[2].score + + +def test_matched_on_breakdown(): + top = score_candidates(_candidates(), want_role="button", + want_name="Save")[0] + assert top.matched_on["role"] == pytest.approx(1.0) + assert top.matched_on["name"] == pytest.approx(1.0) + assert top.matched_on["enabled"] == pytest.approx(1.0) + + +def test_disabled_wrong_role_ranks_last(): + ranked = score_candidates(_candidates(), want_role="button", want_name="Save") + assert ranked[-1].element["role"] == "link" # disabled + wrong role + + +def test_injected_similarity_is_used(): + calls = [] + + def sim(a, b): + calls.append((a, b)) + return 1.0 if a == b else 0.0 + + score_candidates(_candidates(), want_name="Save", name_similarity=sim, + prefer_enabled=False) + assert calls and all(call[0] == "Save" for call in calls) + + +def test_best_candidate_and_empty(): + assert best_candidate(_candidates(), want_name="Save").element["name"] == "Save" + assert best_candidate([], want_name="x") is None + + +def test_scored_candidate_to_dict(): + top = score_candidates(_candidates(), want_role="button")[0] + assert isinstance(top, ScoredCandidate) + assert set(top.to_dict()) == {"element", "score", "matched_on"} + + +# --- wiring --------------------------------------------------------------- + +def test_wiring(): + known = set(ac.executor.known_commands()) + assert {"AC_score_candidates", "AC_best_candidate"} <= known + from je_auto_control.utils.mcp_server.tools import build_default_tool_registry + names = {t.name for t in build_default_tool_registry()} + assert {"ac_score_candidates", "ac_best_candidate"} <= names + from je_auto_control.gui.script_builder.command_schema import _build_specs + specs = {s.command for s in _build_specs()} + assert {"AC_score_candidates", "AC_best_candidate"} <= specs + + +def test_facade_exports(): + for attr in ("score_candidates", "best_candidate", "ScoredCandidate"): + assert hasattr(ac, attr) and attr in ac.__all__