From a49aed58f1bf4b6b503e80d4249bc8027cdbad3e Mon Sep 17 00:00:00 2001 From: JeffreyChen Date: Tue, 23 Jun 2026 06:52:45 +0800 Subject: [PATCH] Add element-box fusion and reading-order for set-of-marks --- README/WHATS_NEW_zh-CN.md | 6 ++ README/WHATS_NEW_zh-TW.md | 6 ++ WHATS_NEW.md | 6 ++ .../doc/new_features/v138_features_doc.rst | 46 ++++++++ docs/source/Eng/eng_index.rst | 1 + .../Zh/doc/new_features/v138_features_doc.rst | 38 +++++++ docs/source/Zh/zh_index.rst | 1 + je_auto_control/__init__.py | 8 ++ .../gui/script_builder/command_schema.py | 21 ++++ .../utils/element_parse/__init__.py | 6 ++ .../utils/element_parse/element_parse.py | 100 ++++++++++++++++++ .../utils/executor/action_executor.py | 28 +++++ .../utils/mcp_server/tools/_factories.py | 35 +++++- .../utils/mcp_server/tools/_handlers.py | 10 ++ .../headless/test_element_parse_batch.py | 88 +++++++++++++++ 15 files changed, 399 insertions(+), 1 deletion(-) create mode 100644 docs/source/Eng/doc/new_features/v138_features_doc.rst create mode 100644 docs/source/Zh/doc/new_features/v138_features_doc.rst create mode 100644 je_auto_control/utils/element_parse/__init__.py create mode 100644 je_auto_control/utils/element_parse/element_parse.py create mode 100644 test/unit_test/headless/test_element_parse_batch.py diff --git a/README/WHATS_NEW_zh-CN.md b/README/WHATS_NEW_zh-CN.md index 758136c0..e565fb5d 100644 --- a/README/WHATS_NEW_zh-CN.md +++ b/README/WHATS_NEW_zh-CN.md @@ -1,5 +1,11 @@ # 本次更新 — AutoControl +## 本次更新 (2026-06-23) — 融合并排序屏幕元素框 + +把原始的 OCR + 图标 + a11y 框转成一份干净、已编号的元素列表。完整参考:[`docs/source/Zh/doc/new_features/v138_features_doc.rst`](../docs/source/Zh/doc/new_features/v138_features_doc.rst)。 + +- **`iou` / `merge_boxes` / `fuse_elements` / `reading_order`**(`AC_fuse_elements`、`AC_reading_order`):`set_of_marks` 为干净的元素列表编号,但没有任何功能*产生*它——真实画面解析会产出三个彼此重叠、有重复且无顺序的来源。本功能补上这一步:依 IoU 去除近重复框、并集 OCR/icon/a11y 并在重叠时保留最可信来源(`source_priority` a11y > ocr > icon)、再由上到下 / 由左到右排序并给予稳定 `index`。纯 `dict` 框 → 纯标准库、完全无头可测;直接与 `set_of_marks` 搭配。 + ## 本次更新 (2026-06-23) — 可操作性闸门(操作前先等待就绪) 目标真正就绪前不要点击。完整参考:[`docs/source/Zh/doc/new_features/v137_features_doc.rst`](../docs/source/Zh/doc/new_features/v137_features_doc.rst)。 diff --git a/README/WHATS_NEW_zh-TW.md b/README/WHATS_NEW_zh-TW.md index 97d830c2..b0cbc148 100644 --- a/README/WHATS_NEW_zh-TW.md +++ b/README/WHATS_NEW_zh-TW.md @@ -1,5 +1,11 @@ # 本次更新 — AutoControl +## 本次更新 (2026-06-23) — 融合並排序螢幕元素框 + +把原始的 OCR + 圖示 + a11y 框轉成一份乾淨、已編號的元素清單。完整參考:[`docs/source/Zh/doc/new_features/v138_features_doc.rst`](../docs/source/Zh/doc/new_features/v138_features_doc.rst)。 + +- **`iou` / `merge_boxes` / `fuse_elements` / `reading_order`**(`AC_fuse_elements`、`AC_reading_order`):`set_of_marks` 為乾淨的元素清單編號,但沒有任何功能*產生*它——真實畫面解析會產出三個彼此重疊、有重複且無順序的來源。本功能補上這一步:依 IoU 去除近重複框、聯集 OCR/icon/a11y 並在重疊時保留最可信來源(`source_priority` a11y > ocr > icon)、再由上到下 / 由左到右排序並給予穩定 `index`。純 `dict` 框 → 純標準函式庫、完全無頭可測;直接與 `set_of_marks` 搭配。 + ## 本次更新 (2026-06-23) — 可操作性閘門(操作前先等待就緒) 目標真正就緒前不要點擊。完整參考:[`docs/source/Zh/doc/new_features/v137_features_doc.rst`](../docs/source/Zh/doc/new_features/v137_features_doc.rst)。 diff --git a/WHATS_NEW.md b/WHATS_NEW.md index d3b82083..1e6e3775 100644 --- a/WHATS_NEW.md +++ b/WHATS_NEW.md @@ -1,5 +1,11 @@ # What's New — AutoControl +## What's new (2026-06-23) — Fuse & Order On-Screen Element Boxes + +Turn raw OCR + icon + a11y boxes into one clean, numbered element list. Full reference: [`docs/source/Eng/doc/new_features/v138_features_doc.rst`](docs/source/Eng/doc/new_features/v138_features_doc.rst). + +- **`iou` / `merge_boxes` / `fuse_elements` / `reading_order`** (`AC_fuse_elements`, `AC_reading_order`): `set_of_marks` numbers a clean element list but nothing *produced* it — a real screen parse yields three overlapping sources with duplicates and no order. These supply the missing step: drop near-duplicate boxes by IoU, union OCR/icon/a11y keeping the most trustworthy source on overlap (`source_priority` a11y > ocr > icon), and sort top-to-bottom/left-to-right with a stable `index`. Plain `dict` boxes → pure-stdlib, fully headless-testable; pairs directly with `set_of_marks`. + ## What's new (2026-06-23) — Actionability Gate (Wait Until Ready Before Acting) Don't click until the target is genuinely ready. Full reference: [`docs/source/Eng/doc/new_features/v137_features_doc.rst`](docs/source/Eng/doc/new_features/v137_features_doc.rst). diff --git a/docs/source/Eng/doc/new_features/v138_features_doc.rst b/docs/source/Eng/doc/new_features/v138_features_doc.rst new file mode 100644 index 00000000..02e95405 --- /dev/null +++ b/docs/source/Eng/doc/new_features/v138_features_doc.rst @@ -0,0 +1,46 @@ +Fuse & Order On-Screen Element Boxes +==================================== + +``set_of_marks.mark_elements`` numbers a single, already-clean element list — but +nothing *produces* that list. A real screen parse yields three overlapping sources +(OCR text boxes, icon/shape boxes, accessibility boxes) with heavy duplication and no +consistent order. This module is the missing connective tissue between the locators +(``locate_text``, ``find_shapes``, the a11y tree) and ``set_of_marks``: de-duplicate +by overlap, union the sources keeping the most trustworthy box, and sort into reading +order with a stable index. + +Every box is a plain ``dict`` with ``x, y, width, height`` (plus any extra keys such +as ``text`` / ``source`` / ``score``), so the whole module is pure-stdlib and fully +unit-testable. Imports no ``PySide6``. + +Headless API +------------ + +.. code-block:: python + + from je_auto_control import iou, merge_boxes, fuse_elements, reading_order + + iou(box_a, box_b) # overlap of two boxes, 0..1 + deduped = merge_boxes(raw_boxes, iou_threshold=0.9) + + # Union three detector outputs; on overlap the a11y box wins, then OCR, then icon. + elements = fuse_elements(ocr_boxes=ocr, icon_boxes=icons, a11y_boxes=tree) + + # Sort top-to-bottom, left-to-right and add an "index" to each. + for el in reading_order(elements): + print(el["index"], el.get("text"), el["x"], el["y"]) + +``iou`` returns the intersection-over-union of two boxes. ``merge_boxes`` keeps the +largest of any cluster overlapping above ``iou_threshold``. ``fuse_elements`` tags +each input with its ``source``, then drops cross-source overlaps preferring +``source_priority`` (default ``a11y`` > ``ocr`` > ``icon``, then larger area). +``reading_order`` bands rows within ``row_tol`` pixels, orders by ``x`` within each +row, and returns new dicts carrying a sequential ``index``. + +Executor commands +----------------- + +``AC_fuse_elements`` (``ocr`` / ``icon`` / ``a11y`` JSON arrays + ``iou_threshold`` +→ ``{count, elements}``) and ``AC_reading_order`` (``elements`` + ``row_tol`` → +``{count, elements}``). They are exposed as the MCP tools ``ac_fuse_elements`` / +``ac_reading_order`` and as Script Builder commands under **Image**. diff --git a/docs/source/Eng/eng_index.rst b/docs/source/Eng/eng_index.rst index 30e3bb55..e73b2af6 100644 --- a/docs/source/Eng/eng_index.rst +++ b/docs/source/Eng/eng_index.rst @@ -160,6 +160,7 @@ Comprehensive guides for all AutoControl features. doc/new_features/v135_features_doc doc/new_features/v136_features_doc doc/new_features/v137_features_doc + doc/new_features/v138_features_doc doc/ocr_backends/ocr_backends_doc doc/observability/observability_doc doc/operations_layer/operations_layer_doc diff --git a/docs/source/Zh/doc/new_features/v138_features_doc.rst b/docs/source/Zh/doc/new_features/v138_features_doc.rst new file mode 100644 index 00000000..2e913482 --- /dev/null +++ b/docs/source/Zh/doc/new_features/v138_features_doc.rst @@ -0,0 +1,38 @@ +融合並排序螢幕元素框 +==================== + +``set_of_marks.mark_elements`` 為單一、已乾淨的元素清單編號——但沒有任何功能*產生*那份清單。真實的畫面解析會 +產出三個彼此重疊的來源(OCR 文字框、圖示 / 形狀框、無障礙樹框),有大量重複且無一致順序。本模組是定位器 +(``locate_text``、``find_shapes``、a11y 樹)與 ``set_of_marks`` 之間缺少的連接組織:依重疊去重、聯集各來源並 +保留最可信的框、再排成閱讀順序並給予穩定索引。 + +每個框都是帶 ``x, y, width, height`` 的純 ``dict``(可附帶 ``text`` / ``source`` / ``score`` 等鍵),因此整個模組 +皆為純標準函式庫且完全可單元測試。不匯入 ``PySide6``。 + +無頭 API +-------- + +.. code-block:: python + + from je_auto_control import iou, merge_boxes, fuse_elements, reading_order + + iou(box_a, box_b) # 兩框的重疊度,0..1 + deduped = merge_boxes(raw_boxes, iou_threshold=0.9) + + # 聯集三個偵測器輸出;重疊時 a11y 框勝出,其次 OCR,再其次 icon。 + elements = fuse_elements(ocr_boxes=ocr, icon_boxes=icons, a11y_boxes=tree) + + # 由上到下、由左到右排序並為每個元素加上 "index"。 + for el in reading_order(elements): + print(el["index"], el.get("text"), el["x"], el["y"]) + +``iou`` 回傳兩框的交集除以聯集。``merge_boxes`` 在任一群重疊超過 ``iou_threshold`` 時保留最大者。``fuse_elements`` +為每個輸入標記 ``source``,再依 ``source_priority``(預設 ``a11y`` > ``ocr`` > ``icon``,其後較大面積)丟棄跨來源 +重疊。``reading_order`` 將相距 ``row_tol`` 像素內的元素歸為同列、列內依 ``x`` 排序,並回傳帶有遞增 ``index`` 的新字典。 + +執行器命令 +---------- + +``AC_fuse_elements``(``ocr`` / ``icon`` / ``a11y`` JSON 陣列 + ``iou_threshold`` → ``{count, elements}``)與 +``AC_reading_order``(``elements`` + ``row_tol`` → ``{count, elements}``)。它們以 MCP 工具 ``ac_fuse_elements`` / +``ac_reading_order`` 以及 Script Builder 中 **Image** 分類下的命令提供。 diff --git a/docs/source/Zh/zh_index.rst b/docs/source/Zh/zh_index.rst index 2fcb5579..ff1ca6ec 100644 --- a/docs/source/Zh/zh_index.rst +++ b/docs/source/Zh/zh_index.rst @@ -160,6 +160,7 @@ AutoControl 所有功能的完整使用指南。 doc/new_features/v135_features_doc doc/new_features/v136_features_doc doc/new_features/v137_features_doc + doc/new_features/v138_features_doc doc/ocr_backends/ocr_backends_doc doc/observability/observability_doc doc/operations_layer/operations_layer_doc diff --git a/je_auto_control/__init__.py b/je_auto_control/__init__.py index cf1890a9..12e4f679 100644 --- a/je_auto_control/__init__.py +++ b/je_auto_control/__init__.py @@ -312,6 +312,10 @@ from je_auto_control.utils.actionability import ( ActionabilityReport, GateConfig, act_when_ready, wait_actionable, ) +# Fuse and order on-screen element boxes (IoU, merge, fuse sources, reading order) +from je_auto_control.utils.element_parse import ( + fuse_elements, iou, merge_boxes, reading_order, +) # CI workflow annotations (GitHub Actions) from je_auto_control.utils.ci_annotations import ( emit_annotations, format_annotation, @@ -1141,6 +1145,10 @@ def start_autocontrol_gui(*args, **kwargs): "act_when_ready", "ActionabilityReport", "GateConfig", + "iou", + "merge_boxes", + "fuse_elements", + "reading_order", "emit_annotations", "format_annotation", "ClipboardHistory", "default_clipboard_history", "analyze_heal_log", "heal_stats", "scan_secrets", diff --git a/je_auto_control/gui/script_builder/command_schema.py b/je_auto_control/gui/script_builder/command_schema.py index c1b69f35..3bbdb75d 100644 --- a/je_auto_control/gui/script_builder/command_schema.py +++ b/je_auto_control/gui/script_builder/command_schema.py @@ -391,6 +391,27 @@ def _add_image_specs(specs: List[CommandSpec]) -> None: ), description="Clean up an image for OCR / matching (grayscale/binarize/…).", )) + specs.append(CommandSpec( + "AC_fuse_elements", "Image", "Fuse Element Boxes", + fields=( + FieldSpec("ocr", FieldType.STRING, optional=True, + placeholder='[{"x":..,"y":..,"width":..,"height":..}]'), + FieldSpec("icon", FieldType.STRING, optional=True), + FieldSpec("a11y", FieldType.STRING, optional=True), + FieldSpec("iou_threshold", FieldType.FLOAT, optional=True, default=0.9, + min_value=0.0, max_value=1.0), + ), + description="Union OCR/icon/a11y boxes, dropping cross-source duplicates.", + )) + specs.append(CommandSpec( + "AC_reading_order", "Image", "Reading Order", + fields=( + FieldSpec("elements", FieldType.STRING, + placeholder='[{"x":..,"y":..,"width":..,"height":..}]'), + FieldSpec("row_tol", FieldType.INT, optional=True, default=12), + ), + description="Order element boxes top-to-bottom, left-to-right (+ index).", + )) def _add_ocr_specs(specs: List[CommandSpec]) -> None: diff --git a/je_auto_control/utils/element_parse/__init__.py b/je_auto_control/utils/element_parse/__init__.py new file mode 100644 index 00000000..38caada2 --- /dev/null +++ b/je_auto_control/utils/element_parse/__init__.py @@ -0,0 +1,6 @@ +"""Fuse and order on-screen element boxes (IoU, merge, fuse sources, reading order).""" +from je_auto_control.utils.element_parse.element_parse import ( + fuse_elements, iou, merge_boxes, reading_order, +) + +__all__ = ["fuse_elements", "iou", "merge_boxes", "reading_order"] diff --git a/je_auto_control/utils/element_parse/element_parse.py b/je_auto_control/utils/element_parse/element_parse.py new file mode 100644 index 00000000..fee457db --- /dev/null +++ b/je_auto_control/utils/element_parse/element_parse.py @@ -0,0 +1,100 @@ +"""Fuse and order on-screen element boxes — the step before set-of-marks numbering. + +``set_of_marks.mark_elements`` assumes a single, already-deduplicated element list +and just numbers it; nothing produces that list. A real screen parse yields *three* +overlapping sources — OCR text boxes, icon/shape boxes, accessibility boxes — with +heavy overlap and no consistent order. This module supplies the missing connective +tissue: ``iou`` / ``merge_boxes`` drop near-duplicates, ``fuse_elements`` unions the +three sources keeping the most trustworthy one on overlap, and ``reading_order`` +sorts the survivors top-to-bottom, left-to-right and assigns a stable index. + +Every box is a plain ``dict`` with ``x, y, width, height`` (plus any extra keys such +as ``text`` / ``source`` / ``score``), so this is pure-stdlib and fully unit-testable. +Imports no ``PySide6``. +""" +from typing import Any, Dict, List, Optional, Sequence, Tuple + +Box = Dict[str, Any] +_DEFAULT_PRIORITY = ("a11y", "ocr", "icon") + + +def _xywh(box: Box) -> Tuple[int, int, int, int]: + return (int(box["x"]), int(box["y"]), int(box["width"]), int(box["height"])) + + +def _area(box: Box) -> int: + _x, _y, width, height = _xywh(box) + return width * height + + +def iou(box_a: Box, box_b: Box) -> float: + """Return the intersection-over-union (0..1) of two ``{x,y,width,height}`` boxes.""" + ax, ay, aw, ah = _xywh(box_a) + bx, by, bw, bh = _xywh(box_b) + left, top = max(ax, bx), max(ay, by) + right, bottom = min(ax + aw, bx + bw), min(ay + ah, by + bh) + inter = max(0, right - left) * max(0, bottom - top) + if inter == 0: + return 0.0 + union = aw * ah + bw * bh - inter + return inter / union if union else 0.0 + + +def _dedup(boxes: Sequence[Box], iou_threshold: float) -> List[Box]: + kept: List[Box] = [] + for box in boxes: + if all(iou(box, other) <= iou_threshold for other in kept): + kept.append(box) + return kept + + +def merge_boxes(boxes: Sequence[Box], *, iou_threshold: float = 0.9) -> List[Box]: + """Drop near-duplicate boxes (IoU above ``iou_threshold``), largest kept first.""" + ordered = sorted(boxes, key=_area, reverse=True) + return _dedup(ordered, float(iou_threshold)) + + +def fuse_elements(ocr_boxes: Optional[Sequence[Box]] = None, + icon_boxes: Optional[Sequence[Box]] = None, + a11y_boxes: Optional[Sequence[Box]] = None, *, + iou_threshold: float = 0.9, + source_priority: Sequence[str] = _DEFAULT_PRIORITY + ) -> List[Box]: + """Union OCR + icon + accessibility boxes, dropping cross-source duplicates. + + On overlap (IoU above ``iou_threshold``) the box from the higher-priority source + wins (``source_priority`` default a11y > ocr > icon, then larger area). Each box + is tagged with its ``source``. + """ + rank = {name: index for index, name in enumerate(source_priority)} + tagged: List[Box] = [] + for source, boxes in (("a11y", a11y_boxes), ("ocr", ocr_boxes), + ("icon", icon_boxes)): + for box in boxes or (): + item = dict(box) + item.setdefault("source", source) + tagged.append(item) + tagged.sort(key=lambda box: (rank.get(box.get("source"), len(rank)), + -_area(box))) + return _dedup(tagged, float(iou_threshold)) + + +def reading_order(elements: Sequence[Box], *, row_tol: int = 12) -> List[Box]: + """Return the elements sorted top-to-bottom, left-to-right, with an ``index`` key. + + Elements whose tops are within ``row_tol`` pixels are treated as the same row and + ordered by ``x`` within it. + """ + rows: List[Dict[str, Any]] = [] + for element in sorted(elements, key=lambda box: _xywh(box)[1]): + top = _xywh(element)[1] + row = next((candidate for candidate in rows + if abs(top - candidate["top"]) <= int(row_tol)), None) + if row is None: + rows.append({"top": top, "items": [element]}) + else: + row["items"].append(element) + ordered: List[Box] = [] + for row in rows: + ordered.extend(sorted(row["items"], key=lambda box: _xywh(box)[0])) + return [dict(element, index=index) for index, element in enumerate(ordered)] diff --git a/je_auto_control/utils/executor/action_executor.py b/je_auto_control/utils/executor/action_executor.py index d606248c..fe76826c 100644 --- a/je_auto_control/utils/executor/action_executor.py +++ b/je_auto_control/utils/executor/action_executor.py @@ -3479,6 +3479,32 @@ def locate(): return report.to_dict() +def _fuse_elements(ocr: Any = None, icon: Any = None, a11y: Any = None, + iou_threshold: Any = 0.9) -> Dict[str, Any]: + """Adapter: union OCR / icon / a11y element boxes, dropping duplicates.""" + import json + from je_auto_control.utils.element_parse import fuse_elements + + def parse(value: Any) -> list: + if isinstance(value, str): + return json.loads(value) if value.strip() else [] + return list(value) if value else [] + + elements = fuse_elements(parse(ocr), parse(icon), parse(a11y), + iou_threshold=float(iou_threshold)) + return {"count": len(elements), "elements": elements} + + +def _reading_order(elements: Any, row_tol: Any = 12) -> Dict[str, Any]: + """Adapter: order element boxes top-to-bottom, left-to-right, with an index.""" + import json + from je_auto_control.utils.element_parse import reading_order + if isinstance(elements, str): + elements = json.loads(elements) + ordered = reading_order(list(elements), row_tol=int(row_tol)) + return {"count": len(ordered), "elements": ordered} + + def _with_modifiers(modifiers: Any, actions: Any) -> Dict[str, Any]: """Adapter: run nested actions while modifier keys are held down.""" import json @@ -5209,6 +5235,8 @@ def __init__(self): "AC_enumerate_monitors": _enumerate_monitors, "AC_monitor_at_point": _monitor_at_point, "AC_wait_actionable": _wait_actionable, + "AC_fuse_elements": _fuse_elements, + "AC_reading_order": _reading_order, "AC_tile_rect": _tile_rect, "AC_grid_rects": _grid_rects, "AC_cascade_rects": _cascade_rects, diff --git a/je_auto_control/utils/mcp_server/tools/_factories.py b/je_auto_control/utils/mcp_server/tools/_factories.py index 3dacc1c3..8b74943d 100644 --- a/je_auto_control/utils/mcp_server/tools/_factories.py +++ b/je_auto_control/utils/mcp_server/tools/_factories.py @@ -2867,6 +2867,38 @@ def actionability_tools() -> List[MCPTool]: ] +def element_parse_tools() -> List[MCPTool]: + return [ + MCPTool( + name="ac_fuse_elements", + description=("Union OCR + icon + accessibility element boxes into one " + "de-duplicated list (drop cross-source overlaps > " + "'iou_threshold'; higher-priority source wins). Each box " + "is {x,y,width,height,...}. Returns {count, elements}."), + input_schema=schema({ + "ocr": {"type": "array", "items": {"type": "object"}}, + "icon": {"type": "array", "items": {"type": "object"}}, + "a11y": {"type": "array", "items": {"type": "object"}}, + "iou_threshold": {"type": "number"}}, + required=[]), + handler=h.fuse_elements, + annotations=READ_ONLY, + ), + MCPTool( + name="ac_reading_order", + description=("Sort element boxes top-to-bottom, left-to-right and add a " + "stable 'index' to each (elements within 'row_tol' px of " + "each other count as one row). Returns {count, elements}."), + input_schema=schema({ + "elements": {"type": "array", "items": {"type": "object"}}, + "row_tol": {"type": "integer"}}, + required=["elements"]), + handler=h.reading_order, + annotations=READ_ONLY, + ), + ] + + def ssim_tools() -> List[MCPTool]: return [ MCPTool( @@ -6370,7 +6402,8 @@ def media_assert_tools() -> List[MCPTool]: modifier_state_tools, grid_locator_tools, visual_match_tools, color_region_tools, ssim_tools, feature_match_tools, shape_locator_tools, window_layout_tools, window_arrange_tools, preprocess_tools, - monitor_layout_tools, actionability_tools, plugin_sdk_tools, governance_tools, + monitor_layout_tools, actionability_tools, element_parse_tools, + plugin_sdk_tools, governance_tools, credential_lease_tools, egress_tools, approval_testing_tools, trajectory_eval_tools, compliance_tools, agent_trace_tools, video_report_tools, fuzzy_tools, artifact_store_tools, image_dedup_tools, diff --git a/je_auto_control/utils/mcp_server/tools/_handlers.py b/je_auto_control/utils/mcp_server/tools/_handlers.py index 8b39ef7c..6093a542 100644 --- a/je_auto_control/utils/mcp_server/tools/_handlers.py +++ b/je_auto_control/utils/mcp_server/tools/_handlers.py @@ -2178,6 +2178,16 @@ def wait_actionable(template, timeout_s=5.0, stable_for_s=0.3, min_score=0.8, return _wait_actionable(template, timeout_s, stable_for_s, min_score, region) +def fuse_elements(ocr=None, icon=None, a11y=None, iou_threshold=0.9): + from je_auto_control.utils.executor.action_executor import _fuse_elements + return _fuse_elements(ocr, icon, a11y, iou_threshold) + + +def reading_order(elements, row_tol=12): + from je_auto_control.utils.executor.action_executor import _reading_order + return _reading_order(elements, row_tol) + + def detect_drift(reference, current, threshold=0.25, bins=10): from je_auto_control.utils.executor.action_executor import _detect_drift return _detect_drift(reference, current, threshold, bins) diff --git a/test/unit_test/headless/test_element_parse_batch.py b/test/unit_test/headless/test_element_parse_batch.py new file mode 100644 index 00000000..6e315a05 --- /dev/null +++ b/test/unit_test/headless/test_element_parse_batch.py @@ -0,0 +1,88 @@ +"""Headless tests for element-box fusion / ordering. No Qt.""" +import pytest + +import je_auto_control as ac +from je_auto_control.utils.element_parse import ( + fuse_elements, iou, merge_boxes, reading_order, +) + + +def _b(x, y, w, h, **extra): + return dict(x=x, y=y, width=w, height=h, **extra) + + +def test_iou_full_half_none(): + assert iou(_b(0, 0, 10, 10), _b(0, 0, 10, 10)) == pytest.approx(1.0) + assert iou(_b(0, 0, 10, 10), _b(5, 0, 10, 10)) == pytest.approx(1 / 3) + assert iou(_b(0, 0, 10, 10), _b(100, 100, 10, 10)) == pytest.approx(0.0, abs=1e-9) + + +def test_merge_drops_near_duplicates(): + merged = merge_boxes([_b(0, 0, 100, 100), _b(1, 1, 100, 100), + _b(500, 500, 20, 20)]) + assert len(merged) == 2 + assert merged[0]["width"] == 100 # largest kept first + + +def test_merge_threshold_controls_strictness(): + boxes = [_b(0, 0, 100, 100), _b(40, 0, 100, 100)] # ~0.43 IoU + assert len(merge_boxes(boxes, iou_threshold=0.9)) == 2 # kept as distinct + assert len(merge_boxes(boxes, iou_threshold=0.3)) == 1 # merged + + +def test_fuse_prefers_higher_priority_source(): + fused = fuse_elements( + ocr_boxes=[_b(0, 0, 50, 20, text="OK")], + a11y_boxes=[_b(1, 0, 50, 20, text="OK", role="button")]) + assert len(fused) == 1 + assert fused[0]["source"] == "a11y" and fused[0]["role"] == "button" + + +def test_fuse_keeps_disjoint_boxes_from_all_sources(): + fused = fuse_elements( + ocr_boxes=[_b(0, 0, 20, 20)], + icon_boxes=[_b(100, 0, 20, 20)], + a11y_boxes=[_b(200, 0, 20, 20)]) + assert len(fused) == 3 + assert {f["source"] for f in fused} == {"ocr", "icon", "a11y"} + + +def test_fuse_custom_priority(): + fused = fuse_elements( + ocr_boxes=[_b(0, 0, 50, 20, text="from-ocr")], + icon_boxes=[_b(0, 0, 50, 20, text="from-icon")], + source_priority=("icon", "ocr", "a11y")) + assert len(fused) == 1 and fused[0]["source"] == "icon" + + +def test_reading_order_rows_then_columns(): + elements = [_b(100, 0, 20, 20, id="b"), _b(0, 0, 20, 20, id="a"), + _b(0, 100, 20, 20, id="c")] + ordered = reading_order(elements) + assert [(e["id"], e["index"]) for e in ordered] == [("a", 0), ("b", 1), + ("c", 2)] + + +def test_reading_order_row_tolerance_bands(): + # tops differ by 5px -> same row when tol >= 5 + elements = [_b(50, 5, 10, 10, id="r"), _b(0, 0, 10, 10, id="l")] + ordered = reading_order(elements, row_tol=8) + assert [e["id"] for e in ordered] == ["l", "r"] + + +# --- wiring --------------------------------------------------------------- + +def test_wiring(): + known = set(ac.executor.known_commands()) + assert {"AC_fuse_elements", "AC_reading_order"} <= known + from je_auto_control.utils.mcp_server.tools import build_default_tool_registry + names = {t.name for t in build_default_tool_registry()} + assert {"ac_fuse_elements", "ac_reading_order"} <= names + from je_auto_control.gui.script_builder.command_schema import _build_specs + specs = {s.command for s in _build_specs()} + assert {"AC_fuse_elements", "AC_reading_order"} <= specs + + +def test_facade_exports(): + for attr in ("iou", "merge_boxes", "fuse_elements", "reading_order"): + assert hasattr(ac, attr) and attr in ac.__all__