From 51515b476861f5c082299809ae550379c1a58313 Mon Sep 17 00:00:00 2001 From: JeffreyChen Date: Wed, 24 Jun 2026 03:50:43 +0800 Subject: [PATCH] Add form_fields: multi-direction label/value association + checkbox state ocr/structure only pairs a label: with the immediately next cell, so it can't handle label-above-value, two-column key/value, right-aligned values or non-text widgets, and has no checkbox notion. Pair each label with the nearest aligned value across directions (right/below) within max_gap, match free-standing widgets to their nearest label, and read checkbox state from the box's dark-pixel fill ratio. --- README/WHATS_NEW_zh-CN.md | 6 + README/WHATS_NEW_zh-TW.md | 6 + WHATS_NEW.md | 6 + .../doc/new_features/v166_features_doc.rst | 47 +++++++ docs/source/Eng/eng_index.rst | 1 + .../Zh/doc/new_features/v166_features_doc.rst | 41 ++++++ docs/source/Zh/zh_index.rst | 1 + je_auto_control/__init__.py | 7 + .../gui/script_builder/command_schema.py | 23 ++++ .../utils/executor/action_executor.py | 29 +++++ je_auto_control/utils/form_fields/__init__.py | 6 + .../utils/form_fields/form_fields.py | 122 ++++++++++++++++++ .../utils/mcp_server/tools/_factories.py | 27 ++++ .../utils/mcp_server/tools/_handlers.py | 11 ++ .../headless/test_form_fields_batch.py | 71 ++++++++++ 15 files changed, 404 insertions(+) create mode 100644 docs/source/Eng/doc/new_features/v166_features_doc.rst create mode 100644 docs/source/Zh/doc/new_features/v166_features_doc.rst create mode 100644 je_auto_control/utils/form_fields/__init__.py create mode 100644 je_auto_control/utils/form_fields/form_fields.py create mode 100644 test/unit_test/headless/test_form_fields_batch.py diff --git a/README/WHATS_NEW_zh-CN.md b/README/WHATS_NEW_zh-CN.md index 541ba0ba..0fa68253 100644 --- a/README/WHATS_NEW_zh-CN.md +++ b/README/WHATS_NEW_zh-CN.md @@ -1,5 +1,11 @@ # 本次更新 — AutoControl +## 本次更新 (2026-06-24) — 表单字段关联(多方向)+ 复选框状态 + +即使值在下方或右对齐也能把标签与值配对,并读取复选框状态。完整参考:[`docs/source/Zh/doc/new_features/v166_features_doc.rst`](../docs/source/Zh/doc/new_features/v166_features_doc.rst)。 + +- **`associate_fields` / `match_labels_to_widgets` / `checkbox_state`**(`AC_associate_fields`、`AC_match_labels_to_widgets`):`ocr/structure` 只把 `label:` 与*紧接的下一格*配对——无法处理标签在上、双列 key/value、右对齐值或非文字 widget,且无复选框概念。本功能把每个标签与多*方向*(右 / 下)中 `max_gap` 内最近的对齐值配对,把独立 widget(复选框 / 单选钮 / 输入框)配到最近标签,并由框内暗像素填充比例读取复选框状态。关联部分纯标准库;只有 `checkbox_state` 触及像素(隔离在 `visual_match` 灰阶加载器之后)。不导入 `PySide6`。 + ## 本次更新 (2026-06-24) — 留白投影列推断(无框线表格) 靠留白间隙推断列来读取无框线表格。完整参考:[`docs/source/Zh/doc/new_features/v165_features_doc.rst`](../docs/source/Zh/doc/new_features/v165_features_doc.rst)。 diff --git a/README/WHATS_NEW_zh-TW.md b/README/WHATS_NEW_zh-TW.md index 8cca4a52..e40c36d9 100644 --- a/README/WHATS_NEW_zh-TW.md +++ b/README/WHATS_NEW_zh-TW.md @@ -1,5 +1,11 @@ # 本次更新 — AutoControl +## 本次更新 (2026-06-24) — 表單欄位關聯(多方向)+ 核取方塊狀態 + +即使值在下方或右對齊也能把標籤與值配對,並讀取核取方塊狀態。完整參考:[`docs/source/Zh/doc/new_features/v166_features_doc.rst`](../docs/source/Zh/doc/new_features/v166_features_doc.rst)。 + +- **`associate_fields` / `match_labels_to_widgets` / `checkbox_state`**(`AC_associate_fields`、`AC_match_labels_to_widgets`):`ocr/structure` 只把 `label:` 與*緊接的下一格*配對——無法處理標籤在上、雙欄 key/value、右對齊值或非文字 widget,且無核取方塊概念。本功能把每個標籤與多*方向*(右 / 下)中 `max_gap` 內最近的對齊值配對,把獨立 widget(核取方塊 / 單選鈕 / 輸入框)配到最近標籤,並由框內暗像素填充比例讀取核取方塊狀態。關聯部分純標準函式庫;只有 `checkbox_state` 觸及像素(隔離在 `visual_match` 灰階載入器之後)。不匯入 `PySide6`。 + ## 本次更新 (2026-06-24) — 留白投影欄位偵測(無框線表格) 靠留白間隙推導欄位來讀取無框線表格。完整參考:[`docs/source/Zh/doc/new_features/v165_features_doc.rst`](../docs/source/Zh/doc/new_features/v165_features_doc.rst)。 diff --git a/WHATS_NEW.md b/WHATS_NEW.md index 664ab738..c95e5708 100644 --- a/WHATS_NEW.md +++ b/WHATS_NEW.md @@ -1,5 +1,11 @@ # What's New — AutoControl +## What's new (2026-06-24) — Form Field Association (Multi-Direction) + Checkbox State + +Pair form labels with values even when the value is below or right-aligned, and read checkbox state. Full reference: [`docs/source/Eng/doc/new_features/v166_features_doc.rst`](docs/source/Eng/doc/new_features/v166_features_doc.rst). + +- **`associate_fields` / `match_labels_to_widgets` / `checkbox_state`** (`AC_associate_fields`, `AC_match_labels_to_widgets`): `ocr/structure` only pairs a `label:` with the *immediately next* cell — it can't handle label-above-value, two-column key/value, right-aligned values, or non-text widgets, and has no checkbox notion. This pairs each label with the nearest aligned value across *directions* (right / below) within `max_gap`, matches free-standing widgets (checkbox/radio/input) to their nearest label, and reads checkbox state from the box's dark-pixel fill ratio. Association is pure-stdlib; only `checkbox_state` touches pixels (behind the `visual_match` gray loader). No `PySide6`. + ## What's new (2026-06-24) — Whitespace-Projection Columns (Borderless Tables) Read borderless tables by inferring columns from the whitespace gaps. Full reference: [`docs/source/Eng/doc/new_features/v165_features_doc.rst`](docs/source/Eng/doc/new_features/v165_features_doc.rst). diff --git a/docs/source/Eng/doc/new_features/v166_features_doc.rst b/docs/source/Eng/doc/new_features/v166_features_doc.rst new file mode 100644 index 00000000..c35bce63 --- /dev/null +++ b/docs/source/Eng/doc/new_features/v166_features_doc.rst @@ -0,0 +1,47 @@ +Form Field Association (Multi-Direction) + Checkbox State +========================================================= + +``ocr/structure`` recognises a label only if its text ends in ``:`` and pairs it with the +*immediately next* cell — it cannot handle a label sitting *above* its value, a two-column +key/value layout, right-aligned values, or any widget that isn't a text cell, and it has no +notion of checkbox / radio state at all. ``form_fields`` generalises this: it pairs each +label with the nearest aligned value in any of several *directions* (right, below), matches +free-standing widgets (checkboxes, radios, inputs) to their nearest label, and reads a +checkbox's checked state from its fill ratio. + +The association is pure-stdlib over plain box dicts (fully unit-testable, no image); only +``checkbox_state`` touches pixels, isolated behind the shared ``visual_match`` gray loader so +tests can pass a synthetic array. Reuses ``table_grid_fill``'s box-bounds reader. Imports no +``PySide6``. + +Headless API +------------ + +.. code-block:: python + + from je_auto_control import (associate_fields, match_labels_to_widgets, + checkbox_state) + + fields = associate_fields(ocr_boxes, directions=("right", "below")) + # [{"label": "Name", "value": "Ann", "direction": "right", "gap": 20, ...}] + + pairs = match_labels_to_widgets(label_boxes, checkbox_boxes) + # [{"widget": {...}, "label": "Accept terms", "distance": 22}] + + state = checkbox_state(screenshot, checkbox_box) # "checked" | "unchecked" + +``associate_fields`` treats boxes whose text ends in ``:`` as labels and pairs each with the +nearest value box in the requested directions (within ``max_gap``), returning +``{label, value, direction, gap, label_box, value_box}``. ``match_labels_to_widgets`` assigns +each widget to its nearest label by centre distance. ``checkbox_state`` returns +``"checked"`` / ``"unchecked"`` from the dark-pixel fill ratio of the box (image is injectable +— path / ndarray / PIL). + +Executor commands +----------------- + +``AC_associate_fields`` (``text_boxes`` / ``directions`` / ``max_gap`` → ``{count, fields}``) +and ``AC_match_labels_to_widgets`` (``labels`` / ``widgets`` → ``{count, pairs}``). They are +exposed as the MCP tools ``ac_associate_fields`` / ``ac_match_labels_to_widgets`` (read-only) +and as the Script Builder commands **Associate Form Fields** / **Match Labels To Widgets** +under **OCR**. diff --git a/docs/source/Eng/eng_index.rst b/docs/source/Eng/eng_index.rst index e48b63c1..9a1ad891 100644 --- a/docs/source/Eng/eng_index.rst +++ b/docs/source/Eng/eng_index.rst @@ -188,6 +188,7 @@ Comprehensive guides for all AutoControl features. doc/new_features/v163_features_doc doc/new_features/v164_features_doc doc/new_features/v165_features_doc + doc/new_features/v166_features_doc doc/ocr_backends/ocr_backends_doc doc/observability/observability_doc doc/operations_layer/operations_layer_doc diff --git a/docs/source/Zh/doc/new_features/v166_features_doc.rst b/docs/source/Zh/doc/new_features/v166_features_doc.rst new file mode 100644 index 00000000..49b28fe1 --- /dev/null +++ b/docs/source/Zh/doc/new_features/v166_features_doc.rst @@ -0,0 +1,41 @@ +表單欄位關聯(多方向)+ 核取方塊狀態 +====================================== + +``ocr/structure`` 只有在框文字以 ``:`` 結尾時才視為標籤,並只與*緊接的下一格*配對——無法 +處理標籤在值*上方*、雙欄 key/value 版面、右對齊值,或任何非文字格的 widget,且完全沒有 +核取方塊 / 單選鈕狀態的概念。``form_fields`` 將其一般化:把每個標籤與多個*方向*(右、下)中 +最近的對齊值配對,把獨立的 widget(核取方塊、單選鈕、輸入框)配到最近的標籤,並由填充比例 +讀取核取方塊的勾選狀態。 + +關聯部分為純標準函式庫,作用於純框字典(可完整單元測試、不需影像);只有 ``checkbox_state`` +觸及像素,且隔離在共用的 ``visual_match`` 灰階載入器之後,讓測試可傳入合成陣列。重用 +``table_grid_fill`` 的框邊界讀取器。不匯入 ``PySide6``。 + +無頭 API +-------- + +.. code-block:: python + + from je_auto_control import (associate_fields, match_labels_to_widgets, + checkbox_state) + + fields = associate_fields(ocr_boxes, directions=("right", "below")) + # [{"label": "Name", "value": "Ann", "direction": "right", "gap": 20, ...}] + + pairs = match_labels_to_widgets(label_boxes, checkbox_boxes) + # [{"widget": {...}, "label": "Accept terms", "distance": 22}] + + state = checkbox_state(screenshot, checkbox_box) # "checked" | "unchecked" + +``associate_fields`` 把文字以 ``:`` 結尾的框視為標籤,並在指定方向中(``max_gap`` 內)配到 +最近的值框,回傳 ``{label, value, direction, gap, label_box, value_box}``。 +``match_labels_to_widgets`` 依中心距離把每個 widget 配到最近標籤。``checkbox_state`` 由框內 +暗像素填充比例回傳 ``"checked"`` / ``"unchecked"``(image 可注入——路徑 / ndarray / PIL)。 + +執行器指令 +---------- + +``AC_associate_fields``(``text_boxes`` / ``directions`` / ``max_gap`` → ``{count, fields}``) +與 ``AC_match_labels_to_widgets``(``labels`` / ``widgets`` → ``{count, pairs}``)。兩者以 +MCP 工具 ``ac_associate_fields`` / ``ac_match_labels_to_widgets``(唯讀)及 Script Builder 指令 +**Associate Form Fields** / **Match Labels To Widgets**(位於 **OCR** 分類下)形式提供。 diff --git a/docs/source/Zh/zh_index.rst b/docs/source/Zh/zh_index.rst index 717a4f42..01a17ed5 100644 --- a/docs/source/Zh/zh_index.rst +++ b/docs/source/Zh/zh_index.rst @@ -188,6 +188,7 @@ AutoControl 所有功能的完整使用指南。 doc/new_features/v163_features_doc doc/new_features/v164_features_doc doc/new_features/v165_features_doc + doc/new_features/v166_features_doc doc/ocr_backends/ocr_backends_doc doc/observability/observability_doc doc/operations_layer/operations_layer_doc diff --git a/je_auto_control/__init__.py b/je_auto_control/__init__.py index 11d52148..df228909 100644 --- a/je_auto_control/__init__.py +++ b/je_auto_control/__init__.py @@ -307,6 +307,10 @@ from je_auto_control.utils.column_layout import ( assign_columns, column_gutters, detect_borderless_table, vertical_projection, ) +# Associate form labels with values (multi-direction) + checkbox state +from je_auto_control.utils.form_fields import ( + associate_fields, checkbox_state, match_labels_to_widgets, +) # Locate on-screen regions by colour (mask + connected components) from je_auto_control.utils.color_region import ( find_color_region, find_color_regions, @@ -1238,6 +1242,9 @@ def start_autocontrol_gui(*args, **kwargs): "column_gutters", "assign_columns", "detect_borderless_table", + "associate_fields", + "match_labels_to_widgets", + "checkbox_state", "find_color_region", "find_color_regions", "ssim_compare", diff --git a/je_auto_control/gui/script_builder/command_schema.py b/je_auto_control/gui/script_builder/command_schema.py index 67eb954f..b51defc7 100644 --- a/je_auto_control/gui/script_builder/command_schema.py +++ b/je_auto_control/gui/script_builder/command_schema.py @@ -729,6 +729,29 @@ def _add_ocr_specs(specs: List[CommandSpec]) -> None: ), description="Find borderless-table column separators by whitespace projection.", )) + specs.append(CommandSpec( + "AC_associate_fields", "OCR", "Associate Form Fields", + fields=( + FieldSpec("text_boxes", FieldType.STRING, + placeholder='[{"x":0,"y":0,"width":60,"height":20,' + '"text":"Name:"}]'), + FieldSpec("directions", FieldType.STRING, optional=True, + placeholder='["right", "below"]'), + FieldSpec("max_gap", FieldType.INT, optional=True, default=150), + ), + description="Pair 'label:' boxes with the nearest aligned value (right/below).", + )) + specs.append(CommandSpec( + "AC_match_labels_to_widgets", "OCR", "Match Labels To Widgets", + fields=( + FieldSpec("labels", FieldType.STRING, + placeholder='[{"x":0,"y":0,"width":60,"height":20,' + '"text":"Accept"}]'), + FieldSpec("widgets", FieldType.STRING, + placeholder='[{"x":120,"y":0,"width":16,"height":16}]'), + ), + description="Match each checkbox/radio/input to its nearest label by centre.", + )) specs.append(CommandSpec( "AC_scroll_to_find", "OCR", "Scroll Until Visible", fields=( diff --git a/je_auto_control/utils/executor/action_executor.py b/je_auto_control/utils/executor/action_executor.py index 9e90ddf6..d3e85ca1 100644 --- a/je_auto_control/utils/executor/action_executor.py +++ b/je_auto_control/utils/executor/action_executor.py @@ -3433,6 +3433,33 @@ def _detect_borderless_table(boxes: Any, page_width: Any = None, min_gap: Any = return {"found": table is not None, "table": table} +def _associate_fields(text_boxes: Any, directions: Any = None, + max_gap: Any = 150) -> Dict[str, Any]: + """Adapter: pair form labels with their nearest aligned value boxes.""" + import json + from je_auto_control.utils.form_fields import associate_fields + if isinstance(text_boxes, str): + text_boxes = json.loads(text_boxes) + if isinstance(directions, str): + directions = json.loads(directions) if directions.strip() else None + fields = associate_fields(text_boxes, + directions=tuple(directions) if directions + else ("right", "below"), max_gap=int(max_gap)) + return {"count": len(fields), "fields": fields} + + +def _match_labels_to_widgets(labels: Any, widgets: Any) -> Dict[str, Any]: + """Adapter: match each widget (checkbox / radio / input) to its nearest label.""" + import json + from je_auto_control.utils.form_fields import match_labels_to_widgets + if isinstance(labels, str): + labels = json.loads(labels) + if isinstance(widgets, str): + widgets = json.loads(widgets) + pairs = match_labels_to_widgets(labels, widgets) + return {"count": len(pairs), "pairs": pairs} + + def _find_color_region(rgb: Any, tolerance: Any = 20, min_area: Any = 50, region: Any = None) -> Dict[str, Any]: """Adapter: locate coloured regions on the screen, largest first.""" @@ -5886,6 +5913,8 @@ def __init__(self): "AC_populate_table": _populate_table, "AC_column_gutters": _column_gutters, "AC_detect_borderless_table": _detect_borderless_table, + "AC_associate_fields": _associate_fields, + "AC_match_labels_to_widgets": _match_labels_to_widgets, "AC_ssim_compare": _ssim_compare, "AC_ssim_changed_regions": _ssim_changed_regions, "AC_feature_match": _feature_match, diff --git a/je_auto_control/utils/form_fields/__init__.py b/je_auto_control/utils/form_fields/__init__.py new file mode 100644 index 00000000..6a3b3274 --- /dev/null +++ b/je_auto_control/utils/form_fields/__init__.py @@ -0,0 +1,6 @@ +"""Associate form labels with values (multi-direction) and read checkbox state.""" +from je_auto_control.utils.form_fields.form_fields import ( + associate_fields, checkbox_state, match_labels_to_widgets, +) + +__all__ = ["associate_fields", "match_labels_to_widgets", "checkbox_state"] diff --git a/je_auto_control/utils/form_fields/form_fields.py b/je_auto_control/utils/form_fields/form_fields.py new file mode 100644 index 00000000..5fb646df --- /dev/null +++ b/je_auto_control/utils/form_fields/form_fields.py @@ -0,0 +1,122 @@ +"""Associate form labels with their values and read checkbox state. + +``ocr/structure`` recognises a label only if its text ends in ``:`` and pairs it with the +*immediately next* cell — it cannot handle a label sitting *above* its value, a two-column +key/value layout, right-aligned values, or any widget that isn't a text cell, and it has no +notion of checkbox / radio state at all. ``form_fields`` generalises this: it pairs each +label with the nearest aligned value in any of several *directions* (right, below), matches +free-standing widgets (checkboxes, radios, inputs) to their nearest label, and reads a +checkbox's checked state from its fill ratio. + +The association is pure-stdlib over plain box dicts (fully unit-testable, no image); only +``checkbox_state`` touches pixels, isolated behind the shared ``visual_match`` gray loader so +tests can pass a synthetic array. Reuses ``table_grid_fill``'s box-bounds reader. Imports no +``PySide6``. +""" +from typing import Any, Callable, Dict, List, Optional, Sequence, Tuple + +from je_auto_control.utils.table_grid_fill.table_grid_fill import _box_bounds + +Box = Dict[str, Any] + + +def _overlap_1d(a0: int, a1: int, b0: int, b1: int) -> int: + return max(0, min(a1, b1) - max(a0, b0)) + + +def _right_value(label: Box, values: Sequence[Box], max_gap: int): + """Nearest value to the right of ``label`` that shares a row, or ``None``.""" + left, top, right, bottom = _box_bounds(label) + best: Optional[Tuple[Box, int]] = None + for value in values: + vl, vt, _, vb = _box_bounds(value) + gap = vl - right + if vl >= right and _overlap_1d(top, bottom, vt, vb) > 0 and 0 <= gap <= max_gap \ + and (best is None or gap < best[1]): + best = (value, gap) + return best + + +def _below_value(label: Box, values: Sequence[Box], max_gap: int): + """Nearest value below ``label`` that shares a column, or ``None``.""" + left, top, right, bottom = _box_bounds(label) + best: Optional[Tuple[Box, int]] = None + for value in values: + vl, vt, vr, _ = _box_bounds(value) + gap = vt - bottom + if vt >= bottom and _overlap_1d(left, right, vl, vr) > 0 and 0 <= gap <= max_gap \ + and (best is None or gap < best[1]): + best = (value, gap) + return best + + +_DIRECTIONS: Dict[str, Callable[[Box, Sequence[Box], int], Any]] = { + "right": _right_value, "below": _below_value} + + +def _clean_label(box: Box) -> str: + return str(box.get("text", "")).strip().rstrip(":").strip() + + +def _best_value(label: Box, values: Sequence[Box], directions: Sequence[str], + max_gap: int): + """Pick the nearest value across the requested directions, or ``None``.""" + best = None + for direction in directions: + candidate = _DIRECTIONS[direction](label, values, max_gap) + if candidate is not None and (best is None or candidate[1] < best[1]): + best = (candidate[0], candidate[1], direction) + return best + + +def associate_fields(text_boxes: Sequence[Box], *, + directions: Sequence[str] = ("right", "below"), + max_gap: int = 150) -> List[Dict[str, Any]]: + """Pair each ``label:`` box with its nearest aligned value box. + + Labels are boxes whose text ends in ``:``; every other box is a candidate value. + Returns ``{label, value, direction, gap, label_box, value_box}`` per matched label. + """ + labels = [b for b in text_boxes if str(b.get("text", "")).strip().endswith(":")] + label_ids = {id(b) for b in labels} + values = [b for b in text_boxes if id(b) not in label_ids] + fields: List[Dict[str, Any]] = [] + for label in labels: + best = _best_value(label, values, directions, int(max_gap)) + if best is not None: + fields.append({"label": _clean_label(label), + "value": str(best[0].get("text", "")), + "direction": best[2], "gap": best[1], + "label_box": label, "value_box": best[0]}) + return fields + + +def match_labels_to_widgets(labels: Sequence[Box], + widgets: Sequence[Box]) -> List[Dict[str, Any]]: + """Match each widget (checkbox / radio / input) to its nearest label by centre.""" + pairs: List[Dict[str, Any]] = [] + for widget in widgets: + wl, wt, wr, wb = _box_bounds(widget) + wcx, wcy = (wl + wr) // 2, (wt + wb) // 2 + best: Optional[Tuple[Box, int]] = None + for label in labels: + ll, lt, lr, lb = _box_bounds(label) + dist = abs(wcx - (ll + lr) // 2) + abs(wcy - (lt + lb) // 2) + if best is None or dist < best[1]: + best = (label, dist) + if best is not None: + pairs.append({"widget": widget, "label": str(best[0].get("text", "")), + "distance": best[1]}) + return pairs + + +def checkbox_state(image: Any, box: Box, *, fill_threshold: float = 0.15) -> str: + """Return ``"checked"`` / ``"unchecked"`` from the dark-pixel fill ratio of ``box``.""" + import numpy as np + from je_auto_control.utils.visual_match.visual_match import _to_gray + left, top, right, bottom = _box_bounds(box) + patch = _to_gray(image)[top:bottom, left:right] + if patch.size == 0: + return "unchecked" + filled = float(np.count_nonzero(patch < 128)) / patch.size + return "checked" if filled >= float(fill_threshold) else "unchecked" diff --git a/je_auto_control/utils/mcp_server/tools/_factories.py b/je_auto_control/utils/mcp_server/tools/_factories.py index f1ecfabd..10eef6c5 100644 --- a/je_auto_control/utils/mcp_server/tools/_factories.py +++ b/je_auto_control/utils/mcp_server/tools/_factories.py @@ -3761,6 +3761,33 @@ def screen_grid_tools() -> List[MCPTool]: handler=h.detect_borderless_table, annotations=READ_ONLY, ), + MCPTool( + name="ac_associate_fields", + description=("Pair form labels with their values: for each 'text_boxes' " + "entry whose text ends in ':', find the nearest aligned value " + "box in 'directions' (right / below). Returns {count, fields:" + "[{label,value,direction,gap,label_box,value_box}]}. " + "'max_gap' caps the distance."), + input_schema=schema({ + "text_boxes": {"type": "array", "items": {"type": "object"}}, + "directions": {"type": "array", "items": {"type": "string"}}, + "max_gap": {"type": "integer"}}, + required=["text_boxes"]), + handler=h.associate_fields, + annotations=READ_ONLY, + ), + MCPTool( + name="ac_match_labels_to_widgets", + description=("Match each widget (checkbox / radio / input box) to its " + "nearest 'labels' entry by centre distance. Returns {count, " + "pairs:[{widget,label,distance}]}."), + input_schema=schema({ + "labels": {"type": "array", "items": {"type": "object"}}, + "widgets": {"type": "array", "items": {"type": "object"}}}, + required=["labels", "widgets"]), + handler=h.match_labels_to_widgets, + annotations=READ_ONLY, + ), ] diff --git a/je_auto_control/utils/mcp_server/tools/_handlers.py b/je_auto_control/utils/mcp_server/tools/_handlers.py index 0ab8f82c..06e4eb01 100644 --- a/je_auto_control/utils/mcp_server/tools/_handlers.py +++ b/je_auto_control/utils/mcp_server/tools/_handlers.py @@ -2158,6 +2158,17 @@ def detect_borderless_table(boxes, page_width=None, min_gap=8, min_cols=2, return _detect_borderless_table(boxes, page_width, min_gap, min_cols, min_rows) +def associate_fields(text_boxes, directions=None, max_gap=150): + from je_auto_control.utils.executor.action_executor import _associate_fields + return _associate_fields(text_boxes, directions, max_gap) + + +def match_labels_to_widgets(labels, widgets): + from je_auto_control.utils.executor.action_executor import ( + _match_labels_to_widgets) + return _match_labels_to_widgets(labels, widgets) + + def find_color_region(rgb, tolerance=20, min_area=50, region=None): from je_auto_control.utils.executor.action_executor import ( _find_color_region) diff --git a/test/unit_test/headless/test_form_fields_batch.py b/test/unit_test/headless/test_form_fields_batch.py new file mode 100644 index 00000000..b8726498 --- /dev/null +++ b/test/unit_test/headless/test_form_fields_batch.py @@ -0,0 +1,71 @@ +"""Headless tests for form label/value association + checkbox state.""" +import pytest + +import je_auto_control as ac +from je_auto_control.utils.form_fields import ( + associate_fields, checkbox_state, match_labels_to_widgets, +) + + +def _box(x, y, text, w=60, h=20): + return {"x": x, "y": y, "width": w, "height": h, "text": text} + + +def test_label_to_value_on_the_right(): + boxes = [_box(0, 0, "Name:"), _box(80, 0, "Ann")] + fields = associate_fields(boxes) + assert len(fields) == 1 + assert fields[0]["label"] == "Name" + assert fields[0]["value"] == "Ann" and fields[0]["direction"] == "right" + + +def test_label_above_value_below(): + # value directly below the label, nothing to the right + boxes = [_box(0, 0, "Email:"), _box(0, 40, "a@b.com")] + fields = associate_fields(boxes, directions=("right", "below")) + assert fields[0]["value"] == "a@b.com" and fields[0]["direction"] == "below" + + +def test_nearest_value_wins_and_gap_within_max(): + boxes = [_box(0, 0, "City:"), _box(70, 0, "Taipei"), _box(400, 0, "Far")] + fields = associate_fields(boxes, max_gap=150) + assert fields[0]["value"] == "Taipei" + + +def test_value_beyond_max_gap_unmatched(): + boxes = [_box(0, 0, "Lonely:"), _box(900, 0, "TooFar")] + assert associate_fields(boxes, max_gap=150) == [] + + +def test_match_labels_to_widgets_nearest(): + labels = [_box(0, 0, "Accept"), _box(0, 100, "Decline")] + widgets = [_box(120, 0, "", w=16, h=16)] + pairs = match_labels_to_widgets(labels, widgets) + assert pairs[0]["label"] == "Accept" + + +def test_checkbox_state_from_fill(): + np = pytest.importorskip("numpy") + image = np.full((40, 40), 255, dtype=np.uint8) # white + image[5:35, 5:35] = 0 # dark fill inside the box + box = {"x": 0, "y": 0, "width": 40, "height": 40} + assert checkbox_state(image, box) == "checked" + assert checkbox_state(np.full((40, 40), 255, np.uint8), box) == "unchecked" + + +# --- wiring --------------------------------------------------------------- + +def test_wiring(): + known = set(ac.executor.known_commands()) + assert {"AC_associate_fields", "AC_match_labels_to_widgets"} <= known + from je_auto_control.utils.mcp_server.tools import build_default_tool_registry + names = {t.name for t in build_default_tool_registry()} + assert {"ac_associate_fields", "ac_match_labels_to_widgets"} <= names + from je_auto_control.gui.script_builder.command_schema import _build_specs + specs = {s.command for s in _build_specs()} + assert {"AC_associate_fields", "AC_match_labels_to_widgets"} <= specs + + +def test_facade_exports(): + for name in ("associate_fields", "match_labels_to_widgets", "checkbox_state"): + assert hasattr(ac, name) and name in ac.__all__