From e0075e27c7c6daf92065076b4660dac16fdfa0e8 Mon Sep 17 00:00:00 2001 From: JeffreyChen Date: Wed, 24 Jun 2026 08:21:00 +0800 Subject: [PATCH 1/2] Add control_patterns: extended UIA patterns (Expand/Select/Range/Scroll) The accessibility backend had only Value/Invoke/Toggle/Grid-read, so treeviews, listboxes/combos, sliders and off-screen rows had no native pattern path. Add ExpandCollapse / SelectionItem / RangeValue / ScrollItem actions on top of the existing backend ABC, dispatched through the injectable get_backend() seam (headless-testable via a fake backend; real UIA calls in the Windows backend). --- README/WHATS_NEW_zh-CN.md | 6 + README/WHATS_NEW_zh-TW.md | 6 + WHATS_NEW.md | 6 + .../doc/new_features/v181_features_doc.rst | 46 +++++++ docs/source/Eng/eng_index.rst | 1 + .../Zh/doc/new_features/v181_features_doc.rst | 42 +++++++ docs/source/Zh/zh_index.rst | 1 + je_auto_control/__init__.py | 8 ++ .../gui/script_builder/command_schema.py | 35 ++++++ .../utils/accessibility/backends/base.py | 46 ++++++- .../accessibility/backends/windows_backend.py | 73 +++++++++++- .../utils/control_patterns/__init__.py | 11 ++ .../control_patterns/control_patterns.py | 77 ++++++++++++ .../utils/executor/action_executor.py | 71 +++++++++++ .../utils/mcp_server/tools/_factories.py | 58 +++++++++ .../utils/mcp_server/tools/_handlers.py | 39 ++++++ .../headless/test_control_patterns_batch.py | 112 ++++++++++++++++++ 17 files changed, 636 insertions(+), 2 deletions(-) create mode 100644 docs/source/Eng/doc/new_features/v181_features_doc.rst create mode 100644 docs/source/Zh/doc/new_features/v181_features_doc.rst create mode 100644 je_auto_control/utils/control_patterns/__init__.py create mode 100644 je_auto_control/utils/control_patterns/control_patterns.py create mode 100644 test/unit_test/headless/test_control_patterns_batch.py diff --git a/README/WHATS_NEW_zh-CN.md b/README/WHATS_NEW_zh-CN.md index d4c92bce..519be585 100644 --- a/README/WHATS_NEW_zh-CN.md +++ b/README/WHATS_NEW_zh-CN.md @@ -1,5 +1,11 @@ # 本次更新 — AutoControl +## 本次更新 (2026-06-24) — 扩充 UIA 控制模式(展开 / 选取 / 范围 / 滚动) + +以原生模式驱动树节点、列表/下拉项目、滑块与滚动,而非像素猜测。完整参考:[`docs/source/Zh/doc/new_features/v181_features_doc.rst`](../docs/source/Zh/doc/new_features/v181_features_doc.rst)。 + +- **`expand_control` / `collapse_control` / `control_expand_state` / `select_control_item` / `control_range` / `set_control_range` / `scroll_control_into_view`**(`AC_expand_control`、`AC_select_control_item`、`AC_set_control_range` 等):无障碍后端原本只有 Value/Invoke/Toggle/Grid-read 模式,故树状视图、列表/下拉、滑块与屏幕外行都没有原生调用路径。本功能在既有后端 ABC 之上补上 ExpandCollapse / SelectionItem / RangeValue / ScrollItem 模式,通过可注入的 `accessibility.backends.get_backend()` 接缝分派(以 fake backend 无头测试;真正 UIA 调用在 Windows 后端)。不导入 `PySide6`。 + ## 本次更新 (2026-06-24) — 匹配前安定门 + 命中稳定性 避免在动画进行中匹配,并确认命中跨帧维持稳定。完整参考:[`docs/source/Zh/doc/new_features/v180_features_doc.rst`](../docs/source/Zh/doc/new_features/v180_features_doc.rst)。 diff --git a/README/WHATS_NEW_zh-TW.md b/README/WHATS_NEW_zh-TW.md index bb67dab1..bfd2f407 100644 --- a/README/WHATS_NEW_zh-TW.md +++ b/README/WHATS_NEW_zh-TW.md @@ -1,5 +1,11 @@ # 本次更新 — AutoControl +## 本次更新 (2026-06-24) — 擴充 UIA 控制模式(展開 / 選取 / 範圍 / 捲動) + +以原生模式驅動樹節點、清單/下拉項目、滑桿與捲動,而非像素猜測。完整參考:[`docs/source/Zh/doc/new_features/v181_features_doc.rst`](../docs/source/Zh/doc/new_features/v181_features_doc.rst)。 + +- **`expand_control` / `collapse_control` / `control_expand_state` / `select_control_item` / `control_range` / `set_control_range` / `scroll_control_into_view`**(`AC_expand_control`、`AC_select_control_item`、`AC_set_control_range` 等):無障礙後端原本只有 Value/Invoke/Toggle/Grid-read 模式,故樹狀檢視、清單/下拉、滑桿與螢幕外列都沒有原生呼叫路徑。本功能在既有後端 ABC 之上補上 ExpandCollapse / SelectionItem / RangeValue / ScrollItem 模式,透過可注入的 `accessibility.backends.get_backend()` 接縫分派(以 fake backend 無頭測試;真正 UIA 呼叫在 Windows 後端)。不匯入 `PySide6`。 + ## 本次更新 (2026-06-24) — 比對前安定閘 + 命中穩定性 避免在動畫進行中比對,並確認命中跨幀維持穩定。完整參考:[`docs/source/Zh/doc/new_features/v180_features_doc.rst`](../docs/source/Zh/doc/new_features/v180_features_doc.rst)。 diff --git a/WHATS_NEW.md b/WHATS_NEW.md index 1104cf7a..ffc28c4e 100644 --- a/WHATS_NEW.md +++ b/WHATS_NEW.md @@ -1,5 +1,11 @@ # What's New — AutoControl +## What's new (2026-06-24) — Extended UIA Control Patterns (Expand / Select / Range / Scroll) + +Drive tree nodes, list/combo items, sliders and scroll natively, not by pixel guessing. Full reference: [`docs/source/Eng/doc/new_features/v181_features_doc.rst`](docs/source/Eng/doc/new_features/v181_features_doc.rst). + +- **`expand_control` / `collapse_control` / `control_expand_state` / `select_control_item` / `control_range` / `set_control_range` / `scroll_control_into_view`** (`AC_expand_control`, `AC_select_control_item`, `AC_set_control_range`, …): the accessibility backend had only Value/Invoke/Toggle/Grid-read patterns, so treeviews, listboxes/combos, sliders and off-screen rows had no native call path. This adds ExpandCollapse / SelectionItem / RangeValue / ScrollItem patterns on top of the existing backend ABC, dispatched through the injectable `accessibility.backends.get_backend()` seam (headless-testable via a fake backend; real UIA calls in the Windows backend). No `PySide6`. + ## What's new (2026-06-24) — Pre-Match Settle Gating + Match Persistence Avoid matching mid-animation, and confirm a hit holds steady across frames. Full reference: [`docs/source/Eng/doc/new_features/v180_features_doc.rst`](docs/source/Eng/doc/new_features/v180_features_doc.rst). diff --git a/docs/source/Eng/doc/new_features/v181_features_doc.rst b/docs/source/Eng/doc/new_features/v181_features_doc.rst new file mode 100644 index 00000000..77c32734 --- /dev/null +++ b/docs/source/Eng/doc/new_features/v181_features_doc.rst @@ -0,0 +1,46 @@ +Extended UIA Control Patterns (Expand / Select / Range / Scroll) +=============================================================== + +The accessibility backend shipped only four control patterns — Value, Invoke, Toggle and a +read-only Grid dump. That left the controls automation hits most often undriveable by their +*native* pattern: a treeview node could not be expanded, a listbox / combobox item could not be +selected (SelectionItemPattern), a slider could not be set (RangeValuePattern), and a control +could not be scrolled into view (ScrollItemPattern) — those fell back to fragile pixel guessing. +``control_patterns`` adds those object-level actions on top of the existing accessibility +backend ABC. + +Each function is a thin dispatch onto the injectable ``accessibility.backends.get_backend()`` +seam (the same seam the rest of the accessibility module uses), so the headless core is +unit-testable on any platform by injecting a fake backend; the real UI Automation calls live in +the Windows backend (ExpandCollapse / SelectionItem / RangeValue / ScrollItem patterns). +Backends that don't implement a pattern raise ``AccessibilityNotAvailableError``. Imports no +``PySide6``. + +Headless API +------------ + +.. code-block:: python + + from je_auto_control import (expand_control, collapse_control, + control_expand_state, select_control_item, + control_range, set_control_range, + scroll_control_into_view) + + expand_control(name="Documents", role="treeitem") # open a tree node + select_control_item(name="Option B") # pick a list/combo item + set_control_range(75, name="Volume") # set a slider + print(control_range(name="Volume")) # {"value": 75.0, "minimum": 0, "maximum": 100} + scroll_control_into_view(name="Row 200") # bring a row on-screen + +All locate the control by ``name`` / ``role`` / ``app_name`` / ``automation_id`` (same as the +existing ``control_invoke`` / ``control_toggle``). The expand/select/scroll/set actions return +``bool``; ``control_expand_state`` returns ``expanded`` / ``collapsed`` / ``partial`` / ``leaf`` +(or ``None``); ``control_range`` returns ``{value, minimum, maximum}`` (or ``None``). + +Executor commands +----------------- + +``AC_expand_control`` / ``AC_collapse_control`` / ``AC_control_expand_state`` / +``AC_select_control_item`` / ``AC_control_range`` / ``AC_set_control_range`` / +``AC_scroll_control_into_view``. They are exposed as the matching ``ac_*`` MCP tools (the action +ones destructive, the reads read-only) and as Script Builder commands under **Native UI**. diff --git a/docs/source/Eng/eng_index.rst b/docs/source/Eng/eng_index.rst index e2c5eed5..10f84a13 100644 --- a/docs/source/Eng/eng_index.rst +++ b/docs/source/Eng/eng_index.rst @@ -203,6 +203,7 @@ Comprehensive guides for all AutoControl features. doc/new_features/v178_features_doc doc/new_features/v179_features_doc doc/new_features/v180_features_doc + doc/new_features/v181_features_doc doc/ocr_backends/ocr_backends_doc doc/observability/observability_doc doc/operations_layer/operations_layer_doc diff --git a/docs/source/Zh/doc/new_features/v181_features_doc.rst b/docs/source/Zh/doc/new_features/v181_features_doc.rst new file mode 100644 index 00000000..a2b1f7d8 --- /dev/null +++ b/docs/source/Zh/doc/new_features/v181_features_doc.rst @@ -0,0 +1,42 @@ +擴充 UIA 控制模式(展開 / 選取 / 範圍 / 捲動) +=============================================== + +無障礙後端原本只提供四種控制模式——Value、Invoke、Toggle 與唯讀的 Grid dump。這使得自動化 +最常遇到的控制項無法以其*原生*模式驅動:樹節點無法展開、清單 / 下拉項目無法選取 +(SelectionItemPattern)、滑桿無法設定(RangeValuePattern)、控制項無法捲入視野 +(ScrollItemPattern)——這些只能退回脆弱的像素猜測。``control_patterns`` 在既有的無障礙後端 +ABC 之上補上這些物件層級動作。 + +每個函式都是對可注入的 ``accessibility.backends.get_backend()`` 接縫的薄分派(與無障礙模組 +其餘部分相同的接縫),因此無頭核心可在任何平台透過注入 fake backend 單元測試;真正的 +UI Automation 呼叫位於 Windows 後端(ExpandCollapse / SelectionItem / RangeValue / ScrollItem +模式)。未實作某模式的後端會拋出 ``AccessibilityNotAvailableError``。不匯入 ``PySide6``。 + +無頭 API +-------- + +.. code-block:: python + + from je_auto_control import (expand_control, collapse_control, + control_expand_state, select_control_item, + control_range, set_control_range, + scroll_control_into_view) + + expand_control(name="Documents", role="treeitem") # 展開樹節點 + select_control_item(name="Option B") # 選取清單/下拉項目 + set_control_range(75, name="Volume") # 設定滑桿 + print(control_range(name="Volume")) # {"value": 75.0, "minimum": 0, "maximum": 100} + scroll_control_into_view(name="Row 200") # 把某列帶上螢幕 + +全部以 ``name`` / ``role`` / ``app_name`` / ``automation_id`` 定位控制項(與既有 +``control_invoke`` / ``control_toggle`` 相同)。展開/選取/捲動/設定動作回傳 ``bool``; +``control_expand_state`` 回傳 ``expanded`` / ``collapsed`` / ``partial`` / ``leaf``(或 +``None``);``control_range`` 回傳 ``{value, minimum, maximum}``(或 ``None``)。 + +執行器指令 +---------- + +``AC_expand_control`` / ``AC_collapse_control`` / ``AC_control_expand_state`` / +``AC_select_control_item`` / ``AC_control_range`` / ``AC_set_control_range`` / +``AC_scroll_control_into_view``。皆以對應的 ``ac_*`` MCP 工具(動作類為破壞性、讀取類為唯讀) +及 Script Builder 指令(位於 **Native UI** 分類下)形式提供。 diff --git a/docs/source/Zh/zh_index.rst b/docs/source/Zh/zh_index.rst index 261eedf0..3c90f5b6 100644 --- a/docs/source/Zh/zh_index.rst +++ b/docs/source/Zh/zh_index.rst @@ -203,6 +203,7 @@ AutoControl 所有功能的完整使用指南。 doc/new_features/v178_features_doc doc/new_features/v179_features_doc doc/new_features/v180_features_doc + doc/new_features/v181_features_doc doc/ocr_backends/ocr_backends_doc doc/observability/observability_doc doc/operations_layer/operations_layer_doc diff --git a/je_auto_control/__init__.py b/je_auto_control/__init__.py index 5a741775..ebd319fc 100644 --- a/je_auto_control/__init__.py +++ b/je_auto_control/__init__.py @@ -48,6 +48,11 @@ find_accessibility_element, list_accessibility_elements, read_control_table, ) +# Extended UIA control patterns (Expand / Select / Range / Scroll) +from je_auto_control.utils.control_patterns import ( + collapse_control, control_expand_state, control_range, expand_control, + scroll_control_into_view, select_control_item, set_control_range, +) # VLM element locator (headless) from je_auto_control.utils.vision import ( VLMNotAvailableError, click_by_description, locate_by_description, @@ -1609,6 +1614,9 @@ def start_autocontrol_gui(*args, **kwargs): "find_accessibility_element", "list_accessibility_elements", "control_get_value", "control_set_value", "control_invoke", "control_toggle", "read_control_table", + "expand_control", "collapse_control", "control_expand_state", + "select_control_item", "control_range", "set_control_range", + "scroll_control_into_view", # VLM locator "VLMNotAvailableError", "locate_by_description", "click_by_description", "verify_description", diff --git a/je_auto_control/gui/script_builder/command_schema.py b/je_auto_control/gui/script_builder/command_schema.py index c46132e9..7d776cc7 100644 --- a/je_auto_control/gui/script_builder/command_schema.py +++ b/je_auto_control/gui/script_builder/command_schema.py @@ -1488,6 +1488,41 @@ def _add_native_control_specs(specs: List[CommandSpec]) -> None: fields=fields, description="Read a grid/table/list control as rows of cell strings.", )) + specs.append(CommandSpec( + "AC_expand_control", "Native UI", "Expand Control", + fields=fields, + description="Expand a tree node / combobox (ExpandCollapsePattern).", + )) + specs.append(CommandSpec( + "AC_collapse_control", "Native UI", "Collapse Control", + fields=fields, + description="Collapse a tree node / combobox (ExpandCollapsePattern).", + )) + specs.append(CommandSpec( + "AC_control_expand_state", "Native UI", "Control Expand State", + fields=fields, + description="Read expanded/collapsed/partial/leaf state of a control.", + )) + specs.append(CommandSpec( + "AC_select_control_item", "Native UI", "Select Control Item", + fields=fields, + description="Select a list / tree / tab item (SelectionItemPattern).", + )) + specs.append(CommandSpec( + "AC_control_range", "Native UI", "Get Control Range", + fields=fields, + description="Read a slider / progress range (RangeValuePattern).", + )) + specs.append(CommandSpec( + "AC_set_control_range", "Native UI", "Set Control Range", + fields=(FieldSpec("value", FieldType.FLOAT),) + fields, + description="Set a slider / progress / spinner value (RangeValuePattern).", + )) + specs.append(CommandSpec( + "AC_scroll_control_into_view", "Native UI", "Scroll Control Into View", + fields=fields, + description="Scroll a control into view (ScrollItemPattern).", + )) def _add_misc_specs(specs: List[CommandSpec]) -> None: diff --git a/je_auto_control/utils/accessibility/backends/base.py b/je_auto_control/utils/accessibility/backends/base.py index 2ca8ef7e..017666fb 100644 --- a/je_auto_control/utils/accessibility/backends/base.py +++ b/je_auto_control/utils/accessibility/backends/base.py @@ -1,5 +1,5 @@ """Abstract accessibility backend.""" -from typing import List, Optional +from typing import Any, Dict, List, Optional from je_auto_control.utils.accessibility.element import ( AccessibilityElement, AccessibilityNotAvailableError, @@ -56,6 +56,50 @@ def read_table(self, name: Optional[str] = None, role: Optional[str] = None, """Read a grid/table/list control as rows of cell strings.""" self._unsupported("read_table") + # --- extended control patterns (Expand / Selection / Range / Scroll) ---- + + def expand(self, name: Optional[str] = None, role: Optional[str] = None, + app_name: Optional[str] = None, + automation_id: Optional[str] = None) -> bool: + """Expand the matched control (ExpandCollapsePattern); True on success.""" + self._unsupported("expand") + + def collapse(self, name: Optional[str] = None, role: Optional[str] = None, + app_name: Optional[str] = None, + automation_id: Optional[str] = None) -> bool: + """Collapse the matched control (ExpandCollapsePattern); True on success.""" + self._unsupported("collapse") + + def expand_state(self, name: Optional[str] = None, role: Optional[str] = None, + app_name: Optional[str] = None, + automation_id: Optional[str] = None) -> Optional[str]: + """Return ``expanded`` / ``collapsed`` / ``partial`` / ``leaf``, or None.""" + self._unsupported("expand_state") + + def select_item(self, name: Optional[str] = None, role: Optional[str] = None, + app_name: Optional[str] = None, + automation_id: Optional[str] = None) -> bool: + """Select the matched item (SelectionItemPattern); True on success.""" + self._unsupported("select_item") + + def get_range(self, name: Optional[str] = None, role: Optional[str] = None, + app_name: Optional[str] = None, + automation_id: Optional[str] = None) -> Optional[Dict[str, Any]]: + """Return ``{value, minimum, maximum}`` (RangeValuePattern), or None.""" + self._unsupported("get_range") + + def set_range_value(self, value: float, name: Optional[str] = None, + role: Optional[str] = None, app_name: Optional[str] = None, + automation_id: Optional[str] = None) -> bool: + """Set a slider / progress value (RangeValuePattern); True on success.""" + self._unsupported("set_range_value") + + def scroll_into_view(self, name: Optional[str] = None, + role: Optional[str] = None, app_name: Optional[str] = None, + automation_id: Optional[str] = None) -> bool: + """Scroll the matched control into view (ScrollItemPattern); True on success.""" + self._unsupported("scroll_into_view") + def _unsupported(self, operation: str): """Raise a clear error for an action this backend can't perform.""" raise AccessibilityNotAvailableError( diff --git a/je_auto_control/utils/accessibility/backends/windows_backend.py b/je_auto_control/utils/accessibility/backends/windows_backend.py index be8c3c41..fd3f0657 100644 --- a/je_auto_control/utils/accessibility/backends/windows_backend.py +++ b/je_auto_control/utils/accessibility/backends/windows_backend.py @@ -8,7 +8,7 @@ Only ``is_control_element=True`` nodes are surfaced to avoid millions of decorative text children. """ -from typing import List, Optional +from typing import Any, Dict, List, Optional from je_auto_control.utils.accessibility.backends.base import ( AccessibilityBackend, @@ -25,6 +25,11 @@ _UIA_INVOKE_PATTERN_ID = 10000 _UIA_TOGGLE_PATTERN_ID = 10015 _UIA_GRID_PATTERN_ID = 10006 +_UIA_EXPANDCOLLAPSE_PATTERN_ID = 10005 +_UIA_SELECTIONITEM_PATTERN_ID = 10010 +_UIA_RANGEVALUE_PATTERN_ID = 10003 +_UIA_SCROLLITEM_PATTERN_ID = 10017 +_EXPAND_STATES = {0: "collapsed", 1: "expanded", 2: "partial", 3: "leaf"} def _is_available() -> bool: @@ -192,6 +197,72 @@ def read_table(self, name=None, role=None, app_name=None, return [] return [self._read_row(pattern, r, cols) for r in range(rows)] + def _invoke_pattern_method(self, name, role, app_name, automation_id, + pattern_id, interface_name, action): + """Find a control, query a pattern, run ``action(pattern)`` → bool.""" + raw = self._find_raw(name, role, app_name, automation_id) + pattern = self._pattern(raw, pattern_id, interface_name) if raw else None + if pattern is None: + return False + try: + action(pattern) + return True + except (OSError, AttributeError): + return False + + def expand(self, name=None, role=None, app_name=None, automation_id=None): + return self._invoke_pattern_method( + name, role, app_name, automation_id, _UIA_EXPANDCOLLAPSE_PATTERN_ID, + "IUIAutomationExpandCollapsePattern", lambda p: p.Expand()) + + def collapse(self, name=None, role=None, app_name=None, automation_id=None): + return self._invoke_pattern_method( + name, role, app_name, automation_id, _UIA_EXPANDCOLLAPSE_PATTERN_ID, + "IUIAutomationExpandCollapsePattern", lambda p: p.Collapse()) + + def expand_state(self, name=None, role=None, app_name=None, + automation_id=None) -> Optional[str]: + raw = self._find_raw(name, role, app_name, automation_id) + pattern = self._pattern(raw, _UIA_EXPANDCOLLAPSE_PATTERN_ID, + "IUIAutomationExpandCollapsePattern") if raw else None + if pattern is None: + return None + try: + return _EXPAND_STATES.get(int(pattern.CurrentExpandCollapseState)) + except (OSError, AttributeError, ValueError, TypeError): + return None + + def select_item(self, name=None, role=None, app_name=None, automation_id=None): + return self._invoke_pattern_method( + name, role, app_name, automation_id, _UIA_SELECTIONITEM_PATTERN_ID, + "IUIAutomationSelectionItemPattern", lambda p: p.Select()) + + def set_range_value(self, value, name=None, role=None, app_name=None, + automation_id=None): + return self._invoke_pattern_method( + name, role, app_name, automation_id, _UIA_RANGEVALUE_PATTERN_ID, + "IUIAutomationRangeValuePattern", lambda p: p.SetValue(float(value))) + + def scroll_into_view(self, name=None, role=None, app_name=None, + automation_id=None): + return self._invoke_pattern_method( + name, role, app_name, automation_id, _UIA_SCROLLITEM_PATTERN_ID, + "IUIAutomationScrollItemPattern", lambda p: p.ScrollIntoView()) + + def get_range(self, name=None, role=None, app_name=None, + automation_id=None) -> Optional[Dict[str, Any]]: + raw = self._find_raw(name, role, app_name, automation_id) + pattern = self._pattern(raw, _UIA_RANGEVALUE_PATTERN_ID, + "IUIAutomationRangeValuePattern") if raw else None + if pattern is None: + return None + try: + return {"value": float(pattern.CurrentValue), + "minimum": float(pattern.CurrentMinimum), + "maximum": float(pattern.CurrentMaximum)} + except (OSError, AttributeError, ValueError, TypeError): + return None + @staticmethod def _read_row(pattern, row: int, cols: int): """Read one grid row into a list of cell strings.""" diff --git a/je_auto_control/utils/control_patterns/__init__.py b/je_auto_control/utils/control_patterns/__init__.py new file mode 100644 index 00000000..548541dd --- /dev/null +++ b/je_auto_control/utils/control_patterns/__init__.py @@ -0,0 +1,11 @@ +"""Extended UI Automation control-pattern actions (Expand / Select / Range / Scroll).""" +from je_auto_control.utils.control_patterns.control_patterns import ( + collapse_control, control_expand_state, control_range, expand_control, + scroll_control_into_view, select_control_item, set_control_range, +) + +__all__ = [ + "expand_control", "collapse_control", "control_expand_state", + "select_control_item", "control_range", "set_control_range", + "scroll_control_into_view", +] diff --git a/je_auto_control/utils/control_patterns/control_patterns.py b/je_auto_control/utils/control_patterns/control_patterns.py new file mode 100644 index 00000000..83e9f2e3 --- /dev/null +++ b/je_auto_control/utils/control_patterns/control_patterns.py @@ -0,0 +1,77 @@ +"""Extended UI Automation control-pattern actions (Expand / Select / Range / Scroll). + +The accessibility backend ships only four control patterns — Value, Invoke, Toggle and a +read-only Grid dump. That leaves the controls automation hits most often undriveable by their +*native* pattern: a treeview node can't be expanded, a listbox / combobox item can't be +selected (SelectionItemPattern), a slider can't be set (RangeValuePattern), and a control can't +be scrolled into view (ScrollItemPattern) — today those fall back to fragile pixel guessing. +``control_patterns`` adds those object-level actions on top of the existing backend ABC. + +Each function is a thin dispatch onto the injectable ``accessibility.backends.get_backend()`` +seam (the same seam the rest of the accessibility module uses), so the headless core is +unit-testable on any platform by injecting a fake backend; the real UIA calls live in the +Windows backend. Imports no ``PySide6``. +""" +from typing import Any, Dict, Optional + + +def _backend(): + from je_auto_control.utils.accessibility.backends import get_backend + return get_backend() + + +def expand_control(name: Optional[str] = None, role: Optional[str] = None, + app_name: Optional[str] = None, + automation_id: Optional[str] = None) -> bool: + """Expand a tree node / combobox / expander (ExpandCollapsePattern).""" + return _backend().expand(name=name, role=role, app_name=app_name, + automation_id=automation_id) + + +def collapse_control(name: Optional[str] = None, role: Optional[str] = None, + app_name: Optional[str] = None, + automation_id: Optional[str] = None) -> bool: + """Collapse a tree node / combobox / expander (ExpandCollapsePattern).""" + return _backend().collapse(name=name, role=role, app_name=app_name, + automation_id=automation_id) + + +def control_expand_state(name: Optional[str] = None, role: Optional[str] = None, + app_name: Optional[str] = None, + automation_id: Optional[str] = None) -> Optional[str]: + """Return ``expanded`` / ``collapsed`` / ``partial`` / ``leaf`` for a control.""" + return _backend().expand_state(name=name, role=role, app_name=app_name, + automation_id=automation_id) + + +def select_control_item(name: Optional[str] = None, role: Optional[str] = None, + app_name: Optional[str] = None, + automation_id: Optional[str] = None) -> bool: + """Select a list / tree / tab item (SelectionItemPattern).""" + return _backend().select_item(name=name, role=role, app_name=app_name, + automation_id=automation_id) + + +def control_range(name: Optional[str] = None, role: Optional[str] = None, + app_name: Optional[str] = None, + automation_id: Optional[str] = None) -> Optional[Dict[str, Any]]: + """Return ``{value, minimum, maximum}`` of a slider / progress (RangeValuePattern).""" + return _backend().get_range(name=name, role=role, app_name=app_name, + automation_id=automation_id) + + +def set_control_range(value: float, name: Optional[str] = None, + role: Optional[str] = None, app_name: Optional[str] = None, + automation_id: Optional[str] = None) -> bool: + """Set a slider / progress / spinner value (RangeValuePattern).""" + return _backend().set_range_value(float(value), name=name, role=role, + app_name=app_name, + automation_id=automation_id) + + +def scroll_control_into_view(name: Optional[str] = None, role: Optional[str] = None, + app_name: Optional[str] = None, + automation_id: Optional[str] = None) -> bool: + """Scroll a control into view before acting on it (ScrollItemPattern).""" + return _backend().scroll_into_view(name=name, role=role, app_name=app_name, + automation_id=automation_id) diff --git a/je_auto_control/utils/executor/action_executor.py b/je_auto_control/utils/executor/action_executor.py index 3534e300..de3c3236 100644 --- a/je_auto_control/utils/executor/action_executor.py +++ b/je_auto_control/utils/executor/action_executor.py @@ -2352,6 +2352,70 @@ def _control_toggle(name: Optional[str] = None, role: Optional[str] = None, automation_id=automation_id) +def _expand_control(name: Optional[str] = None, role: Optional[str] = None, + app_name: Optional[str] = None, + automation_id: Optional[str] = None) -> bool: + """Adapter: expand a tree node / combobox (ExpandCollapsePattern).""" + from je_auto_control.utils.control_patterns import expand_control + return expand_control(name=name, role=role, app_name=app_name, + automation_id=automation_id) + + +def _collapse_control(name: Optional[str] = None, role: Optional[str] = None, + app_name: Optional[str] = None, + automation_id: Optional[str] = None) -> bool: + """Adapter: collapse a tree node / combobox (ExpandCollapsePattern).""" + from je_auto_control.utils.control_patterns import collapse_control + return collapse_control(name=name, role=role, app_name=app_name, + automation_id=automation_id) + + +def _control_expand_state(name: Optional[str] = None, role: Optional[str] = None, + app_name: Optional[str] = None, + automation_id: Optional[str] = None) -> Dict[str, Any]: + """Adapter: the expand/collapse state of a control.""" + from je_auto_control.utils.control_patterns import control_expand_state + return {"state": control_expand_state(name=name, role=role, app_name=app_name, + automation_id=automation_id)} + + +def _select_control_item(name: Optional[str] = None, role: Optional[str] = None, + app_name: Optional[str] = None, + automation_id: Optional[str] = None) -> bool: + """Adapter: select a list / tree / tab item (SelectionItemPattern).""" + from je_auto_control.utils.control_patterns import select_control_item + return select_control_item(name=name, role=role, app_name=app_name, + automation_id=automation_id) + + +def _control_range(name: Optional[str] = None, role: Optional[str] = None, + app_name: Optional[str] = None, + automation_id: Optional[str] = None) -> Dict[str, Any]: + """Adapter: read a slider / progress range (RangeValuePattern).""" + from je_auto_control.utils.control_patterns import control_range + info = control_range(name=name, role=role, app_name=app_name, + automation_id=automation_id) + return {"found": info is not None, "range": info} + + +def _set_control_range(value: Any, name: Optional[str] = None, + role: Optional[str] = None, app_name: Optional[str] = None, + automation_id: Optional[str] = None) -> bool: + """Adapter: set a slider / progress / spinner value (RangeValuePattern).""" + from je_auto_control.utils.control_patterns import set_control_range + return set_control_range(float(value), name=name, role=role, + app_name=app_name, automation_id=automation_id) + + +def _scroll_control_into_view(name: Optional[str] = None, role: Optional[str] = None, + app_name: Optional[str] = None, + automation_id: Optional[str] = None) -> bool: + """Adapter: scroll a control into view (ScrollItemPattern).""" + from je_auto_control.utils.control_patterns import scroll_control_into_view + return scroll_control_into_view(name=name, role=role, app_name=app_name, + automation_id=automation_id) + + def _read_table(name: Optional[str] = None, role: Optional[str] = None, app_name: Optional[str] = None, automation_id: Optional[str] = None) -> List[List[str]]: @@ -6021,6 +6085,13 @@ def __init__(self): "AC_control_set_value": _control_set_value, "AC_control_invoke": _control_invoke, "AC_control_toggle": _control_toggle, + "AC_expand_control": _expand_control, + "AC_collapse_control": _collapse_control, + "AC_control_expand_state": _control_expand_state, + "AC_select_control_item": _select_control_item, + "AC_control_range": _control_range, + "AC_set_control_range": _set_control_range, + "AC_scroll_control_into_view": _scroll_control_into_view, "AC_read_table": _read_table, "AC_watchdog_add": _watchdog_add, "AC_watchdog_start": _watchdog_start, diff --git a/je_auto_control/utils/mcp_server/tools/_factories.py b/je_auto_control/utils/mcp_server/tools/_factories.py index 0db7d0a3..68e57b09 100644 --- a/je_auto_control/utils/mcp_server/tools/_factories.py +++ b/je_auto_control/utils/mcp_server/tools/_factories.py @@ -1099,6 +1099,64 @@ def a11y_control_tools() -> List[MCPTool]: handler=h.read_table, annotations=READ_ONLY, ), + MCPTool( + name="ac_expand_control", + description=("Expand a tree node / combobox / expander natively " + "(ExpandCollapsePattern). Located by name/role/app_name/" + "automation_id. Returns True on success."), + input_schema=schema(dict(_M)), + handler=h.expand_control, + annotations=DESTRUCTIVE, + ), + MCPTool( + name="ac_collapse_control", + description=("Collapse a tree node / combobox / expander natively " + "(ExpandCollapsePattern). Returns True on success."), + input_schema=schema(dict(_M)), + handler=h.collapse_control, + annotations=DESTRUCTIVE, + ), + MCPTool( + name="ac_control_expand_state", + description=("Read a control's expand state via ExpandCollapsePattern: " + "{state: expanded|collapsed|partial|leaf|null}."), + input_schema=schema(dict(_M)), + handler=h.control_expand_state, + annotations=READ_ONLY, + ), + MCPTool( + name="ac_select_control_item", + description=("Select a list / tree / tab item natively " + "(SelectionItemPattern). Returns True on success."), + input_schema=schema(dict(_M)), + handler=h.select_control_item, + annotations=DESTRUCTIVE, + ), + MCPTool( + name="ac_control_range", + description=("Read a slider / progress / spinner range via " + "RangeValuePattern: {found, range:{value,minimum,maximum}}."), + input_schema=schema(dict(_M)), + handler=h.control_range, + annotations=READ_ONLY, + ), + MCPTool( + name="ac_set_control_range", + description=("Set a slider / progress / spinner 'value' natively " + "(RangeValuePattern). Returns True on success."), + input_schema=schema({"value": {"type": "number"}, **_M}, + required=["value"]), + handler=h.set_control_range, + annotations=DESTRUCTIVE, + ), + MCPTool( + name="ac_scroll_control_into_view", + description=("Scroll a control into view before acting " + "(ScrollItemPattern). Returns True on success."), + input_schema=schema(dict(_M)), + handler=h.scroll_control_into_view, + annotations=DESTRUCTIVE, + ), ] diff --git a/je_auto_control/utils/mcp_server/tools/_handlers.py b/je_auto_control/utils/mcp_server/tools/_handlers.py index 5d7a8051..6e109873 100644 --- a/je_auto_control/utils/mcp_server/tools/_handlers.py +++ b/je_auto_control/utils/mcp_server/tools/_handlers.py @@ -751,6 +751,45 @@ def read_table(name=None, role=None, app_name=None, automation_id=None): automation_id=automation_id) +def expand_control(name=None, role=None, app_name=None, automation_id=None): + from je_auto_control.utils.executor.action_executor import _expand_control + return _expand_control(name, role, app_name, automation_id) + + +def collapse_control(name=None, role=None, app_name=None, automation_id=None): + from je_auto_control.utils.executor.action_executor import _collapse_control + return _collapse_control(name, role, app_name, automation_id) + + +def control_expand_state(name=None, role=None, app_name=None, automation_id=None): + from je_auto_control.utils.executor.action_executor import ( + _control_expand_state) + return _control_expand_state(name, role, app_name, automation_id) + + +def select_control_item(name=None, role=None, app_name=None, automation_id=None): + from je_auto_control.utils.executor.action_executor import _select_control_item + return _select_control_item(name, role, app_name, automation_id) + + +def control_range(name=None, role=None, app_name=None, automation_id=None): + from je_auto_control.utils.executor.action_executor import _control_range + return _control_range(name, role, app_name, automation_id) + + +def set_control_range(value, name=None, role=None, app_name=None, + automation_id=None): + from je_auto_control.utils.executor.action_executor import _set_control_range + return _set_control_range(value, name, role, app_name, automation_id) + + +def scroll_control_into_view(name=None, role=None, app_name=None, + automation_id=None): + from je_auto_control.utils.executor.action_executor import ( + _scroll_control_into_view) + return _scroll_control_into_view(name, role, app_name, automation_id) + + def watchdog_add(title, action="close", case_sensitive=False, name=None): from je_auto_control.utils.watchdog import default_popup_watchdog default_popup_watchdog.add_window_rule( diff --git a/test/unit_test/headless/test_control_patterns_batch.py b/test/unit_test/headless/test_control_patterns_batch.py new file mode 100644 index 00000000..f6c922c5 --- /dev/null +++ b/test/unit_test/headless/test_control_patterns_batch.py @@ -0,0 +1,112 @@ +"""Headless tests for extended control patterns (fake backend via the seam).""" +import je_auto_control as ac +from je_auto_control.utils.accessibility.backends import base as backend_base +from je_auto_control.utils.control_patterns import ( + collapse_control, control_expand_state, control_range, expand_control, + scroll_control_into_view, select_control_item, set_control_range, +) + + +class _FakeBackend(backend_base.AccessibilityBackend): + name = "fake" + available = True + + def __init__(self): + self.calls = [] + + def expand(self, **kw): + self.calls.append(("expand", kw)) + return True + + def collapse(self, **kw): + self.calls.append(("collapse", kw)) + return True + + def expand_state(self, **kw): + return "collapsed" + + def select_item(self, **kw): + self.calls.append(("select", kw)) + return True + + def get_range(self, **kw): + return {"value": 30.0, "minimum": 0.0, "maximum": 100.0} + + def set_range_value(self, value, **kw): + self.calls.append(("set_range", value)) + return True + + def scroll_into_view(self, **kw): + return True + + +def _inject(monkeypatch, backend): + import je_auto_control.utils.accessibility.backends as backends + monkeypatch.setattr(backends, "_cached_backend", backend, raising=False) + + +def test_expand_and_collapse_dispatch(monkeypatch): + fake = _FakeBackend() + _inject(monkeypatch, fake) + assert expand_control(name="Node", role="treeitem") is True + assert collapse_control(automation_id="tree1") is True + assert ("expand", {"name": "Node", "role": "treeitem", "app_name": None, + "automation_id": None}) in fake.calls + + +def test_expand_state(monkeypatch): + _inject(monkeypatch, _FakeBackend()) + assert control_expand_state(name="Node") == "collapsed" + + +def test_select_item(monkeypatch): + fake = _FakeBackend() + _inject(monkeypatch, fake) + assert select_control_item(name="Option B") is True + assert fake.calls[0][0] == "select" + + +def test_range_get_and_set(monkeypatch): + fake = _FakeBackend() + _inject(monkeypatch, fake) + assert control_range(name="Volume") == {"value": 30.0, "minimum": 0.0, + "maximum": 100.0} + assert set_control_range(75, name="Volume") is True + assert ("set_range", 75.0) in fake.calls + + +def test_scroll_into_view(monkeypatch): + _inject(monkeypatch, _FakeBackend()) + assert scroll_control_into_view(name="Row 50") is True + + +def test_unsupported_backend_raises(monkeypatch): + from je_auto_control.utils.accessibility.element import ( + AccessibilityNotAvailableError) + _inject(monkeypatch, backend_base.AccessibilityBackend()) # all _unsupported + try: + expand_control(name="x") + raised = False + except AccessibilityNotAvailableError: + raised = True + assert raised is True + + +# --- wiring --------------------------------------------------------------- + +def test_wiring(): + known = set(ac.executor.known_commands()) + assert {"AC_expand_control", "AC_set_control_range"} <= known + from je_auto_control.utils.mcp_server.tools import build_default_tool_registry + names = {t.name for t in build_default_tool_registry()} + assert {"ac_expand_control", "ac_set_control_range"} <= names + from je_auto_control.gui.script_builder.command_schema import _build_specs + specs = {s.command for s in _build_specs()} + assert {"AC_expand_control", "AC_set_control_range"} <= specs + + +def test_facade_exports(): + for name in ("expand_control", "collapse_control", "control_expand_state", + "select_control_item", "control_range", "set_control_range", + "scroll_control_into_view"): + assert hasattr(ac, name) and name in ac.__all__ From eea4ae66cc9a45eaef006d774b052c78855d3145 Mon Sep 17 00:00:00 2001 From: JeffreyChen Date: Wed, 24 Jun 2026 08:31:19 +0800 Subject: [PATCH 2/2] Match fake backend signatures to the ABC (Pylint W0221) The test's _FakeBackend overrode pattern methods with **kw instead of the explicit name/role/app_name/automation_id signature, which Codacy/Pylint flags as arguments-differ. Use the same signatures as AccessibilityBackend. --- .../headless/test_control_patterns_batch.py | 28 ++++++++++++------- 1 file changed, 18 insertions(+), 10 deletions(-) diff --git a/test/unit_test/headless/test_control_patterns_batch.py b/test/unit_test/headless/test_control_patterns_batch.py index f6c922c5..495e98ad 100644 --- a/test/unit_test/headless/test_control_patterns_batch.py +++ b/test/unit_test/headless/test_control_patterns_batch.py @@ -14,29 +14,37 @@ class _FakeBackend(backend_base.AccessibilityBackend): def __init__(self): self.calls = [] - def expand(self, **kw): - self.calls.append(("expand", kw)) + def expand(self, name=None, role=None, app_name=None, automation_id=None): + self.calls.append(("expand", {"name": name, "role": role, + "app_name": app_name, + "automation_id": automation_id})) return True - def collapse(self, **kw): - self.calls.append(("collapse", kw)) + def collapse(self, name=None, role=None, app_name=None, automation_id=None): + self.calls.append(("collapse", {"name": name, "role": role, + "app_name": app_name, + "automation_id": automation_id})) return True - def expand_state(self, **kw): + def expand_state(self, name=None, role=None, app_name=None, + automation_id=None): return "collapsed" - def select_item(self, **kw): - self.calls.append(("select", kw)) + def select_item(self, name=None, role=None, app_name=None, + automation_id=None): + self.calls.append(("select", name)) return True - def get_range(self, **kw): + def get_range(self, name=None, role=None, app_name=None, automation_id=None): return {"value": 30.0, "minimum": 0.0, "maximum": 100.0} - def set_range_value(self, value, **kw): + def set_range_value(self, value, name=None, role=None, app_name=None, + automation_id=None): self.calls.append(("set_range", value)) return True - def scroll_into_view(self, **kw): + def scroll_into_view(self, name=None, role=None, app_name=None, + automation_id=None): return True