Add AC_read_table: read grid/table/list controls as rows

JE-Chen · JE-Chen · commit 8a213d439026 · 2026-06-19T04:05:00.000+08:00
Complete the native control driver with table reading — the agent
research's other top desktop gap (data scraping). Adds read_table to the
backend interface, a Windows UIAutomation Grid-pattern implementation,
read_control_table in the facade, the AC_read_table executor command, the
ac_read_table MCP tool, a Script Builder entry, fake-backend tests, and
the v7 docs / README sections.
diff --git a/README.md b/README.md
@@ -66,6 +66,7 @@ Object-level desktop automation: read and drive native controls through the OS a
 
 - **Read / set value** — `control_get_value` / `control_set_value` (`AC_control_get_value` / `AC_control_set_value`): read a textbox/combo value (no OCR) and set it in one call (no per-key typing).
 - **Invoke / toggle** — `control_invoke` / `control_toggle` (`AC_control_invoke` / `AC_control_toggle`): press a button or flip a checkbox via its control pattern.
+- **Read a table/grid** — `read_control_table` (`AC_read_table`): scrape a grid/list/table control into rows of cell strings — desktop data extraction without OCR.
 - Targets a control by `name` / `role` / `app_name` / `automation_id` (the stable Windows identifier), so it survives layout/localization changes.
 
 ## What's new (2026-06-19)
diff --git a/README/README_zh-CN.md b/README/README_zh-CN.md
@@ -65,6 +65,7 @@
 
 - **读取 / 设置值** — `control_get_value` / `control_set_value`(`AC_control_get_value` / `AC_control_set_value`):读 textbox/combo 值(不用 OCR),一次设置值(不必逐键输入)。
 - **调用 / 切换** — `control_invoke` / `control_toggle`(`AC_control_invoke` / `AC_control_toggle`):通过控件模式按按钮或切换复选框。
+- **读取表格/列表** — `read_control_table`(`AC_read_table`):把 grid/list/table 控件抓成逐行单元格字符串——不用 OCR 的桌面数据提取。
 - 以 `name` / `role` / `app_name` / `automation_id`(Windows 稳定标识符)定位,版面/本地化改变也不坏。
 
 ## 本次更新 (2026-06-19)
diff --git a/README/README_zh-TW.md b/README/README_zh-TW.md
@@ -65,6 +65,7 @@
 
 - **讀取 / 設定值** — `control_get_value` / `control_set_value`(`AC_control_get_value` / `AC_control_set_value`):讀 textbox/combo 值(不用 OCR),一次設定值(不必逐鍵輸入)。
 - **呼叫 / 切換** — `control_invoke` / `control_toggle`(`AC_control_invoke` / `AC_control_toggle`):透過控制模式按按鈕或切換核取方塊。
+- **讀取表格/清單** — `read_control_table`(`AC_read_table`):把 grid/list/table 控制項抓成逐列儲存格字串——不用 OCR 的桌面資料擷取。
 - 以 `name` / `role` / `app_name` / `automation_id`(Windows 穩定識別碼)定位,版面/在地化改變也不壞。
 
 ## 本次更新 (2026-06-19)
diff --git a/docs/source/Eng/doc/new_features/v7_features_doc.rst b/docs/source/Eng/doc/new_features/v7_features_doc.rst
@@ -56,6 +56,22 @@ Invoking and toggling
 Executor commands: ``AC_control_invoke``, ``AC_control_toggle``.
 
 
+Reading tables / grids
+====================
+
+::
+
+    from je_auto_control import read_control_table
+
+    rows = read_control_table(name="Results", app_name="myapp.exe")
+    # -> [["Sam", "30"], ["Lee", "25"], ...]
+
+``read_control_table`` reads a grid/table/list control into rows of cell
+strings via the Grid pattern — reliable desktop data scraping without OCR.
+
+Executor command: ``AC_read_table``.
+
+
 Targeting controls
 =================
 
diff --git a/docs/source/Zh/doc/new_features/v7_features_doc.rst b/docs/source/Zh/doc/new_features/v7_features_doc.rst
@@ -52,6 +52,22 @@ Builder),並提供 Windows UIAutomation 後端;無法執行該動作的後端會
 執行器指令:``AC_control_invoke``、``AC_control_toggle``。
 
 
+讀取表格 / 清單
+================
+
+::
+
+    from je_auto_control import read_control_table
+
+    rows = read_control_table(name="Results", app_name="myapp.exe")
+    # -> [["Sam", "30"], ["Lee", "25"], ...]
+
+``read_control_table`` 透過 Grid pattern 把 grid/table/list 控制項讀成
+逐列的儲存格字串——不用 OCR 的可靠桌面資料抓取。
+
+執行器指令:``AC_read_table``。
+
+
 定位控制項
 ==========
 
diff --git a/je_auto_control/__init__.py b/je_auto_control/__init__.py
@@ -46,6 +46,7 @@
     click_accessibility_element, control_get_value, control_invoke,
     control_set_value, control_toggle, dump_accessibility_tree,
     find_accessibility_element, list_accessibility_elements,
+    read_control_table,
 )
 # VLM element locator (headless)
 from je_auto_control.utils.vision import (
@@ -546,7 +547,7 @@ def start_autocontrol_gui(*args, **kwargs):
     "click_accessibility_element", "dump_accessibility_tree",
     "find_accessibility_element", "list_accessibility_elements",
     "control_get_value", "control_set_value", "control_invoke",
-    "control_toggle",
+    "control_toggle", "read_control_table",
     # VLM locator
     "VLMNotAvailableError", "locate_by_description", "click_by_description",
     "verify_description",
diff --git a/je_auto_control/gui/script_builder/command_schema.py b/je_auto_control/gui/script_builder/command_schema.py
@@ -596,6 +596,11 @@ def _add_native_control_specs(specs: List[CommandSpec]) -> None:
         fields=fields,
         description="Toggle a native control (e.g. a checkbox).",
     ))
+    specs.append(CommandSpec(
+        "AC_read_table", "Native UI", "Read Table / Grid",
+        fields=fields,
+        description="Read a grid/table/list control as rows of cell strings.",
+    ))
 
 
 def _add_misc_specs(specs: List[CommandSpec]) -> None:
diff --git a/je_auto_control/utils/accessibility/__init__.py b/je_auto_control/utils/accessibility/__init__.py
@@ -4,6 +4,7 @@
     click_accessibility_element, control_get_value, control_invoke,
     control_set_value, control_toggle, dump_accessibility_tree,
     find_accessibility_element, list_accessibility_elements,
+    read_control_table,
 )
 from je_auto_control.utils.accessibility.recorder import (
     AXRecorderEvent, AccessibilityRecorder,
@@ -20,5 +21,5 @@
     "dump_accessibility_tree", "find_accessibility_element",
     "list_accessibility_elements", "max_depth",
     "control_get_value", "control_set_value", "control_invoke",
-    "control_toggle",
+    "control_toggle", "read_control_table",
 ]
diff --git a/je_auto_control/utils/accessibility/accessibility_api.py b/je_auto_control/utils/accessibility/accessibility_api.py
@@ -126,11 +126,20 @@ def control_toggle(name: Optional[str] = None, role: Optional[str] = None,
         name=name, role=role, app_name=app_name, automation_id=automation_id)
 
 
+def read_control_table(name: Optional[str] = None, role: Optional[str] = None,
+                       app_name: Optional[str] = None,
+                       automation_id: Optional[str] = None,
+                       ) -> List[List[str]]:
+    """Read a grid/table/list control as rows of cell strings."""
+    return get_backend().read_table(
+        name=name, role=role, app_name=app_name, automation_id=automation_id)
+
+
 __all__ = [
     "AccessibilityElement", "AccessibilityNotAvailableError",
     "AXTreeNode",
     "click_accessibility_element", "dump_accessibility_tree",
     "find_accessibility_element", "list_accessibility_elements",
     "control_get_value", "control_set_value", "control_invoke",
-    "control_toggle",
+    "control_toggle", "read_control_table",
 ]
diff --git a/je_auto_control/utils/accessibility/backends/base.py b/je_auto_control/utils/accessibility/backends/base.py
@@ -49,6 +49,13 @@ def toggle(self, name: Optional[str] = None, role: Optional[str] = None,
         """Toggle the matched control (e.g. a checkbox)."""
         self._unsupported("toggle")
 
+    def read_table(self, name: Optional[str] = None, role: Optional[str] = None,
+                   app_name: Optional[str] = None,
+                   automation_id: Optional[str] = None,
+                   ) -> List[List[str]]:
+        """Read a grid/table/list control as rows of cell strings."""
+        self._unsupported("read_table")
+
     def _unsupported(self, operation: str):
         """Raise a clear error for an action this backend can't perform."""
         raise AccessibilityNotAvailableError(
diff --git a/je_auto_control/utils/accessibility/backends/windows_backend.py b/je_auto_control/utils/accessibility/backends/windows_backend.py
@@ -24,6 +24,7 @@
 _UIA_VALUE_PATTERN_ID = 10002
 _UIA_INVOKE_PATTERN_ID = 10000
 _UIA_TOGGLE_PATTERN_ID = 10015
+_UIA_GRID_PATTERN_ID = 10006
 
 
 def _is_available() -> bool:
@@ -177,6 +178,32 @@ def toggle(self, name=None, role=None, app_name=None,
         except (OSError, AttributeError):
             return False
 
+    def read_table(self, name=None, role=None, app_name=None,
+                   automation_id=None):
+        raw = self._find_raw(name, role, app_name, automation_id)
+        pattern = self._pattern(raw, _UIA_GRID_PATTERN_ID,
+                                "IUIAutomationGridPattern") if raw else None
+        if pattern is None:
+            return []
+        try:
+            rows = int(pattern.CurrentRowCount or 0)
+            cols = int(pattern.CurrentColumnCount or 0)
+        except (OSError, AttributeError):
+            return []
+        return [self._read_row(pattern, r, cols) for r in range(rows)]
+
+    @staticmethod
+    def _read_row(pattern, row: int, cols: int):
+        """Read one grid row into a list of cell strings."""
+        cells = []
+        for col in range(cols):
+            try:
+                cell = pattern.GetItem(row, col)
+                cells.append(str(cell.CurrentName or "") if cell else "")
+            except (OSError, AttributeError):
+                cells.append("")
+        return cells
+
 
 def _convert_uia(raw) -> Optional[AccessibilityElement]:
     try:
diff --git a/je_auto_control/utils/executor/action_executor.py b/je_auto_control/utils/executor/action_executor.py
@@ -2248,6 +2248,15 @@ def _control_toggle(name: Optional[str] = None, role: Optional[str] = None,
                           automation_id=automation_id)
 
 
+def _read_table(name: Optional[str] = None, role: Optional[str] = None,
+                app_name: Optional[str] = None,
+                automation_id: Optional[str] = None) -> List[List[str]]:
+    """Adapter: read a grid/table/list control as rows of cell strings."""
+    from je_auto_control.utils.accessibility import read_control_table
+    return read_control_table(name=name, role=role, app_name=app_name,
+                              automation_id=automation_id)
+
+
 class Executor:
     """
     Executor
@@ -2399,6 +2408,7 @@ def __init__(self):
             "AC_control_set_value": _control_set_value,
             "AC_control_invoke": _control_invoke,
             "AC_control_toggle": _control_toggle,
+            "AC_read_table": _read_table,
             "AC_a11y_record_start": _a11y_record_start,
             "AC_a11y_record_stop": _a11y_record_stop,
             "AC_a11y_record_events": _a11y_record_events,
diff --git a/je_auto_control/utils/mcp_server/tools/_factories.py b/je_auto_control/utils/mcp_server/tools/_factories.py
@@ -1089,6 +1089,16 @@ def a11y_control_tools() -> List[MCPTool]:
             handler=h.control_toggle,
             annotations=DESTRUCTIVE,
         ),
+        MCPTool(
+            name="ac_read_table",
+            description=("Read a grid/table/list control as rows of cell "
+                         "strings via the accessibility Grid pattern. Located "
+                         "by name/role/app_name/automation_id. Reliable "
+                         "desktop data scraping without OCR."),
+            input_schema=schema(dict(_M)),
+            handler=h.read_table,
+            annotations=READ_ONLY,
+        ),
     ]
 
 
diff --git a/je_auto_control/utils/mcp_server/tools/_handlers.py b/je_auto_control/utils/mcp_server/tools/_handlers.py
@@ -745,6 +745,12 @@ def control_toggle(name=None, role=None, app_name=None, automation_id=None):
               automation_id=automation_id)
 
 
+def read_table(name=None, role=None, app_name=None, automation_id=None):
+    from je_auto_control.utils.accessibility import read_control_table as _r
+    return _r(name=name, role=role, app_name=app_name,
+              automation_id=automation_id)
+
+
 def vlm_locate(description: str,
                screen_region: Optional[List[int]] = None,
                model: Optional[str] = None) -> Optional[List[int]]:
diff --git a/test/unit_test/headless/test_native_control.py b/test/unit_test/headless/test_native_control.py
@@ -50,6 +50,10 @@ def toggle(self, name=None, role=None, app_name=None, automation_id=None):
         self.toggled.append(name)
         return True
 
+    def read_table(self, name=None, role=None, app_name=None,
+                   automation_id=None):
+        return [["Sam", "30"], ["Lee", "25"]]
+
 
 @pytest.fixture()
 def fake(monkeypatch):
@@ -89,25 +93,35 @@ def test_executor_commands(fake):
     assert "Stay signed in" in fake.toggled
 
 
+def test_read_table(fake):
+    rows = ac.read_control_table(name="Grid")
+    assert rows == [["Sam", "30"], ["Lee", "25"]]
+    record = ac.execute_action([["AC_read_table", {"name": "Grid"}]])
+    assert any("Sam" in str(v) for v in record.values())
+
+
 def test_facade_and_executor_registered(fake):
     assert ac.control_get_value is api.control_get_value
     assert {"AC_control_get_value", "AC_control_set_value",
-            "AC_control_invoke", "AC_control_toggle"} <= ac.executor.known_commands()
+            "AC_control_invoke", "AC_control_toggle",
+            "AC_read_table"} <= ac.executor.known_commands()
 
 
 def test_mcp_tools_registered():
     from je_auto_control.utils.mcp_server.tools import build_default_tool_registry
     names = {t.name for t in build_default_tool_registry()}
     assert {"ac_control_get_value", "ac_control_set_value",
-            "ac_control_invoke", "ac_control_toggle"} <= names
+            "ac_control_invoke", "ac_control_toggle",
+            "ac_read_table"} <= names
 
 
 def test_unsupported_backend_raises_clearly():
     backend = NullAccessibilityBackend()
     for call in (lambda: backend.get_value(name="x"),
                  lambda: backend.set_value("v", name="x"),
                  lambda: backend.invoke(name="x"),
-                 lambda: backend.toggle(name="x")):
+                 lambda: backend.toggle(name="x"),
+                 lambda: backend.read_table(name="x")):
         with pytest.raises(AccessibilityNotAvailableError):
             call()
 
@@ -116,7 +130,7 @@ def test_builder_specs_present_and_wired():
     from je_auto_control.gui.script_builder.command_schema import _build_specs
     known = ac.executor.known_commands()
     cmds = {s.command for s in _build_specs()}
-    assert {"AC_control_get_value", "AC_control_set_value",
-            "AC_control_invoke", "AC_control_toggle"} <= cmds
-    assert {"AC_control_get_value", "AC_control_set_value",
-            "AC_control_invoke", "AC_control_toggle"} <= known
+    wanted = {"AC_control_get_value", "AC_control_set_value",
+              "AC_control_invoke", "AC_control_toggle", "AC_read_table"}
+    assert wanted <= cmds
+    assert wanted <= known

Original file line number	Diff line number	Diff line change
`@@ -4,6 +4,7 @@`
`4`	`4`	`click_accessibility_element, control_get_value, control_invoke,`
`5`	`5`	`control_set_value, control_toggle, dump_accessibility_tree,`
`6`	`6`	`find_accessibility_element, list_accessibility_elements,`
	`7`	`+ read_control_table,`
`7`	`8`	`)`
`8`	`9`	`from je_auto_control.utils.accessibility.recorder import (`
`9`	`10`	`AXRecorderEvent, AccessibilityRecorder,`
`@@ -20,5 +21,5 @@`
`20`	`21`	`"dump_accessibility_tree", "find_accessibility_element",`
`21`	`22`	`"list_accessibility_elements", "max_depth",`
`22`	`23`	`"control_get_value", "control_set_value", "control_invoke",`
`23`		`- "control_toggle",`
	`24`	`+ "control_toggle", "read_control_table",`
`24`	`25`	`]`