From cc3e2b68040352992d5c94c3628b3597db0249ee Mon Sep 17 00:00:00 2001 From: JeffreyChen Date: Tue, 23 Jun 2026 01:10:56 +0800 Subject: [PATCH] Add relative mouse movement (move by delta) --- README/WHATS_NEW_zh-CN.md | 6 +++ README/WHATS_NEW_zh-TW.md | 6 +++ WHATS_NEW.md | 6 +++ .../doc/new_features/v120_features_doc.rst | 35 ++++++++++++ docs/source/Eng/eng_index.rst | 1 + .../Zh/doc/new_features/v120_features_doc.rst | 29 ++++++++++ docs/source/Zh/zh_index.rst | 1 + je_auto_control/__init__.py | 6 +++ .../gui/script_builder/command_schema.py | 8 +++ .../utils/executor/action_executor.py | 7 +++ .../utils/mcp_server/tools/_factories.py | 17 +++++- .../utils/mcp_server/tools/_handlers.py | 5 ++ .../utils/mouse_relative/__init__.py | 6 +++ .../utils/mouse_relative/mouse_relative.py | 53 ++++++++++++++++++ .../headless/test_mouse_relative_batch.py | 54 +++++++++++++++++++ 15 files changed, 239 insertions(+), 1 deletion(-) create mode 100644 docs/source/Eng/doc/new_features/v120_features_doc.rst create mode 100644 docs/source/Zh/doc/new_features/v120_features_doc.rst create mode 100644 je_auto_control/utils/mouse_relative/__init__.py create mode 100644 je_auto_control/utils/mouse_relative/mouse_relative.py create mode 100644 test/unit_test/headless/test_mouse_relative_batch.py diff --git a/README/WHATS_NEW_zh-CN.md b/README/WHATS_NEW_zh-CN.md index b8e95cee..ed021acd 100644 --- a/README/WHATS_NEW_zh-CN.md +++ b/README/WHATS_NEW_zh-CN.md @@ -1,5 +1,11 @@ # 本次更新 — AutoControl +## 本次更新 (2026-06-23) — 相对鼠标移动 + +从当前位置将指针位移一个增量。完整参考:[`docs/source/Zh/doc/new_features/v120_features_doc.rst`](../docs/source/Zh/doc/new_features/v120_features_doc.rst)。 + +- **`move_mouse_relative` / `relative_target`**(`AC_move_mouse_relative`):鼠标 wrapper 只有绝对的 `set_mouse_position`——没有给相对指针 / 画布 / FPS 应用与渐进式拖曳用的 `moveRel(dx, dy)`。本功能读取实时位置并依增量移动;`relative_target` 为纯算术,getter/setter 可注入以供无头测试。纯标准库、确定。 + ## 本次更新 (2026-06-23) — 按住按键 / 自动重复 按住一个键一段时间,或以固定频率自动重复。完整参考:[`docs/source/Zh/doc/new_features/v119_features_doc.rst`](../docs/source/Zh/doc/new_features/v119_features_doc.rst)。 diff --git a/README/WHATS_NEW_zh-TW.md b/README/WHATS_NEW_zh-TW.md index ae61b4bd..c2c5b1e5 100644 --- a/README/WHATS_NEW_zh-TW.md +++ b/README/WHATS_NEW_zh-TW.md @@ -1,5 +1,11 @@ # 本次更新 — AutoControl +## 本次更新 (2026-06-23) — 相對滑鼠移動 + +從目前位置將指標位移一個增量。完整參考:[`docs/source/Zh/doc/new_features/v120_features_doc.rst`](../docs/source/Zh/doc/new_features/v120_features_doc.rst)。 + +- **`move_mouse_relative` / `relative_target`**(`AC_move_mouse_relative`):滑鼠 wrapper 只有絕對的 `set_mouse_position`——沒有給相對指標 / 畫布 / FPS 應用與漸進式拖曳用的 `moveRel(dx, dy)`。本功能讀取即時位置並依增量移動;`relative_target` 為純算術,getter/setter 可注入以供無頭測試。純標準函式庫、具決定性。 + ## 本次更新 (2026-06-23) — 按住按鍵 / 自動重複 按住一個鍵一段時間,或以固定頻率自動重複。完整參考:[`docs/source/Zh/doc/new_features/v119_features_doc.rst`](../docs/source/Zh/doc/new_features/v119_features_doc.rst)。 diff --git a/WHATS_NEW.md b/WHATS_NEW.md index e9005de0..b19d3407 100644 --- a/WHATS_NEW.md +++ b/WHATS_NEW.md @@ -1,5 +1,11 @@ # What's New — AutoControl +## What's new (2026-06-23) — Relative Mouse Movement + +Nudge the pointer by a delta from where it is. Full reference: [`docs/source/Eng/doc/new_features/v120_features_doc.rst`](docs/source/Eng/doc/new_features/v120_features_doc.rst). + +- **`move_mouse_relative` / `relative_target`** (`AC_move_mouse_relative`): the mouse wrapper only had absolute `set_mouse_position` — no `moveRel(dx, dy)` for relative-pointer / canvas / FPS apps and incremental drags. Reads the live position and moves by the delta; `relative_target` is the pure arithmetic, and the getter/setter are injectable for headless tests. Pure-stdlib, deterministic. + ## What's new (2026-06-23) — Hold Key / Auto-Repeat Hold a key for a duration, or auto-repeat it at a fixed rate. Full reference: [`docs/source/Eng/doc/new_features/v119_features_doc.rst`](docs/source/Eng/doc/new_features/v119_features_doc.rst). diff --git a/docs/source/Eng/doc/new_features/v120_features_doc.rst b/docs/source/Eng/doc/new_features/v120_features_doc.rst new file mode 100644 index 00000000..83100e65 --- /dev/null +++ b/docs/source/Eng/doc/new_features/v120_features_doc.rst @@ -0,0 +1,35 @@ +Relative Mouse Movement +======================= + +The mouse wrapper exposes only absolute ``set_mouse_position`` — there was no +"nudge the pointer by ``(dx, dy)``" (the pynput / PyAutoGUI ``moveRel`` staple), +which relative-pointer / canvas / FPS-style apps and incremental drags need. + +:func:`relative_target` is the pure arithmetic (current + delta) and is +unit-testable; :func:`move_mouse_relative` reads the live position and sets the +new one, with both the getter and setter injectable so it is tested without a +real pointer. Imports no ``PySide6``. + +Headless API +------------ + +.. code-block:: python + + from je_auto_control import move_mouse_relative, relative_target + + move_mouse_relative(-40, 12) # nudge left 40, down 12 from where it is + # {'from': [200, 200], 'to': [160, 212], 'delta': [-40, 12]} + + relative_target((100, 100), 10, -5) # (110, 95) — pure, no I/O + +``move_mouse_relative`` reads the current position (raising +``AutoControlMouseException`` if it cannot), adds the delta, and moves there. +``get_position`` / ``set_position`` default to the real mouse wrapper but are +injectable for headless tests. + +Executor commands +----------------- + +``AC_move_mouse_relative`` takes ``dx`` / ``dy`` and returns ``{from, to, +delta}``. It is exposed as the MCP tool ``ac_move_mouse_relative`` and as a +Script Builder command under **Mouse**. diff --git a/docs/source/Eng/eng_index.rst b/docs/source/Eng/eng_index.rst index 5945bab3..28c6c940 100644 --- a/docs/source/Eng/eng_index.rst +++ b/docs/source/Eng/eng_index.rst @@ -142,6 +142,7 @@ Comprehensive guides for all AutoControl features. doc/new_features/v117_features_doc doc/new_features/v118_features_doc doc/new_features/v119_features_doc + doc/new_features/v120_features_doc doc/ocr_backends/ocr_backends_doc doc/observability/observability_doc doc/operations_layer/operations_layer_doc diff --git a/docs/source/Zh/doc/new_features/v120_features_doc.rst b/docs/source/Zh/doc/new_features/v120_features_doc.rst new file mode 100644 index 00000000..95b698f4 --- /dev/null +++ b/docs/source/Zh/doc/new_features/v120_features_doc.rst @@ -0,0 +1,29 @@ +相對滑鼠移動 +============ + +滑鼠 wrapper 只提供絕對的 ``set_mouse_position``——先前沒有「將指標位移 ``(dx, dy)``」(pynput / PyAutoGUI 的 +``moveRel`` 慣用法),而相對指標 / 畫布 / FPS 類應用與漸進式拖曳都需要它。 + +:func:`relative_target` 為純算術(目前位置 + 位移),可單元測試;:func:`move_mouse_relative` 讀取即時位置並 +設定新位置,getter 與 setter 皆可注入,因此可在無真實指標下測試。不匯入 ``PySide6``。 + +無頭 API +-------- + +.. code-block:: python + + from je_auto_control import move_mouse_relative, relative_target + + move_mouse_relative(-40, 12) # 從目前位置往左 40、往下 12 + # {'from': [200, 200], 'to': [160, 212], 'delta': [-40, 12]} + + relative_target((100, 100), 10, -5) # (110, 95) — 純函式、無 I/O + +``move_mouse_relative`` 讀取目前位置(若無法讀取則拋出 ``AutoControlMouseException``)、加上位移、再移動過去。 +``get_position`` / ``set_position`` 預設為真實滑鼠 wrapper,但可注入以供無頭測試。 + +執行器命令 +---------- + +``AC_move_mouse_relative`` 接受 ``dx`` / ``dy`` 並回傳 ``{from, to, delta}``。它以 MCP 工具 +``ac_move_mouse_relative`` 以及 Script Builder 中 **Mouse** 分類下的命令提供。 diff --git a/docs/source/Zh/zh_index.rst b/docs/source/Zh/zh_index.rst index 7f5c98a5..5b507654 100644 --- a/docs/source/Zh/zh_index.rst +++ b/docs/source/Zh/zh_index.rst @@ -142,6 +142,7 @@ AutoControl 所有功能的完整使用指南。 doc/new_features/v117_features_doc doc/new_features/v118_features_doc doc/new_features/v119_features_doc + doc/new_features/v120_features_doc doc/ocr_backends/ocr_backends_doc doc/observability/observability_doc doc/operations_layer/operations_layer_doc diff --git a/je_auto_control/__init__.py b/je_auto_control/__init__.py index ae9dacb0..ee64490a 100644 --- a/je_auto_control/__init__.py +++ b/je_auto_control/__init__.py @@ -259,6 +259,10 @@ from je_auto_control.utils.field_entry import plan_field_set, set_field_text # Hold a key for a duration / auto-repeat at a fixed rate from je_auto_control.utils.key_hold import hold_key, plan_key_hold +# Relative mouse movement (move by a delta from the current position) +from je_auto_control.utils.mouse_relative import ( + move_mouse_relative, relative_target, +) # CI workflow annotations (GitHub Actions) from je_auto_control.utils.ci_annotations import ( emit_annotations, format_annotation, @@ -1038,6 +1042,8 @@ def start_autocontrol_gui(*args, **kwargs): "set_field_text", "plan_key_hold", "hold_key", + "move_mouse_relative", + "relative_target", "emit_annotations", "format_annotation", "ClipboardHistory", "default_clipboard_history", "analyze_heal_log", "heal_stats", "scan_secrets", diff --git a/je_auto_control/gui/script_builder/command_schema.py b/je_auto_control/gui/script_builder/command_schema.py index 1c11146e..93c18018 100644 --- a/je_auto_control/gui/script_builder/command_schema.py +++ b/je_auto_control/gui/script_builder/command_schema.py @@ -758,6 +758,14 @@ def _add_misc_specs(specs: List[CommandSpec]) -> None: ), description="Press, drag through a polyline of waypoints, release.", )) + specs.append(CommandSpec( + "AC_move_mouse_relative", "Mouse", "Move Relative", + fields=( + FieldSpec("dx", FieldType.INT, placeholder="-40"), + FieldSpec("dy", FieldType.INT, placeholder="12"), + ), + description="Move the pointer by (dx, dy) from its current position.", + )) specs.append(CommandSpec( "AC_list_plugins", "Tools", "List Plugin Commands", fields=(FieldSpec("group", FieldType.STRING, optional=True, diff --git a/je_auto_control/utils/executor/action_executor.py b/je_auto_control/utils/executor/action_executor.py index aa21e73b..3711be1c 100644 --- a/je_auto_control/utils/executor/action_executor.py +++ b/je_auto_control/utils/executor/action_executor.py @@ -3131,6 +3131,12 @@ def _hold_key(key: str, duration_s: Any = 1.0, return hold_key(key, float(duration_s), rate_hz=rate) +def _move_mouse_relative(dx: Any, dy: Any) -> Dict[str, Any]: + """Adapter: move the pointer by a delta from its current position.""" + from je_auto_control.utils.mouse_relative import move_mouse_relative + return move_mouse_relative(int(dx), int(dy)) + + def _cas_put(name: str, key: str, value: Any, expected_version: Any = None) -> Dict[str, Any]: """Adapter: optimistic put into a named versioned store.""" @@ -4830,6 +4836,7 @@ def __init__(self): "AC_drag_path": _drag_path, "AC_set_field_text": _set_field_text, "AC_hold_key": _hold_key, + "AC_move_mouse_relative": _move_mouse_relative, "AC_detect_drift": _detect_drift, "AC_categorical_drift": _categorical_drift, "AC_diff_rows": _diff_rows, diff --git a/je_auto_control/utils/mcp_server/tools/_factories.py b/je_auto_control/utils/mcp_server/tools/_factories.py index b62f009a..9207f965 100644 --- a/je_auto_control/utils/mcp_server/tools/_factories.py +++ b/je_auto_control/utils/mcp_server/tools/_factories.py @@ -2578,6 +2578,21 @@ def tween_drag_tools() -> List[MCPTool]: ] +def mouse_relative_tools() -> List[MCPTool]: + return [ + MCPTool( + name="ac_move_mouse_relative", + description=("Move the pointer by ('dx', 'dy') relative to its " + "current position. Returns {from, to, delta}."), + input_schema=schema( + {"dx": {"type": "integer"}, "dy": {"type": "integer"}}, + required=["dx", "dy"]), + handler=h.move_mouse_relative, + annotations=SIDE_EFFECT_ONLY, + ), + ] + + def key_hold_tools() -> List[MCPTool]: return [ MCPTool( @@ -5896,7 +5911,7 @@ def media_assert_tools() -> List[MCPTool]: input_macro_tools, resilience_tools, ci_annotation_tools, clipboard_history_tools, audit_analysis_tools, process_doc_tools, tween_drag_tools, mouse_path_tools, field_entry_tools, - key_hold_tools, plugin_sdk_tools, governance_tools, + key_hold_tools, mouse_relative_tools, plugin_sdk_tools, governance_tools, credential_lease_tools, egress_tools, approval_testing_tools, trajectory_eval_tools, compliance_tools, agent_trace_tools, video_report_tools, fuzzy_tools, artifact_store_tools, image_dedup_tools, diff --git a/je_auto_control/utils/mcp_server/tools/_handlers.py b/je_auto_control/utils/mcp_server/tools/_handlers.py index 1d25ee3b..228a3a7d 100644 --- a/je_auto_control/utils/mcp_server/tools/_handlers.py +++ b/je_auto_control/utils/mcp_server/tools/_handlers.py @@ -2048,6 +2048,11 @@ def hold_key(key, duration_s=1.0, rate_hz=None): return _hold_key(key, duration_s, rate_hz) +def move_mouse_relative(dx, dy): + from je_auto_control.utils.executor.action_executor import _move_mouse_relative + return _move_mouse_relative(dx, dy) + + def detect_drift(reference, current, threshold=0.25, bins=10): from je_auto_control.utils.executor.action_executor import _detect_drift return _detect_drift(reference, current, threshold, bins) diff --git a/je_auto_control/utils/mouse_relative/__init__.py b/je_auto_control/utils/mouse_relative/__init__.py new file mode 100644 index 00000000..ffdd7a7c --- /dev/null +++ b/je_auto_control/utils/mouse_relative/__init__.py @@ -0,0 +1,6 @@ +"""Relative mouse movement — move by a delta from the current position.""" +from je_auto_control.utils.mouse_relative.mouse_relative import ( + move_mouse_relative, relative_target, +) + +__all__ = ["move_mouse_relative", "relative_target"] diff --git a/je_auto_control/utils/mouse_relative/mouse_relative.py b/je_auto_control/utils/mouse_relative/mouse_relative.py new file mode 100644 index 00000000..088f36df --- /dev/null +++ b/je_auto_control/utils/mouse_relative/mouse_relative.py @@ -0,0 +1,53 @@ +"""Relative mouse movement — move by a delta from the current position. + +The mouse wrapper exposes only absolute ``set_mouse_position``; there is no +"nudge the pointer by (dx, dy)" (pynput / PyAutoGUI ``moveRel`` staple), which is +what relative-pointer / canvas / FPS-style apps and incremental drags need. + +:func:`relative_target` is the pure arithmetic (current + delta) and is +unit-testable; :func:`move_mouse_relative` reads the live position and sets the +new one, with both the getter and setter injectable so it is tested without a +real pointer. Imports no ``PySide6``. +""" +from typing import Any, Callable, Dict, Optional, Tuple + +from je_auto_control.utils.exception.exceptions import AutoControlMouseException + +PositionGetter = Callable[[], Optional[Tuple[int, int]]] +PositionSetter = Callable[[int, int], Any] + + +def relative_target(current: Tuple[int, int], dx: int, dy: int) -> Tuple[int, int]: + """Return ``current`` offset by ``(dx, dy)`` as an integer ``(x, y)``.""" + return int(current[0]) + int(dx), int(current[1]) + int(dy) + + +def move_mouse_relative(dx: int, dy: int, *, + get_position: Optional[PositionGetter] = None, + set_position: Optional[PositionSetter] = None, + ) -> Dict[str, Any]: + """Move the pointer by ``(dx, dy)`` relative to where it is now. + + ``get_position`` / ``set_position`` default to the real mouse wrapper but are + injectable for headless tests. Raises :class:`AutoControlMouseException` if + the current position cannot be read. + """ + getter = get_position or _default_get_position + setter = set_position or _default_set_position + current = getter() + if current is None: + raise AutoControlMouseException("could not read the current mouse position") + target = relative_target((int(current[0]), int(current[1])), dx, dy) + setter(target[0], target[1]) + return {"from": [int(current[0]), int(current[1])], + "to": [target[0], target[1]], "delta": [int(dx), int(dy)]} + + +def _default_get_position() -> Optional[Tuple[int, int]]: + from je_auto_control.wrapper.auto_control_mouse import get_mouse_position + return get_mouse_position() + + +def _default_set_position(x: int, y: int) -> Any: + from je_auto_control.wrapper.auto_control_mouse import set_mouse_position + return set_mouse_position(x, y) diff --git a/test/unit_test/headless/test_mouse_relative_batch.py b/test/unit_test/headless/test_mouse_relative_batch.py new file mode 100644 index 00000000..576d9055 --- /dev/null +++ b/test/unit_test/headless/test_mouse_relative_batch.py @@ -0,0 +1,54 @@ +"""Headless tests for relative mouse movement. No Qt.""" +import je_auto_control as ac +from je_auto_control.utils.exception.exceptions import AutoControlMouseException +from je_auto_control.utils.mouse_relative import ( + move_mouse_relative, relative_target, +) + + +def test_relative_target_arithmetic(): + assert relative_target((100, 100), -40, 12) == (60, 112) + assert relative_target((0, 0), 0, 0) == (0, 0) + + +def test_move_uses_current_position_plus_delta(): + moves = [] + result = move_mouse_relative( + 10, -5, get_position=lambda: (200, 200), + set_position=lambda x, y: moves.append((x, y))) + assert moves == [(210, 195)] + assert result == {"from": [200, 200], "to": [210, 195], "delta": [10, -5]} + + +def test_raises_when_position_unreadable(): + try: + move_mouse_relative(1, 1, get_position=lambda: None, + set_position=lambda x, y: None) + except AutoControlMouseException: + pass + else: # pragma: no cover + raise AssertionError("expected AutoControlMouseException") + + +# --- wiring --------------------------------------------------------------- + +def test_executor_adapter_planning(): + # the default backend get/set is device-bound; exercise the pure arithmetic + # the adapter relies on instead. + assert relative_target((5, 5), 3, 4) == (8, 9) + + +def test_wiring(): + known = ac.executor.known_commands() + assert "AC_move_mouse_relative" in set(known) + from je_auto_control.utils.mcp_server.tools import build_default_tool_registry + names = {t.name for t in build_default_tool_registry()} + assert "ac_move_mouse_relative" in names + from je_auto_control.gui.script_builder.command_schema import _build_specs + specs = {s.command for s in _build_specs()} + assert "AC_move_mouse_relative" in specs + + +def test_facade_exports(): + for attr in ("move_mouse_relative", "relative_target"): + assert hasattr(ac, attr) and attr in ac.__all__