From 2ed4f865522cfe4ba646e495b52c0107e546a92f Mon Sep 17 00:00:00 2001 From: JeffreyChen Date: Tue, 23 Jun 2026 01:54:06 +0800 Subject: [PATCH] Add held-modifiers context across an action group --- README/WHATS_NEW_zh-CN.md | 6 ++ README/WHATS_NEW_zh-TW.md | 6 ++ WHATS_NEW.md | 6 ++ .../doc/new_features/v123_features_doc.rst | 42 ++++++++++++ docs/source/Eng/eng_index.rst | 1 + .../Zh/doc/new_features/v123_features_doc.rst | 37 +++++++++++ docs/source/Zh/zh_index.rst | 1 + je_auto_control/__init__.py | 6 ++ .../gui/script_builder/command_schema.py | 9 +++ .../utils/executor/action_executor.py | 15 +++++ .../utils/mcp_server/tools/_factories.py | 22 ++++++- .../utils/mcp_server/tools/_handlers.py | 5 ++ .../utils/modifier_state/__init__.py | 6 ++ .../utils/modifier_state/modifier_state.py | 57 ++++++++++++++++ .../headless/test_modifier_state_batch.py | 65 +++++++++++++++++++ 15 files changed, 282 insertions(+), 2 deletions(-) create mode 100644 docs/source/Eng/doc/new_features/v123_features_doc.rst create mode 100644 docs/source/Zh/doc/new_features/v123_features_doc.rst create mode 100644 je_auto_control/utils/modifier_state/__init__.py create mode 100644 je_auto_control/utils/modifier_state/modifier_state.py create mode 100644 test/unit_test/headless/test_modifier_state_batch.py diff --git a/README/WHATS_NEW_zh-CN.md b/README/WHATS_NEW_zh-CN.md index df3b421b..e86b3680 100644 --- a/README/WHATS_NEW_zh-CN.md +++ b/README/WHATS_NEW_zh-CN.md @@ -1,5 +1,11 @@ # 本次更新 — AutoControl +## 本次更新 (2026-06-23) — 在动作组中持续按住修饰键 + +在多个动作之间持续按住 ctrl/shift,即使出错也会放开。完整参考:[`docs/source/Zh/doc/new_features/v123_features_doc.rst`](../docs/source/Zh/doc/new_features/v123_features_doc.rst)。 + +- **`hold_modifiers` / `plan_with_modifiers`**(`AC_with_modifiers`):`hotkey` 会立即放开按键——先前无法在多个独立动作之间持续按住修饰键(shift 连点范围选取、ctrl 连点多选)并保证放开。`hold_modifiers` 是 context manager,进入时按下、离开时(在 `finally`)以反向放开,因此不会泄漏;`plan_with_modifiers` 为纯计划。可注入 sink、确定。 + ## 本次更新 (2026-06-23) — Unicode 文本输入(emoji / CJK) 输入 `write` 无法处理的任何 Unicode(emoji / CJK / 重音)。完整参考:[`docs/source/Zh/doc/new_features/v122_features_doc.rst`](../docs/source/Zh/doc/new_features/v122_features_doc.rst)。 diff --git a/README/WHATS_NEW_zh-TW.md b/README/WHATS_NEW_zh-TW.md index 612bf539..3408beb7 100644 --- a/README/WHATS_NEW_zh-TW.md +++ b/README/WHATS_NEW_zh-TW.md @@ -1,5 +1,11 @@ # 本次更新 — AutoControl +## 本次更新 (2026-06-23) — 在動作群組中持續按住修飾鍵 + +在多個動作之間持續按住 ctrl/shift,即使出錯也會放開。完整參考:[`docs/source/Zh/doc/new_features/v123_features_doc.rst`](../docs/source/Zh/doc/new_features/v123_features_doc.rst)。 + +- **`hold_modifiers` / `plan_with_modifiers`**(`AC_with_modifiers`):`hotkey` 會立即放開按鍵——先前無法在多個獨立動作之間持續按住修飾鍵(shift 連點範圍選取、ctrl 連點多選)並保證放開。`hold_modifiers` 是 context manager,進入時按下、離開時(在 `finally`)以反向放開,因此不會外洩;`plan_with_modifiers` 為純計畫。可注入 sink、具決定性。 + ## 本次更新 (2026-06-23) — Unicode 文字輸入(emoji / CJK) 輸入 `write` 無法處理的任何 Unicode(emoji / CJK / 重音)。完整參考:[`docs/source/Zh/doc/new_features/v122_features_doc.rst`](../docs/source/Zh/doc/new_features/v122_features_doc.rst)。 diff --git a/WHATS_NEW.md b/WHATS_NEW.md index c4d1bc82..7f6429b6 100644 --- a/WHATS_NEW.md +++ b/WHATS_NEW.md @@ -1,5 +1,11 @@ # What's New — AutoControl +## What's new (2026-06-23) — Held Modifiers Across an Action Group + +Hold ctrl/shift down across several actions, released even on error. Full reference: [`docs/source/Eng/doc/new_features/v123_features_doc.rst`](docs/source/Eng/doc/new_features/v123_features_doc.rst). + +- **`hold_modifiers` / `plan_with_modifiers`** (`AC_with_modifiers`): `hotkey` releases its keys immediately — there was no way to hold a modifier down across several independent actions (shift-click range select, ctrl-click multi-select) with a guaranteed release. `hold_modifiers` is a context manager that presses on enter and releases in reverse on exit (in a `finally`, so nothing leaks); `plan_with_modifiers` is the pure plan. Injectable sink, deterministic. + ## What's new (2026-06-23) — Unicode Text Entry (Emoji / CJK) Type any Unicode (emoji / CJK / accented) that `write` can't. Full reference: [`docs/source/Eng/doc/new_features/v122_features_doc.rst`](docs/source/Eng/doc/new_features/v122_features_doc.rst). diff --git a/docs/source/Eng/doc/new_features/v123_features_doc.rst b/docs/source/Eng/doc/new_features/v123_features_doc.rst new file mode 100644 index 00000000..3fd7232a --- /dev/null +++ b/docs/source/Eng/doc/new_features/v123_features_doc.rst @@ -0,0 +1,42 @@ +Held Modifiers Across an Action Group +===================================== + +``hotkey`` presses a set of keys and releases them immediately — fine for a +one-shot chord, but there was no way to hold ``ctrl`` (or ``shift``) *down across +several independent actions* (range-select with shift-clicks, ctrl-clicks to +multi-select) and be sure the modifiers are released even if one of those actions +raises. + +:func:`plan_with_modifiers` wraps an op-step list with press / release steps and +is pure / unit-testable; :func:`hold_modifiers` is a context manager that presses +on enter and releases (in reverse) on exit — including on exception — +dispatching through an injectable ``sink``. Imports no ``PySide6``. + +Headless API +------------ + +.. code-block:: python + + from je_auto_control import hold_modifiers, plan_with_modifiers + from je_auto_control import click_mouse + + # shift-held range select: every click happens with shift down + with hold_modifiers(["shift"]): + click_mouse("mouse_left", 100, 100) + click_mouse("mouse_left", 100, 300) + # shift is released here — even if a click raised + + plan_with_modifiers([{"op": "click"}], ["ctrl", "shift"]) + # press ctrl, press shift, click, release shift, release ctrl + +Modifiers are pressed in order on entry and released in *reverse* order on exit, +in a ``finally`` block, so a stuck modifier can never leak. ``plan_with_modifiers`` +is the pure plan for any op-step list. + +Executor commands +----------------- + +``AC_with_modifiers`` runs a nested JSON action list while ``modifiers`` (e.g. +``["ctrl"]`` or ``"ctrl+shift"``) are held, releasing them even if an action +fails. It is exposed as the MCP tool ``ac_with_modifiers`` and as a Script +Builder command under **Keyboard**. diff --git a/docs/source/Eng/eng_index.rst b/docs/source/Eng/eng_index.rst index d52d3d31..3638c676 100644 --- a/docs/source/Eng/eng_index.rst +++ b/docs/source/Eng/eng_index.rst @@ -145,6 +145,7 @@ Comprehensive guides for all AutoControl features. doc/new_features/v120_features_doc doc/new_features/v121_features_doc doc/new_features/v122_features_doc + doc/new_features/v123_features_doc doc/ocr_backends/ocr_backends_doc doc/observability/observability_doc doc/operations_layer/operations_layer_doc diff --git a/docs/source/Zh/doc/new_features/v123_features_doc.rst b/docs/source/Zh/doc/new_features/v123_features_doc.rst new file mode 100644 index 00000000..0bb21775 --- /dev/null +++ b/docs/source/Zh/doc/new_features/v123_features_doc.rst @@ -0,0 +1,37 @@ +在動作群組中持續按住修飾鍵 +========================== + +``hotkey`` 按下一組鍵後立即放開——適合一次性的組合鍵,但先前無法在*多個獨立動作之間持續按住* ``ctrl`` +(或 ``shift``)(以 shift 連點做範圍選取、以 ctrl 連點做多選),也無法確保即使其中某個動作拋例外時修飾鍵仍會 +被放開。 + +:func:`plan_with_modifiers` 以 press / release 步驟包覆操作步驟清單,為純函式、可單元測試;:func:`hold_modifiers` +是一個 context manager,進入時按下、離開時(含例外情況)以反向順序放開,並透過可注入的 ``sink`` 派發。 +不匯入 ``PySide6``。 + +無頭 API +-------- + +.. code-block:: python + + from je_auto_control import hold_modifiers, plan_with_modifiers + from je_auto_control import click_mouse + + # 按住 shift 做範圍選取:每次點擊都在 shift 按下狀態進行 + with hold_modifiers(["shift"]): + click_mouse("mouse_left", 100, 100) + click_mouse("mouse_left", 100, 300) + # shift 在此放開——即使某次點擊拋例外也是 + + plan_with_modifiers([{"op": "click"}], ["ctrl", "shift"]) + # 按 ctrl、按 shift、click、放 shift、放 ctrl + +修飾鍵進入時依序按下、離開時在 ``finally`` 區塊以*反向*順序放開,因此卡住的修飾鍵絕不會外洩。 +``plan_with_modifiers`` 是任意操作步驟清單的純計畫。 + +執行器命令 +---------- + +``AC_with_modifiers`` 在按住 ``modifiers``(如 ``["ctrl"]`` 或 ``"ctrl+shift"``)時執行巢狀 JSON 動作清單, +即使某動作失敗也會放開修飾鍵。它以 MCP 工具 ``ac_with_modifiers`` 以及 Script Builder 中 **Keyboard** 分類下 +的命令提供。 diff --git a/docs/source/Zh/zh_index.rst b/docs/source/Zh/zh_index.rst index 159dfe01..03877ac1 100644 --- a/docs/source/Zh/zh_index.rst +++ b/docs/source/Zh/zh_index.rst @@ -145,6 +145,7 @@ AutoControl 所有功能的完整使用指南。 doc/new_features/v120_features_doc doc/new_features/v121_features_doc doc/new_features/v122_features_doc + doc/new_features/v123_features_doc doc/ocr_backends/ocr_backends_doc doc/observability/observability_doc doc/operations_layer/operations_layer_doc diff --git a/je_auto_control/__init__.py b/je_auto_control/__init__.py index 350cf419..f6af54db 100644 --- a/je_auto_control/__init__.py +++ b/je_auto_control/__init__.py @@ -267,6 +267,10 @@ from je_auto_control.utils.text_unicode import ( plan_paste, type_unicode, unicode_code_units, ) +# Hold modifier keys across a group of actions (release-on-error) +from je_auto_control.utils.modifier_state import ( + hold_modifiers, plan_with_modifiers, +) # CI workflow annotations (GitHub Actions) from je_auto_control.utils.ci_annotations import ( emit_annotations, format_annotation, @@ -1052,6 +1056,8 @@ def start_autocontrol_gui(*args, **kwargs): "type_unicode", "plan_paste", "unicode_code_units", + "hold_modifiers", + "plan_with_modifiers", "emit_annotations", "format_annotation", "ClipboardHistory", "default_clipboard_history", "analyze_heal_log", "heal_stats", "scan_secrets", diff --git a/je_auto_control/gui/script_builder/command_schema.py b/je_auto_control/gui/script_builder/command_schema.py index f33a256c..71505ef1 100644 --- a/je_auto_control/gui/script_builder/command_schema.py +++ b/je_auto_control/gui/script_builder/command_schema.py @@ -194,6 +194,15 @@ def _add_keyboard_specs(specs: List[CommandSpec]) -> None: ), description="Enter any Unicode text via clipboard paste (write can't).", )) + specs.append(CommandSpec( + "AC_with_modifiers", "Keyboard", "With Modifiers Held", + fields=( + FieldSpec("modifiers", FieldType.STRING, placeholder="ctrl+shift"), + FieldSpec("actions", FieldType.STRING, + placeholder='[["AC_click_mouse", {...}], ...]'), + ), + description="Run nested actions while modifiers are held (release-safe).", + )) specs.append(CommandSpec( "AC_hotkey", "Keyboard", "Hotkey", fields=( diff --git a/je_auto_control/utils/executor/action_executor.py b/je_auto_control/utils/executor/action_executor.py index 3a7cfea9..e533a7ba 100644 --- a/je_auto_control/utils/executor/action_executor.py +++ b/je_auto_control/utils/executor/action_executor.py @@ -3161,6 +3161,20 @@ def _type_unicode(text: str, modifier: str = "ctrl") -> Dict[str, Any]: return type_unicode(text, modifier=modifier) +def _with_modifiers(modifiers: Any, actions: Any) -> Dict[str, Any]: + """Adapter: run nested actions while modifier keys are held down.""" + import json + from je_auto_control.utils.modifier_state import hold_modifiers + if isinstance(modifiers, str): + modifiers = (json.loads(modifiers) if modifiers.strip().startswith("[") + else [part.strip() for part in modifiers.split("+")]) + if isinstance(actions, str): + actions = json.loads(actions) + with hold_modifiers(list(modifiers)): + record = executor.execute_action(list(actions), raise_on_error=True) + return {"modifiers": list(modifiers), "record": record} + + def _cas_put(name: str, key: str, value: Any, expected_version: Any = None) -> Dict[str, Any]: """Adapter: optimistic put into a named versioned store.""" @@ -4862,6 +4876,7 @@ def __init__(self): "AC_hold_key": _hold_key, "AC_move_mouse_relative": _move_mouse_relative, "AC_type_unicode": _type_unicode, + "AC_with_modifiers": _with_modifiers, "AC_detect_drift": _detect_drift, "AC_categorical_drift": _categorical_drift, "AC_diff_rows": _diff_rows, diff --git a/je_auto_control/utils/mcp_server/tools/_factories.py b/je_auto_control/utils/mcp_server/tools/_factories.py index 02fb398c..bdaf5517 100644 --- a/je_auto_control/utils/mcp_server/tools/_factories.py +++ b/je_auto_control/utils/mcp_server/tools/_factories.py @@ -2596,6 +2596,24 @@ def tween_drag_tools() -> List[MCPTool]: ] +def modifier_state_tools() -> List[MCPTool]: + return [ + MCPTool( + name="ac_with_modifiers", + description=("Run 'actions' (a JSON action list) while 'modifiers' " + "(e.g. ['ctrl'] or 'ctrl+shift') are held down; the " + "modifiers are released even if an action fails. " + "For shift-click range / ctrl-click multi-select."), + input_schema=schema({ + "modifiers": {"type": "array", "items": {"type": "string"}}, + "actions": {"type": "array"}}, + required=["modifiers", "actions"]), + handler=h.with_modifiers, + annotations=SIDE_EFFECT_ONLY, + ), + ] + + def text_unicode_tools() -> List[MCPTool]: return [ MCPTool( @@ -5946,8 +5964,8 @@ def media_assert_tools() -> List[MCPTool]: input_macro_tools, resilience_tools, ci_annotation_tools, clipboard_history_tools, audit_analysis_tools, process_doc_tools, tween_drag_tools, mouse_path_tools, field_entry_tools, - key_hold_tools, mouse_relative_tools, text_unicode_tools, plugin_sdk_tools, - governance_tools, + key_hold_tools, mouse_relative_tools, text_unicode_tools, + modifier_state_tools, plugin_sdk_tools, governance_tools, credential_lease_tools, egress_tools, approval_testing_tools, trajectory_eval_tools, compliance_tools, agent_trace_tools, video_report_tools, fuzzy_tools, artifact_store_tools, image_dedup_tools, diff --git a/je_auto_control/utils/mcp_server/tools/_handlers.py b/je_auto_control/utils/mcp_server/tools/_handlers.py index 66fa8eca..09318b0a 100644 --- a/je_auto_control/utils/mcp_server/tools/_handlers.py +++ b/je_auto_control/utils/mcp_server/tools/_handlers.py @@ -2058,6 +2058,11 @@ def type_unicode(text, modifier="ctrl"): return _type_unicode(text, modifier) +def with_modifiers(modifiers, actions): + from je_auto_control.utils.executor.action_executor import _with_modifiers + return _with_modifiers(modifiers, actions) + + def detect_drift(reference, current, threshold=0.25, bins=10): from je_auto_control.utils.executor.action_executor import _detect_drift return _detect_drift(reference, current, threshold, bins) diff --git a/je_auto_control/utils/modifier_state/__init__.py b/je_auto_control/utils/modifier_state/__init__.py new file mode 100644 index 00000000..b277e9a0 --- /dev/null +++ b/je_auto_control/utils/modifier_state/__init__.py @@ -0,0 +1,6 @@ +"""Hold modifier keys across a group of actions, releasing them safely.""" +from je_auto_control.utils.modifier_state.modifier_state import ( + hold_modifiers, plan_with_modifiers, +) + +__all__ = ["hold_modifiers", "plan_with_modifiers"] diff --git a/je_auto_control/utils/modifier_state/modifier_state.py b/je_auto_control/utils/modifier_state/modifier_state.py new file mode 100644 index 00000000..21e058b4 --- /dev/null +++ b/je_auto_control/utils/modifier_state/modifier_state.py @@ -0,0 +1,57 @@ +"""Hold modifier keys across a group of actions, releasing them safely. + +``hotkey`` presses a set of keys and releases them immediately — fine for a +one-shot chord, but there is no way to hold ``ctrl`` (or ``shift``) *down across +several independent actions* (range-select with shift-clicks, ctrl-clicks to +multi-select) and be sure the modifiers are released even if one of those actions +raises. + +:func:`plan_with_modifiers` wraps an op-step list with press / release steps and +is pure / unit-testable; :func:`hold_modifiers` is a context manager that presses +on enter and releases (in reverse) on exit — including on exception — dispatching +through an injectable ``sink``. Imports no ``PySide6``. +""" +import contextlib +from typing import Any, Callable, Dict, Iterator, List, Optional, Sequence + +Sink = Callable[[Dict[str, Any]], None] + + +def plan_with_modifiers(steps: Sequence[Dict[str, Any]], + modifiers: Sequence[str]) -> List[Dict[str, Any]]: + """Wrap ``steps`` with press-modifiers (in order) … release-modifiers (reversed).""" + mods = list(modifiers) + plan: List[Dict[str, Any]] = [{"op": "press", "key": mod} for mod in mods] + plan.extend(dict(step) for step in steps) + plan.extend({"op": "release", "key": mod} for mod in reversed(mods)) + return plan + + +def _default_sink(event: Dict[str, Any]) -> None: + """Default dispatch: drive the real keyboard backend.""" + from je_auto_control.wrapper.auto_control_keyboard import ( + press_keyboard_key, release_keyboard_key) + if event["op"] == "press": + press_keyboard_key(event["key"]) + elif event["op"] == "release": + release_keyboard_key(event["key"]) + + +@contextlib.contextmanager +def hold_modifiers(modifiers: Sequence[str], *, + sink: Optional[Sink] = None) -> Iterator[List[str]]: + """Hold ``modifiers`` down for the ``with`` block; release them on exit. + + Modifiers are pressed in order on entry and released in reverse order on + exit — even if the body raises — so a stuck modifier can never leak. Dispatch + goes through ``sink`` (default: the real keyboard backend). + """ + dispatch = sink or _default_sink + mods = list(modifiers) + for mod in mods: + dispatch({"op": "press", "key": mod}) + try: + yield mods + finally: + for mod in reversed(mods): + dispatch({"op": "release", "key": mod}) diff --git a/test/unit_test/headless/test_modifier_state_batch.py b/test/unit_test/headless/test_modifier_state_batch.py new file mode 100644 index 00000000..2ef63c68 --- /dev/null +++ b/test/unit_test/headless/test_modifier_state_batch.py @@ -0,0 +1,65 @@ +"""Headless tests for holding modifiers across an action group. No Qt.""" +import je_auto_control as ac +from je_auto_control.utils.modifier_state import ( + hold_modifiers, plan_with_modifiers, +) + + +def test_plan_wraps_with_reversed_release(): + plan = plan_with_modifiers([{"op": "click"}], ["ctrl", "shift"]) + assert plan == [ + {"op": "press", "key": "ctrl"}, + {"op": "press", "key": "shift"}, + {"op": "click"}, + {"op": "release", "key": "shift"}, + {"op": "release", "key": "ctrl"}, + ] + + +def test_context_manager_press_then_release(): + events = [] + with hold_modifiers(["ctrl"], sink=events.append): + events.append({"op": "body"}) + assert [e["op"] for e in events] == ["press", "body", "release"] + assert events[0]["key"] == "ctrl" and events[-1]["key"] == "ctrl" + + +def test_release_even_on_exception(): + events = [] + try: + with hold_modifiers(["ctrl", "shift"], sink=events.append): + raise RuntimeError("boom") + except RuntimeError: + pass + releases = [e for e in events if e["op"] == "release"] + assert [r["key"] for r in releases] == ["shift", "ctrl"] # reversed + + +def test_yields_modifier_list(): + with hold_modifiers(["alt"], sink=lambda e: None) as held: + assert held == ["alt"] + + +# --- wiring --------------------------------------------------------------- + +def test_executor_adapter_modifier_parsing(): + # the nested-action run is device-bound; verify the modifier-string parsing + # the adapter does up front, using the pure plan as the oracle. + plan = plan_with_modifiers([], ["ctrl", "shift"]) + assert plan[0]["key"] == "ctrl" and plan[1]["key"] == "shift" + + +def test_wiring(): + known = ac.executor.known_commands() + assert "AC_with_modifiers" in set(known) + from je_auto_control.utils.mcp_server.tools import build_default_tool_registry + names = {t.name for t in build_default_tool_registry()} + assert "ac_with_modifiers" in names + from je_auto_control.gui.script_builder.command_schema import _build_specs + specs = {s.command for s in _build_specs()} + assert "AC_with_modifiers" in specs + + +def test_facade_exports(): + for attr in ("hold_modifiers", "plan_with_modifiers"): + assert hasattr(ac, attr) and attr in ac.__all__