From f98e0107e4acf29c4623eaca38d7b8e4a6471660 Mon Sep 17 00:00:00 2001 From: JeffreyChen Date: Mon, 22 Jun 2026 22:26:42 +0800 Subject: [PATCH 01/39] Add check-digit algorithms (Luhn, Verhoeff, Damm, ISO 7064 MOD 97-10) --- README/WHATS_NEW_zh-CN.md | 6 + README/WHATS_NEW_zh-TW.md | 6 + WHATS_NEW.md | 6 + .../doc/new_features/v115_features_doc.rst | 49 +++++++ docs/source/Eng/eng_index.rst | 1 + .../Zh/doc/new_features/v115_features_doc.rst | 41 ++++++ docs/source/Zh/zh_index.rst | 1 + je_auto_control/__init__.py | 14 ++ .../gui/script_builder/command_schema.py | 18 +++ je_auto_control/utils/checksum/__init__.py | 12 ++ je_auto_control/utils/checksum/checksum.py | 120 ++++++++++++++++++ .../utils/executor/action_executor.py | 24 ++++ .../utils/mcp_server/tools/_factories.py | 28 +++- .../utils/mcp_server/tools/_handlers.py | 10 ++ .../unit_test/headless/test_checksum_batch.py | 88 +++++++++++++ 15 files changed, 423 insertions(+), 1 deletion(-) create mode 100644 docs/source/Eng/doc/new_features/v115_features_doc.rst create mode 100644 docs/source/Zh/doc/new_features/v115_features_doc.rst create mode 100644 je_auto_control/utils/checksum/__init__.py create mode 100644 je_auto_control/utils/checksum/checksum.py create mode 100644 test/unit_test/headless/test_checksum_batch.py diff --git a/README/WHATS_NEW_zh-CN.md b/README/WHATS_NEW_zh-CN.md index 97777681..46ae914c 100644 --- a/README/WHATS_NEW_zh-CN.md +++ b/README/WHATS_NEW_zh-CN.md @@ -1,5 +1,11 @@ # 本次更新 — AutoControl +## 本次更新 (2026-06-22) — 校验位算法 + +计算/验证 Luhn、Verhoeff、Damm 与 ISO 7064 MOD 97-10 校验位。完整参考:[`docs/source/Zh/doc/new_features/v115_features_doc.rst`](../docs/source/Zh/doc/new_features/v115_features_doc.rst)。 + +- **`luhn_validate` / `luhn_check_digit` / `verhoeff_*` / `damm_*` / `mod97_10_*`**(`AC_checksum_validate`、`AC_checksum_digit`):`pii_text` 以正则检测卡号/IBAN 形状、`data_quality` 做正则验证,但没有任何功能计算或验证*校验位*。本功能加入多数标识符背后的四种方案(卡号/IMEI、身份证号、IBAN)——`identifier_validate` 所依据的共用引擎。纯标准库、确定。 + ## 本次更新 (2026-06-22) — 移动平均平滑 平滑噪声值序列。完整参考:[`docs/source/Zh/doc/new_features/v102_features_doc.rst`](../docs/source/Zh/doc/new_features/v102_features_doc.rst)。 diff --git a/README/WHATS_NEW_zh-TW.md b/README/WHATS_NEW_zh-TW.md index 4f8f3d0d..def51dfa 100644 --- a/README/WHATS_NEW_zh-TW.md +++ b/README/WHATS_NEW_zh-TW.md @@ -1,5 +1,11 @@ # 本次更新 — AutoControl +## 本次更新 (2026-06-22) — 檢查碼演算法 + +計算/驗證 Luhn、Verhoeff、Damm 與 ISO 7064 MOD 97-10 檢查碼。完整參考:[`docs/source/Zh/doc/new_features/v115_features_doc.rst`](../docs/source/Zh/doc/new_features/v115_features_doc.rst)。 + +- **`luhn_validate` / `luhn_check_digit` / `verhoeff_*` / `damm_*` / `mod97_10_*`**(`AC_checksum_validate`、`AC_checksum_digit`):`pii_text` 以正則偵測卡號/IBAN 形狀、`data_quality` 做正則驗證,但沒有任何功能計算或驗證*檢查碼*。本功能加入多數識別碼背後的四種方案(卡號/IMEI、國民身分碼、IBAN)——`identifier_validate` 所依據的共用引擎。純標準函式庫、具決定性。 + ## 本次更新 (2026-06-22) — 移動平均平滑 平滑雜訊值序列。完整參考:[`docs/source/Zh/doc/new_features/v102_features_doc.rst`](../docs/source/Zh/doc/new_features/v102_features_doc.rst)。 diff --git a/WHATS_NEW.md b/WHATS_NEW.md index b60ca25d..cb4f5df7 100644 --- a/WHATS_NEW.md +++ b/WHATS_NEW.md @@ -1,5 +1,11 @@ # What's New — AutoControl +## What's new (2026-06-22) — Check-Digit Algorithms + +Compute / verify Luhn, Verhoeff, Damm and ISO 7064 MOD 97-10 check digits. Full reference: [`docs/source/Eng/doc/new_features/v115_features_doc.rst`](docs/source/Eng/doc/new_features/v115_features_doc.rst). + +- **`luhn_validate` / `luhn_check_digit` / `verhoeff_*` / `damm_*` / `mod97_10_*`** (`AC_checksum_validate`, `AC_checksum_digit`): `pii_text` detects card/IBAN shapes by regex and `data_quality` does regex validation, but nothing computed or verified a *check digit*. This adds the four schemes behind most identifiers (cards/IMEI, national IDs, IBAN) — the shared engine `identifier_validate` builds on. Pure-stdlib, deterministic. + ## What's new (2026-06-22) — GNU gettext Catalog I/O (.po / .mo) Read/compile the de-facto translation format. Full reference: [`docs/source/Eng/doc/new_features/v114_features_doc.rst`](docs/source/Eng/doc/new_features/v114_features_doc.rst). diff --git a/docs/source/Eng/doc/new_features/v115_features_doc.rst b/docs/source/Eng/doc/new_features/v115_features_doc.rst new file mode 100644 index 00000000..0b2355e7 --- /dev/null +++ b/docs/source/Eng/doc/new_features/v115_features_doc.rst @@ -0,0 +1,49 @@ +Check-Digit Algorithms +====================== + +``pii_text`` detects credit-card and IBAN *shapes* by regex and ``data_quality`` +does type / range / regex validation, but nothing actually computes or verifies a +*check digit*. This adds the shared arithmetic engine for the four schemes behind +most real-world identifiers — and the primitive that account-number, card, IBAN, +ISBN and EAN validation build on. + +Pure standard library (integer arithmetic; the Verhoeff and Damm tables are small +embedded constants). Every function is pure (string in, bool / str out), so it is +fully deterministic in CI. + +Headless API +------------ + +.. code-block:: python + + from je_auto_control import ( + luhn_validate, luhn_check_digit, + verhoeff_validate, verhoeff_check_digit, + damm_validate, damm_check_digit, + mod97_10_validate, mod97_10_check_digits, + ) + + luhn_validate("4111111111111111") # True (credit-card / IMEI) + luhn_check_digit("7992739871") # '3' -> 79927398713 + verhoeff_validate("2363") # True (catches transpositions) + damm_check_digit("572") # '4' + mod97_10_validate("3214282912345698765432161182") # True (IBAN engine) + +- **Luhn** (mod 10): credit cards, IMEI, many national IDs — catches all + single-digit errors and most adjacent transpositions. +- **Verhoeff** and **Damm**: decimal schemes that catch *all* single-digit and + adjacent-transposition errors (stronger than Luhn). +- **ISO 7064 MOD 97-10**: the two-check-digit scheme behind IBAN and similar. + +Each scheme exposes ``*_validate(number)`` (does the value incl. its check digit +verify?) and ``*_check_digit`` / ``*_check_digits`` (what digit(s) to append to a +bare payload?). Non-digit characters are ignored, so spaced/grouped input works. + +Executor commands +----------------- + +``AC_checksum_validate`` takes a ``scheme`` (``luhn`` / ``verhoeff`` / ``damm`` / +``mod97``) plus a ``number`` and returns ``{valid}``; ``AC_checksum_digit`` returns +``{check_digit}`` for a ``partial``. Both are exposed as MCP tools +(``ac_checksum_validate`` / ``ac_checksum_digit``) and as Script Builder commands +under **Data**. diff --git a/docs/source/Eng/eng_index.rst b/docs/source/Eng/eng_index.rst index 77ff21a5..8e2192c9 100644 --- a/docs/source/Eng/eng_index.rst +++ b/docs/source/Eng/eng_index.rst @@ -137,6 +137,7 @@ Comprehensive guides for all AutoControl features. doc/new_features/v112_features_doc doc/new_features/v113_features_doc doc/new_features/v114_features_doc + doc/new_features/v115_features_doc doc/ocr_backends/ocr_backends_doc doc/observability/observability_doc doc/operations_layer/operations_layer_doc diff --git a/docs/source/Zh/doc/new_features/v115_features_doc.rst b/docs/source/Zh/doc/new_features/v115_features_doc.rst new file mode 100644 index 00000000..5a0216b0 --- /dev/null +++ b/docs/source/Zh/doc/new_features/v115_features_doc.rst @@ -0,0 +1,41 @@ +檢查碼演算法 +============ + +``pii_text`` 以正則偵測信用卡與 IBAN 的*形狀*、``data_quality`` 做型別/範圍/正則驗證,但沒有任何功能實際 +計算或驗證*檢查碼*。本功能加入多數真實世界識別碼背後四種方案的共用運算引擎——也是帳號、卡號、IBAN、 +ISBN、EAN 驗證所依據的基本元件。 + +純標準函式庫(整數運算;Verhoeff 與 Damm 表為小型內嵌常數)。每個函式皆為純函式(字串進、bool/str 出), +因此在 CI 中完全具決定性。 + +無頭 API +-------- + +.. code-block:: python + + from je_auto_control import ( + luhn_validate, luhn_check_digit, + verhoeff_validate, verhoeff_check_digit, + damm_validate, damm_check_digit, + mod97_10_validate, mod97_10_check_digits, + ) + + luhn_validate("4111111111111111") # True (信用卡 / IMEI) + luhn_check_digit("7992739871") # '3' -> 79927398713 + verhoeff_validate("2363") # True (可抓出換位錯誤) + damm_check_digit("572") # '4' + mod97_10_validate("3214282912345698765432161182") # True (IBAN 引擎) + +- **Luhn**(mod 10):信用卡、IMEI、多種國民身分碼——可抓出所有單一數字錯誤與多數相鄰換位。 +- **Verhoeff** 與 **Damm**:十進位方案,可抓出*所有*單一數字與相鄰換位錯誤(比 Luhn 更強)。 +- **ISO 7064 MOD 97-10**:IBAN 等使用的雙檢查碼方案。 + +每個方案提供 ``*_validate(number)``(含檢查碼的值是否驗證通過?)與 ``*_check_digit`` / ``*_check_digits`` +(對裸負載應附加哪些檢查碼?)。非數字字元會被忽略,因此含空格/分組的輸入也適用。 + +執行器命令 +---------- + +``AC_checksum_validate`` 接受 ``scheme``(``luhn`` / ``verhoeff`` / ``damm`` / ``mod97``)與 ``number`` 並回傳 +``{valid}``;``AC_checksum_digit`` 對 ``partial`` 回傳 ``{check_digit}``。兩者皆以 MCP 工具 +(``ac_checksum_validate`` / ``ac_checksum_digit``)以及 Script Builder 中 **Data** 分類下的命令提供。 diff --git a/docs/source/Zh/zh_index.rst b/docs/source/Zh/zh_index.rst index 6f85b45d..b03d2ce3 100644 --- a/docs/source/Zh/zh_index.rst +++ b/docs/source/Zh/zh_index.rst @@ -137,6 +137,7 @@ AutoControl 所有功能的完整使用指南。 doc/new_features/v112_features_doc doc/new_features/v113_features_doc doc/new_features/v114_features_doc + doc/new_features/v115_features_doc doc/ocr_backends/ocr_backends_doc doc/observability/observability_doc doc/operations_layer/operations_layer_doc diff --git a/je_auto_control/__init__.py b/je_auto_control/__init__.py index 61aff45b..d09b71d0 100644 --- a/je_auto_control/__init__.py +++ b/je_auto_control/__init__.py @@ -245,6 +245,12 @@ from je_auto_control.utils.gettext_catalog import ( GettextCatalog, parse_po, parse_po_file, read_mo, read_mo_file, ) +# Check-digit algorithms (Luhn / Verhoeff / Damm / ISO 7064 MOD 97-10) +from je_auto_control.utils.checksum import ( + damm_check_digit, damm_validate, luhn_check_digit, luhn_validate, + mod97_10_check_digits, mod97_10_validate, verhoeff_check_digit, + verhoeff_validate, +) # CI workflow annotations (GitHub Actions) from je_auto_control.utils.ci_annotations import ( emit_annotations, format_annotation, @@ -1007,6 +1013,14 @@ def start_autocontrol_gui(*args, **kwargs): "parse_po_file", "read_mo", "read_mo_file", + "luhn_validate", + "luhn_check_digit", + "verhoeff_validate", + "verhoeff_check_digit", + "damm_validate", + "damm_check_digit", + "mod97_10_validate", + "mod97_10_check_digits", "emit_annotations", "format_annotation", "ClipboardHistory", "default_clipboard_history", "analyze_heal_log", "heal_stats", "scan_secrets", diff --git a/je_auto_control/gui/script_builder/command_schema.py b/je_auto_control/gui/script_builder/command_schema.py index e2a66b3d..8535c216 100644 --- a/je_auto_control/gui/script_builder/command_schema.py +++ b/je_auto_control/gui/script_builder/command_schema.py @@ -2170,6 +2170,24 @@ def _add_resilience_specs(specs: List[CommandSpec]) -> None: ), description="Pick the plural-correct translation for count n.", )) + specs.append(CommandSpec( + "AC_checksum_validate", "Data", "Checksum: Validate", + fields=( + FieldSpec("scheme", FieldType.STRING, + placeholder="luhn | verhoeff | damm | mod97"), + FieldSpec("number", FieldType.STRING, placeholder="4111111111111111"), + ), + description="Validate a number's check digit (Luhn/Verhoeff/Damm/mod97).", + )) + specs.append(CommandSpec( + "AC_checksum_digit", "Data", "Checksum: Check Digit", + fields=( + FieldSpec("scheme", FieldType.STRING, + placeholder="luhn | verhoeff | damm | mod97"), + FieldSpec("partial", FieldType.STRING, placeholder="799273987"), + ), + description="Compute the check digit(s) to append to a value.", + )) specs.append(CommandSpec( "AC_diff_rows", "Data", "Dataset Diff: Rows by Key", fields=( diff --git a/je_auto_control/utils/checksum/__init__.py b/je_auto_control/utils/checksum/__init__.py new file mode 100644 index 00000000..4edddc3c --- /dev/null +++ b/je_auto_control/utils/checksum/__init__.py @@ -0,0 +1,12 @@ +"""Check-digit algorithms: Luhn, Verhoeff, Damm, ISO 7064 MOD 97-10.""" +from je_auto_control.utils.checksum.checksum import ( + damm_check_digit, damm_validate, luhn_check_digit, luhn_validate, + mod97_10_check_digits, mod97_10_validate, verhoeff_check_digit, + verhoeff_validate, +) + +__all__ = [ + "damm_check_digit", "damm_validate", "luhn_check_digit", "luhn_validate", + "mod97_10_check_digits", "mod97_10_validate", "verhoeff_check_digit", + "verhoeff_validate", +] diff --git a/je_auto_control/utils/checksum/checksum.py b/je_auto_control/utils/checksum/checksum.py new file mode 100644 index 00000000..aa162b2b --- /dev/null +++ b/je_auto_control/utils/checksum/checksum.py @@ -0,0 +1,120 @@ +"""Check-digit algorithms: Luhn, Verhoeff, Damm, ISO 7064 MOD 97-10. + +``pii_text`` detects credit-card and IBAN *shapes* by regex and ``data_quality`` +does type/range/regex validation, but nothing in the project actually computes or +verifies a *check digit*. This is the shared arithmetic engine that catches the +single-digit and adjacent-transposition typos those formats are designed to +detect, and the primitive that ``identifier_validate`` (IBAN / ISBN / EAN / card) +builds on. + +Pure standard library (integer arithmetic only; the Verhoeff and Damm tables are +small embedded constants). Every function is pure (string in, bool/str out), so it +is fully deterministic in CI. +""" +from typing import List + +# Verhoeff dihedral-group multiplication, permutation and inverse tables. +_VERHOEFF_D = ( + (0, 1, 2, 3, 4, 5, 6, 7, 8, 9), (1, 2, 3, 4, 0, 6, 7, 8, 9, 5), + (2, 3, 4, 0, 1, 7, 8, 9, 5, 6), (3, 4, 0, 1, 2, 8, 9, 5, 6, 7), + (4, 0, 1, 2, 3, 9, 5, 6, 7, 8), (5, 9, 8, 7, 6, 0, 4, 3, 2, 1), + (6, 5, 9, 8, 7, 1, 0, 4, 3, 2), (7, 6, 5, 9, 8, 2, 1, 0, 4, 3), + (8, 7, 6, 5, 9, 3, 2, 1, 0, 4), (9, 8, 7, 6, 5, 4, 3, 2, 1, 0), +) +_VERHOEFF_P = ( + (0, 1, 2, 3, 4, 5, 6, 7, 8, 9), (1, 5, 7, 6, 2, 8, 3, 0, 9, 4), + (5, 8, 0, 3, 7, 9, 6, 1, 4, 2), (8, 9, 1, 6, 0, 4, 3, 5, 2, 7), + (9, 4, 5, 3, 1, 2, 6, 8, 7, 0), (4, 2, 8, 6, 5, 7, 3, 9, 0, 1), + (2, 7, 9, 3, 8, 0, 6, 4, 1, 5), (7, 0, 4, 6, 9, 1, 3, 2, 5, 8), +) +_VERHOEFF_INV = (0, 4, 3, 2, 1, 5, 6, 7, 8, 9) + +# Damm quasigroup table (totally anti-symmetric). +_DAMM = ( + (0, 3, 1, 7, 5, 9, 8, 6, 4, 2), (7, 0, 9, 2, 1, 5, 4, 8, 6, 3), + (4, 2, 0, 6, 8, 7, 1, 3, 5, 9), (1, 7, 5, 0, 9, 8, 3, 4, 2, 6), + (6, 1, 2, 3, 0, 4, 5, 9, 7, 8), (3, 6, 7, 4, 2, 0, 9, 5, 8, 1), + (5, 8, 6, 9, 7, 2, 0, 1, 3, 4), (8, 9, 4, 5, 3, 6, 2, 0, 1, 7), + (9, 4, 3, 8, 6, 1, 7, 2, 0, 5), (2, 5, 8, 1, 4, 3, 6, 7, 9, 0), +) + + +def _digits(value: object) -> List[int]: + """Extract the decimal digits of a value as a list of ints.""" + return [int(ch) for ch in str(value) if ch.isdigit()] + + +# --- Luhn (mod 10) -------------------------------------------------------- + +def _luhn_sum(digits: List[int]) -> int: + total = 0 + for index, digit in enumerate(reversed(digits)): + if index % 2 == 1: + digit *= 2 + if digit > 9: + digit -= 9 + total += digit + return total + + +def luhn_validate(number: object) -> bool: + """Whether ``number`` (incl. its trailing check digit) passes Luhn.""" + digits = _digits(number) + return bool(digits) and _luhn_sum(digits) % 10 == 0 + + +def luhn_check_digit(partial: object) -> str: + """Return the Luhn check digit to append to ``partial`` (no check digit).""" + total = _luhn_sum(_digits(partial) + [0]) + return str((10 - total % 10) % 10) + + +# --- Verhoeff ------------------------------------------------------------- + +def verhoeff_validate(number: object) -> bool: + """Whether ``number`` (incl. check digit) passes the Verhoeff scheme.""" + check = 0 + for index, digit in enumerate(reversed(_digits(number))): + check = _VERHOEFF_D[check][_VERHOEFF_P[index % 8][digit]] + return check == 0 + + +def verhoeff_check_digit(partial: object) -> str: + """Return the Verhoeff check digit to append to ``partial``.""" + check = 0 + for index, digit in enumerate(reversed(_digits(partial))): + check = _VERHOEFF_D[check][_VERHOEFF_P[(index + 1) % 8][digit]] + return str(_VERHOEFF_INV[check]) + + +# --- Damm ----------------------------------------------------------------- + +def damm_validate(number: object) -> bool: + """Whether ``number`` (incl. check digit) passes the Damm scheme.""" + interim = 0 + for digit in _digits(number): + interim = _DAMM[interim][digit] + return interim == 0 + + +def damm_check_digit(partial: object) -> str: + """Return the Damm check digit to append to ``partial``.""" + interim = 0 + for digit in _digits(partial): + interim = _DAMM[interim][digit] + return str(interim) + + +# --- ISO 7064 MOD 97-10 (the IBAN engine) --------------------------------- + +def mod97_10_validate(number: object) -> bool: + """Whether the numeric string ``number`` satisfies ISO 7064 MOD 97-10.""" + digits = "".join(ch for ch in str(number) if ch.isdigit()) + return bool(digits) and int(digits) % 97 == 1 + + +def mod97_10_check_digits(partial: object) -> str: + """Return the two MOD 97-10 check digits to append to ``partial``.""" + digits = "".join(ch for ch in str(partial) if ch.isdigit()) + value = int(digits) if digits else 0 + return f"{(1 - value * 100) % 97:02d}" diff --git a/je_auto_control/utils/executor/action_executor.py b/je_auto_control/utils/executor/action_executor.py index 65539211..c80bd280 100644 --- a/je_auto_control/utils/executor/action_executor.py +++ b/je_auto_control/utils/executor/action_executor.py @@ -3047,6 +3047,28 @@ def _gettext_ngettext(po: str, msgid: str, msgid_plural: str, return {"text": catalog.ngettext(msgid, msgid_plural, int(n))} +def _checksum_validate(scheme: str, number: str) -> Dict[str, Any]: + """Adapter: validate a number's check digit under a named scheme.""" + from je_auto_control.utils import checksum as cs + validators = {"luhn": cs.luhn_validate, "verhoeff": cs.verhoeff_validate, + "damm": cs.damm_validate, "mod97": cs.mod97_10_validate} + func = validators.get(scheme) + if func is None: + raise AutoControlActionException(f"unknown checksum scheme: {scheme!r}") + return {"valid": func(number)} + + +def _checksum_digit(scheme: str, partial: str) -> Dict[str, Any]: + """Adapter: compute the check digit(s) for a value under a named scheme.""" + from je_auto_control.utils import checksum as cs + digits = {"luhn": cs.luhn_check_digit, "verhoeff": cs.verhoeff_check_digit, + "damm": cs.damm_check_digit, "mod97": cs.mod97_10_check_digits} + func = digits.get(scheme) + if func is None: + raise AutoControlActionException(f"unknown checksum scheme: {scheme!r}") + return {"check_digit": func(partial)} + + def _cas_put(name: str, key: str, value: Any, expected_version: Any = None) -> Dict[str, Any]: """Adapter: optimistic put into a named versioned store.""" @@ -4740,6 +4762,8 @@ def __init__(self): "AC_format_message": _format_message, "AC_gettext_translate": _gettext_translate, "AC_gettext_ngettext": _gettext_ngettext, + "AC_checksum_validate": _checksum_validate, + "AC_checksum_digit": _checksum_digit, "AC_detect_drift": _detect_drift, "AC_categorical_drift": _categorical_drift, "AC_diff_rows": _diff_rows, diff --git a/je_auto_control/utils/mcp_server/tools/_factories.py b/je_auto_control/utils/mcp_server/tools/_factories.py index be267fbc..4ce0f72b 100644 --- a/je_auto_control/utils/mcp_server/tools/_factories.py +++ b/je_auto_control/utils/mcp_server/tools/_factories.py @@ -3709,6 +3709,32 @@ def gettext_catalog_tools() -> List[MCPTool]: ] +def checksum_tools() -> List[MCPTool]: + return [ + MCPTool( + name="ac_checksum_validate", + description=("Validate a number's check digit. 'scheme' is " + "luhn|verhoeff|damm|mod97. Returns {valid}."), + input_schema=schema( + {"scheme": {"type": "string"}, "number": {"type": "string"}}, + ["scheme", "number"]), + handler=h.checksum_validate, + annotations=READ_ONLY, + ), + MCPTool( + name="ac_checksum_digit", + description=("Compute the check digit(s) to append to 'partial'. " + "'scheme' is luhn|verhoeff|damm|mod97. " + "Returns {check_digit}."), + input_schema=schema( + {"scheme": {"type": "string"}, "partial": {"type": "string"}}, + ["scheme", "partial"]), + handler=h.checksum_digit, + annotations=READ_ONLY, + ), + ] + + def message_format_tools() -> List[MCPTool]: return [ MCPTool( @@ -5789,7 +5815,7 @@ def media_assert_tools() -> List[MCPTool]: dedup_window_tools, sequence_gap_tools, optimistic_tools, outbox_tools, locale_collation_tools, confusables_tools, readability_tools, bidi_check_tools, list_format_tools, message_format_tools, - gettext_catalog_tools, + gettext_catalog_tools, checksum_tools, dataset_diff_tools, referential_tools, link_header_tools, multipart_tools, http_content_tools, cookie_jar_tools, http_conditional_tools, saga_tools, decision_table_tools, locator_repair_tools, diff --git a/je_auto_control/utils/mcp_server/tools/_handlers.py b/je_auto_control/utils/mcp_server/tools/_handlers.py index 882db17b..6a8d7e8f 100644 --- a/je_auto_control/utils/mcp_server/tools/_handlers.py +++ b/je_auto_control/utils/mcp_server/tools/_handlers.py @@ -2017,6 +2017,16 @@ def gettext_ngettext(po, msgid, msgid_plural, n): return _gettext_ngettext(po, msgid, msgid_plural, n) +def checksum_validate(scheme, number): + from je_auto_control.utils.executor.action_executor import _checksum_validate + return _checksum_validate(scheme, number) + + +def checksum_digit(scheme, partial): + from je_auto_control.utils.executor.action_executor import _checksum_digit + return _checksum_digit(scheme, partial) + + def detect_drift(reference, current, threshold=0.25, bins=10): from je_auto_control.utils.executor.action_executor import _detect_drift return _detect_drift(reference, current, threshold, bins) diff --git a/test/unit_test/headless/test_checksum_batch.py b/test/unit_test/headless/test_checksum_batch.py new file mode 100644 index 00000000..a8da2425 --- /dev/null +++ b/test/unit_test/headless/test_checksum_batch.py @@ -0,0 +1,88 @@ +"""Headless tests for check-digit algorithms. No Qt.""" +import je_auto_control as ac +from je_auto_control.utils.checksum import ( + damm_check_digit, damm_validate, luhn_check_digit, luhn_validate, + mod97_10_check_digits, mod97_10_validate, verhoeff_check_digit, + verhoeff_validate, +) + + +def test_luhn(): + assert luhn_validate("4111111111111111") is True + assert luhn_validate("4111111111111112") is False + assert luhn_check_digit("7992739871") == "3" # 79927398713 is valid + assert luhn_validate("79927398713") is True + assert luhn_validate("") is False # no digits + + +def test_verhoeff(): + assert verhoeff_check_digit("236") == "3" + assert verhoeff_validate("2363") is True + assert verhoeff_validate("2364") is False # wrong check digit + assert verhoeff_validate("2633") is False # transposition caught + + +def test_damm(): + assert damm_check_digit("572") == "4" + assert damm_validate("5724") is True + assert damm_validate("5720") is False + assert damm_validate("7524") is False # transposition caught + + +def test_mod97_10(): + # GB82WEST12345698765432 rearranged + letters->numbers ends ...% 97 == 1 + rearranged = "3214282912345698765432161182" + assert mod97_10_validate(rearranged) is True + assert mod97_10_validate("3214282912345698765432161183") is False + # appended check digits make the value satisfy MOD 97-10 + body = "123456789" + check = mod97_10_check_digits(body) + assert mod97_10_validate(body + check) is True + assert len(check) == 2 + + +def test_check_digit_round_trips(): + for scheme_validate, scheme_digit in [ + (luhn_validate, luhn_check_digit), + (verhoeff_validate, verhoeff_check_digit), + (damm_validate, damm_check_digit)]: + base = "12345678" + assert scheme_validate(base + scheme_digit(base)) is True + + +# --- wiring --------------------------------------------------------------- + +def test_executor_round_trip(): + rec = ac.execute_action([[ + "AC_checksum_validate", + {"scheme": "luhn", "number": "4111111111111111"}]]) + out = next(v for v in rec.values() if isinstance(v, dict)) + assert out["valid"] is True + rec2 = ac.execute_action([[ + "AC_checksum_digit", {"scheme": "damm", "partial": "572"}]]) + assert next(v for v in rec2.values() if isinstance(v, dict))["check_digit"] == "4" + + +def test_executor_unknown_scheme_errors(): + rec = ac.execute_action([[ + "AC_checksum_validate", {"scheme": "nope", "number": "1"}]]) + # the executor records the failure rather than returning a {valid} dict + assert not any(isinstance(v, dict) and v.get("valid") for v in rec.values()) + + +def test_wiring(): + known = ac.executor.known_commands() + assert {"AC_checksum_validate", "AC_checksum_digit"} <= set(known) + from je_auto_control.utils.mcp_server.tools import build_default_tool_registry + names = {t.name for t in build_default_tool_registry()} + assert {"ac_checksum_validate", "ac_checksum_digit"} <= names + from je_auto_control.gui.script_builder.command_schema import _build_specs + specs = {s.command for s in _build_specs()} + assert {"AC_checksum_validate", "AC_checksum_digit"} <= specs + + +def test_facade_exports(): + for attr in ("luhn_validate", "luhn_check_digit", "verhoeff_validate", + "verhoeff_check_digit", "damm_validate", "damm_check_digit", + "mod97_10_validate", "mod97_10_check_digits"): + assert hasattr(ac, attr) and attr in ac.__all__ From 390f8ab8aadabcb251795b310a64818754deb927 Mon Sep 17 00:00:00 2001 From: JeffreyChen Date: Mon, 22 Jun 2026 22:37:58 +0800 Subject: [PATCH 02/39] Add IMEI and spaced-input Luhn test vectors --- test/unit_test/headless/test_checksum_batch.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test/unit_test/headless/test_checksum_batch.py b/test/unit_test/headless/test_checksum_batch.py index a8da2425..f5697ea3 100644 --- a/test/unit_test/headless/test_checksum_batch.py +++ b/test/unit_test/headless/test_checksum_batch.py @@ -13,6 +13,8 @@ def test_luhn(): assert luhn_check_digit("7992739871") == "3" # 79927398713 is valid assert luhn_validate("79927398713") is True assert luhn_validate("") is False # no digits + assert luhn_validate("49015420323751") is True # a valid IMEI + assert luhn_validate("3566 0020 2036 0505") is True # spaced input ignored def test_verhoeff(): From 7139f0d83a0b69b2b00740b42cf8c7114df7ee0f Mon Sep 17 00:00:00 2001 From: JeffreyChen Date: Mon, 22 Jun 2026 23:50:14 +0800 Subject: [PATCH 03/39] Add multi-waypoint mouse gestures (move/drag through a polyline) --- README/WHATS_NEW_zh-CN.md | 6 ++ README/WHATS_NEW_zh-TW.md | 6 ++ WHATS_NEW.md | 6 ++ .../doc/new_features/v116_features_doc.rst | 43 ++++++++++ docs/source/Eng/eng_index.rst | 1 + .../Zh/doc/new_features/v116_features_doc.rst | 35 ++++++++ docs/source/Zh/zh_index.rst | 1 + je_auto_control/__init__.py | 8 ++ .../gui/script_builder/command_schema.py | 30 +++++++ .../utils/executor/action_executor.py | 25 ++++++ .../utils/mcp_server/tools/_factories.py | 39 ++++++++- .../utils/mcp_server/tools/_handlers.py | 11 +++ je_auto_control/utils/mouse_path/__init__.py | 6 ++ .../utils/mouse_path/mouse_path.py | 86 +++++++++++++++++++ .../headless/test_mouse_path_batch.py | 77 +++++++++++++++++ 15 files changed, 379 insertions(+), 1 deletion(-) create mode 100644 docs/source/Eng/doc/new_features/v116_features_doc.rst create mode 100644 docs/source/Zh/doc/new_features/v116_features_doc.rst create mode 100644 je_auto_control/utils/mouse_path/__init__.py create mode 100644 je_auto_control/utils/mouse_path/mouse_path.py create mode 100644 test/unit_test/headless/test_mouse_path_batch.py diff --git a/README/WHATS_NEW_zh-CN.md b/README/WHATS_NEW_zh-CN.md index 46ae914c..1a4c438c 100644 --- a/README/WHATS_NEW_zh-CN.md +++ b/README/WHATS_NEW_zh-CN.md @@ -1,5 +1,11 @@ # 本次更新 — AutoControl +## 本次更新 (2026-06-22) — 多路径点鼠标手势 + +让指针沿着路径点折线移动或拖曳。完整参考:[`docs/source/Zh/doc/new_features/v116_features_doc.rst`](../docs/source/Zh/doc/new_features/v116_features_doc.rst)。 + +- **`plan_path` / `move_along_path` / `drag_path` / `path_easings`**(`AC_move_along_path`、`AC_drag_path`):`humanize` 与 `tween_drag` 只在单一起点→终点之间插值——先前无法驱动任意的路径点链(签名、框选、多停靠点拖曳)并在整段路径中按住按键。`plan_path` 为纯缓动点运算(重用 `tween_drag` 的缓动、交接点去重);移动/拖曳通过可注入的 sink 派发以供无头测试。纯标准库、确定。 + ## 本次更新 (2026-06-22) — 校验位算法 计算/验证 Luhn、Verhoeff、Damm 与 ISO 7064 MOD 97-10 校验位。完整参考:[`docs/source/Zh/doc/new_features/v115_features_doc.rst`](../docs/source/Zh/doc/new_features/v115_features_doc.rst)。 diff --git a/README/WHATS_NEW_zh-TW.md b/README/WHATS_NEW_zh-TW.md index def51dfa..1e5aeb88 100644 --- a/README/WHATS_NEW_zh-TW.md +++ b/README/WHATS_NEW_zh-TW.md @@ -1,5 +1,11 @@ # 本次更新 — AutoControl +## 本次更新 (2026-06-22) — 多路徑點滑鼠手勢 + +讓指標沿著路徑點折線移動或拖曳。完整參考:[`docs/source/Zh/doc/new_features/v116_features_doc.rst`](../docs/source/Zh/doc/new_features/v116_features_doc.rst)。 + +- **`plan_path` / `move_along_path` / `drag_path` / `path_easings`**(`AC_move_along_path`、`AC_drag_path`):`humanize` 與 `tween_drag` 只在單一起點→終點之間插值——先前無法驅動任意的路徑點鏈(簽名、框選、多停靠點拖曳)並在整段路徑中按住按鍵。`plan_path` 為純緩動點運算(重用 `tween_drag` 的緩動、交接點去重);移動/拖曳透過可注入的 sink 派發以供無頭測試。純標準函式庫、具決定性。 + ## 本次更新 (2026-06-22) — 檢查碼演算法 計算/驗證 Luhn、Verhoeff、Damm 與 ISO 7064 MOD 97-10 檢查碼。完整參考:[`docs/source/Zh/doc/new_features/v115_features_doc.rst`](../docs/source/Zh/doc/new_features/v115_features_doc.rst)。 diff --git a/WHATS_NEW.md b/WHATS_NEW.md index cb4f5df7..2015723d 100644 --- a/WHATS_NEW.md +++ b/WHATS_NEW.md @@ -1,5 +1,11 @@ # What's New — AutoControl +## What's new (2026-06-22) — Multi-Waypoint Mouse Gestures + +Move or drag the pointer through a polyline of waypoints. Full reference: [`docs/source/Eng/doc/new_features/v116_features_doc.rst`](docs/source/Eng/doc/new_features/v116_features_doc.rst). + +- **`plan_path` / `move_along_path` / `drag_path` / `path_easings`** (`AC_move_along_path`, `AC_drag_path`): `humanize` and `tween_drag` only interpolate a single start→end hop — there was no way to drive an arbitrary chain of waypoints (signatures, marquee selects, multi-stop drags) with the button held across the whole path. `plan_path` is pure eased point math (reusing `tween_drag`'s easings, junctions de-duplicated); the move/drag dispatch through an injectable sink for headless testing. Pure-stdlib, deterministic. + ## What's new (2026-06-22) — Check-Digit Algorithms Compute / verify Luhn, Verhoeff, Damm and ISO 7064 MOD 97-10 check digits. Full reference: [`docs/source/Eng/doc/new_features/v115_features_doc.rst`](docs/source/Eng/doc/new_features/v115_features_doc.rst). diff --git a/docs/source/Eng/doc/new_features/v116_features_doc.rst b/docs/source/Eng/doc/new_features/v116_features_doc.rst new file mode 100644 index 00000000..3b39a672 --- /dev/null +++ b/docs/source/Eng/doc/new_features/v116_features_doc.rst @@ -0,0 +1,43 @@ +Multi-Waypoint Mouse Gestures +============================= + +``humanize.humanized_path`` and ``tween_drag`` only interpolate a *single* +start → end hop. Real gestures — signatures, marquee / rubber-band selections, +dragging through several drop targets, shape gestures — need an arbitrary chain +of waypoints, with the button optionally held down across the whole path. + +:func:`plan_path` is pure point math (reusing the named easings from +``tween_drag``) and is unit-testable on its own; :func:`move_along_path` and +:func:`drag_path` dispatch through an injectable ``sink`` so the gesture is +tested without real input. Imports no ``PySide6``. + +Headless API +------------ + +.. code-block:: python + + from je_auto_control import ( + plan_path, move_along_path, drag_path, path_easings, + ) + + # the eased point list through every waypoint (junctions de-duplicated) + plan_path([(100, 100), (400, 150), (400, 500)], per_segment_steps=20) + + move_along_path([(100, 100), (400, 150), (400, 500)]) # hover a polyline + drag_path([(50, 50), (300, 50), (300, 300)], button="mouse_left") # L-drag + +``plan_path`` interpolates each consecutive pair with ``per_segment_steps`` eased +steps (``easing`` is any name from ``path_easings()`` — ``linear`` / +``ease_in_out_quad`` / ``ease_out_cubic`` / ``ease_in_cubic``) and does not +duplicate the shared junction points. ``move_along_path`` emits move events +through the path; ``drag_path`` presses at the first waypoint, moves through the +whole path, and releases at the last — for multi-stop drags. Both take a ``sink`` +override for headless testing. + +Executor commands +----------------- + +``AC_move_along_path`` and ``AC_drag_path`` take ``waypoints`` (a JSON +``[[x, y], ...]`` list) plus ``easing`` / ``per_segment_steps`` (and ``button`` +for the drag). Both are exposed as MCP tools (``ac_move_along_path`` / +``ac_drag_path``) and as Script Builder commands under **Mouse**. diff --git a/docs/source/Eng/eng_index.rst b/docs/source/Eng/eng_index.rst index 8e2192c9..6e8a2e24 100644 --- a/docs/source/Eng/eng_index.rst +++ b/docs/source/Eng/eng_index.rst @@ -138,6 +138,7 @@ Comprehensive guides for all AutoControl features. doc/new_features/v113_features_doc doc/new_features/v114_features_doc doc/new_features/v115_features_doc + doc/new_features/v116_features_doc doc/ocr_backends/ocr_backends_doc doc/observability/observability_doc doc/operations_layer/operations_layer_doc diff --git a/docs/source/Zh/doc/new_features/v116_features_doc.rst b/docs/source/Zh/doc/new_features/v116_features_doc.rst new file mode 100644 index 00000000..bd958ffa --- /dev/null +++ b/docs/source/Zh/doc/new_features/v116_features_doc.rst @@ -0,0 +1,35 @@ +多路徑點滑鼠手勢 +================ + +``humanize.humanized_path`` 與 ``tween_drag`` 只在*單一*起點 → 終點之間插值。真實手勢——簽名、框選 +(marquee / rubber-band)、拖曳經過多個放置目標、形狀手勢——需要任意的路徑點鏈,且可在整段路徑中持續按住按鍵。 + +:func:`plan_path` 為純點運算(重用 ``tween_drag`` 的具名緩動),本身即可單元測試;:func:`move_along_path` 與 +:func:`drag_path` 透過可注入的 ``sink`` 派發,因此手勢可在無真實輸入下測試。不匯入 ``PySide6``。 + +無頭 API +-------- + +.. code-block:: python + + from je_auto_control import ( + plan_path, move_along_path, drag_path, path_easings, + ) + + # 經過每個路徑點的緩動點列(交接點不重複) + plan_path([(100, 100), (400, 150), (400, 500)], per_segment_steps=20) + + move_along_path([(100, 100), (400, 150), (400, 500)]) # 沿折線移動 + drag_path([(50, 50), (300, 50), (300, 300)], button="mouse_left") # L 形拖曳 + +``plan_path`` 以 ``per_segment_steps`` 個緩動步驟對每個相鄰點對插值(``easing`` 為 ``path_easings()`` 中任一 +名稱——``linear`` / ``ease_in_out_quad`` / ``ease_out_cubic`` / ``ease_in_cubic``),且不重複共用的交接點。 +``move_along_path`` 沿路徑發出移動事件;``drag_path`` 在第一個路徑點按下、移動經過整段路徑、在最後一點放開—— +用於多停靠點拖曳。兩者皆可傳入 ``sink`` 以供無頭測試。 + +執行器命令 +---------- + +``AC_move_along_path`` 與 ``AC_drag_path`` 接受 ``waypoints``(JSON ``[[x, y], ...]`` 列表)以及 ``easing`` / +``per_segment_steps``(拖曳另含 ``button``)。兩者皆以 MCP 工具(``ac_move_along_path`` / ``ac_drag_path``) +以及 Script Builder 中 **Mouse** 分類下的命令提供。 diff --git a/docs/source/Zh/zh_index.rst b/docs/source/Zh/zh_index.rst index b03d2ce3..a69899e9 100644 --- a/docs/source/Zh/zh_index.rst +++ b/docs/source/Zh/zh_index.rst @@ -138,6 +138,7 @@ AutoControl 所有功能的完整使用指南。 doc/new_features/v113_features_doc doc/new_features/v114_features_doc doc/new_features/v115_features_doc + doc/new_features/v116_features_doc doc/ocr_backends/ocr_backends_doc doc/observability/observability_doc doc/operations_layer/operations_layer_doc diff --git a/je_auto_control/__init__.py b/je_auto_control/__init__.py index d09b71d0..38de2b16 100644 --- a/je_auto_control/__init__.py +++ b/je_auto_control/__init__.py @@ -251,6 +251,10 @@ mod97_10_check_digits, mod97_10_validate, verhoeff_check_digit, verhoeff_validate, ) +# Multi-waypoint mouse gestures (move / drag through a polyline of points) +from je_auto_control.utils.mouse_path import ( + drag_path, move_along_path, path_easings, plan_path, +) # CI workflow annotations (GitHub Actions) from je_auto_control.utils.ci_annotations import ( emit_annotations, format_annotation, @@ -1021,6 +1025,10 @@ def start_autocontrol_gui(*args, **kwargs): "damm_check_digit", "mod97_10_validate", "mod97_10_check_digits", + "plan_path", + "move_along_path", + "drag_path", + "path_easings", "emit_annotations", "format_annotation", "ClipboardHistory", "default_clipboard_history", "analyze_heal_log", "heal_stats", "scan_secrets", diff --git a/je_auto_control/gui/script_builder/command_schema.py b/je_auto_control/gui/script_builder/command_schema.py index 8535c216..77c687f4 100644 --- a/je_auto_control/gui/script_builder/command_schema.py +++ b/je_auto_control/gui/script_builder/command_schema.py @@ -681,6 +681,36 @@ def _add_misc_specs(specs: List[CommandSpec]) -> None: description="Drag along an eased path; 'start'/'end' [x,y] via JSON " "view.", )) + specs.append(CommandSpec( + "AC_move_along_path", "Mouse", "Move Along Path", + fields=( + FieldSpec("waypoints", FieldType.STRING, + placeholder="[[100,100],[400,150],[400,500]]"), + FieldSpec("easing", FieldType.ENUM, + choices=("linear", "ease_in_out_quad", "ease_out_cubic", + "ease_in_cubic"), + optional=True, default="linear"), + FieldSpec("per_segment_steps", FieldType.INT, optional=True, + default=20), + ), + description="Move the pointer through a polyline of waypoints.", + )) + specs.append(CommandSpec( + "AC_drag_path", "Mouse", "Drag Along Path", + fields=( + FieldSpec("waypoints", FieldType.STRING, + placeholder="[[50,50],[300,50],[300,300]]"), + FieldSpec("button", FieldType.ENUM, choices=_MOUSE_BUTTONS, + optional=True, default="mouse_left"), + FieldSpec("easing", FieldType.ENUM, + choices=("linear", "ease_in_out_quad", "ease_out_cubic", + "ease_in_cubic"), + optional=True, default="linear"), + FieldSpec("per_segment_steps", FieldType.INT, optional=True, + default=20), + ), + description="Press, drag through a polyline of waypoints, release.", + )) specs.append(CommandSpec( "AC_list_plugins", "Tools", "List Plugin Commands", fields=(FieldSpec("group", FieldType.STRING, optional=True, diff --git a/je_auto_control/utils/executor/action_executor.py b/je_auto_control/utils/executor/action_executor.py index c80bd280..1c88529c 100644 --- a/je_auto_control/utils/executor/action_executor.py +++ b/je_auto_control/utils/executor/action_executor.py @@ -3069,6 +3069,29 @@ def _checksum_digit(scheme: str, partial: str) -> Dict[str, Any]: return {"check_digit": func(partial)} +def _waypoints(value: Any) -> Any: + """Coerce a JSON string of waypoints into a list.""" + import json + return json.loads(value) if isinstance(value, str) else value + + +def _move_along_path(waypoints: Any, easing: str = "linear", + per_segment_steps: Any = 20) -> Dict[str, Any]: + """Adapter: move the pointer through a polyline of waypoints.""" + from je_auto_control.utils.mouse_path import move_along_path + return move_along_path(_waypoints(waypoints), easing=easing, + per_segment_steps=int(per_segment_steps)) + + +def _drag_path(waypoints: Any, button: str = "mouse_left", + easing: str = "linear", + per_segment_steps: Any = 20) -> Dict[str, Any]: + """Adapter: press, drag through a polyline of waypoints, release.""" + from je_auto_control.utils.mouse_path import drag_path + return drag_path(_waypoints(waypoints), button=button, easing=easing, + per_segment_steps=int(per_segment_steps)) + + def _cas_put(name: str, key: str, value: Any, expected_version: Any = None) -> Dict[str, Any]: """Adapter: optimistic put into a named versioned store.""" @@ -4764,6 +4787,8 @@ def __init__(self): "AC_gettext_ngettext": _gettext_ngettext, "AC_checksum_validate": _checksum_validate, "AC_checksum_digit": _checksum_digit, + "AC_move_along_path": _move_along_path, + "AC_drag_path": _drag_path, "AC_detect_drift": _detect_drift, "AC_categorical_drift": _categorical_drift, "AC_diff_rows": _diff_rows, diff --git a/je_auto_control/utils/mcp_server/tools/_factories.py b/je_auto_control/utils/mcp_server/tools/_factories.py index 4ce0f72b..cace7ba2 100644 --- a/je_auto_control/utils/mcp_server/tools/_factories.py +++ b/je_auto_control/utils/mcp_server/tools/_factories.py @@ -2550,6 +2550,42 @@ def tween_drag_tools() -> List[MCPTool]: ] +def mouse_path_tools() -> List[MCPTool]: + return [ + MCPTool( + name="ac_move_along_path", + description=("Move the pointer through 'waypoints' ([[x,y],...]) as " + "an eased polyline. 'per_segment_steps' + 'easing' " + "(linear / ease_*). Returns {points, path}."), + input_schema=schema({ + "waypoints": {"type": "array", + "items": {"type": "array", + "items": {"type": "integer"}}}, + "easing": {"type": "string"}, + "per_segment_steps": {"type": "integer"}}, + required=["waypoints"]), + handler=h.move_along_path, + annotations=SIDE_EFFECT_ONLY, + ), + MCPTool( + name="ac_drag_path", + description=("Press at the first of 'waypoints' ([[x,y],...]), drag " + "through them, release at the last. 'button', 'easing', " + "'per_segment_steps'. Returns {points, path}."), + input_schema=schema({ + "waypoints": {"type": "array", + "items": {"type": "array", + "items": {"type": "integer"}}}, + "button": {"type": "string"}, + "easing": {"type": "string"}, + "per_segment_steps": {"type": "integer"}}, + required=["waypoints"]), + handler=h.drag_path, + annotations=SIDE_EFFECT_ONLY, + ), + ] + + def plugin_sdk_tools() -> List[MCPTool]: _G = {"group": {"type": "string"}} return [ @@ -5795,7 +5831,8 @@ def media_assert_tools() -> List[MCPTool]: checkpoint_tools, set_of_marks_tools, screen_state_tools, input_macro_tools, resilience_tools, ci_annotation_tools, clipboard_history_tools, audit_analysis_tools, - process_doc_tools, tween_drag_tools, plugin_sdk_tools, governance_tools, + process_doc_tools, tween_drag_tools, mouse_path_tools, plugin_sdk_tools, + governance_tools, credential_lease_tools, egress_tools, approval_testing_tools, trajectory_eval_tools, compliance_tools, agent_trace_tools, video_report_tools, fuzzy_tools, artifact_store_tools, image_dedup_tools, diff --git a/je_auto_control/utils/mcp_server/tools/_handlers.py b/je_auto_control/utils/mcp_server/tools/_handlers.py index 6a8d7e8f..551a8bdf 100644 --- a/je_auto_control/utils/mcp_server/tools/_handlers.py +++ b/je_auto_control/utils/mcp_server/tools/_handlers.py @@ -2027,6 +2027,17 @@ def checksum_digit(scheme, partial): return _checksum_digit(scheme, partial) +def move_along_path(waypoints, easing="linear", per_segment_steps=20): + from je_auto_control.utils.executor.action_executor import _move_along_path + return _move_along_path(waypoints, easing, per_segment_steps) + + +def drag_path(waypoints, button="mouse_left", easing="linear", + per_segment_steps=20): + from je_auto_control.utils.executor.action_executor import _drag_path + return _drag_path(waypoints, button, easing, per_segment_steps) + + def detect_drift(reference, current, threshold=0.25, bins=10): from je_auto_control.utils.executor.action_executor import _detect_drift return _detect_drift(reference, current, threshold, bins) diff --git a/je_auto_control/utils/mouse_path/__init__.py b/je_auto_control/utils/mouse_path/__init__.py new file mode 100644 index 00000000..369cc9a1 --- /dev/null +++ b/je_auto_control/utils/mouse_path/__init__.py @@ -0,0 +1,6 @@ +"""Multi-waypoint mouse gestures (move or drag through a polyline of points).""" +from je_auto_control.utils.mouse_path.mouse_path import ( + drag_path, move_along_path, path_easings, plan_path, +) + +__all__ = ["drag_path", "move_along_path", "path_easings", "plan_path"] diff --git a/je_auto_control/utils/mouse_path/mouse_path.py b/je_auto_control/utils/mouse_path/mouse_path.py new file mode 100644 index 00000000..121ca93e --- /dev/null +++ b/je_auto_control/utils/mouse_path/mouse_path.py @@ -0,0 +1,86 @@ +"""Multi-waypoint mouse gestures: move or drag through a polyline of points. + +``humanize.humanized_path`` and ``tween_drag`` only interpolate a *single* +start -> end hop. Real gestures — signatures, marquee/rubber-band selections, +drag-through-multiple-drop-targets, shape gestures — need an arbitrary chain of +waypoints with the button optionally held down across the whole path. + +:func:`plan_path` is pure point math (reusing the named easings from +``tween_drag``) and is unit-testable on its own; :func:`move_along_path` and +:func:`drag_path` dispatch through an injectable ``sink`` so the gesture is +tested without real input. Imports no ``PySide6``. +""" +from typing import Any, Callable, Dict, List, Optional, Sequence + +from je_auto_control.utils.tween_drag.tween_drag import easing_names, tween_points + +Point = Sequence[int] +Sink = Callable[[Dict[str, Any]], None] + + +def plan_path(waypoints: Sequence[Point], *, easing: str = "linear", + per_segment_steps: int = 20) -> List[List[int]]: + """Return the eased point list passing through every waypoint in order. + + Each consecutive pair is interpolated with ``per_segment_steps`` eased steps + (named easings from ``tween_drag``); shared junction points are not + duplicated. Fewer than two waypoints yields the points unchanged. + """ + points: List[List[int]] = [] + if not waypoints: + return points + if len(waypoints) == 1: + first = waypoints[0] + return [[int(first[0]), int(first[1])]] + for index in range(len(waypoints) - 1): + segment = tween_points(waypoints[index], waypoints[index + 1], + per_segment_steps, easing) + points.extend(segment[1:] if index else segment) + return points + + +def _default_sink(event: Dict[str, Any]) -> None: + """Default dispatch: drive the real mouse backend.""" + from je_auto_control.wrapper.auto_control_mouse import ( + press_mouse, release_mouse, set_mouse_position) + x, y = int(event["x"]), int(event["y"]) + set_mouse_position(x, y) + op = event["op"] + if op == "press": + press_mouse(event.get("button", "mouse_left"), x, y) + elif op == "release": + release_mouse(event.get("button", "mouse_left"), x, y) + + +def move_along_path(waypoints: Sequence[Point], *, easing: str = "linear", + per_segment_steps: int = 20, + sink: Optional[Sink] = None) -> Dict[str, Any]: + """Move the pointer through ``waypoints`` (no button held).""" + points = plan_path(waypoints, easing=easing, + per_segment_steps=per_segment_steps) + dispatch = sink or _default_sink + for x, y in points: + dispatch({"op": "move", "x": x, "y": y}) + return {"points": len(points), "path": points} + + +def drag_path(waypoints: Sequence[Point], *, button: str = "mouse_left", + easing: str = "linear", per_segment_steps: int = 20, + sink: Optional[Sink] = None) -> Dict[str, Any]: + """Press at the first waypoint, move through the path, release at the last.""" + points = plan_path(waypoints, easing=easing, + per_segment_steps=per_segment_steps) + if not points: + return {"points": 0, "path": points} + dispatch = sink or _default_sink + first, last = points[0], points[-1] + dispatch({"op": "press", "button": button, "x": first[0], "y": first[1]}) + for x, y in points: + dispatch({"op": "move", "x": x, "y": y}) + dispatch({"op": "release", "button": button, "x": last[0], "y": last[1]}) + return {"points": len(points), "path": points} + + +def path_easings() -> List[str]: + """Return the available easing names (shared with ``tween_drag``).""" + return easing_names() diff --git a/test/unit_test/headless/test_mouse_path_batch.py b/test/unit_test/headless/test_mouse_path_batch.py new file mode 100644 index 00000000..62411d8a --- /dev/null +++ b/test/unit_test/headless/test_mouse_path_batch.py @@ -0,0 +1,77 @@ +"""Headless tests for multi-waypoint mouse gestures. No Qt.""" +import json + +import je_auto_control as ac +from je_auto_control.utils.mouse_path import ( + drag_path, move_along_path, path_easings, plan_path, +) + + +def test_plan_path_through_waypoints_dedups_junctions(): + points = plan_path([(0, 0), (10, 0), (10, 10)], per_segment_steps=5) + assert points[0] == [0, 0] and points[-1] == [10, 10] + assert len(points) == 11 # 5 + 5 + shared junction once + assert points.count([10, 0]) == 1 # junction not duplicated + + +def test_plan_path_single_and_empty(): + assert plan_path([(3, 4)]) == [[3, 4]] + assert plan_path([]) == [] + + +def test_move_along_path_emits_only_moves(): + events = [] + result = move_along_path([(0, 0), (2, 2)], per_segment_steps=2, + sink=events.append) + assert all(e["op"] == "move" for e in events) + assert result["points"] == len(events) == 3 + + +def test_drag_path_press_first_release_last(): + events = [] + drag_path([(0, 0), (5, 5), (5, 0)], per_segment_steps=2, + sink=events.append) + assert events[0]["op"] == "press" and (events[0]["x"], events[0]["y"]) == (0, 0) + assert events[-1]["op"] == "release" and (events[-1]["x"], events[-1]["y"]) == (5, 0) + assert [e["op"] for e in events[1:-1]] == ["move"] * (len(events) - 2) + + +def test_drag_path_empty_waypoints_noop(): + events = [] + result = drag_path([], sink=events.append) + assert result["points"] == 0 and events == [] + + +def test_easing_changes_intermediate_points(): + linear = plan_path([(0, 0), (100, 0)], easing="linear", per_segment_steps=10) + eased = plan_path([(0, 0), (100, 0)], easing="ease_in_cubic", + per_segment_steps=10) + assert linear[0] == eased[0] and linear[-1] == eased[-1] + assert linear[5] != eased[5] # different curve mid-path + assert "linear" in path_easings() + + +# --- wiring --------------------------------------------------------------- + +def test_executor_waypoint_coercion(): + # the executor's backend dispatch is device-bound; exercise the adapter's + # JSON-string -> list coercion (the part that runs before any real input). + from je_auto_control.utils.executor.action_executor import _waypoints + assert _waypoints(json.dumps([[0, 0], [4, 4]])) == [[0, 0], [4, 4]] + assert _waypoints([[1, 2]]) == [[1, 2]] # already a list + + +def test_wiring(): + known = ac.executor.known_commands() + assert {"AC_move_along_path", "AC_drag_path"} <= set(known) + from je_auto_control.utils.mcp_server.tools import build_default_tool_registry + names = {t.name for t in build_default_tool_registry()} + assert {"ac_move_along_path", "ac_drag_path"} <= names + from je_auto_control.gui.script_builder.command_schema import _build_specs + specs = {s.command for s in _build_specs()} + assert {"AC_move_along_path", "AC_drag_path"} <= specs + + +def test_facade_exports(): + for attr in ("plan_path", "move_along_path", "drag_path", "path_easings"): + assert hasattr(ac, attr) and attr in ac.__all__ From c4e140d5daeac1d3dfe66a42e3ba2d45d03f81fd Mon Sep 17 00:00:00 2001 From: JeffreyChen Date: Tue, 23 Jun 2026 00:04:09 +0800 Subject: [PATCH 04/39] Add clear-then-type field entry (Playwright fill idiom) --- README/WHATS_NEW_zh-CN.md | 6 ++ README/WHATS_NEW_zh-TW.md | 6 ++ WHATS_NEW.md | 6 ++ .../doc/new_features/v117_features_doc.rst | 43 +++++++++++ docs/source/Eng/eng_index.rst | 1 + .../Zh/doc/new_features/v117_features_doc.rst | 37 ++++++++++ docs/source/Zh/zh_index.rst | 1 + je_auto_control/__init__.py | 4 ++ .../gui/script_builder/command_schema.py | 12 ++++ .../utils/executor/action_executor.py | 9 +++ je_auto_control/utils/field_entry/__init__.py | 6 ++ .../utils/field_entry/field_entry.py | 70 ++++++++++++++++++ .../utils/mcp_server/tools/_factories.py | 22 +++++- .../utils/mcp_server/tools/_handlers.py | 5 ++ .../headless/test_field_entry_batch.py | 71 +++++++++++++++++++ 15 files changed, 297 insertions(+), 2 deletions(-) create mode 100644 docs/source/Eng/doc/new_features/v117_features_doc.rst create mode 100644 docs/source/Zh/doc/new_features/v117_features_doc.rst create mode 100644 je_auto_control/utils/field_entry/__init__.py create mode 100644 je_auto_control/utils/field_entry/field_entry.py create mode 100644 test/unit_test/headless/test_field_entry_batch.py diff --git a/README/WHATS_NEW_zh-CN.md b/README/WHATS_NEW_zh-CN.md index 1a4c438c..b492e979 100644 --- a/README/WHATS_NEW_zh-CN.md +++ b/README/WHATS_NEW_zh-CN.md @@ -1,5 +1,11 @@ # 本次更新 — AutoControl +## 本次更新 (2026-06-23) — 清空再输入字段 + +可靠地设定文本字段的值(Playwright 的 `fill` 惯用法)。完整参考:[`docs/source/Zh/doc/new_features/v117_features_doc.rst`](../docs/source/Zh/doc/new_features/v117_features_doc.rst)。 + +- **`set_field_text` / `plan_field_set`**(`AC_set_field_text`):先前没有单一的「聚焦 → 清空 → 设值」基本元件,且 `write` 对 emoji/CJK 会抛异常。本功能清空字段(全选 + 删除)后再输入文本——可选择通过剪贴板(`paste=True`),这是 `write` 无法处理之 Unicode 的安全途径。`modifier` 为平台命令键(`ctrl`/`command`)。纯计划 + 可注入 sink、确定。 + ## 本次更新 (2026-06-22) — 多路径点鼠标手势 让指针沿着路径点折线移动或拖曳。完整参考:[`docs/source/Zh/doc/new_features/v116_features_doc.rst`](../docs/source/Zh/doc/new_features/v116_features_doc.rst)。 diff --git a/README/WHATS_NEW_zh-TW.md b/README/WHATS_NEW_zh-TW.md index 1e5aeb88..c50b73dc 100644 --- a/README/WHATS_NEW_zh-TW.md +++ b/README/WHATS_NEW_zh-TW.md @@ -1,5 +1,11 @@ # 本次更新 — AutoControl +## 本次更新 (2026-06-23) — 清空再輸入欄位 + +可靠地設定文字欄位的值(Playwright 的 `fill` 慣用法)。完整參考:[`docs/source/Zh/doc/new_features/v117_features_doc.rst`](../docs/source/Zh/doc/new_features/v117_features_doc.rst)。 + +- **`set_field_text` / `plan_field_set`**(`AC_set_field_text`):先前沒有單一的「聚焦 → 清空 → 設值」基本元件,且 `write` 對 emoji/CJK 會拋例外。本功能清空欄位(全選 + 刪除)後再輸入文字——可選擇透過剪貼簿(`paste=True`),這是 `write` 無法處理之 Unicode 的安全途徑。`modifier` 為平台指令鍵(`ctrl`/`command`)。純計畫 + 可注入 sink、具決定性。 + ## 本次更新 (2026-06-22) — 多路徑點滑鼠手勢 讓指標沿著路徑點折線移動或拖曳。完整參考:[`docs/source/Zh/doc/new_features/v116_features_doc.rst`](../docs/source/Zh/doc/new_features/v116_features_doc.rst)。 diff --git a/WHATS_NEW.md b/WHATS_NEW.md index 2015723d..78991a24 100644 --- a/WHATS_NEW.md +++ b/WHATS_NEW.md @@ -1,5 +1,11 @@ # What's New — AutoControl +## What's new (2026-06-23) — Clear-Then-Type Field Entry + +Reliably set a text field's value (the Playwright `fill` idiom). Full reference: [`docs/source/Eng/doc/new_features/v117_features_doc.rst`](docs/source/Eng/doc/new_features/v117_features_doc.rst). + +- **`set_field_text` / `plan_field_set`** (`AC_set_field_text`): there was no single "focus → clear → set value" primitive, and `write` raises on emoji/CJK. This clears the field (select-all + delete) then enters the text — optionally via the clipboard (`paste=True`) which is the Unicode-safe path `write` can't do. `modifier` is the platform command key (`ctrl`/`command`). Pure-planning + injectable sink, deterministic. + ## What's new (2026-06-22) — Multi-Waypoint Mouse Gestures Move or drag the pointer through a polyline of waypoints. Full reference: [`docs/source/Eng/doc/new_features/v116_features_doc.rst`](docs/source/Eng/doc/new_features/v116_features_doc.rst). diff --git a/docs/source/Eng/doc/new_features/v117_features_doc.rst b/docs/source/Eng/doc/new_features/v117_features_doc.rst new file mode 100644 index 00000000..e36f20c5 --- /dev/null +++ b/docs/source/Eng/doc/new_features/v117_features_doc.rst @@ -0,0 +1,43 @@ +Clear-Then-Type Field Entry +=========================== + +Setting a field's value reliably means *clearing* whatever is there first, then +entering the new text — otherwise automation appends to or corrupts the existing +content. The framework has ``write`` (types, but raises on emoji / CJK / chars +outside the layout table) and ``set_clipboard`` / ``hotkey`` separately, but no +single "focus → clear → set value" primitive and no paste strategy for text that +``write`` cannot type. This adds the Playwright ``fill`` idiom. + +:func:`plan_field_set` builds the deterministic op-plan (pure, unit-testable); +:func:`set_field_text` dispatches it through an injectable ``sink`` so it is +tested without real input. Imports no ``PySide6``. + +Headless API +------------ + +.. code-block:: python + + from je_auto_control import set_field_text, plan_field_set + + set_field_text("new value") # select-all, delete, type + set_field_text("café 🚀", paste=True) # via clipboard (Unicode-safe) + set_field_text("appended", clear="none") # no clear, just type + set_field_text("値", paste=True, modifier="command") # macOS + + plan_field_set("hi") + # [{'op': 'hotkey', 'keys': ['ctrl', 'a']}, + # {'op': 'key', 'key': 'delete'}, + # {'op': 'type', 'text': 'hi'}] + +``clear`` is ``"select_all"`` (the ``modifier``+A then Delete clear) or +``"none"``. ``paste=True`` enters the text through the clipboard (``modifier``+V) +— the reliable path for Unicode / emoji / CJK that ``write`` cannot type — rather +than typing key by key. ``modifier`` is the platform command key (``"ctrl"``; use +``"command"`` on macOS). An unknown ``clear`` mode raises ``ValueError``. + +Executor commands +----------------- + +``AC_set_field_text`` takes ``text`` plus ``clear`` / ``paste`` / ``modifier`` and +returns ``{ops, plan}``. It is exposed as the MCP tool ``ac_set_field_text`` and +as a Script Builder command under **Keyboard**. diff --git a/docs/source/Eng/eng_index.rst b/docs/source/Eng/eng_index.rst index 6e8a2e24..13d9a99c 100644 --- a/docs/source/Eng/eng_index.rst +++ b/docs/source/Eng/eng_index.rst @@ -139,6 +139,7 @@ Comprehensive guides for all AutoControl features. doc/new_features/v114_features_doc doc/new_features/v115_features_doc doc/new_features/v116_features_doc + doc/new_features/v117_features_doc doc/ocr_backends/ocr_backends_doc doc/observability/observability_doc doc/operations_layer/operations_layer_doc diff --git a/docs/source/Zh/doc/new_features/v117_features_doc.rst b/docs/source/Zh/doc/new_features/v117_features_doc.rst new file mode 100644 index 00000000..4c527bda --- /dev/null +++ b/docs/source/Zh/doc/new_features/v117_features_doc.rst @@ -0,0 +1,37 @@ +清空再輸入欄位 +============== + +可靠地設定欄位值,必須先*清空*既有內容再輸入新文字——否則自動化會附加到或破壞既有內容。框架分別有 ``write`` +(可輸入,但對 emoji / CJK / 不在版面表內的字元會拋例外)與 ``set_clipboard`` / ``hotkey``,但沒有單一的 +「聚焦 → 清空 → 設值」基本元件,也沒有可輸入 ``write`` 無法處理之文字的貼上策略。本功能加入 Playwright 的 +``fill`` 慣用法。 + +:func:`plan_field_set` 建立決定性的操作計畫(純函式、可單元測試);:func:`set_field_text` 透過可注入的 ``sink`` +派發,因此可在無真實輸入下測試。不匯入 ``PySide6``。 + +無頭 API +-------- + +.. code-block:: python + + from je_auto_control import set_field_text, plan_field_set + + set_field_text("new value") # 全選、刪除、輸入 + set_field_text("café 🚀", paste=True) # 透過剪貼簿(Unicode 安全) + set_field_text("appended", clear="none") # 不清空,直接輸入 + set_field_text("値", paste=True, modifier="command") # macOS + + plan_field_set("hi") + # [{'op': 'hotkey', 'keys': ['ctrl', 'a']}, + # {'op': 'key', 'key': 'delete'}, + # {'op': 'type', 'text': 'hi'}] + +``clear`` 為 ``"select_all"``(``modifier``+A 再 Delete 的清空)或 ``"none"``。``paste=True`` 透過剪貼簿 +(``modifier``+V)輸入文字——這是 ``write`` 無法輸入之 Unicode / emoji / CJK 的可靠途徑——而非逐鍵輸入。 +``modifier`` 為平台指令鍵(``"ctrl"``;macOS 用 ``"command"``)。未知的 ``clear`` 模式會拋出 ``ValueError``。 + +執行器命令 +---------- + +``AC_set_field_text`` 接受 ``text`` 以及 ``clear`` / ``paste`` / ``modifier``,並回傳 ``{ops, plan}``。它以 +MCP 工具 ``ac_set_field_text`` 以及 Script Builder 中 **Keyboard** 分類下的命令提供。 diff --git a/docs/source/Zh/zh_index.rst b/docs/source/Zh/zh_index.rst index a69899e9..705a2ca8 100644 --- a/docs/source/Zh/zh_index.rst +++ b/docs/source/Zh/zh_index.rst @@ -139,6 +139,7 @@ AutoControl 所有功能的完整使用指南。 doc/new_features/v114_features_doc doc/new_features/v115_features_doc doc/new_features/v116_features_doc + doc/new_features/v117_features_doc doc/ocr_backends/ocr_backends_doc doc/observability/observability_doc doc/operations_layer/operations_layer_doc diff --git a/je_auto_control/__init__.py b/je_auto_control/__init__.py index 38de2b16..3b5276d3 100644 --- a/je_auto_control/__init__.py +++ b/je_auto_control/__init__.py @@ -255,6 +255,8 @@ from je_auto_control.utils.mouse_path import ( drag_path, move_along_path, path_easings, plan_path, ) +# Clear-then-type a text field (Playwright `fill` idiom; paste for Unicode) +from je_auto_control.utils.field_entry import plan_field_set, set_field_text # CI workflow annotations (GitHub Actions) from je_auto_control.utils.ci_annotations import ( emit_annotations, format_annotation, @@ -1029,6 +1031,8 @@ def start_autocontrol_gui(*args, **kwargs): "move_along_path", "drag_path", "path_easings", + "plan_field_set", + "set_field_text", "emit_annotations", "format_annotation", "ClipboardHistory", "default_clipboard_history", "analyze_heal_log", "heal_stats", "scan_secrets", diff --git a/je_auto_control/gui/script_builder/command_schema.py b/je_auto_control/gui/script_builder/command_schema.py index 77c687f4..852d35a2 100644 --- a/je_auto_control/gui/script_builder/command_schema.py +++ b/je_auto_control/gui/script_builder/command_schema.py @@ -162,6 +162,18 @@ def _add_keyboard_specs(specs: List[CommandSpec]) -> None: ), description="Type text with randomized per-key delays.", )) + specs.append(CommandSpec( + "AC_set_field_text", "Keyboard", "Set Field Text", + fields=( + FieldSpec("text", FieldType.STRING, placeholder="new value"), + FieldSpec("clear", FieldType.ENUM, choices=("select_all", "none"), + optional=True, default="select_all"), + FieldSpec("paste", FieldType.BOOL, optional=True, default=False), + FieldSpec("modifier", FieldType.STRING, optional=True, + default="ctrl", placeholder="ctrl | command"), + ), + description="Clear the focused field then enter text (paste for Unicode).", + )) specs.append(CommandSpec( "AC_hotkey", "Keyboard", "Hotkey", fields=( diff --git a/je_auto_control/utils/executor/action_executor.py b/je_auto_control/utils/executor/action_executor.py index 1c88529c..87b81248 100644 --- a/je_auto_control/utils/executor/action_executor.py +++ b/je_auto_control/utils/executor/action_executor.py @@ -3092,6 +3092,14 @@ def _drag_path(waypoints: Any, button: str = "mouse_left", per_segment_steps=int(per_segment_steps)) +def _set_field_text(text: str, clear: str = "select_all", paste: Any = False, + modifier: str = "ctrl") -> Dict[str, Any]: + """Adapter: clear the focused field and enter text.""" + from je_auto_control.utils.field_entry import set_field_text + return set_field_text(text, clear=clear, paste=bool(paste), + modifier=modifier) + + def _cas_put(name: str, key: str, value: Any, expected_version: Any = None) -> Dict[str, Any]: """Adapter: optimistic put into a named versioned store.""" @@ -4789,6 +4797,7 @@ def __init__(self): "AC_checksum_digit": _checksum_digit, "AC_move_along_path": _move_along_path, "AC_drag_path": _drag_path, + "AC_set_field_text": _set_field_text, "AC_detect_drift": _detect_drift, "AC_categorical_drift": _categorical_drift, "AC_diff_rows": _diff_rows, diff --git a/je_auto_control/utils/field_entry/__init__.py b/je_auto_control/utils/field_entry/__init__.py new file mode 100644 index 00000000..be31039e --- /dev/null +++ b/je_auto_control/utils/field_entry/__init__.py @@ -0,0 +1,6 @@ +"""Clear-then-type a text field (the Playwright ``fill`` idiom).""" +from je_auto_control.utils.field_entry.field_entry import ( + plan_field_set, set_field_text, +) + +__all__ = ["plan_field_set", "set_field_text"] diff --git a/je_auto_control/utils/field_entry/field_entry.py b/je_auto_control/utils/field_entry/field_entry.py new file mode 100644 index 00000000..991c30d4 --- /dev/null +++ b/je_auto_control/utils/field_entry/field_entry.py @@ -0,0 +1,70 @@ +"""Clear-then-type a text field (the Playwright ``fill`` idiom). + +Setting a field's value reliably means *clearing* whatever is there first, then +entering the new text — otherwise automation appends to or corrupts the existing +content. The framework has ``write`` (types, but raises on emoji / CJK / chars +outside the layout table) and ``set_clipboard`` / ``hotkey`` separately, but no +single "focus → clear → set value" primitive, and no paste strategy for text +``write`` cannot type. + +:func:`plan_field_set` builds the deterministic op-plan (pure, unit-testable); +:func:`set_field_text` dispatches it through an injectable ``sink`` so it is +tested without real input. Imports no ``PySide6``. +""" +from typing import Any, Callable, Dict, List, Optional + +_CLEAR_MODES = ("select_all", "none") +Sink = Callable[[Dict[str, Any]], None] + + +def plan_field_set(text: str, *, clear: str = "select_all", paste: bool = False, + modifier: str = "ctrl") -> List[Dict[str, Any]]: + """Return the op-plan to set a focused field to ``text``. + + ``clear`` is ``"select_all"`` (``modifier``+A then Delete) or ``"none"``. + ``paste=True`` enters the text via the clipboard (``modifier``+V) — the + reliable path for Unicode / emoji / CJK that ``write`` cannot type — instead + of typing it key by key. ``modifier`` is the platform command key + (``"ctrl"``; use ``"command"`` on macOS). Raises ``ValueError`` on an unknown + ``clear`` mode. + """ + if clear not in _CLEAR_MODES: + raise ValueError(f"unknown clear mode: {clear!r}") + plan: List[Dict[str, Any]] = [] + if clear == "select_all": + plan.append({"op": "hotkey", "keys": [modifier, "a"]}) + plan.append({"op": "key", "key": "delete"}) + if paste: + plan.append({"op": "set_clipboard", "text": text}) + plan.append({"op": "hotkey", "keys": [modifier, "v"]}) + else: + plan.append({"op": "type", "text": text}) + return plan + + +def _default_sink(event: Dict[str, Any]) -> None: + """Default dispatch: drive the real keyboard / clipboard backend.""" + op = event["op"] + if op == "hotkey": + from je_auto_control.wrapper.auto_control_keyboard import hotkey + hotkey(list(event["keys"])) + elif op == "key": + from je_auto_control.wrapper.auto_control_keyboard import type_keyboard + type_keyboard(event["key"]) + elif op == "type": + from je_auto_control.wrapper.auto_control_keyboard import write + write(event["text"]) + elif op == "set_clipboard": + from je_auto_control.utils.clipboard.clipboard import set_clipboard + set_clipboard(event["text"]) + + +def set_field_text(text: str, *, clear: str = "select_all", paste: bool = False, + modifier: str = "ctrl", + sink: Optional[Sink] = None) -> Dict[str, Any]: + """Clear the focused field and enter ``text``; return the dispatched plan.""" + plan = plan_field_set(text, clear=clear, paste=paste, modifier=modifier) + dispatch = sink or _default_sink + for event in plan: + dispatch(event) + return {"ops": len(plan), "plan": plan} diff --git a/je_auto_control/utils/mcp_server/tools/_factories.py b/je_auto_control/utils/mcp_server/tools/_factories.py index cace7ba2..32a2f6d8 100644 --- a/je_auto_control/utils/mcp_server/tools/_factories.py +++ b/je_auto_control/utils/mcp_server/tools/_factories.py @@ -2550,6 +2550,24 @@ def tween_drag_tools() -> List[MCPTool]: ] +def field_entry_tools() -> List[MCPTool]: + return [ + MCPTool( + name="ac_set_field_text", + description=("Clear the focused field and enter 'text' (Playwright " + "fill). 'clear' select_all|none; 'paste' True for " + "Unicode/emoji via clipboard; 'modifier' ctrl|command. " + "Returns {ops, plan}."), + input_schema=schema({ + "text": {"type": "string"}, "clear": {"type": "string"}, + "paste": {"type": "boolean"}, "modifier": {"type": "string"}}, + required=["text"]), + handler=h.set_field_text, + annotations=SIDE_EFFECT_ONLY, + ), + ] + + def mouse_path_tools() -> List[MCPTool]: return [ MCPTool( @@ -5831,8 +5849,8 @@ def media_assert_tools() -> List[MCPTool]: checkpoint_tools, set_of_marks_tools, screen_state_tools, input_macro_tools, resilience_tools, ci_annotation_tools, clipboard_history_tools, audit_analysis_tools, - process_doc_tools, tween_drag_tools, mouse_path_tools, plugin_sdk_tools, - governance_tools, + process_doc_tools, tween_drag_tools, mouse_path_tools, field_entry_tools, + plugin_sdk_tools, governance_tools, credential_lease_tools, egress_tools, approval_testing_tools, trajectory_eval_tools, compliance_tools, agent_trace_tools, video_report_tools, fuzzy_tools, artifact_store_tools, image_dedup_tools, diff --git a/je_auto_control/utils/mcp_server/tools/_handlers.py b/je_auto_control/utils/mcp_server/tools/_handlers.py index 551a8bdf..9be960c7 100644 --- a/je_auto_control/utils/mcp_server/tools/_handlers.py +++ b/je_auto_control/utils/mcp_server/tools/_handlers.py @@ -2038,6 +2038,11 @@ def drag_path(waypoints, button="mouse_left", easing="linear", return _drag_path(waypoints, button, easing, per_segment_steps) +def set_field_text(text, clear="select_all", paste=False, modifier="ctrl"): + from je_auto_control.utils.executor.action_executor import _set_field_text + return _set_field_text(text, clear, paste, modifier) + + def detect_drift(reference, current, threshold=0.25, bins=10): from je_auto_control.utils.executor.action_executor import _detect_drift return _detect_drift(reference, current, threshold, bins) diff --git a/test/unit_test/headless/test_field_entry_batch.py b/test/unit_test/headless/test_field_entry_batch.py new file mode 100644 index 00000000..881b4786 --- /dev/null +++ b/test/unit_test/headless/test_field_entry_batch.py @@ -0,0 +1,71 @@ +"""Headless tests for clear-then-type field entry. No Qt.""" +import je_auto_control as ac +from je_auto_control.utils.field_entry import plan_field_set, set_field_text + + +def test_default_plan_clears_then_types(): + plan = plan_field_set("hello") + assert plan == [ + {"op": "hotkey", "keys": ["ctrl", "a"]}, + {"op": "key", "key": "delete"}, + {"op": "type", "text": "hello"}, + ] + + +def test_paste_plan_uses_clipboard(): + plan = plan_field_set("café 🚀", paste=True) + ops = [event["op"] for event in plan] + assert ops == ["hotkey", "key", "set_clipboard", "hotkey"] + assert plan[2] == {"op": "set_clipboard", "text": "café 🚀"} + assert plan[-1]["keys"] == ["ctrl", "v"] + + +def test_clear_none_skips_clear(): + assert plan_field_set("x", clear="none") == [{"op": "type", "text": "x"}] + + +def test_modifier_override_for_macos(): + plan = plan_field_set("y", modifier="command") + assert plan[0]["keys"] == ["command", "a"] + + +def test_unknown_clear_raises(): + try: + plan_field_set("z", clear="bogus") + except ValueError as error: + assert "clear" in str(error) + else: # pragma: no cover + raise AssertionError("expected ValueError") + + +def test_set_field_text_dispatches_plan(): + events = [] + result = set_field_text("hi", sink=events.append) + assert result["ops"] == 3 + assert [e["op"] for e in events] == ["hotkey", "key", "type"] + + +# --- wiring --------------------------------------------------------------- + +def test_executor_adapter_planning(): + # the executor's default dispatch is device-bound; exercise the adapter + # planning + JSON-free path by calling the underlying with an injected sink. + events = [] + set_field_text("v", paste=True, sink=events.append) + assert events[-1] == {"op": "hotkey", "keys": ["ctrl", "v"]} + + +def test_wiring(): + known = ac.executor.known_commands() + assert "AC_set_field_text" in set(known) + from je_auto_control.utils.mcp_server.tools import build_default_tool_registry + names = {t.name for t in build_default_tool_registry()} + assert "ac_set_field_text" in names + from je_auto_control.gui.script_builder.command_schema import _build_specs + specs = {s.command for s in _build_specs()} + assert "AC_set_field_text" in specs + + +def test_facade_exports(): + for attr in ("plan_field_set", "set_field_text"): + assert hasattr(ac, attr) and attr in ac.__all__ From e16f409a070df742f471bbacd630f4f683147b84 Mon Sep 17 00:00:00 2001 From: JeffreyChen Date: Tue, 23 Jun 2026 00:18:45 +0800 Subject: [PATCH 05/39] Add blocking wait-until-vanish (wait_until_gone / image / text) --- README/WHATS_NEW_zh-CN.md | 6 ++ README/WHATS_NEW_zh-TW.md | 6 ++ WHATS_NEW.md | 6 ++ .../doc/new_features/v118_features_doc.rst | 41 +++++++++++ docs/source/Eng/eng_index.rst | 1 + .../Zh/doc/new_features/v118_features_doc.rst | 35 ++++++++++ docs/source/Zh/zh_index.rst | 1 + je_auto_control/__init__.py | 6 +- .../gui/script_builder/command_schema.py | 24 +++++++ .../utils/executor/action_executor.py | 25 +++++++ .../utils/mcp_server/tools/_factories.py | 28 ++++++++ .../utils/mcp_server/tools/_handlers.py | 15 ++++ je_auto_control/utils/smart_waits/__init__.py | 13 ++-- je_auto_control/utils/smart_waits/waits.py | 68 +++++++++++++++++++ .../headless/test_wait_gone_batch.py | 68 +++++++++++++++++++ 15 files changed, 335 insertions(+), 8 deletions(-) create mode 100644 docs/source/Eng/doc/new_features/v118_features_doc.rst create mode 100644 docs/source/Zh/doc/new_features/v118_features_doc.rst create mode 100644 test/unit_test/headless/test_wait_gone_batch.py diff --git a/README/WHATS_NEW_zh-CN.md b/README/WHATS_NEW_zh-CN.md index b492e979..76883031 100644 --- a/README/WHATS_NEW_zh-CN.md +++ b/README/WHATS_NEW_zh-CN.md @@ -1,5 +1,11 @@ # 本次更新 — AutoControl +## 本次更新 (2026-06-23) — 等待消失(阻塞式 vanish 等待) + +阻塞直到转圈圈 / toast / 对话框消失。完整参考:[`docs/source/Zh/doc/new_features/v118_features_doc.rst`](../docs/source/Zh/doc/new_features/v118_features_doc.rst)。 + +- **`wait_until_gone` / `wait_until_image_gone` / `wait_until_text_gone`**(`AC_wait_image_gone`、`AC_wait_text_gone`):`wait_for_image`/`wait_for_text` 只阻塞到某物*出现*,`observer` 则以异步回调在消失时触发——先前没有*阻塞式*的「等到此图像/文本消失再继续」。通用的 `wait_until_gone` 接受任意谓词(可无头测试);图像/文本辅助函数从定位函数建立。`gone_for_s` 可消抖。返回 `WaitOutcome`。纯标准库。 + ## 本次更新 (2026-06-23) — 清空再输入字段 可靠地设定文本字段的值(Playwright 的 `fill` 惯用法)。完整参考:[`docs/source/Zh/doc/new_features/v117_features_doc.rst`](../docs/source/Zh/doc/new_features/v117_features_doc.rst)。 diff --git a/README/WHATS_NEW_zh-TW.md b/README/WHATS_NEW_zh-TW.md index c50b73dc..ec7eb96f 100644 --- a/README/WHATS_NEW_zh-TW.md +++ b/README/WHATS_NEW_zh-TW.md @@ -1,5 +1,11 @@ # 本次更新 — AutoControl +## 本次更新 (2026-06-23) — 等待消失(阻塞式 vanish 等待) + +阻塞直到轉圈圈 / toast / 對話框消失。完整參考:[`docs/source/Zh/doc/new_features/v118_features_doc.rst`](../docs/source/Zh/doc/new_features/v118_features_doc.rst)。 + +- **`wait_until_gone` / `wait_until_image_gone` / `wait_until_text_gone`**(`AC_wait_image_gone`、`AC_wait_text_gone`):`wait_for_image`/`wait_for_text` 只阻塞到某物*出現*,`observer` 則以非同步回呼在消失時觸發——先前沒有*阻塞式*的「等到此影像/文字消失再繼續」。通用的 `wait_until_gone` 接受任意述詞(可無頭測試);影像/文字輔助函式從定位函式建立。`gone_for_s` 可消抖。回傳 `WaitOutcome`。純標準函式庫。 + ## 本次更新 (2026-06-23) — 清空再輸入欄位 可靠地設定文字欄位的值(Playwright 的 `fill` 慣用法)。完整參考:[`docs/source/Zh/doc/new_features/v117_features_doc.rst`](../docs/source/Zh/doc/new_features/v117_features_doc.rst)。 diff --git a/WHATS_NEW.md b/WHATS_NEW.md index 78991a24..1c299998 100644 --- a/WHATS_NEW.md +++ b/WHATS_NEW.md @@ -1,5 +1,11 @@ # What's New — AutoControl +## What's new (2026-06-23) — Wait Until Gone (Blocking Vanish Waits) + +Block until a spinner / toast / dialog disappears. Full reference: [`docs/source/Eng/doc/new_features/v118_features_doc.rst`](docs/source/Eng/doc/new_features/v118_features_doc.rst). + +- **`wait_until_gone` / `wait_until_image_gone` / `wait_until_text_gone`** (`AC_wait_image_gone`, `AC_wait_text_gone`): `wait_for_image`/`wait_for_text` only block until something *appears*, and `observer` fires async callbacks on vanish — there was no *blocking* "wait until this image/text disappears then continue" call. The generic `wait_until_gone` takes any predicate (headless-testable); the image/text helpers build it from the locate functions. `gone_for_s` debounces flicker. Returns a `WaitOutcome`. Pure-stdlib. + ## What's new (2026-06-23) — Clear-Then-Type Field Entry Reliably set a text field's value (the Playwright `fill` idiom). Full reference: [`docs/source/Eng/doc/new_features/v117_features_doc.rst`](docs/source/Eng/doc/new_features/v117_features_doc.rst). diff --git a/docs/source/Eng/doc/new_features/v118_features_doc.rst b/docs/source/Eng/doc/new_features/v118_features_doc.rst new file mode 100644 index 00000000..5b6b3c88 --- /dev/null +++ b/docs/source/Eng/doc/new_features/v118_features_doc.rst @@ -0,0 +1,41 @@ +Wait Until Gone (Blocking Vanish Waits) +======================================= + +``wait_for_image`` / ``wait_for_text`` block until something *appears*, and the +``observer`` fires async callbacks on vanish — but there was no *blocking* "wait +until this spinner / toast / dialog **disappears** then continue" call for an +image or text. ``wait_until_window_closed`` covers windows only. This adds the +missing vanish waits to the ``smart_waits`` family. + +The generic :func:`wait_until_gone` takes any predicate, so its loop is +headless-testable without a real screen; the image / text helpers build that +predicate from the locate functions. Imports no ``PySide6``. + +Headless API +------------ + +.. code-block:: python + + from je_auto_control import ( + wait_until_gone, wait_until_image_gone, wait_until_text_gone, + ) + + # generic: wait until any predicate has been falsey + wait_until_gone(lambda: spinner_is_visible(), timeout_s=15) + + wait_until_image_gone("spinner.png", timeout_s=15) # image left the screen + wait_until_text_gone("Loading...", timeout_s=15) # OCR text disappeared + +Each returns a ``WaitOutcome`` (``succeeded`` / ``reason`` / ``elapsed_s`` / +``samples_taken``) — the same result type as the other smart waits. ``gone_for_s`` +requires the target to stay absent for that long before succeeding (debounces a +flickering element); ``poll_interval_s`` / ``timeout_s`` bound the loop. + +Executor commands +----------------- + +``AC_wait_image_gone`` and ``AC_wait_text_gone`` take the target plus +``timeout_s`` / ``poll_interval_s`` / ``gone_for_s`` (and ``detect_threshold`` for +the image) and return the ``WaitOutcome`` dict. Both are exposed as MCP tools +(``ac_wait_image_gone`` / ``ac_wait_text_gone``) and as Script Builder commands +under **Flow**. diff --git a/docs/source/Eng/eng_index.rst b/docs/source/Eng/eng_index.rst index 13d9a99c..b9203ae9 100644 --- a/docs/source/Eng/eng_index.rst +++ b/docs/source/Eng/eng_index.rst @@ -140,6 +140,7 @@ Comprehensive guides for all AutoControl features. doc/new_features/v115_features_doc doc/new_features/v116_features_doc doc/new_features/v117_features_doc + doc/new_features/v118_features_doc doc/ocr_backends/ocr_backends_doc doc/observability/observability_doc doc/operations_layer/operations_layer_doc diff --git a/docs/source/Zh/doc/new_features/v118_features_doc.rst b/docs/source/Zh/doc/new_features/v118_features_doc.rst new file mode 100644 index 00000000..d39d4d59 --- /dev/null +++ b/docs/source/Zh/doc/new_features/v118_features_doc.rst @@ -0,0 +1,35 @@ +等待消失(阻塞式 vanish 等待) +============================== + +``wait_for_image`` / ``wait_for_text`` 會阻塞直到某物*出現*,``observer`` 則以非同步回呼在消失時觸發——但先前 +沒有針對影像或文字的*阻塞式*「等到這個轉圈圈 / toast / 對話框**消失**再繼續」呼叫。``wait_until_window_closed`` +只涵蓋視窗。本功能為 ``smart_waits`` 家族補上缺少的 vanish 等待。 + +通用的 :func:`wait_until_gone` 接受任意述詞,因此其迴圈可在無真實螢幕下做無頭測試;影像 / 文字輔助函式則 +從定位函式建立該述詞。不匯入 ``PySide6``。 + +無頭 API +-------- + +.. code-block:: python + + from je_auto_control import ( + wait_until_gone, wait_until_image_gone, wait_until_text_gone, + ) + + # 通用:等到任意述詞變為 falsey + wait_until_gone(lambda: spinner_is_visible(), timeout_s=15) + + wait_until_image_gone("spinner.png", timeout_s=15) # 影像離開螢幕 + wait_until_text_gone("Loading...", timeout_s=15) # OCR 文字消失 + +每個皆回傳 ``WaitOutcome``(``succeeded`` / ``reason`` / ``elapsed_s`` / ``samples_taken``)——與其他 smart +waits 相同的結果型別。``gone_for_s`` 要求目標需持續缺席該段時間才算成功(可消抖閃爍的元素); +``poll_interval_s`` / ``timeout_s`` 界定迴圈。 + +執行器命令 +---------- + +``AC_wait_image_gone`` 與 ``AC_wait_text_gone`` 接受目標以及 ``timeout_s`` / ``poll_interval_s`` / +``gone_for_s``(影像另含 ``detect_threshold``),並回傳 ``WaitOutcome`` dict。兩者皆以 MCP 工具 +(``ac_wait_image_gone`` / ``ac_wait_text_gone``)以及 Script Builder 中 **Flow** 分類下的命令提供。 diff --git a/docs/source/Zh/zh_index.rst b/docs/source/Zh/zh_index.rst index 705a2ca8..a1152f4c 100644 --- a/docs/source/Zh/zh_index.rst +++ b/docs/source/Zh/zh_index.rst @@ -140,6 +140,7 @@ AutoControl 所有功能的完整使用指南。 doc/new_features/v115_features_doc doc/new_features/v116_features_doc doc/new_features/v117_features_doc + doc/new_features/v118_features_doc doc/ocr_backends/ocr_backends_doc doc/observability/observability_doc doc/operations_layer/operations_layer_doc diff --git a/je_auto_control/__init__.py b/je_auto_control/__init__.py index 3b5276d3..1494b92b 100644 --- a/je_auto_control/__init__.py +++ b/je_auto_control/__init__.py @@ -605,8 +605,9 @@ # Smart waits (frame-diff replacements for time.sleep) from je_auto_control.utils.smart_waits import ( WaitOutcome, wait_until_clipboard_changes, wait_until_file, - wait_until_pixel_changes, wait_until_port, wait_until_process, - wait_until_region_idle, wait_until_screen_stable, wait_until_window_closed, + wait_until_gone, wait_until_image_gone, wait_until_pixel_changes, + wait_until_port, wait_until_process, wait_until_region_idle, + wait_until_screen_stable, wait_until_text_gone, wait_until_window_closed, ) # Visual regression (golden-image comparison) from je_auto_control.utils.visual_regression import ( @@ -1230,6 +1231,7 @@ def start_autocontrol_gui(*args, **kwargs): "wait_until_region_idle", "wait_until_screen_stable", "wait_until_clipboard_changes", "wait_until_window_closed", "wait_until_file", "wait_until_port", "wait_until_process", + "wait_until_gone", "wait_until_image_gone", "wait_until_text_gone", # Visual regression + state machine "take_golden", "compare_to_golden", "image_difference", "DiffResult", "MaskRegion", diff --git a/je_auto_control/gui/script_builder/command_schema.py b/je_auto_control/gui/script_builder/command_schema.py index 852d35a2..18b01c1a 100644 --- a/je_auto_control/gui/script_builder/command_schema.py +++ b/je_auto_control/gui/script_builder/command_schema.py @@ -443,6 +443,30 @@ def _add_flow_specs(specs: List[CommandSpec]) -> None: ), description="Wait until the clipboard changes or matches target.", )) + specs.append(CommandSpec( + "AC_wait_image_gone", "Flow", "Wait for Image to Vanish", + fields=( + FieldSpec("image", FieldType.STRING, placeholder="path/to/spinner.png"), + FieldSpec("detect_threshold", FieldType.FLOAT, optional=True, + default=1.0), + FieldSpec("timeout_s", FieldType.FLOAT, optional=True, default=10.0), + FieldSpec("poll_interval_s", FieldType.FLOAT, optional=True, + default=0.2, min_value=0.01), + FieldSpec("gone_for_s", FieldType.FLOAT, optional=True, default=0.0), + ), + description="Block until an image (spinner/toast) leaves the screen.", + )) + specs.append(CommandSpec( + "AC_wait_text_gone", "Flow", "Wait for Text to Vanish", + fields=( + FieldSpec("text", FieldType.STRING, placeholder="Loading..."), + FieldSpec("timeout_s", FieldType.FLOAT, optional=True, default=10.0), + FieldSpec("poll_interval_s", FieldType.FLOAT, optional=True, + default=0.2, min_value=0.01), + FieldSpec("gone_for_s", FieldType.FLOAT, optional=True, default=0.0), + ), + description="Block until on-screen text (OCR) disappears.", + )) specs.append(CommandSpec( "AC_loop", "Flow", "Loop (N times)", fields=( diff --git a/je_auto_control/utils/executor/action_executor.py b/je_auto_control/utils/executor/action_executor.py index 87b81248..ede6aca7 100644 --- a/je_auto_control/utils/executor/action_executor.py +++ b/je_auto_control/utils/executor/action_executor.py @@ -361,6 +361,29 @@ def _wait_clipboard_change(baseline: Optional[str] = None, ).to_dict() +def _wait_image_gone(image: Any, detect_threshold: float = 1.0, + timeout_s: float = 10.0, poll_interval_s: float = 0.2, + gone_for_s: float = 0.0) -> Dict[str, Any]: + """Executor adapter: wait until an image is no longer on screen.""" + from je_auto_control.utils.smart_waits import wait_until_image_gone + return wait_until_image_gone( + image, detect_threshold=float(detect_threshold), + timeout_s=float(timeout_s), poll_interval_s=float(poll_interval_s), + gone_for_s=float(gone_for_s), + ).to_dict() + + +def _wait_text_gone(text: str, timeout_s: float = 10.0, + poll_interval_s: float = 0.2, + gone_for_s: float = 0.0) -> Dict[str, Any]: + """Executor adapter: wait until text is no longer on screen (OCR).""" + from je_auto_control.utils.smart_waits import wait_until_text_gone + return wait_until_text_gone( + text, timeout_s=float(timeout_s), + poll_interval_s=float(poll_interval_s), gone_for_s=float(gone_for_s), + ).to_dict() + + def _wait_window_closed(title: str, case_sensitive: bool = False, timeout_s: float = 10.0, poll_interval_s: float = 0.2) -> Dict[str, Any]: @@ -4971,6 +4994,8 @@ def __init__(self): "AC_wait_for_port": _wait_for_port, "AC_wait_for_process": _wait_for_process, "AC_wait_clipboard_change": _wait_clipboard_change, + "AC_wait_image_gone": _wait_image_gone, + "AC_wait_text_gone": _wait_text_gone, "AC_wait_window_closed": _wait_window_closed, # Cost telemetry (LLM token + USD tracking) diff --git a/je_auto_control/utils/mcp_server/tools/_factories.py b/je_auto_control/utils/mcp_server/tools/_factories.py index 32a2f6d8..76c83357 100644 --- a/je_auto_control/utils/mcp_server/tools/_factories.py +++ b/je_auto_control/utils/mcp_server/tools/_factories.py @@ -1277,6 +1277,34 @@ def cost_telemetry_tools() -> List[MCPTool]: def smart_wait_tools() -> List[MCPTool]: return [ + MCPTool( + name="ac_wait_image_gone", + description=("Block until 'image' is no longer found on screen " + "(spinner/toast/dialog vanished). 'detect_threshold', " + "'timeout_s', 'poll_interval_s', 'gone_for_s'."), + input_schema=schema({ + "image": {"type": "string"}, + "detect_threshold": {"type": "number"}, + "timeout_s": {"type": "number"}, + "poll_interval_s": {"type": "number"}, + "gone_for_s": {"type": "number"}}, + required=["image"]), + handler=h.wait_image_gone, + annotations=READ_ONLY, + ), + MCPTool( + name="ac_wait_text_gone", + description=("Block until 'text' is no longer found on screen (OCR). " + "'timeout_s', 'poll_interval_s', 'gone_for_s'."), + input_schema=schema({ + "text": {"type": "string"}, + "timeout_s": {"type": "number"}, + "poll_interval_s": {"type": "number"}, + "gone_for_s": {"type": "number"}}, + required=["text"]), + handler=h.wait_text_gone, + annotations=READ_ONLY, + ), MCPTool( name="ac_wait_screen_stable", description=("Block until the screen stops moving (consecutive " diff --git a/je_auto_control/utils/mcp_server/tools/_handlers.py b/je_auto_control/utils/mcp_server/tools/_handlers.py index 9be960c7..c685a332 100644 --- a/je_auto_control/utils/mcp_server/tools/_handlers.py +++ b/je_auto_control/utils/mcp_server/tools/_handlers.py @@ -2418,6 +2418,21 @@ def wait_screen_stable(region: Optional[List[int]] = None, ).to_dict() +def wait_image_gone(image, detect_threshold: float = 1.0, + timeout_s: float = 10.0, poll_interval_s: float = 0.2, + gone_for_s: float = 0.0) -> Dict[str, Any]: + from je_auto_control.utils.executor.action_executor import _wait_image_gone + return _wait_image_gone(image, detect_threshold, timeout_s, + poll_interval_s, gone_for_s) + + +def wait_text_gone(text: str, timeout_s: float = 10.0, + poll_interval_s: float = 0.2, + gone_for_s: float = 0.0) -> Dict[str, Any]: + from je_auto_control.utils.executor.action_executor import _wait_text_gone + return _wait_text_gone(text, timeout_s, poll_interval_s, gone_for_s) + + def wait_for_file(path: str, timeout_s: float = 30.0, poll_interval_s: float = 0.25, stable_for_s: float = 1.0, diff --git a/je_auto_control/utils/smart_waits/__init__.py b/je_auto_control/utils/smart_waits/__init__.py index e71bb355..24e51427 100644 --- a/je_auto_control/utils/smart_waits/__init__.py +++ b/je_auto_control/utils/smart_waits/__init__.py @@ -10,8 +10,9 @@ from je_auto_control.utils.smart_waits.waits import ( ClipboardReader, FileStatReader, Frame, PortConnector, ProcessLister, ScreenSampler, WaitOutcome, WindowFinder, wait_until_clipboard_changes, - wait_until_file, wait_until_pixel_changes, wait_until_port, - wait_until_process, wait_until_region_idle, wait_until_screen_stable, + wait_until_file, wait_until_gone, wait_until_image_gone, + wait_until_pixel_changes, wait_until_port, wait_until_process, + wait_until_region_idle, wait_until_screen_stable, wait_until_text_gone, wait_until_window_closed, ) @@ -19,8 +20,8 @@ __all__ = [ "ClipboardReader", "FileStatReader", "Frame", "PortConnector", "ProcessLister", "ScreenSampler", "WaitOutcome", "WindowFinder", - "wait_until_clipboard_changes", "wait_until_file", - "wait_until_pixel_changes", "wait_until_port", "wait_until_process", - "wait_until_region_idle", "wait_until_screen_stable", - "wait_until_window_closed", + "wait_until_clipboard_changes", "wait_until_file", "wait_until_gone", + "wait_until_image_gone", "wait_until_pixel_changes", "wait_until_port", + "wait_until_process", "wait_until_region_idle", "wait_until_screen_stable", + "wait_until_text_gone", "wait_until_window_closed", ] diff --git a/je_auto_control/utils/smart_waits/waits.py b/je_auto_control/utils/smart_waits/waits.py index 44d89808..87212545 100644 --- a/je_auto_control/utils/smart_waits/waits.py +++ b/je_auto_control/utils/smart_waits/waits.py @@ -378,6 +378,74 @@ def wait_until_process(name: str, *, present: bool = True, started, samples) +def wait_until_gone(present: Callable[[], bool], *, + timeout_s: float = 10.0, poll_interval_s: float = 0.2, + gone_for_s: float = 0.0) -> WaitOutcome: + """Return once ``present()`` has been falsey for ``gone_for_s`` seconds. + + The blocking complement of ``wait_for_image`` / ``wait_for_text``: wait for a + spinner / toast / dialog to *disappear*. ``present`` is any predicate (e.g. + "is this image still on screen"); it is polled every ``poll_interval_s`` up to + ``timeout_s``. Injecting ``present`` keeps the loop headless-testable. + """ + if timeout_s <= 0: + raise ValueError(_TIMEOUT_POSITIVE) + if poll_interval_s <= 0: + raise ValueError(_POLL_POSITIVE) + if gone_for_s < 0: + raise ValueError("gone_for_s must be >= 0") + started = time.monotonic() + deadline = started + float(timeout_s) + samples = 0 + gone_since: Optional[float] = None + while time.monotonic() < deadline: + samples += 1 + if present(): + gone_since = None + else: + if gone_since is None: + gone_since = time.monotonic() + if time.monotonic() - gone_since >= float(gone_for_s): + return _finish(True, "target gone", started, samples) + time.sleep(float(poll_interval_s)) + return _finish(False, "timeout while waiting for target to vanish", + started, samples) + + +def _image_present(image: Any, detect_threshold: float) -> bool: + """Whether ``image`` is currently locatable on screen.""" + from je_auto_control.utils.exception.exceptions import ImageNotFoundException + from je_auto_control.wrapper.auto_control_image import locate_image_center + try: + locate_image_center(image, detect_threshold=detect_threshold) + return True + except ImageNotFoundException: + return False + + +def wait_until_image_gone(image: Any, *, detect_threshold: float = 1.0, + timeout_s: float = 10.0, poll_interval_s: float = 0.2, + gone_for_s: float = 0.0) -> WaitOutcome: + """Wait until ``image`` is no longer found on screen.""" + return wait_until_gone(lambda: _image_present(image, detect_threshold), + timeout_s=timeout_s, poll_interval_s=poll_interval_s, + gone_for_s=gone_for_s) + + +def _text_present(text: str) -> bool: + """Whether ``text`` is currently found on screen via OCR.""" + from je_auto_control.utils.ocr.ocr_engine import find_text_matches + return bool(find_text_matches(text)) + + +def wait_until_text_gone(text: str, *, timeout_s: float = 10.0, + poll_interval_s: float = 0.2, + gone_for_s: float = 0.0) -> WaitOutcome: + """Wait until ``text`` is no longer found on screen (OCR).""" + return wait_until_gone(lambda: _text_present(text), timeout_s=timeout_s, + poll_interval_s=poll_interval_s, gone_for_s=gone_for_s) + + def _default_process_lister(name: str) -> List[str]: """List running process names matching ``name`` (requires psutil).""" from je_auto_control.utils.assertion.assertions import _running_process_names diff --git a/test/unit_test/headless/test_wait_gone_batch.py b/test/unit_test/headless/test_wait_gone_batch.py new file mode 100644 index 00000000..dabb2419 --- /dev/null +++ b/test/unit_test/headless/test_wait_gone_batch.py @@ -0,0 +1,68 @@ +"""Headless tests for blocking wait-until-vanish. No Qt.""" +import je_auto_control as ac +from je_auto_control.utils.smart_waits import WaitOutcome, wait_until_gone + + +def test_returns_when_predicate_becomes_false(): + # present() is True for the first 2 polls, then False. + calls = {"n": 0} + + def present(): + calls["n"] += 1 + return calls["n"] <= 2 + + outcome = wait_until_gone(present, timeout_s=5.0, poll_interval_s=0.001) + assert isinstance(outcome, WaitOutcome) + assert outcome.succeeded is True + assert outcome.reason == "target gone" + assert outcome.samples_taken == 3 + + +def test_already_gone_returns_immediately(): + outcome = wait_until_gone(lambda: False, timeout_s=5.0, + poll_interval_s=0.001) + assert outcome.succeeded is True and outcome.samples_taken == 1 + + +def test_timeout_when_always_present(): + outcome = wait_until_gone(lambda: True, timeout_s=0.05, + poll_interval_s=0.001) + assert outcome.succeeded is False + assert "timeout" in outcome.reason + + +def test_validation(): + for bad in ({"timeout_s": 0}, {"poll_interval_s": 0}, {"gone_for_s": -1}): + try: + wait_until_gone(lambda: False, **bad) + except ValueError: + pass + else: # pragma: no cover + raise AssertionError(f"expected ValueError for {bad}") + + +def test_gone_for_requires_sustained_absence(): + # flips False once then True again -> must NOT count as gone with gone_for_s + seq = iter([True, False, True, True, True, True, True, True]) + outcome = wait_until_gone(lambda: next(seq, True), timeout_s=0.05, + poll_interval_s=0.001, gone_for_s=10.0) + assert outcome.succeeded is False # never gone long enough + + +# --- wiring --------------------------------------------------------------- + +def test_wiring(): + known = ac.executor.known_commands() + assert {"AC_wait_image_gone", "AC_wait_text_gone"} <= set(known) + from je_auto_control.utils.mcp_server.tools import build_default_tool_registry + names = {t.name for t in build_default_tool_registry()} + assert {"ac_wait_image_gone", "ac_wait_text_gone"} <= names + from je_auto_control.gui.script_builder.command_schema import _build_specs + specs = {s.command for s in _build_specs()} + assert {"AC_wait_image_gone", "AC_wait_text_gone"} <= specs + + +def test_facade_exports(): + for attr in ("wait_until_gone", "wait_until_image_gone", + "wait_until_text_gone"): + assert hasattr(ac, attr) and attr in ac.__all__ From 69fee0cef7dc85179eac57df7a1a54a04442016f Mon Sep 17 00:00:00 2001 From: JeffreyChen Date: Tue, 23 Jun 2026 00:52:05 +0800 Subject: [PATCH 06/39] Add hold-key / auto-repeat key input --- README/WHATS_NEW_zh-CN.md | 6 ++ README/WHATS_NEW_zh-TW.md | 6 ++ WHATS_NEW.md | 6 ++ .../doc/new_features/v119_features_doc.rst | 39 ++++++++++ docs/source/Eng/eng_index.rst | 1 + .../Zh/doc/new_features/v119_features_doc.rst | 34 +++++++++ docs/source/Zh/zh_index.rst | 1 + je_auto_control/__init__.py | 4 + .../gui/script_builder/command_schema.py | 11 +++ .../utils/executor/action_executor.py | 9 +++ je_auto_control/utils/key_hold/__init__.py | 4 + je_auto_control/utils/key_hold/key_hold.py | 73 ++++++++++++++++++ .../utils/mcp_server/tools/_factories.py | 20 ++++- .../utils/mcp_server/tools/_handlers.py | 5 ++ .../unit_test/headless/test_key_hold_batch.py | 75 +++++++++++++++++++ 15 files changed, 293 insertions(+), 1 deletion(-) create mode 100644 docs/source/Eng/doc/new_features/v119_features_doc.rst create mode 100644 docs/source/Zh/doc/new_features/v119_features_doc.rst create mode 100644 je_auto_control/utils/key_hold/__init__.py create mode 100644 je_auto_control/utils/key_hold/key_hold.py create mode 100644 test/unit_test/headless/test_key_hold_batch.py diff --git a/README/WHATS_NEW_zh-CN.md b/README/WHATS_NEW_zh-CN.md index 76883031..b8e95cee 100644 --- a/README/WHATS_NEW_zh-CN.md +++ b/README/WHATS_NEW_zh-CN.md @@ -1,5 +1,11 @@ # 本次更新 — AutoControl +## 本次更新 (2026-06-23) — 按住按键 / 自动重复 + +按住一个键一段时间,或以固定频率自动重复。完整参考:[`docs/source/Zh/doc/new_features/v119_features_doc.rst`](../docs/source/Zh/doc/new_features/v119_features_doc.rst)。 + +- **`hold_key` / `plan_key_hold`**(`AC_hold_key`):`type_keyboard` 是瞬间按下+放开——先前没有「按住此键 N 秒」(游戏移动、按住滚动)或「每秒送 R 次」(自动重复)。`plan_key_hold` 建立确定性操作计划(按下/等待/放开,或为 `rate_hz` 产生 N 个间隔按键事件);`hold_key` 将等待导向可注入的 `sleep`、按键导向可注入的 `sink`。纯计划、确定。 + ## 本次更新 (2026-06-23) — 等待消失(阻塞式 vanish 等待) 阻塞直到转圈圈 / toast / 对话框消失。完整参考:[`docs/source/Zh/doc/new_features/v118_features_doc.rst`](../docs/source/Zh/doc/new_features/v118_features_doc.rst)。 diff --git a/README/WHATS_NEW_zh-TW.md b/README/WHATS_NEW_zh-TW.md index ec7eb96f..ae61b4bd 100644 --- a/README/WHATS_NEW_zh-TW.md +++ b/README/WHATS_NEW_zh-TW.md @@ -1,5 +1,11 @@ # 本次更新 — AutoControl +## 本次更新 (2026-06-23) — 按住按鍵 / 自動重複 + +按住一個鍵一段時間,或以固定頻率自動重複。完整參考:[`docs/source/Zh/doc/new_features/v119_features_doc.rst`](../docs/source/Zh/doc/new_features/v119_features_doc.rst)。 + +- **`hold_key` / `plan_key_hold`**(`AC_hold_key`):`type_keyboard` 是瞬間按下+放開——先前沒有「按住此鍵 N 秒」(遊戲移動、按住捲動)或「每秒送 R 次」(自動重複)。`plan_key_hold` 建立決定性操作計畫(按下/等待/放開,或為 `rate_hz` 產生 N 個間隔按鍵事件);`hold_key` 將等待導向可注入的 `sleep`、按鍵導向可注入的 `sink`。純計畫、具決定性。 + ## 本次更新 (2026-06-23) — 等待消失(阻塞式 vanish 等待) 阻塞直到轉圈圈 / toast / 對話框消失。完整參考:[`docs/source/Zh/doc/new_features/v118_features_doc.rst`](../docs/source/Zh/doc/new_features/v118_features_doc.rst)。 diff --git a/WHATS_NEW.md b/WHATS_NEW.md index 1c299998..e9005de0 100644 --- a/WHATS_NEW.md +++ b/WHATS_NEW.md @@ -1,5 +1,11 @@ # What's New — AutoControl +## What's new (2026-06-23) — Hold Key / Auto-Repeat + +Hold a key for a duration, or auto-repeat it at a fixed rate. Full reference: [`docs/source/Eng/doc/new_features/v119_features_doc.rst`](docs/source/Eng/doc/new_features/v119_features_doc.rst). + +- **`hold_key` / `plan_key_hold`** (`AC_hold_key`): `type_keyboard` is an instant down+up — there was no "hold this key for N seconds" (game movement, hold-to-scroll) or "send it at R presses/second" (auto-repeat). `plan_key_hold` builds the deterministic op-plan (press/wait/release, or N spaced key events for `rate_hz`); `hold_key` routes waits to an injectable `sleep` and keys to an injectable `sink`. Pure-planning, deterministic. + ## What's new (2026-06-23) — Wait Until Gone (Blocking Vanish Waits) Block until a spinner / toast / dialog disappears. Full reference: [`docs/source/Eng/doc/new_features/v118_features_doc.rst`](docs/source/Eng/doc/new_features/v118_features_doc.rst). diff --git a/docs/source/Eng/doc/new_features/v119_features_doc.rst b/docs/source/Eng/doc/new_features/v119_features_doc.rst new file mode 100644 index 00000000..68edaa18 --- /dev/null +++ b/docs/source/Eng/doc/new_features/v119_features_doc.rst @@ -0,0 +1,39 @@ +Hold Key / Auto-Repeat +====================== + +``type_keyboard`` is an instant down+up and ``input_macro.run_sequence`` can +hand-roll a press / wait / release, but there was no primitive for "hold this key +for N seconds" (game movement, hold-to-scroll) or "send it at R presses per +second" (auto-repeat). + +:func:`plan_key_hold` builds the deterministic op-plan (pure, unit-testable); +:func:`hold_key` dispatches it through an injectable ``sink`` and ``sleep`` so it +is tested without real input or real waiting. Imports no ``PySide6``. + +Headless API +------------ + +.. code-block:: python + + from je_auto_control import hold_key, plan_key_hold + + hold_key("key_d", duration_s=1.5) # press, hold 1.5s, release + hold_key("key_down", duration_s=2.0, rate_hz=20) # 40 key events @ 50ms + + plan_key_hold("space", 1.0) + # [{'op': 'press', 'key': 'space'}, + # {'op': 'wait', 'seconds': 1.0}, + # {'op': 'release', 'key': 'space'}] + +With ``rate_hz`` unset the key is pressed, held for ``duration_s``, then released. +With ``rate_hz`` set it is sent as ``round(duration_s * rate_hz)`` discrete key +events spaced ``1 / rate_hz`` apart — simulated auto-repeat for movement / scroll +loops. A non-positive duration or rate raises ``ValueError``. ``hold_key`` routes +the ``wait`` steps to ``sleep`` and the key steps to ``sink``, both injectable. + +Executor commands +----------------- + +``AC_hold_key`` takes ``key`` plus ``duration_s`` and an optional ``rate_hz`` and +returns ``{ops, plan}``. It is exposed as the MCP tool ``ac_hold_key`` and as a +Script Builder command under **Keyboard**. diff --git a/docs/source/Eng/eng_index.rst b/docs/source/Eng/eng_index.rst index b9203ae9..5945bab3 100644 --- a/docs/source/Eng/eng_index.rst +++ b/docs/source/Eng/eng_index.rst @@ -141,6 +141,7 @@ Comprehensive guides for all AutoControl features. doc/new_features/v116_features_doc doc/new_features/v117_features_doc doc/new_features/v118_features_doc + doc/new_features/v119_features_doc doc/ocr_backends/ocr_backends_doc doc/observability/observability_doc doc/operations_layer/operations_layer_doc diff --git a/docs/source/Zh/doc/new_features/v119_features_doc.rst b/docs/source/Zh/doc/new_features/v119_features_doc.rst new file mode 100644 index 00000000..10df75cf --- /dev/null +++ b/docs/source/Zh/doc/new_features/v119_features_doc.rst @@ -0,0 +1,34 @@ +按住按鍵 / 自動重複 +================== + +``type_keyboard`` 是瞬間的按下+放開,``input_macro.run_sequence`` 雖可手動拼出按下 / 等待 / 放開,但先前沒有 +「按住此鍵 N 秒」(遊戲移動、按住捲動)或「每秒送 R 次」(自動重複)的基本元件。 + +:func:`plan_key_hold` 建立決定性的操作計畫(純函式、可單元測試);:func:`hold_key` 透過可注入的 ``sink`` +與 ``sleep`` 派發,因此可在無真實輸入、無真實等待下測試。不匯入 ``PySide6``。 + +無頭 API +-------- + +.. code-block:: python + + from je_auto_control import hold_key, plan_key_hold + + hold_key("key_d", duration_s=1.5) # 按下、按住 1.5 秒、放開 + hold_key("key_down", duration_s=2.0, rate_hz=20) # 40 個按鍵事件 @ 50ms + + plan_key_hold("space", 1.0) + # [{'op': 'press', 'key': 'space'}, + # {'op': 'wait', 'seconds': 1.0}, + # {'op': 'release', 'key': 'space'}] + +未設定 ``rate_hz`` 時,鍵會被按下、按住 ``duration_s``、再放開。設定 ``rate_hz`` 時,會送出 +``round(duration_s * rate_hz)`` 個相隔 ``1 / rate_hz`` 的離散按鍵事件——用於移動 / 捲動迴圈的模擬自動重複。 +非正數的時長或頻率會拋出 ``ValueError``。``hold_key`` 將 ``wait`` 步驟導向 ``sleep``、按鍵步驟導向 ``sink``, +兩者皆可注入。 + +執行器命令 +---------- + +``AC_hold_key`` 接受 ``key`` 以及 ``duration_s`` 與選用的 ``rate_hz``,並回傳 ``{ops, plan}``。它以 MCP 工具 +``ac_hold_key`` 以及 Script Builder 中 **Keyboard** 分類下的命令提供。 diff --git a/docs/source/Zh/zh_index.rst b/docs/source/Zh/zh_index.rst index a1152f4c..7f5c98a5 100644 --- a/docs/source/Zh/zh_index.rst +++ b/docs/source/Zh/zh_index.rst @@ -141,6 +141,7 @@ AutoControl 所有功能的完整使用指南。 doc/new_features/v116_features_doc doc/new_features/v117_features_doc doc/new_features/v118_features_doc + doc/new_features/v119_features_doc doc/ocr_backends/ocr_backends_doc doc/observability/observability_doc doc/operations_layer/operations_layer_doc diff --git a/je_auto_control/__init__.py b/je_auto_control/__init__.py index 1494b92b..ae9dacb0 100644 --- a/je_auto_control/__init__.py +++ b/je_auto_control/__init__.py @@ -257,6 +257,8 @@ ) # Clear-then-type a text field (Playwright `fill` idiom; paste for Unicode) from je_auto_control.utils.field_entry import plan_field_set, set_field_text +# Hold a key for a duration / auto-repeat at a fixed rate +from je_auto_control.utils.key_hold import hold_key, plan_key_hold # CI workflow annotations (GitHub Actions) from je_auto_control.utils.ci_annotations import ( emit_annotations, format_annotation, @@ -1034,6 +1036,8 @@ def start_autocontrol_gui(*args, **kwargs): "path_easings", "plan_field_set", "set_field_text", + "plan_key_hold", + "hold_key", "emit_annotations", "format_annotation", "ClipboardHistory", "default_clipboard_history", "analyze_heal_log", "heal_stats", "scan_secrets", diff --git a/je_auto_control/gui/script_builder/command_schema.py b/je_auto_control/gui/script_builder/command_schema.py index 18b01c1a..1c11146e 100644 --- a/je_auto_control/gui/script_builder/command_schema.py +++ b/je_auto_control/gui/script_builder/command_schema.py @@ -174,6 +174,17 @@ def _add_keyboard_specs(specs: List[CommandSpec]) -> None: ), description="Clear the focused field then enter text (paste for Unicode).", )) + specs.append(CommandSpec( + "AC_hold_key", "Keyboard", "Hold Key", + fields=( + FieldSpec("key", FieldType.STRING, placeholder="e.g. key_d, space"), + FieldSpec("duration_s", FieldType.FLOAT, default=1.0, + min_value=0.01), + FieldSpec("rate_hz", FieldType.FLOAT, optional=True, + placeholder="auto-repeat presses/sec (blank = hold)"), + ), + description="Hold a key for a duration, or auto-repeat it at rate_hz.", + )) specs.append(CommandSpec( "AC_hotkey", "Keyboard", "Hotkey", fields=( diff --git a/je_auto_control/utils/executor/action_executor.py b/je_auto_control/utils/executor/action_executor.py index ede6aca7..aa21e73b 100644 --- a/je_auto_control/utils/executor/action_executor.py +++ b/je_auto_control/utils/executor/action_executor.py @@ -3123,6 +3123,14 @@ def _set_field_text(text: str, clear: str = "select_all", paste: Any = False, modifier=modifier) +def _hold_key(key: str, duration_s: Any = 1.0, + rate_hz: Any = None) -> Dict[str, Any]: + """Adapter: hold a key for a duration (or auto-repeat at rate_hz).""" + from je_auto_control.utils.key_hold import hold_key + rate = float(rate_hz) if rate_hz not in (None, "") else None + return hold_key(key, float(duration_s), rate_hz=rate) + + def _cas_put(name: str, key: str, value: Any, expected_version: Any = None) -> Dict[str, Any]: """Adapter: optimistic put into a named versioned store.""" @@ -4821,6 +4829,7 @@ def __init__(self): "AC_move_along_path": _move_along_path, "AC_drag_path": _drag_path, "AC_set_field_text": _set_field_text, + "AC_hold_key": _hold_key, "AC_detect_drift": _detect_drift, "AC_categorical_drift": _categorical_drift, "AC_diff_rows": _diff_rows, diff --git a/je_auto_control/utils/key_hold/__init__.py b/je_auto_control/utils/key_hold/__init__.py new file mode 100644 index 00000000..4fb6c7be --- /dev/null +++ b/je_auto_control/utils/key_hold/__init__.py @@ -0,0 +1,4 @@ +"""Hold a key for a duration, or auto-repeat it at a fixed rate.""" +from je_auto_control.utils.key_hold.key_hold import hold_key, plan_key_hold + +__all__ = ["hold_key", "plan_key_hold"] diff --git a/je_auto_control/utils/key_hold/key_hold.py b/je_auto_control/utils/key_hold/key_hold.py new file mode 100644 index 00000000..cdc79102 --- /dev/null +++ b/je_auto_control/utils/key_hold/key_hold.py @@ -0,0 +1,73 @@ +"""Hold a key for a duration, or auto-repeat it at a fixed rate. + +``type_keyboard`` is an instant down+up and ``input_macro.run_sequence`` can hand- +roll a press / wait / release, but there is no primitive for "hold this key for N +seconds" (game movement, hold-to-scroll) or "send it at R presses per second" +(auto-repeat). :func:`plan_key_hold` builds the deterministic op-plan (pure, +unit-testable); :func:`hold_key` dispatches it through an injectable ``sink`` and +``sleep`` so it is tested without real input or real waiting. Imports no +``PySide6``. +""" +import time +from typing import Any, Callable, Dict, List, Optional + +Sink = Callable[[Dict[str, Any]], None] + + +def plan_key_hold(key: str, duration_s: float, *, + rate_hz: Optional[float] = None) -> List[Dict[str, Any]]: + """Return the op-plan to hold (or auto-repeat) ``key`` for ``duration_s``. + + With ``rate_hz`` unset the key is pressed, held for ``duration_s``, then + released. With ``rate_hz`` set it is sent as ``round(duration_s * rate_hz)`` + discrete key events spaced ``1 / rate_hz`` apart (simulated auto-repeat). + Raises ``ValueError`` on a non-positive duration or rate. + """ + if duration_s <= 0: + raise ValueError("duration_s must be positive") + if rate_hz is None: + return [{"op": "press", "key": key}, + {"op": "wait", "seconds": float(duration_s)}, + {"op": "release", "key": key}] + if rate_hz <= 0: + raise ValueError("rate_hz must be positive") + interval = 1.0 / float(rate_hz) + count = max(1, round(float(duration_s) * float(rate_hz))) + plan: List[Dict[str, Any]] = [] + for index in range(count): + plan.append({"op": "key", "key": key}) + if index != count - 1: + plan.append({"op": "wait", "seconds": interval}) + return plan + + +def _default_sink(event: Dict[str, Any]) -> None: + """Default dispatch: drive the real keyboard backend.""" + from je_auto_control.wrapper.auto_control_keyboard import ( + press_keyboard_key, release_keyboard_key, type_keyboard) + op = event["op"] + if op == "press": + press_keyboard_key(event["key"]) + elif op == "release": + release_keyboard_key(event["key"]) + elif op == "key": + type_keyboard(event["key"]) + + +def hold_key(key: str, duration_s: float, *, rate_hz: Optional[float] = None, + sink: Optional[Sink] = None, + sleep: Optional[Callable[[float], None]] = None) -> Dict[str, Any]: + """Hold or auto-repeat ``key`` for ``duration_s``; return the dispatched plan. + + ``wait`` ops go to ``sleep`` (default :func:`time.sleep`); key ops go to + ``sink`` (default: the real keyboard backend). + """ + plan = plan_key_hold(key, duration_s, rate_hz=rate_hz) + dispatch = sink or _default_sink + pause = sleep or time.sleep + for event in plan: + if event["op"] == "wait": + pause(event["seconds"]) + else: + dispatch(event) + return {"ops": len(plan), "plan": plan} diff --git a/je_auto_control/utils/mcp_server/tools/_factories.py b/je_auto_control/utils/mcp_server/tools/_factories.py index 76c83357..b62f009a 100644 --- a/je_auto_control/utils/mcp_server/tools/_factories.py +++ b/je_auto_control/utils/mcp_server/tools/_factories.py @@ -2578,6 +2578,24 @@ def tween_drag_tools() -> List[MCPTool]: ] +def key_hold_tools() -> List[MCPTool]: + return [ + MCPTool( + name="ac_hold_key", + description=("Hold 'key' for 'duration_s' seconds, or set 'rate_hz' " + "to auto-repeat it at that many presses/second. " + "Returns {ops, plan}."), + input_schema=schema({ + "key": {"type": "string"}, + "duration_s": {"type": "number"}, + "rate_hz": {"type": "number"}}, + required=["key", "duration_s"]), + handler=h.hold_key, + annotations=SIDE_EFFECT_ONLY, + ), + ] + + def field_entry_tools() -> List[MCPTool]: return [ MCPTool( @@ -5878,7 +5896,7 @@ def media_assert_tools() -> List[MCPTool]: input_macro_tools, resilience_tools, ci_annotation_tools, clipboard_history_tools, audit_analysis_tools, process_doc_tools, tween_drag_tools, mouse_path_tools, field_entry_tools, - plugin_sdk_tools, governance_tools, + key_hold_tools, plugin_sdk_tools, governance_tools, credential_lease_tools, egress_tools, approval_testing_tools, trajectory_eval_tools, compliance_tools, agent_trace_tools, video_report_tools, fuzzy_tools, artifact_store_tools, image_dedup_tools, diff --git a/je_auto_control/utils/mcp_server/tools/_handlers.py b/je_auto_control/utils/mcp_server/tools/_handlers.py index c685a332..1d25ee3b 100644 --- a/je_auto_control/utils/mcp_server/tools/_handlers.py +++ b/je_auto_control/utils/mcp_server/tools/_handlers.py @@ -2043,6 +2043,11 @@ def set_field_text(text, clear="select_all", paste=False, modifier="ctrl"): return _set_field_text(text, clear, paste, modifier) +def hold_key(key, duration_s=1.0, rate_hz=None): + from je_auto_control.utils.executor.action_executor import _hold_key + return _hold_key(key, duration_s, rate_hz) + + def detect_drift(reference, current, threshold=0.25, bins=10): from je_auto_control.utils.executor.action_executor import _detect_drift return _detect_drift(reference, current, threshold, bins) diff --git a/test/unit_test/headless/test_key_hold_batch.py b/test/unit_test/headless/test_key_hold_batch.py new file mode 100644 index 00000000..5dcbfc60 --- /dev/null +++ b/test/unit_test/headless/test_key_hold_batch.py @@ -0,0 +1,75 @@ +"""Headless tests for key hold / auto-repeat. No Qt.""" +import je_auto_control as ac +from je_auto_control.utils.key_hold import hold_key, plan_key_hold + + +def test_hold_plan_press_wait_release(): + plan = plan_key_hold("key_d", 1.5) + assert plan == [ + {"op": "press", "key": "key_d"}, + {"op": "wait", "seconds": 1.5}, + {"op": "release", "key": "key_d"}, + ] + + +def test_repeat_plan_emits_n_key_events(): + plan = plan_key_hold("a", 1.0, rate_hz=20) + keys = [event for event in plan if event["op"] == "key"] + waits = [event for event in plan if event["op"] == "wait"] + assert len(keys) == 20 + assert len(waits) == 19 # one fewer gap than events + assert waits[0]["seconds"] == 0.05 # 1 / 20 Hz + + +def test_repeat_rounds_count_min_one(): + assert len([e for e in plan_key_hold("a", 0.01, rate_hz=10) + if e["op"] == "key"]) == 1 # round(0.1) -> 0 -> clamped to 1 + + +def test_hold_key_routes_waits_to_sleep(): + events, slept = [], [] + result = hold_key("a", 0.5, sink=events.append, sleep=slept.append) + assert [e["op"] for e in events] == ["press", "release"] + assert slept == [0.5] + assert result["ops"] == 3 + + +def test_validation(): + for bad in (("a", 0), ("a", -1)): + try: + plan_key_hold(*bad) + except ValueError: + pass + else: # pragma: no cover + raise AssertionError("expected ValueError") + try: + plan_key_hold("a", 1.0, rate_hz=0) + except ValueError: + pass + else: # pragma: no cover + raise AssertionError("expected ValueError for rate_hz=0") + + +# --- wiring --------------------------------------------------------------- + +def test_executor_adapter_planning(): + events, slept = [], [] + from je_auto_control.utils.key_hold import hold_key as _hk + _hk("space", 0.2, sink=events.append, sleep=slept.append) + assert events[0]["op"] == "press" and events[-1]["op"] == "release" + + +def test_wiring(): + known = ac.executor.known_commands() + assert "AC_hold_key" in set(known) + from je_auto_control.utils.mcp_server.tools import build_default_tool_registry + names = {t.name for t in build_default_tool_registry()} + assert "ac_hold_key" in names + from je_auto_control.gui.script_builder.command_schema import _build_specs + specs = {s.command for s in _build_specs()} + assert "AC_hold_key" in specs + + +def test_facade_exports(): + for attr in ("plan_key_hold", "hold_key"): + assert hasattr(ac, attr) and attr in ac.__all__ From 95284da6593dd0daa11578e0b7d4b56c665685ff Mon Sep 17 00:00:00 2001 From: JeffreyChen Date: Tue, 23 Jun 2026 00:59:55 +0800 Subject: [PATCH 07/39] Use pytest.approx + guard list indexing in key_hold tests (S1244/S6466) --- test/unit_test/headless/test_key_hold_batch.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/test/unit_test/headless/test_key_hold_batch.py b/test/unit_test/headless/test_key_hold_batch.py index 5dcbfc60..6cc0fffc 100644 --- a/test/unit_test/headless/test_key_hold_batch.py +++ b/test/unit_test/headless/test_key_hold_batch.py @@ -1,4 +1,6 @@ """Headless tests for key hold / auto-repeat. No Qt.""" +import pytest + import je_auto_control as ac from je_auto_control.utils.key_hold import hold_key, plan_key_hold @@ -18,7 +20,7 @@ def test_repeat_plan_emits_n_key_events(): waits = [event for event in plan if event["op"] == "wait"] assert len(keys) == 20 assert len(waits) == 19 # one fewer gap than events - assert waits[0]["seconds"] == 0.05 # 1 / 20 Hz + assert waits[0]["seconds"] == pytest.approx(0.05) # 1 / 20 Hz def test_repeat_rounds_count_min_one(): @@ -56,7 +58,7 @@ def test_executor_adapter_planning(): events, slept = [], [] from je_auto_control.utils.key_hold import hold_key as _hk _hk("space", 0.2, sink=events.append, sleep=slept.append) - assert events[0]["op"] == "press" and events[-1]["op"] == "release" + assert events and events[0]["op"] == "press" and events[-1]["op"] == "release" def test_wiring(): From cc3e2b68040352992d5c94c3628b3597db0249ee Mon Sep 17 00:00:00 2001 From: JeffreyChen Date: Tue, 23 Jun 2026 01:10:56 +0800 Subject: [PATCH 08/39] Add relative mouse movement (move by delta) --- README/WHATS_NEW_zh-CN.md | 6 +++ README/WHATS_NEW_zh-TW.md | 6 +++ WHATS_NEW.md | 6 +++ .../doc/new_features/v120_features_doc.rst | 35 ++++++++++++ docs/source/Eng/eng_index.rst | 1 + .../Zh/doc/new_features/v120_features_doc.rst | 29 ++++++++++ docs/source/Zh/zh_index.rst | 1 + je_auto_control/__init__.py | 6 +++ .../gui/script_builder/command_schema.py | 8 +++ .../utils/executor/action_executor.py | 7 +++ .../utils/mcp_server/tools/_factories.py | 17 +++++- .../utils/mcp_server/tools/_handlers.py | 5 ++ .../utils/mouse_relative/__init__.py | 6 +++ .../utils/mouse_relative/mouse_relative.py | 53 ++++++++++++++++++ .../headless/test_mouse_relative_batch.py | 54 +++++++++++++++++++ 15 files changed, 239 insertions(+), 1 deletion(-) create mode 100644 docs/source/Eng/doc/new_features/v120_features_doc.rst create mode 100644 docs/source/Zh/doc/new_features/v120_features_doc.rst create mode 100644 je_auto_control/utils/mouse_relative/__init__.py create mode 100644 je_auto_control/utils/mouse_relative/mouse_relative.py create mode 100644 test/unit_test/headless/test_mouse_relative_batch.py diff --git a/README/WHATS_NEW_zh-CN.md b/README/WHATS_NEW_zh-CN.md index b8e95cee..ed021acd 100644 --- a/README/WHATS_NEW_zh-CN.md +++ b/README/WHATS_NEW_zh-CN.md @@ -1,5 +1,11 @@ # 本次更新 — AutoControl +## 本次更新 (2026-06-23) — 相对鼠标移动 + +从当前位置将指针位移一个增量。完整参考:[`docs/source/Zh/doc/new_features/v120_features_doc.rst`](../docs/source/Zh/doc/new_features/v120_features_doc.rst)。 + +- **`move_mouse_relative` / `relative_target`**(`AC_move_mouse_relative`):鼠标 wrapper 只有绝对的 `set_mouse_position`——没有给相对指针 / 画布 / FPS 应用与渐进式拖曳用的 `moveRel(dx, dy)`。本功能读取实时位置并依增量移动;`relative_target` 为纯算术,getter/setter 可注入以供无头测试。纯标准库、确定。 + ## 本次更新 (2026-06-23) — 按住按键 / 自动重复 按住一个键一段时间,或以固定频率自动重复。完整参考:[`docs/source/Zh/doc/new_features/v119_features_doc.rst`](../docs/source/Zh/doc/new_features/v119_features_doc.rst)。 diff --git a/README/WHATS_NEW_zh-TW.md b/README/WHATS_NEW_zh-TW.md index ae61b4bd..c2c5b1e5 100644 --- a/README/WHATS_NEW_zh-TW.md +++ b/README/WHATS_NEW_zh-TW.md @@ -1,5 +1,11 @@ # 本次更新 — AutoControl +## 本次更新 (2026-06-23) — 相對滑鼠移動 + +從目前位置將指標位移一個增量。完整參考:[`docs/source/Zh/doc/new_features/v120_features_doc.rst`](../docs/source/Zh/doc/new_features/v120_features_doc.rst)。 + +- **`move_mouse_relative` / `relative_target`**(`AC_move_mouse_relative`):滑鼠 wrapper 只有絕對的 `set_mouse_position`——沒有給相對指標 / 畫布 / FPS 應用與漸進式拖曳用的 `moveRel(dx, dy)`。本功能讀取即時位置並依增量移動;`relative_target` 為純算術,getter/setter 可注入以供無頭測試。純標準函式庫、具決定性。 + ## 本次更新 (2026-06-23) — 按住按鍵 / 自動重複 按住一個鍵一段時間,或以固定頻率自動重複。完整參考:[`docs/source/Zh/doc/new_features/v119_features_doc.rst`](../docs/source/Zh/doc/new_features/v119_features_doc.rst)。 diff --git a/WHATS_NEW.md b/WHATS_NEW.md index e9005de0..b19d3407 100644 --- a/WHATS_NEW.md +++ b/WHATS_NEW.md @@ -1,5 +1,11 @@ # What's New — AutoControl +## What's new (2026-06-23) — Relative Mouse Movement + +Nudge the pointer by a delta from where it is. Full reference: [`docs/source/Eng/doc/new_features/v120_features_doc.rst`](docs/source/Eng/doc/new_features/v120_features_doc.rst). + +- **`move_mouse_relative` / `relative_target`** (`AC_move_mouse_relative`): the mouse wrapper only had absolute `set_mouse_position` — no `moveRel(dx, dy)` for relative-pointer / canvas / FPS apps and incremental drags. Reads the live position and moves by the delta; `relative_target` is the pure arithmetic, and the getter/setter are injectable for headless tests. Pure-stdlib, deterministic. + ## What's new (2026-06-23) — Hold Key / Auto-Repeat Hold a key for a duration, or auto-repeat it at a fixed rate. Full reference: [`docs/source/Eng/doc/new_features/v119_features_doc.rst`](docs/source/Eng/doc/new_features/v119_features_doc.rst). diff --git a/docs/source/Eng/doc/new_features/v120_features_doc.rst b/docs/source/Eng/doc/new_features/v120_features_doc.rst new file mode 100644 index 00000000..83100e65 --- /dev/null +++ b/docs/source/Eng/doc/new_features/v120_features_doc.rst @@ -0,0 +1,35 @@ +Relative Mouse Movement +======================= + +The mouse wrapper exposes only absolute ``set_mouse_position`` — there was no +"nudge the pointer by ``(dx, dy)``" (the pynput / PyAutoGUI ``moveRel`` staple), +which relative-pointer / canvas / FPS-style apps and incremental drags need. + +:func:`relative_target` is the pure arithmetic (current + delta) and is +unit-testable; :func:`move_mouse_relative` reads the live position and sets the +new one, with both the getter and setter injectable so it is tested without a +real pointer. Imports no ``PySide6``. + +Headless API +------------ + +.. code-block:: python + + from je_auto_control import move_mouse_relative, relative_target + + move_mouse_relative(-40, 12) # nudge left 40, down 12 from where it is + # {'from': [200, 200], 'to': [160, 212], 'delta': [-40, 12]} + + relative_target((100, 100), 10, -5) # (110, 95) — pure, no I/O + +``move_mouse_relative`` reads the current position (raising +``AutoControlMouseException`` if it cannot), adds the delta, and moves there. +``get_position`` / ``set_position`` default to the real mouse wrapper but are +injectable for headless tests. + +Executor commands +----------------- + +``AC_move_mouse_relative`` takes ``dx`` / ``dy`` and returns ``{from, to, +delta}``. It is exposed as the MCP tool ``ac_move_mouse_relative`` and as a +Script Builder command under **Mouse**. diff --git a/docs/source/Eng/eng_index.rst b/docs/source/Eng/eng_index.rst index 5945bab3..28c6c940 100644 --- a/docs/source/Eng/eng_index.rst +++ b/docs/source/Eng/eng_index.rst @@ -142,6 +142,7 @@ Comprehensive guides for all AutoControl features. doc/new_features/v117_features_doc doc/new_features/v118_features_doc doc/new_features/v119_features_doc + doc/new_features/v120_features_doc doc/ocr_backends/ocr_backends_doc doc/observability/observability_doc doc/operations_layer/operations_layer_doc diff --git a/docs/source/Zh/doc/new_features/v120_features_doc.rst b/docs/source/Zh/doc/new_features/v120_features_doc.rst new file mode 100644 index 00000000..95b698f4 --- /dev/null +++ b/docs/source/Zh/doc/new_features/v120_features_doc.rst @@ -0,0 +1,29 @@ +相對滑鼠移動 +============ + +滑鼠 wrapper 只提供絕對的 ``set_mouse_position``——先前沒有「將指標位移 ``(dx, dy)``」(pynput / PyAutoGUI 的 +``moveRel`` 慣用法),而相對指標 / 畫布 / FPS 類應用與漸進式拖曳都需要它。 + +:func:`relative_target` 為純算術(目前位置 + 位移),可單元測試;:func:`move_mouse_relative` 讀取即時位置並 +設定新位置,getter 與 setter 皆可注入,因此可在無真實指標下測試。不匯入 ``PySide6``。 + +無頭 API +-------- + +.. code-block:: python + + from je_auto_control import move_mouse_relative, relative_target + + move_mouse_relative(-40, 12) # 從目前位置往左 40、往下 12 + # {'from': [200, 200], 'to': [160, 212], 'delta': [-40, 12]} + + relative_target((100, 100), 10, -5) # (110, 95) — 純函式、無 I/O + +``move_mouse_relative`` 讀取目前位置(若無法讀取則拋出 ``AutoControlMouseException``)、加上位移、再移動過去。 +``get_position`` / ``set_position`` 預設為真實滑鼠 wrapper,但可注入以供無頭測試。 + +執行器命令 +---------- + +``AC_move_mouse_relative`` 接受 ``dx`` / ``dy`` 並回傳 ``{from, to, delta}``。它以 MCP 工具 +``ac_move_mouse_relative`` 以及 Script Builder 中 **Mouse** 分類下的命令提供。 diff --git a/docs/source/Zh/zh_index.rst b/docs/source/Zh/zh_index.rst index 7f5c98a5..5b507654 100644 --- a/docs/source/Zh/zh_index.rst +++ b/docs/source/Zh/zh_index.rst @@ -142,6 +142,7 @@ AutoControl 所有功能的完整使用指南。 doc/new_features/v117_features_doc doc/new_features/v118_features_doc doc/new_features/v119_features_doc + doc/new_features/v120_features_doc doc/ocr_backends/ocr_backends_doc doc/observability/observability_doc doc/operations_layer/operations_layer_doc diff --git a/je_auto_control/__init__.py b/je_auto_control/__init__.py index ae9dacb0..ee64490a 100644 --- a/je_auto_control/__init__.py +++ b/je_auto_control/__init__.py @@ -259,6 +259,10 @@ from je_auto_control.utils.field_entry import plan_field_set, set_field_text # Hold a key for a duration / auto-repeat at a fixed rate from je_auto_control.utils.key_hold import hold_key, plan_key_hold +# Relative mouse movement (move by a delta from the current position) +from je_auto_control.utils.mouse_relative import ( + move_mouse_relative, relative_target, +) # CI workflow annotations (GitHub Actions) from je_auto_control.utils.ci_annotations import ( emit_annotations, format_annotation, @@ -1038,6 +1042,8 @@ def start_autocontrol_gui(*args, **kwargs): "set_field_text", "plan_key_hold", "hold_key", + "move_mouse_relative", + "relative_target", "emit_annotations", "format_annotation", "ClipboardHistory", "default_clipboard_history", "analyze_heal_log", "heal_stats", "scan_secrets", diff --git a/je_auto_control/gui/script_builder/command_schema.py b/je_auto_control/gui/script_builder/command_schema.py index 1c11146e..93c18018 100644 --- a/je_auto_control/gui/script_builder/command_schema.py +++ b/je_auto_control/gui/script_builder/command_schema.py @@ -758,6 +758,14 @@ def _add_misc_specs(specs: List[CommandSpec]) -> None: ), description="Press, drag through a polyline of waypoints, release.", )) + specs.append(CommandSpec( + "AC_move_mouse_relative", "Mouse", "Move Relative", + fields=( + FieldSpec("dx", FieldType.INT, placeholder="-40"), + FieldSpec("dy", FieldType.INT, placeholder="12"), + ), + description="Move the pointer by (dx, dy) from its current position.", + )) specs.append(CommandSpec( "AC_list_plugins", "Tools", "List Plugin Commands", fields=(FieldSpec("group", FieldType.STRING, optional=True, diff --git a/je_auto_control/utils/executor/action_executor.py b/je_auto_control/utils/executor/action_executor.py index aa21e73b..3711be1c 100644 --- a/je_auto_control/utils/executor/action_executor.py +++ b/je_auto_control/utils/executor/action_executor.py @@ -3131,6 +3131,12 @@ def _hold_key(key: str, duration_s: Any = 1.0, return hold_key(key, float(duration_s), rate_hz=rate) +def _move_mouse_relative(dx: Any, dy: Any) -> Dict[str, Any]: + """Adapter: move the pointer by a delta from its current position.""" + from je_auto_control.utils.mouse_relative import move_mouse_relative + return move_mouse_relative(int(dx), int(dy)) + + def _cas_put(name: str, key: str, value: Any, expected_version: Any = None) -> Dict[str, Any]: """Adapter: optimistic put into a named versioned store.""" @@ -4830,6 +4836,7 @@ def __init__(self): "AC_drag_path": _drag_path, "AC_set_field_text": _set_field_text, "AC_hold_key": _hold_key, + "AC_move_mouse_relative": _move_mouse_relative, "AC_detect_drift": _detect_drift, "AC_categorical_drift": _categorical_drift, "AC_diff_rows": _diff_rows, diff --git a/je_auto_control/utils/mcp_server/tools/_factories.py b/je_auto_control/utils/mcp_server/tools/_factories.py index b62f009a..9207f965 100644 --- a/je_auto_control/utils/mcp_server/tools/_factories.py +++ b/je_auto_control/utils/mcp_server/tools/_factories.py @@ -2578,6 +2578,21 @@ def tween_drag_tools() -> List[MCPTool]: ] +def mouse_relative_tools() -> List[MCPTool]: + return [ + MCPTool( + name="ac_move_mouse_relative", + description=("Move the pointer by ('dx', 'dy') relative to its " + "current position. Returns {from, to, delta}."), + input_schema=schema( + {"dx": {"type": "integer"}, "dy": {"type": "integer"}}, + required=["dx", "dy"]), + handler=h.move_mouse_relative, + annotations=SIDE_EFFECT_ONLY, + ), + ] + + def key_hold_tools() -> List[MCPTool]: return [ MCPTool( @@ -5896,7 +5911,7 @@ def media_assert_tools() -> List[MCPTool]: input_macro_tools, resilience_tools, ci_annotation_tools, clipboard_history_tools, audit_analysis_tools, process_doc_tools, tween_drag_tools, mouse_path_tools, field_entry_tools, - key_hold_tools, plugin_sdk_tools, governance_tools, + key_hold_tools, mouse_relative_tools, plugin_sdk_tools, governance_tools, credential_lease_tools, egress_tools, approval_testing_tools, trajectory_eval_tools, compliance_tools, agent_trace_tools, video_report_tools, fuzzy_tools, artifact_store_tools, image_dedup_tools, diff --git a/je_auto_control/utils/mcp_server/tools/_handlers.py b/je_auto_control/utils/mcp_server/tools/_handlers.py index 1d25ee3b..228a3a7d 100644 --- a/je_auto_control/utils/mcp_server/tools/_handlers.py +++ b/je_auto_control/utils/mcp_server/tools/_handlers.py @@ -2048,6 +2048,11 @@ def hold_key(key, duration_s=1.0, rate_hz=None): return _hold_key(key, duration_s, rate_hz) +def move_mouse_relative(dx, dy): + from je_auto_control.utils.executor.action_executor import _move_mouse_relative + return _move_mouse_relative(dx, dy) + + def detect_drift(reference, current, threshold=0.25, bins=10): from je_auto_control.utils.executor.action_executor import _detect_drift return _detect_drift(reference, current, threshold, bins) diff --git a/je_auto_control/utils/mouse_relative/__init__.py b/je_auto_control/utils/mouse_relative/__init__.py new file mode 100644 index 00000000..ffdd7a7c --- /dev/null +++ b/je_auto_control/utils/mouse_relative/__init__.py @@ -0,0 +1,6 @@ +"""Relative mouse movement — move by a delta from the current position.""" +from je_auto_control.utils.mouse_relative.mouse_relative import ( + move_mouse_relative, relative_target, +) + +__all__ = ["move_mouse_relative", "relative_target"] diff --git a/je_auto_control/utils/mouse_relative/mouse_relative.py b/je_auto_control/utils/mouse_relative/mouse_relative.py new file mode 100644 index 00000000..088f36df --- /dev/null +++ b/je_auto_control/utils/mouse_relative/mouse_relative.py @@ -0,0 +1,53 @@ +"""Relative mouse movement — move by a delta from the current position. + +The mouse wrapper exposes only absolute ``set_mouse_position``; there is no +"nudge the pointer by (dx, dy)" (pynput / PyAutoGUI ``moveRel`` staple), which is +what relative-pointer / canvas / FPS-style apps and incremental drags need. + +:func:`relative_target` is the pure arithmetic (current + delta) and is +unit-testable; :func:`move_mouse_relative` reads the live position and sets the +new one, with both the getter and setter injectable so it is tested without a +real pointer. Imports no ``PySide6``. +""" +from typing import Any, Callable, Dict, Optional, Tuple + +from je_auto_control.utils.exception.exceptions import AutoControlMouseException + +PositionGetter = Callable[[], Optional[Tuple[int, int]]] +PositionSetter = Callable[[int, int], Any] + + +def relative_target(current: Tuple[int, int], dx: int, dy: int) -> Tuple[int, int]: + """Return ``current`` offset by ``(dx, dy)`` as an integer ``(x, y)``.""" + return int(current[0]) + int(dx), int(current[1]) + int(dy) + + +def move_mouse_relative(dx: int, dy: int, *, + get_position: Optional[PositionGetter] = None, + set_position: Optional[PositionSetter] = None, + ) -> Dict[str, Any]: + """Move the pointer by ``(dx, dy)`` relative to where it is now. + + ``get_position`` / ``set_position`` default to the real mouse wrapper but are + injectable for headless tests. Raises :class:`AutoControlMouseException` if + the current position cannot be read. + """ + getter = get_position or _default_get_position + setter = set_position or _default_set_position + current = getter() + if current is None: + raise AutoControlMouseException("could not read the current mouse position") + target = relative_target((int(current[0]), int(current[1])), dx, dy) + setter(target[0], target[1]) + return {"from": [int(current[0]), int(current[1])], + "to": [target[0], target[1]], "delta": [int(dx), int(dy)]} + + +def _default_get_position() -> Optional[Tuple[int, int]]: + from je_auto_control.wrapper.auto_control_mouse import get_mouse_position + return get_mouse_position() + + +def _default_set_position(x: int, y: int) -> Any: + from je_auto_control.wrapper.auto_control_mouse import set_mouse_position + return set_mouse_position(x, y) diff --git a/test/unit_test/headless/test_mouse_relative_batch.py b/test/unit_test/headless/test_mouse_relative_batch.py new file mode 100644 index 00000000..576d9055 --- /dev/null +++ b/test/unit_test/headless/test_mouse_relative_batch.py @@ -0,0 +1,54 @@ +"""Headless tests for relative mouse movement. No Qt.""" +import je_auto_control as ac +from je_auto_control.utils.exception.exceptions import AutoControlMouseException +from je_auto_control.utils.mouse_relative import ( + move_mouse_relative, relative_target, +) + + +def test_relative_target_arithmetic(): + assert relative_target((100, 100), -40, 12) == (60, 112) + assert relative_target((0, 0), 0, 0) == (0, 0) + + +def test_move_uses_current_position_plus_delta(): + moves = [] + result = move_mouse_relative( + 10, -5, get_position=lambda: (200, 200), + set_position=lambda x, y: moves.append((x, y))) + assert moves == [(210, 195)] + assert result == {"from": [200, 200], "to": [210, 195], "delta": [10, -5]} + + +def test_raises_when_position_unreadable(): + try: + move_mouse_relative(1, 1, get_position=lambda: None, + set_position=lambda x, y: None) + except AutoControlMouseException: + pass + else: # pragma: no cover + raise AssertionError("expected AutoControlMouseException") + + +# --- wiring --------------------------------------------------------------- + +def test_executor_adapter_planning(): + # the default backend get/set is device-bound; exercise the pure arithmetic + # the adapter relies on instead. + assert relative_target((5, 5), 3, 4) == (8, 9) + + +def test_wiring(): + known = ac.executor.known_commands() + assert "AC_move_mouse_relative" in set(known) + from je_auto_control.utils.mcp_server.tools import build_default_tool_registry + names = {t.name for t in build_default_tool_registry()} + assert "ac_move_mouse_relative" in names + from je_auto_control.gui.script_builder.command_schema import _build_specs + specs = {s.command for s in _build_specs()} + assert "AC_move_mouse_relative" in specs + + +def test_facade_exports(): + for attr in ("move_mouse_relative", "relative_target"): + assert hasattr(ac, attr) and attr in ac.__all__ From 29bfb10086797ea64c60e32cda8cb27dea804df1 Mon Sep 17 00:00:00 2001 From: JeffreyChen Date: Tue, 23 Jun 2026 01:25:30 +0800 Subject: [PATCH 09/39] Add wait-for-region-colour (appear/vanish) --- .idea/workspace.xml | 96 ++++++++++--------- README/WHATS_NEW_zh-CN.md | 6 ++ README/WHATS_NEW_zh-TW.md | 6 ++ WHATS_NEW.md | 6 ++ .../doc/new_features/v121_features_doc.rst | 39 ++++++++ docs/source/Eng/eng_index.rst | 1 + .../Zh/doc/new_features/v121_features_doc.rst | 35 +++++++ docs/source/Zh/zh_index.rst | 1 + je_auto_control/__init__.py | 10 +- .../gui/script_builder/command_schema.py | 16 ++++ .../utils/executor/action_executor.py | 19 ++++ .../utils/mcp_server/tools/_factories.py | 18 ++++ .../utils/mcp_server/tools/_handlers.py | 7 ++ je_auto_control/utils/smart_waits/__init__.py | 11 ++- je_auto_control/utils/smart_waits/waits.py | 49 ++++++++++ .../headless/test_wait_color_batch.py | 81 ++++++++++++++++ 16 files changed, 347 insertions(+), 54 deletions(-) create mode 100644 docs/source/Eng/doc/new_features/v121_features_doc.rst create mode 100644 docs/source/Zh/doc/new_features/v121_features_doc.rst create mode 100644 test/unit_test/headless/test_wait_color_batch.py diff --git a/.idea/workspace.xml b/.idea/workspace.xml index 50980db1..7b6f36d5 100644 --- a/.idea/workspace.xml +++ b/.idea/workspace.xml @@ -4,7 +4,13 @@