From cacc3239b8fd2bf3453456981049ae8a58eb8aca Mon Sep 17 00:00:00 2001 From: JeffreyChen Date: Wed, 24 Jun 2026 12:41:33 +0800 Subject: [PATCH 1/3] Add clipboard_rich_formats: RTF + CSV/TSV clipboard codecs rich_clipboard added CF_HTML, but RTF (the format rich editors accept for styled paste) and the Csv format Excel reads were still missing. Add both: build_rtf/rtf_to_text build and strip RTF control words and escapes in pure Python with a unit-testable round-trip, and rows_to_csv/ csv_to_rows wrap the stdlib csv module (delimiter-parametrised for TSV). The Win32 get/set share one generic byte-transfer helper; codecs are platform-independent and headless-tested. --- WHATS_NEW.md | 6 + .../doc/new_features/v185_features_doc.rst | 49 ++++ .../Zh/doc/new_features/v185_features_doc.rst | 44 +++ je_auto_control/__init__.py | 8 + .../gui/script_builder/command_schema.py | 25 ++ .../utils/clipboard_rich_formats/__init__.py | 11 + .../clipboard_rich_formats.py | 266 ++++++++++++++++++ .../utils/executor/action_executor.py | 35 +++ .../utils/mcp_server/tools/_factories.py | 40 +++ .../utils/mcp_server/tools/_handlers.py | 20 ++ .../test_clipboard_rich_formats_batch.py | 70 +++++ 11 files changed, 574 insertions(+) create mode 100644 docs/source/Eng/doc/new_features/v185_features_doc.rst create mode 100644 docs/source/Zh/doc/new_features/v185_features_doc.rst create mode 100644 je_auto_control/utils/clipboard_rich_formats/__init__.py create mode 100644 je_auto_control/utils/clipboard_rich_formats/clipboard_rich_formats.py create mode 100644 test/unit_test/headless/test_clipboard_rich_formats_batch.py diff --git a/WHATS_NEW.md b/WHATS_NEW.md index 15cd7df5..47ad558c 100644 --- a/WHATS_NEW.md +++ b/WHATS_NEW.md @@ -1,5 +1,11 @@ # What's New — AutoControl +## What's new (2026-06-24) — Rich Clipboard Formats (RTF and CSV/TSV) + +Put styled text and tables on the clipboard for cross-app paste into Word and Excel. Full reference: [`docs/source/Eng/doc/new_features/v185_features_doc.rst`](docs/source/Eng/doc/new_features/v185_features_doc.rst). + +- **`build_rtf` / `rtf_to_text` / `rows_to_csv` / `csv_to_rows` + `set_clipboard_rtf` / `get_clipboard_rtf` / `set_clipboard_csv` / `get_clipboard_csv`** (`AC_set_clipboard_rtf`, `AC_get_clipboard_rtf`, `AC_set_clipboard_csv`, `AC_get_clipboard_csv`): `rich_clipboard` added CF_HTML, but RTF (the format rich editors accept) and the `Csv` format Excel reads were still missing. This adds both: `build_rtf`/`rtf_to_text` build and strip RTF control words and `\uNNNN` / `\'XX` escapes in pure Python (fully unit-testable round-trip), and `rows_to_csv`/`csv_to_rows` wrap the stdlib `csv` module (delimiter-parametrised, so `\t` gives TSV). The codecs are platform-independent; the Win32 get/set share one generic byte-transfer helper, and the sets seed plain text so plain editors still paste. No `PySide6`. + ## What's new (2026-06-24) — Keyboard Focus Order (Tab sequence / WCAG audit / set-focus) Reason about keyboard navigation: the Tab order, a WCAG focus-order audit, and set-focus. Full reference: [`docs/source/Eng/doc/new_features/v184_features_doc.rst`](docs/source/Eng/doc/new_features/v184_features_doc.rst). diff --git a/docs/source/Eng/doc/new_features/v185_features_doc.rst b/docs/source/Eng/doc/new_features/v185_features_doc.rst new file mode 100644 index 00000000..134cf3fd --- /dev/null +++ b/docs/source/Eng/doc/new_features/v185_features_doc.rst @@ -0,0 +1,49 @@ +Rich Clipboard Formats — RTF and CSV/TSV +======================================== + +``rich_clipboard`` added ``CF_HTML`` for rich paste into Word / Outlook, but two +other cross-application clipboard formats were still missing: + +* **RTF** (``"Rich Text Format"``) — the format almost every rich editor accepts + for styled paste. ``build_rtf`` / ``rtf_to_text`` build and strip RTF control + words and ``\uNNNN`` / ``\'XX`` escapes in pure Python, with a fully + unit-testable round-trip. +* **CSV / TSV** (the registered ``"Csv"`` format Excel reads) — ``rows_to_csv`` / + ``csv_to_rows`` are a thin, delimiter-parametrised wrapper over the stdlib + ``csv`` module, so a table can be put on / read off the clipboard. + +The codecs are platform-independent and headless-testable; only the actual +clipboard I/O is Win32 (raising ``RuntimeError`` elsewhere, like the base +``clipboard`` module), and the byte transfer is a single generic helper shared by +both formats. Imports no ``PySide6``. + +Headless API +------------ + +.. code-block:: python + + from je_auto_control import (build_rtf, rtf_to_text, rows_to_csv, + csv_to_rows, set_clipboard_rtf, set_clipboard_csv) + + rtf = build_rtf("Hello\nWorld") # minimal valid RTF document + rtf_to_text(rtf) # -> "Hello\nWorld" + + rows_to_csv([["a", "b"], ["1", "2"]]) # 'a,b\r\n1,2\r\n' + csv_to_rows("a,b\r\n1,2\r\n") # [["a", "b"], ["1", "2"]] + + set_clipboard_rtf("Paste me as styled text") # Windows + set_clipboard_csv([["Name", "Qty"], ["Pen", "3"]], delimiter="\t") # TSV + +``build_rtf`` escapes braces / backslashes, turns newlines into ``\par`` and +non-ASCII characters into ``\uNNNN?`` escapes (the output is pure ASCII). +``set_clipboard_rtf`` / ``set_clipboard_csv`` also seed plain text by default so +plain editors still paste something; ``get_clipboard_rtf`` returns the raw RTF +string (feed it to ``rtf_to_text``) and ``get_clipboard_csv`` returns rows. + +Executor commands +----------------- + +``AC_set_clipboard_rtf`` / ``AC_get_clipboard_rtf`` / ``AC_set_clipboard_csv`` / +``AC_get_clipboard_csv`` (the sets take ``text`` / ``rows`` + ``delimiter``). They +are exposed as the matching ``ac_*`` MCP tools (the sets side-effect-only, the +gets read-only) and as Script Builder commands under **Data**. diff --git a/docs/source/Zh/doc/new_features/v185_features_doc.rst b/docs/source/Zh/doc/new_features/v185_features_doc.rst new file mode 100644 index 00000000..8d47a2ca --- /dev/null +++ b/docs/source/Zh/doc/new_features/v185_features_doc.rst @@ -0,0 +1,44 @@ +豐富剪貼簿格式——RTF 與 CSV/TSV +============================== + +``rich_clipboard`` 已加入 ``CF_HTML`` 以便把豐富內容貼進 Word / Outlook,但仍缺少另外兩種 +跨應用程式的剪貼簿格式: + +* **RTF**（``"Rich Text Format"``)——幾乎每個豐富編輯器都接受、用於樣式貼上的格式。 + ``build_rtf`` / ``rtf_to_text`` 以純 Python 建立與剝除 RTF 控制字與 ``\uNNNN`` / ``\'XX`` + 轉義,並具備完全可單元測試的往返。 +* **CSV / TSV**（Excel 讀取的已註冊 ``"Csv"`` 格式)——``rows_to_csv`` / ``csv_to_rows`` 是對 + 標準庫 ``csv`` 模組的薄包裝(可指定分隔符),讓表格能放上 / 讀下剪貼簿。 + +這些編解碼器與平台無關且可無頭測試;只有實際的剪貼簿 I/O 為 Win32(在其他平台拋出 +``RuntimeError``,與基礎 ``clipboard`` 模組一致),且位元組傳輸是兩種格式共用的單一泛型輔助 +函式。不匯入 ``PySide6``。 + +無頭 API +-------- + +.. code-block:: python + + from je_auto_control import (build_rtf, rtf_to_text, rows_to_csv, + csv_to_rows, set_clipboard_rtf, set_clipboard_csv) + + rtf = build_rtf("Hello\nWorld") # 最小的有效 RTF 文件 + rtf_to_text(rtf) # -> "Hello\nWorld" + + rows_to_csv([["a", "b"], ["1", "2"]]) # 'a,b\r\n1,2\r\n' + csv_to_rows("a,b\r\n1,2\r\n") # [["a", "b"], ["1", "2"]] + + set_clipboard_rtf("以樣式文字貼上我") # Windows + set_clipboard_csv([["Name", "Qty"], ["Pen", "3"]], delimiter="\t") # TSV + +``build_rtf`` 會轉義大括號 / 反斜線,把換行轉為 ``\par``,並把非 ASCII 字元轉為 ``\uNNNN?`` +轉義(輸出為純 ASCII)。``set_clipboard_rtf`` / ``set_clipboard_csv`` 預設也會種入純文字,讓 +純文字編輯器仍能貼上內容;``get_clipboard_rtf`` 回傳原始 RTF 字串(再餵給 ``rtf_to_text``), +``get_clipboard_csv`` 回傳列。 + +執行器指令 +---------- + +``AC_set_clipboard_rtf`` / ``AC_get_clipboard_rtf`` / ``AC_set_clipboard_csv`` / +``AC_get_clipboard_csv``（set 取 ``text`` / ``rows`` 加 ``delimiter``)。皆以對應的 ``ac_*`` +MCP 工具(set 為僅副作用、get 為唯讀)及 Script Builder 指令(位於 **Data** 分類下)形式提供。 diff --git a/je_auto_control/__init__.py b/je_auto_control/__init__.py index d912ba96..edb09043 100644 --- a/je_auto_control/__init__.py +++ b/je_auto_control/__init__.py @@ -66,6 +66,11 @@ from je_auto_control.utils.focus_order import ( audit_focus_order, focus_control, is_interactive_role, tab_order, ) +# Rich clipboard formats — RTF + CSV/TSV codecs and Windows get / set +from je_auto_control.utils.clipboard_rich_formats import ( + build_rtf, csv_to_rows, get_clipboard_csv, get_clipboard_rtf, rows_to_csv, + rtf_to_text, set_clipboard_csv, set_clipboard_rtf, +) # VLM element locator (headless) from je_auto_control.utils.vision import ( VLMNotAvailableError, click_by_description, locate_by_description, @@ -1634,6 +1639,9 @@ def start_autocontrol_gui(*args, **kwargs): "control_type_name", "humanize_role", "humanize_tree", "assign_node_paths", "find_by_path", "is_interactive_role", "tab_order", "audit_focus_order", "focus_control", + "build_rtf", "rtf_to_text", "rows_to_csv", "csv_to_rows", + "set_clipboard_rtf", "get_clipboard_rtf", + "set_clipboard_csv", "get_clipboard_csv", # VLM locator "VLMNotAvailableError", "locate_by_description", "click_by_description", "verify_description", diff --git a/je_auto_control/gui/script_builder/command_schema.py b/je_auto_control/gui/script_builder/command_schema.py index 4b1b2903..92aae01f 100644 --- a/je_auto_control/gui/script_builder/command_schema.py +++ b/je_auto_control/gui/script_builder/command_schema.py @@ -1597,6 +1597,31 @@ def _add_misc_specs(specs: List[CommandSpec]) -> None: "AC_get_clipboard_files", "Data", "Get Clipboard Files", description="Read the clipboard's file-drop list (CF_HDROP, Windows).", )) + specs.append(CommandSpec( + "AC_set_clipboard_rtf", "Data", "Set Clipboard RTF", + fields=(FieldSpec("text", FieldType.STRING, + placeholder="Styled paste text"),), + description="Put text on the clipboard as Rich Text Format (Windows).", + )) + specs.append(CommandSpec( + "AC_get_clipboard_rtf", "Data", "Get Clipboard RTF", + description="Read the clipboard's RTF document string (Windows).", + )) + specs.append(CommandSpec( + "AC_set_clipboard_csv", "Data", "Set Clipboard CSV/TSV", + fields=( + FieldSpec("rows", FieldType.STRING, + placeholder='[["a", "b"], ["1", "2"]]'), + FieldSpec("delimiter", FieldType.STRING, optional=True, default=","), + ), + description="Put a table on the clipboard as the Csv format (Windows).", + )) + specs.append(CommandSpec( + "AC_get_clipboard_csv", "Data", "Get Clipboard CSV/TSV", + fields=(FieldSpec("delimiter", FieldType.STRING, optional=True, + default=","),), + description="Read the clipboard's Csv content as rows (Windows).", + )) specs.append(CommandSpec( "AC_watchdog_add", "Flow", "Watchdog: Add Popup Rule", fields=( diff --git a/je_auto_control/utils/clipboard_rich_formats/__init__.py b/je_auto_control/utils/clipboard_rich_formats/__init__.py new file mode 100644 index 00000000..57d63070 --- /dev/null +++ b/je_auto_control/utils/clipboard_rich_formats/__init__.py @@ -0,0 +1,11 @@ +"""Rich clipboard formats — RTF and CSV/TSV codecs + Windows get / set.""" +from je_auto_control.utils.clipboard_rich_formats.clipboard_rich_formats import ( + build_rtf, csv_to_rows, get_clipboard_csv, get_clipboard_rtf, rows_to_csv, + rtf_to_text, set_clipboard_csv, set_clipboard_rtf, +) + +__all__ = [ + "build_rtf", "rtf_to_text", "rows_to_csv", "csv_to_rows", + "set_clipboard_rtf", "get_clipboard_rtf", + "set_clipboard_csv", "get_clipboard_csv", +] diff --git a/je_auto_control/utils/clipboard_rich_formats/clipboard_rich_formats.py b/je_auto_control/utils/clipboard_rich_formats/clipboard_rich_formats.py new file mode 100644 index 00000000..8e277b24 --- /dev/null +++ b/je_auto_control/utils/clipboard_rich_formats/clipboard_rich_formats.py @@ -0,0 +1,266 @@ +"""Rich clipboard formats — RTF and CSV/TSV get / set. + +``rich_clipboard`` added ``CF_HTML`` for rich paste into Word / Outlook, but two +other cross-app formats were still missing: + +* **RTF** (``"Rich Text Format"``) — the format almost every rich editor accepts + for styled paste. Building / stripping RTF control words and ``\\uNNNN`` / + ``\\'XX`` escapes is the error-prone part; :func:`build_rtf` / :func:`rtf_to_text` + do it in pure Python with a fully unit-testable round-trip. +* **CSV / TSV** (the registered ``"Csv"`` format Excel reads) — :func:`rows_to_csv` + / :func:`csv_to_rows` are a thin, delimiter-parametrised wrapper over the stdlib + ``csv`` module so a table can be put on / read off the clipboard. + +The codecs are platform-independent and headless-testable; only the actual +clipboard I/O is Win32 (raising ``RuntimeError`` elsewhere, like the base +``clipboard`` module). The Win32 byte transfer is a single generic helper shared +by both formats. Imports no ``PySide6``. +""" +import csv +import io +import sys +from typing import List, Optional, Sequence + +_RTF_FORMAT_NAME = "Rich Text Format" +_CSV_FORMAT_NAME = "Csv" +_GMEM_MOVEABLE = 0x0002 +_RTF_PREAMBLE = "{\\rtf1\\ansi\\ansicpg1252\\deff0{\\fonttbl{\\f0\\fnil Calibri;}}\n" +_RTF_LITERAL_ESCAPE = {"\\": "\\\\", "{": "\\{", "}": "\\}", + "\n": "\\par\n", "\r": "", "\t": "\\tab "} +# RTF destination groups whose textual content is metadata, not document text. +_RTF_DESTINATIONS = frozenset({ + "fonttbl", "colortbl", "stylesheet", "info", "pict", "header", "footer", + "object", "themedata", "datastore", "operator", "generator", "rsidtbl", +}) + + +# --- RTF codec (pure) ------------------------------------------------------ + +def _escape_char(char: str) -> str: + if char in _RTF_LITERAL_ESCAPE: + return _RTF_LITERAL_ESCAPE[char] + if ord(char) > 127: + return f"\\u{ord(char)}?" + return char + + +def build_rtf(text: str) -> str: + """Wrap plain ``text`` in a minimal, valid RTF document. + + Backslash / brace are escaped, newlines become ``\\par`` and non-ASCII + characters become ``\\uNNNN?`` escapes, so the result is pure ASCII. + """ + if not isinstance(text, str): + raise TypeError("build_rtf expects a str") + return _RTF_PREAMBLE + "".join(_escape_char(ch) for ch in text) + "}" + + +def _apply_word(word: str, param: str, stack: List[bool], + result: List[str]) -> int: + """Apply one RTF control word; return how many following chars to skip.""" + if word in _RTF_DESTINATIONS: + stack[-1] = True + return 0 + if stack[-1]: + return 1 if word == "u" else 0 + if word in ("par", "line"): + result.append("\n") + elif word == "tab": + result.append("\t") + elif word == "u" and param: + result.append(chr(int(param) % 0x10000)) + return 1 # the trailing fallback char is consumed + return 0 + + +def _consume_word(text: str, i: int, stack: List[bool], + result: List[str]) -> int: + n = len(text) + j = i + 1 + while j < n and text[j].isalpha(): + j += 1 + k = j + 1 if j < n and text[j] == "-" else j + while k < n and text[k].isdigit(): + k += 1 + param = text[j:k] + if k < n and text[k] == " ": + k += 1 + return k + _apply_word(text[i + 1:j], param, stack, result) + + +def _consume_hex(text: str, i: int, stack: List[bool], + result: List[str]) -> int: + if not stack[-1]: + try: + byte = bytes([int(text[i + 2:i + 4], 16)]) + result.append(byte.decode("cp1252", "replace")) + except ValueError: + pass + return i + 4 + + +def _consume_control(text: str, i: int, stack: List[bool], + result: List[str]) -> int: + nxt = text[i + 1] if i + 1 < len(text) else "" + if nxt in "\\{}": + if not stack[-1]: + result.append(nxt) + return i + 2 + if nxt == "*": + stack[-1] = True + return i + 2 + if nxt == "'": + return _consume_hex(text, i, stack, result) + if nxt.isalpha(): + return _consume_word(text, i, stack, result) + return i + 2 # other control symbol (\~, \-, …) + + +def rtf_to_text(rtf: str) -> str: + """Strip an RTF document to its plain text (inverse of :func:`build_rtf`). + + Drops metadata groups (font / colour / style tables, etc.), converts + ``\\par`` / ``\\line`` to newlines and ``\\tab`` to a tab, and decodes + ``\\uNNNN`` / ``\\'XX`` character escapes. + """ + text = str(rtf) + result: List[str] = [] + stack: List[bool] = [False] + i, n = 0, len(text) + while i < n: + char = text[i] + if char == "{": + stack.append(stack[-1]) + i += 1 + elif char == "}": + if len(stack) > 1: + stack.pop() + i += 1 + elif char == "\\": + i = _consume_control(text, i, stack, result) + elif char in "\r\n": + i += 1 # raw line breaks in the source are insignificant + else: + if not stack[-1]: + result.append(char) + i += 1 + return "".join(result) + + +# --- CSV / TSV codec (pure) ------------------------------------------------ + +def rows_to_csv(rows: Sequence[Sequence[object]], *, delimiter: str = ",") -> str: + """Serialise rows of cells to CSV/TSV text (use ``delimiter="\\t"`` for TSV).""" + buffer = io.StringIO() + writer = csv.writer(buffer, delimiter=delimiter, lineterminator="\r\n") + writer.writerows([[str(cell) for cell in row] for row in rows]) + return buffer.getvalue() + + +def csv_to_rows(text: str, *, delimiter: str = ",") -> List[List[str]]: + """Parse CSV/TSV text into a list of cell-string rows.""" + return [list(row) for row in csv.reader(io.StringIO(str(text)), + delimiter=delimiter)] + + +# --- Win32 clipboard I/O --------------------------------------------------- + +def _format_id(name: str) -> int: + import ctypes + return ctypes.windll.user32.RegisterClipboardFormatW(name) + + +def _win_set_format(format_id: int, payload: bytes, *, + empty_first: bool = True) -> None: + import ctypes + from ctypes import wintypes + user32, kernel32 = ctypes.windll.user32, ctypes.windll.kernel32 + kernel32.GlobalAlloc.restype = wintypes.HGLOBAL + kernel32.GlobalLock.restype = ctypes.c_void_p + if not user32.OpenClipboard(None): + raise RuntimeError("OpenClipboard failed") + try: + if empty_first: + user32.EmptyClipboard() + handle = kernel32.GlobalAlloc(_GMEM_MOVEABLE, len(payload)) + if not handle: + raise RuntimeError("GlobalAlloc failed") + pointer = kernel32.GlobalLock(handle) + ctypes.memmove(pointer, payload, len(payload)) + kernel32.GlobalUnlock(handle) + if not user32.SetClipboardData(format_id, handle): + raise RuntimeError("SetClipboardData failed") + finally: + user32.CloseClipboard() + + +def _win_get_format(format_id: int) -> Optional[bytes]: + import ctypes + from ctypes import wintypes + user32, kernel32 = ctypes.windll.user32, ctypes.windll.kernel32 + user32.GetClipboardData.restype = wintypes.HANDLE + kernel32.GlobalLock.restype = ctypes.c_void_p + if not user32.OpenClipboard(None): + raise RuntimeError("OpenClipboard failed") + try: + handle = user32.GetClipboardData(format_id) + if not handle: + return None + pointer = kernel32.GlobalLock(handle) + size = kernel32.GlobalSize(handle) + data = ctypes.string_at(pointer, size) + kernel32.GlobalUnlock(handle) + return data.split(b"\x00", 1)[0] + finally: + user32.CloseClipboard() + + +def _seed_plaintext(text: str) -> None: + from je_auto_control.utils.clipboard.clipboard import set_clipboard + set_clipboard(text) + + +def set_clipboard_rtf(text: str, *, plaintext: bool = True) -> None: + """Put ``text`` on the clipboard as Rich Text Format (Windows only). + + With ``plaintext`` (default) the raw text is also placed as ``CF_UNICODETEXT`` + so plain editors still paste something. Raises ``RuntimeError`` off Windows. + """ + if not isinstance(text, str): + raise TypeError("set_clipboard_rtf expects a str") + if not sys.platform.startswith("win"): + raise RuntimeError("set_clipboard_rtf is only supported on Windows") + payload = build_rtf(text).encode("ascii", "replace") + b"\x00" + if plaintext: + _seed_plaintext(text) + _win_set_format(_format_id(_RTF_FORMAT_NAME), payload, empty_first=not plaintext) + + +def get_clipboard_rtf() -> Optional[str]: + """Return the clipboard's RTF document string, or ``None`` (Windows only).""" + if not sys.platform.startswith("win"): + raise RuntimeError("get_clipboard_rtf is only supported on Windows") + blob = _win_get_format(_format_id(_RTF_FORMAT_NAME)) + return blob.decode("latin-1") if blob is not None else None + + +def set_clipboard_csv(rows: Sequence[Sequence[object]], *, delimiter: str = ",", + plaintext: bool = True) -> None: + """Put a table on the clipboard as the ``Csv`` format Excel reads (Windows).""" + if not sys.platform.startswith("win"): + raise RuntimeError("set_clipboard_csv is only supported on Windows") + text = rows_to_csv(rows, delimiter=delimiter) + payload = text.encode("utf-8") + b"\x00" + if plaintext: + _seed_plaintext(text) + _win_set_format(_format_id(_CSV_FORMAT_NAME), payload, empty_first=not plaintext) + + +def get_clipboard_csv(*, delimiter: str = ",") -> Optional[List[List[str]]]: + """Return the clipboard's ``Csv`` content as cell-string rows, or ``None``.""" + if not sys.platform.startswith("win"): + raise RuntimeError("get_clipboard_csv is only supported on Windows") + blob = _win_get_format(_format_id(_CSV_FORMAT_NAME)) + if blob is None: + return None + return csv_to_rows(blob.decode("utf-8", "replace"), delimiter=delimiter) diff --git a/je_auto_control/utils/executor/action_executor.py b/je_auto_control/utils/executor/action_executor.py index 4955e76c..f85d4b83 100644 --- a/je_auto_control/utils/executor/action_executor.py +++ b/je_auto_control/utils/executor/action_executor.py @@ -4185,6 +4185,37 @@ def _get_clipboard_files() -> Dict[str, Any]: return {"found": paths is not None, "paths": paths or []} +def _set_clipboard_rtf(text: str) -> Dict[str, Any]: + """Adapter: put text on the clipboard as Rich Text Format (Windows).""" + from je_auto_control.utils.clipboard_rich_formats import set_clipboard_rtf + set_clipboard_rtf(str(text)) + return {"set": True, "length": len(str(text))} + + +def _get_clipboard_rtf() -> Dict[str, Any]: + """Adapter: read the clipboard's RTF document string (Windows).""" + from je_auto_control.utils.clipboard_rich_formats import get_clipboard_rtf + rtf = get_clipboard_rtf() + return {"found": rtf is not None, "rtf": rtf} + + +def _set_clipboard_csv(rows: Any, delimiter: str = ",") -> Dict[str, Any]: + """Adapter: put a table on the clipboard as the Csv format (Windows).""" + import json + from je_auto_control.utils.clipboard_rich_formats import set_clipboard_csv + if isinstance(rows, str): + rows = json.loads(rows) + set_clipboard_csv(rows, delimiter=str(delimiter)) + return {"set": True, "rows": len(rows)} + + +def _get_clipboard_csv(delimiter: str = ",") -> Dict[str, Any]: + """Adapter: read the clipboard's Csv content as rows (Windows).""" + from je_auto_control.utils.clipboard_rich_formats import get_clipboard_csv + rows = get_clipboard_csv(delimiter=str(delimiter)) + return {"found": rows is not None, "rows": rows or []} + + def _image_histogram(source: Any = None, bins: Any = 32, space: str = "hsv", region: Any = None) -> Dict[str, Any]: """Adapter: per-channel colour histogram of an image / the screen.""" @@ -6398,6 +6429,10 @@ def __init__(self): "AC_get_clipboard_html": _get_clipboard_html, "AC_set_clipboard_files": _set_clipboard_files, "AC_get_clipboard_files": _get_clipboard_files, + "AC_set_clipboard_rtf": _set_clipboard_rtf, + "AC_get_clipboard_rtf": _get_clipboard_rtf, + "AC_set_clipboard_csv": _set_clipboard_csv, + "AC_get_clipboard_csv": _get_clipboard_csv, "AC_image_histogram": _image_histogram, "AC_histogram_changed": _histogram_changed, "AC_changed_regions": _changed_regions, diff --git a/je_auto_control/utils/mcp_server/tools/_factories.py b/je_auto_control/utils/mcp_server/tools/_factories.py index 868bd6a4..8d0188aa 100644 --- a/je_auto_control/utils/mcp_server/tools/_factories.py +++ b/je_auto_control/utils/mcp_server/tools/_factories.py @@ -3252,6 +3252,46 @@ def clipboard_files_tools() -> List[MCPTool]: handler=h.get_clipboard_files, annotations=READ_ONLY, ), + MCPTool( + name="ac_set_clipboard_rtf", + description=("Put 'text' on the clipboard as Rich Text Format so it " + "pastes as styled text into Word / rich editors (Windows). " + "Also seeds plain text. Returns {set, length}."), + input_schema=schema({"text": {"type": "string"}}, required=["text"]), + handler=h.set_clipboard_rtf, + annotations=SIDE_EFFECT_ONLY, + ), + MCPTool( + name="ac_get_clipboard_rtf", + description=("Read the clipboard's RTF document string (Windows). " + "Returns {found, rtf}."), + input_schema=schema({}, required=[]), + handler=h.get_clipboard_rtf, + annotations=READ_ONLY, + ), + MCPTool( + name="ac_set_clipboard_csv", + description=("Put a table on the clipboard as the 'Csv' format Excel " + "reads (Windows). 'rows' is a list of row arrays; " + "'delimiter' defaults to ',' (use '\\t' for TSV). " + "Returns {set, rows}."), + input_schema=schema({ + "rows": {"type": "array", + "items": {"type": "array", "items": {"type": "string"}}}, + "delimiter": {"type": "string"}}, + required=["rows"]), + handler=h.set_clipboard_csv, + annotations=SIDE_EFFECT_ONLY, + ), + MCPTool( + name="ac_get_clipboard_csv", + description=("Read the clipboard's 'Csv' content as rows of cell " + "strings (Windows). 'delimiter' defaults to ','. " + "Returns {found, rows}."), + input_schema=schema({"delimiter": {"type": "string"}}), + handler=h.get_clipboard_csv, + annotations=READ_ONLY, + ), ] diff --git a/je_auto_control/utils/mcp_server/tools/_handlers.py b/je_auto_control/utils/mcp_server/tools/_handlers.py index a609c203..55045b29 100644 --- a/je_auto_control/utils/mcp_server/tools/_handlers.py +++ b/je_auto_control/utils/mcp_server/tools/_handlers.py @@ -2464,6 +2464,26 @@ def get_clipboard_files(): return _get_clipboard_files() +def set_clipboard_rtf(text): + from je_auto_control.utils.executor.action_executor import _set_clipboard_rtf + return _set_clipboard_rtf(text) + + +def get_clipboard_rtf(): + from je_auto_control.utils.executor.action_executor import _get_clipboard_rtf + return _get_clipboard_rtf() + + +def set_clipboard_csv(rows, delimiter=","): + from je_auto_control.utils.executor.action_executor import _set_clipboard_csv + return _set_clipboard_csv(rows, delimiter) + + +def get_clipboard_csv(delimiter=","): + from je_auto_control.utils.executor.action_executor import _get_clipboard_csv + return _get_clipboard_csv(delimiter) + + def image_histogram(source=None, bins=32, space="hsv", region=None): from je_auto_control.utils.executor.action_executor import _image_histogram return _image_histogram(source, bins, space, region) diff --git a/test/unit_test/headless/test_clipboard_rich_formats_batch.py b/test/unit_test/headless/test_clipboard_rich_formats_batch.py new file mode 100644 index 00000000..785d8a01 --- /dev/null +++ b/test/unit_test/headless/test_clipboard_rich_formats_batch.py @@ -0,0 +1,70 @@ +"""Headless tests for RTF + CSV/TSV clipboard codecs (pure; Win32 set/get skipped).""" +import je_auto_control as ac +from je_auto_control.utils.clipboard_rich_formats import ( + build_rtf, csv_to_rows, rows_to_csv, rtf_to_text, +) + + +def test_rtf_round_trip_plain_and_breaks(): + for sample in ("Hello world", "Line1\nLine2\nLine3", "Tab\tsep", ""): + assert rtf_to_text(build_rtf(sample)) == sample + + +def test_rtf_round_trip_unicode_and_escapes(): + for sample in ("café crème", "中文字符", "Braces {x} and \\ backslash"): + assert rtf_to_text(build_rtf(sample)) == sample + + +def test_build_rtf_is_ascii_and_wrapped(): + rtf = build_rtf("café") + assert rtf.startswith("{\\rtf1") + assert rtf.endswith("}") + rtf.encode("ascii") # non-ASCII escaped to \uNNNN?, so this must not raise + assert "\\u233?" in rtf # é + + +def test_rtf_to_text_drops_metadata_and_decodes_hex(): + raw = (r"{\rtf1\ansi\deff0{\fonttbl{\f0 Arial;}}" + r"{\colortbl;\red0\green0\blue0;}Caf\'e9 here\par done}") + assert rtf_to_text(raw) == "Café here\ndone" + + +def test_build_rtf_type_error(): + import pytest + with pytest.raises(TypeError): + build_rtf(123) + + +def test_csv_round_trip_with_embedded_specials(): + rows = [["a", "b,c"], ["d", "e\nf"], ["1", "2"]] + assert csv_to_rows(rows_to_csv(rows)) == [["a", "b,c"], ["d", "e\nf"], + ["1", "2"]] + + +def test_tsv_uses_tab_delimiter(): + text = rows_to_csv([["x", "y"], ["1", "2"]], delimiter="\t") + assert text == "x\ty\r\n1\t2\r\n" + assert csv_to_rows(text, delimiter="\t") == [["x", "y"], ["1", "2"]] + + +# --- wiring (Win32 set/get not executed in CI) ----------------------------- + +def test_wiring(): + known = set(ac.executor.known_commands()) + assert {"AC_set_clipboard_rtf", "AC_get_clipboard_rtf", + "AC_set_clipboard_csv", "AC_get_clipboard_csv"} <= known + from je_auto_control.utils.mcp_server.tools import build_default_tool_registry + names = {t.name for t in build_default_tool_registry()} + assert {"ac_set_clipboard_rtf", "ac_get_clipboard_rtf", + "ac_set_clipboard_csv", "ac_get_clipboard_csv"} <= names + from je_auto_control.gui.script_builder.command_schema import _build_specs + specs = {s.command for s in _build_specs()} + assert {"AC_set_clipboard_rtf", "AC_get_clipboard_rtf", + "AC_set_clipboard_csv", "AC_get_clipboard_csv"} <= specs + + +def test_facade_exports(): + for name in ("build_rtf", "rtf_to_text", "rows_to_csv", "csv_to_rows", + "set_clipboard_rtf", "get_clipboard_rtf", + "set_clipboard_csv", "get_clipboard_csv"): + assert hasattr(ac, name) and name in ac.__all__ From 8c4306ba2d6eedcc2010bb3bfef3dc6bf052848e Mon Sep 17 00:00:00 2001 From: JeffreyChen Date: Wed, 24 Jun 2026 12:52:41 +0800 Subject: [PATCH 2/3] Add clipboard_formats: classify + diff the clipboard's available formats The clipboard holds the same content in several formats at once, and nothing reported which formats were present or detected when that set changed. Add classify_format/classify_formats (map standard CF_* ids + registered names to friendly categories), diff_formats (pure monitor primitive returning added/removed/changed), and a Win32 EnumClipboardFormats enumeration. Classifier and diff are pure and headless-tested; only the live enumeration is Win32. --- WHATS_NEW.md | 6 + .../doc/new_features/v186_features_doc.rst | 49 ++++++ .../Zh/doc/new_features/v186_features_doc.rst | 41 +++++ je_auto_control/__init__.py | 7 + .../gui/script_builder/command_schema.py | 18 +++ .../utils/clipboard_formats/__init__.py | 10 ++ .../clipboard_formats/clipboard_formats.py | 140 ++++++++++++++++++ .../utils/executor/action_executor.py | 29 ++++ .../utils/mcp_server/tools/_factories.py | 31 ++++ .../utils/mcp_server/tools/_handlers.py | 15 ++ .../headless/test_clipboard_formats_batch.py | 80 ++++++++++ 11 files changed, 426 insertions(+) create mode 100644 docs/source/Eng/doc/new_features/v186_features_doc.rst create mode 100644 docs/source/Zh/doc/new_features/v186_features_doc.rst create mode 100644 je_auto_control/utils/clipboard_formats/__init__.py create mode 100644 je_auto_control/utils/clipboard_formats/clipboard_formats.py create mode 100644 test/unit_test/headless/test_clipboard_formats_batch.py diff --git a/WHATS_NEW.md b/WHATS_NEW.md index 47ad558c..e9e0adad 100644 --- a/WHATS_NEW.md +++ b/WHATS_NEW.md @@ -1,5 +1,11 @@ # What's New — AutoControl +## What's new (2026-06-24) — Clipboard Format Inspection (classify / diff available formats) + +See which formats are on the clipboard, and detect when its shape changes. Full reference: [`docs/source/Eng/doc/new_features/v186_features_doc.rst`](docs/source/Eng/doc/new_features/v186_features_doc.rst). + +- **`classify_format` / `classify_formats` / `diff_formats` / `list_clipboard_formats` / `clipboard_formats`** (`AC_clipboard_formats`, `AC_classify_formats`, `AC_diff_formats`): the clipboard usually holds the same content in several formats at once (a Word copy = text + HTML + RTF; a file copy = CF_HDROP; a screenshot = CF_DIB). This enumerates the live clipboard (`EnumClipboardFormats`) without consuming anything and classifies each format into a friendly category (text/image/files/html/rtf/csv/audio/…); `diff_formats` is a pure monitor primitive returning `{added, removed, changed}` between two snapshots. The classifier and diff are pure (registered names take priority over dynamic ids); only the live enumeration is Win32. No `PySide6`. + ## What's new (2026-06-24) — Rich Clipboard Formats (RTF and CSV/TSV) Put styled text and tables on the clipboard for cross-app paste into Word and Excel. Full reference: [`docs/source/Eng/doc/new_features/v185_features_doc.rst`](docs/source/Eng/doc/new_features/v185_features_doc.rst). diff --git a/docs/source/Eng/doc/new_features/v186_features_doc.rst b/docs/source/Eng/doc/new_features/v186_features_doc.rst new file mode 100644 index 00000000..7365ce07 --- /dev/null +++ b/docs/source/Eng/doc/new_features/v186_features_doc.rst @@ -0,0 +1,49 @@ +Clipboard Format Inspection (classify / diff available formats) +=============================================================== + +The clipboard usually holds the *same* content in several formats at once — a +copy from Word offers ``CF_UNICODETEXT`` + ``HTML Format`` + ``Rich Text Format``, +a file copy offers ``CF_HDROP``, a screenshot offers ``CF_DIB``. Knowing *which +formats are present* (without consuming any of them) tells an automation what it +can paste, and comparing two snapshots detects when the clipboard's shape +changed. ``clipboard_formats`` adds: + +* :func:`classify_format` / :func:`classify_formats` — map standard ``CF_*`` ids + and registered format names to friendly categories (text / image / files / + html / rtf / csv / audio / …), +* :func:`diff_formats` — a pure monitor primitive: ``{added, removed, changed}`` + between two snapshots, +* :func:`list_clipboard_formats` / :func:`clipboard_formats` — enumerate the live + clipboard (``EnumClipboardFormats``) and classify it. + +The classifier and diff are pure functions (unit-testable on any platform); only +the live enumeration is Win32 (raising ``RuntimeError`` elsewhere). Imports no +``PySide6``. + +Headless API +------------ + +.. code-block:: python + + from je_auto_control import (classify_formats, diff_formats, + clipboard_formats) + + classify_formats([13, {"id": 49383, "name": "HTML Format"}]) + # {"categories": ["html", "text"], "has_text": True, "has_image": False, ...} + + diff_formats([13, 1], [13, 15]) # {"added": [files], "removed": [text], ...} + + clipboard_formats() # live clipboard summary (Windows) + +A descriptor is an id (``13``), an ``{"id": ..., "name": ...}`` dict, or an +``(id, name)`` tuple. A registered ``name`` takes priority over the id, since +registered formats have dynamic ids (``>= 0xC000``). Unrecognised formats are +``"other"``. + +Executor commands +----------------- + +``AC_clipboard_formats`` (live, Windows), ``AC_classify_formats`` (``formats``) +and ``AC_diff_formats`` (``before`` / ``after``) — the latter two are pure and +run anywhere. They are exposed as read-only ``ac_*`` MCP tools and as Script +Builder commands under **Data**. diff --git a/docs/source/Zh/doc/new_features/v186_features_doc.rst b/docs/source/Zh/doc/new_features/v186_features_doc.rst new file mode 100644 index 00000000..7cff1500 --- /dev/null +++ b/docs/source/Zh/doc/new_features/v186_features_doc.rst @@ -0,0 +1,41 @@ +剪貼簿格式檢視(分類 / 比較可用格式) +==================================== + +剪貼簿通常同時以多種格式保存*相同*內容——從 Word 複製會提供 ``CF_UNICODETEXT`` + +``HTML Format`` + ``Rich Text Format``,複製檔案會提供 ``CF_HDROP``,截圖會提供 ``CF_DIB``。 +知道*目前有哪些格式*(且不消耗任何一個)能讓自動化判斷可以貼上什麼,比較兩份快照則能偵測 +剪貼簿的形態何時改變。``clipboard_formats`` 加入: + +* :func:`classify_format` / :func:`classify_formats` ——把標準 ``CF_*`` id 與已註冊格式名稱 + 對應到友善類別(text / image / files / html / rtf / csv / audio……), +* :func:`diff_formats` ——純粹的監看原語:兩份快照之間的 ``{added, removed, changed}``, +* :func:`list_clipboard_formats` / :func:`clipboard_formats` ——列舉存活的剪貼簿 + (``EnumClipboardFormats``)並加以分類。 + +分類器與比較器為純函式(可在任何平台單元測試);只有存活列舉為 Win32(其他平台拋出 +``RuntimeError``)。不匯入 ``PySide6``。 + +無頭 API +-------- + +.. code-block:: python + + from je_auto_control import (classify_formats, diff_formats, + clipboard_formats) + + classify_formats([13, {"id": 49383, "name": "HTML Format"}]) + # {"categories": ["html", "text"], "has_text": True, "has_image": False, ...} + + diff_formats([13, 1], [13, 15]) # {"added": [files], "removed": [text], ...} + + clipboard_formats() # 存活剪貼簿摘要(Windows) + +描述子可為 id(``13``)、``{"id": ..., "name": ...}`` 字典,或 ``(id, name)`` 元組。已註冊的 +``name`` 優先於 id,因為已註冊格式的 id 是動態的(``>= 0xC000``)。未辨識的格式為 ``"other"``。 + +執行器指令 +---------- + +``AC_clipboard_formats``(存活,Windows)、``AC_classify_formats``(``formats``)與 +``AC_diff_formats``(``before`` / ``after``)——後兩者為純函式,可在任何平台執行。皆以唯讀 +``ac_*`` MCP 工具及 Script Builder 指令(位於 **Data** 分類下)形式提供。 diff --git a/je_auto_control/__init__.py b/je_auto_control/__init__.py index edb09043..e9c9e926 100644 --- a/je_auto_control/__init__.py +++ b/je_auto_control/__init__.py @@ -71,6 +71,11 @@ build_rtf, csv_to_rows, get_clipboard_csv, get_clipboard_rtf, rows_to_csv, rtf_to_text, set_clipboard_csv, set_clipboard_rtf, ) +# Clipboard format inspection (classify / diff available formats) +from je_auto_control.utils.clipboard_formats import ( + classify_format, classify_formats, clipboard_formats, diff_formats, + list_clipboard_formats, +) # VLM element locator (headless) from je_auto_control.utils.vision import ( VLMNotAvailableError, click_by_description, locate_by_description, @@ -1642,6 +1647,8 @@ def start_autocontrol_gui(*args, **kwargs): "build_rtf", "rtf_to_text", "rows_to_csv", "csv_to_rows", "set_clipboard_rtf", "get_clipboard_rtf", "set_clipboard_csv", "get_clipboard_csv", + "classify_format", "classify_formats", "diff_formats", + "list_clipboard_formats", "clipboard_formats", # VLM locator "VLMNotAvailableError", "locate_by_description", "click_by_description", "verify_description", diff --git a/je_auto_control/gui/script_builder/command_schema.py b/je_auto_control/gui/script_builder/command_schema.py index 92aae01f..dc6d60eb 100644 --- a/je_auto_control/gui/script_builder/command_schema.py +++ b/je_auto_control/gui/script_builder/command_schema.py @@ -1622,6 +1622,24 @@ def _add_misc_specs(specs: List[CommandSpec]) -> None: default=","),), description="Read the clipboard's Csv content as rows (Windows).", )) + specs.append(CommandSpec( + "AC_clipboard_formats", "Data", "List Clipboard Formats", + description="Enumerate + classify the clipboard's formats (Windows).", + )) + specs.append(CommandSpec( + "AC_classify_formats", "Data", "Classify Clipboard Formats", + fields=(FieldSpec("formats", FieldType.STRING, + placeholder='[1, 13, {"id": 49161, "name": "Csv"}]'),), + description="Classify a provided list of clipboard formats (pure).", + )) + specs.append(CommandSpec( + "AC_diff_formats", "Data", "Diff Clipboard Formats", + fields=( + FieldSpec("before", FieldType.STRING, placeholder="[1, 13]"), + FieldSpec("after", FieldType.STRING, placeholder="[1, 13, 15]"), + ), + description="Diff two clipboard-format snapshots (pure).", + )) specs.append(CommandSpec( "AC_watchdog_add", "Flow", "Watchdog: Add Popup Rule", fields=( diff --git a/je_auto_control/utils/clipboard_formats/__init__.py b/je_auto_control/utils/clipboard_formats/__init__.py new file mode 100644 index 00000000..ac7520be --- /dev/null +++ b/je_auto_control/utils/clipboard_formats/__init__.py @@ -0,0 +1,10 @@ +"""Inspect and classify the clipboard's available formats (pure classify/diff + Win32 enum).""" +from je_auto_control.utils.clipboard_formats.clipboard_formats import ( + classify_format, classify_formats, clipboard_formats, diff_formats, + list_clipboard_formats, +) + +__all__ = [ + "classify_format", "classify_formats", "diff_formats", + "list_clipboard_formats", "clipboard_formats", +] diff --git a/je_auto_control/utils/clipboard_formats/clipboard_formats.py b/je_auto_control/utils/clipboard_formats/clipboard_formats.py new file mode 100644 index 00000000..0527b7e2 --- /dev/null +++ b/je_auto_control/utils/clipboard_formats/clipboard_formats.py @@ -0,0 +1,140 @@ +"""Inspect and classify the clipboard's available formats. + +The clipboard usually holds the *same* content in several formats at once — a +copy from Word offers ``CF_UNICODETEXT`` + ``HTML Format`` + ``Rich Text Format``, +a file copy offers ``CF_HDROP``, a screenshot offers ``CF_DIB``. Knowing *which +formats are present* (without consuming any of them) tells an automation what it +can paste and lets it detect when the clipboard's shape changed. + +``clipboard_formats`` enumerates the live clipboard (``EnumClipboardFormats``) and +classifies each format into a friendly category (text / image / files / html / +rtf / csv / audio / …). The classifier and the snapshot diff are pure functions — +unit-testable on any platform — and only the live enumeration is Win32 (raising +``RuntimeError`` elsewhere, like the base ``clipboard`` module). Imports no +``PySide6``. +""" +from typing import Any, Dict, List, Optional, Sequence, Tuple, Union + +# Standard CF_* clipboard format ids → category. +_CF_CATEGORY = { + 1: "text", 7: "text", 13: "text", # CF_TEXT / OEMTEXT / UNICODETEXT + 2: "image", 8: "image", 17: "image", # CF_BITMAP / DIB / DIBV5 + 3: "image", 6: "image", 14: "image", # METAFILEPICT / TIFF / ENHMETAFILE + 9: "palette", # CF_PALETTE + 11: "audio", 12: "audio", # CF_RIFF / CF_WAVE + 15: "files", # CF_HDROP + 16: "locale", # CF_LOCALE +} +# Registered (named) clipboard formats → category (matched case-insensitively). +_NAME_CATEGORY = { + "html format": "html", + "rich text format": "rtf", + "rich text format without objects": "rtf", + "csv": "csv", + "png": "image", "jfif": "image", "gif": "image", "image/png": "image", + "filenamew": "files", "filename": "files", "filegroupdescriptorw": "files", + "filegroupdescriptor": "files", "shell idlist array": "files", +} +_Format = Union[int, Dict[str, Any], Tuple[int, str]] + + +def _coerce(item: _Format) -> Tuple[int, str]: + """Normalise a format descriptor (int / ``{id,name}`` / ``(id,name)``).""" + if isinstance(item, dict): + return int(item.get("id", 0)), str(item.get("name") or "") + if isinstance(item, (tuple, list)): + return int(item[0]), str(item[1] if len(item) > 1 else "") + return int(item), "" + + +def classify_format(format_id: int, name: Optional[str] = None) -> str: + """Return the friendly category for one clipboard format. + + A registered ``name`` (e.g. ``"HTML Format"``) takes priority; otherwise the + standard ``CF_*`` id is mapped. Anything unrecognised is ``"other"``. + """ + if name: + category = _NAME_CATEGORY.get(name.strip().lower()) + if category is not None: + return category + return _CF_CATEGORY.get(int(format_id), "other") + + +def classify_formats(formats: Sequence[_Format]) -> Dict[str, Any]: + """Classify a list of clipboard formats into a summary. + + Returns ``{categories, formats:[{id,name,category}], has_text, has_image, + has_files}``. + """ + items: List[Dict[str, Any]] = [] + for entry in formats: + format_id, name = _coerce(entry) + items.append({"id": format_id, "name": name, + "category": classify_format(format_id, name)}) + categories = sorted({item["category"] for item in items}) + return {"categories": categories, "formats": items, + "has_text": "text" in categories, + "has_image": "image" in categories, + "has_files": "files" in categories} + + +def diff_formats(before: Sequence[_Format], + after: Sequence[_Format]) -> Dict[str, Any]: + """Diff two clipboard-format snapshots into ``{added, removed, changed}``. + + ``added`` / ``removed`` are classified format dicts; ``changed`` is True when + either is non-empty — a pure monitor primitive over two enumerations. + """ + def keyed(formats: Sequence[_Format]) -> Dict[Tuple[int, str], Dict[str, Any]]: + out: Dict[Tuple[int, str], Dict[str, Any]] = {} + for entry in formats: + format_id, name = _coerce(entry) + out[(format_id, name)] = {"id": format_id, "name": name, + "category": classify_format(format_id, name)} + return out + + before_map, after_map = keyed(before), keyed(after) + added = [after_map[k] for k in after_map if k not in before_map] + removed = [before_map[k] for k in before_map if k not in after_map] + return {"added": added, "removed": removed, + "changed": bool(added or removed)} + + +# --- Win32 live enumeration ------------------------------------------------ + +def _format_name(user32, format_id: int) -> str: + import ctypes + buffer = ctypes.create_unicode_buffer(256) + if user32.GetClipboardFormatNameW(format_id, buffer, 256): + return buffer.value + return "" + + +def list_clipboard_formats() -> List[Dict[str, Any]]: + """Return the formats currently on the clipboard as ``[{id, name}]`` (Windows).""" + import sys + if not sys.platform.startswith("win"): + raise RuntimeError("list_clipboard_formats is only supported on Windows") + import ctypes + user32 = ctypes.windll.user32 + if not user32.OpenClipboard(None): + raise RuntimeError("OpenClipboard failed") + try: + formats: List[Dict[str, Any]] = [] + format_id = user32.EnumClipboardFormats(0) + while format_id: + formats.append({"id": int(format_id), + "name": _format_name(user32, format_id)}) + format_id = user32.EnumClipboardFormats(format_id) + return formats + finally: + user32.CloseClipboard() + + +def clipboard_formats() -> Dict[str, Any]: + """Enumerate and classify the live clipboard's formats (Windows). + + Returns the :func:`classify_formats` summary for whatever is on the + clipboard right now, without consuming any format. + """ + return classify_formats(list_clipboard_formats()) diff --git a/je_auto_control/utils/executor/action_executor.py b/je_auto_control/utils/executor/action_executor.py index f85d4b83..150fb5f7 100644 --- a/je_auto_control/utils/executor/action_executor.py +++ b/je_auto_control/utils/executor/action_executor.py @@ -4216,6 +4216,32 @@ def _get_clipboard_csv(delimiter: str = ",") -> Dict[str, Any]: return {"found": rows is not None, "rows": rows or []} +def _clipboard_formats() -> Dict[str, Any]: + """Adapter: enumerate and classify the live clipboard's formats (Windows).""" + from je_auto_control.utils.clipboard_formats import clipboard_formats + return clipboard_formats() + + +def _classify_formats(formats: Any) -> Dict[str, Any]: + """Adapter: classify a provided list of clipboard formats (pure).""" + import json + from je_auto_control.utils.clipboard_formats import classify_formats + if isinstance(formats, str): + formats = json.loads(formats) + return classify_formats(formats) + + +def _diff_formats(before: Any, after: Any) -> Dict[str, Any]: + """Adapter: diff two clipboard-format snapshots (pure).""" + import json + from je_auto_control.utils.clipboard_formats import diff_formats + if isinstance(before, str): + before = json.loads(before) + if isinstance(after, str): + after = json.loads(after) + return diff_formats(before, after) + + def _image_histogram(source: Any = None, bins: Any = 32, space: str = "hsv", region: Any = None) -> Dict[str, Any]: """Adapter: per-channel colour histogram of an image / the screen.""" @@ -6433,6 +6459,9 @@ def __init__(self): "AC_get_clipboard_rtf": _get_clipboard_rtf, "AC_set_clipboard_csv": _set_clipboard_csv, "AC_get_clipboard_csv": _get_clipboard_csv, + "AC_clipboard_formats": _clipboard_formats, + "AC_classify_formats": _classify_formats, + "AC_diff_formats": _diff_formats, "AC_image_histogram": _image_histogram, "AC_histogram_changed": _histogram_changed, "AC_changed_regions": _changed_regions, diff --git a/je_auto_control/utils/mcp_server/tools/_factories.py b/je_auto_control/utils/mcp_server/tools/_factories.py index 8d0188aa..f9899647 100644 --- a/je_auto_control/utils/mcp_server/tools/_factories.py +++ b/je_auto_control/utils/mcp_server/tools/_factories.py @@ -3292,6 +3292,37 @@ def clipboard_files_tools() -> List[MCPTool]: handler=h.get_clipboard_csv, annotations=READ_ONLY, ), + MCPTool( + name="ac_clipboard_formats", + description=("Enumerate and classify the formats currently on the " + "clipboard without consuming them (Windows). Returns " + "{categories, formats:[{id,name,category}], has_text, " + "has_image, has_files}."), + input_schema=schema({}, required=[]), + handler=h.clipboard_formats, + annotations=READ_ONLY, + ), + MCPTool( + name="ac_classify_formats", + description=("Classify a provided list of clipboard formats (pure). " + "'formats' is a list of ids or {id,name}. Returns the " + "same summary as ac_clipboard_formats."), + input_schema=schema({"formats": {"type": "array"}}, + required=["formats"]), + handler=h.classify_formats, + annotations=READ_ONLY, + ), + MCPTool( + name="ac_diff_formats", + description=("Diff two clipboard-format snapshots (pure): " + "{added, removed, changed}. 'before'/'after' are lists " + "of ids or {id,name}."), + input_schema=schema({"before": {"type": "array"}, + "after": {"type": "array"}}, + required=["before", "after"]), + handler=h.diff_formats, + annotations=READ_ONLY, + ), ] diff --git a/je_auto_control/utils/mcp_server/tools/_handlers.py b/je_auto_control/utils/mcp_server/tools/_handlers.py index 55045b29..f45dcdc5 100644 --- a/je_auto_control/utils/mcp_server/tools/_handlers.py +++ b/je_auto_control/utils/mcp_server/tools/_handlers.py @@ -2484,6 +2484,21 @@ def get_clipboard_csv(delimiter=","): return _get_clipboard_csv(delimiter) +def clipboard_formats(): + from je_auto_control.utils.executor.action_executor import _clipboard_formats + return _clipboard_formats() + + +def classify_formats(formats): + from je_auto_control.utils.executor.action_executor import _classify_formats + return _classify_formats(formats) + + +def diff_formats(before, after): + from je_auto_control.utils.executor.action_executor import _diff_formats + return _diff_formats(before, after) + + def image_histogram(source=None, bins=32, space="hsv", region=None): from je_auto_control.utils.executor.action_executor import _image_histogram return _image_histogram(source, bins, space, region) diff --git a/test/unit_test/headless/test_clipboard_formats_batch.py b/test/unit_test/headless/test_clipboard_formats_batch.py new file mode 100644 index 00000000..aa54c8a0 --- /dev/null +++ b/test/unit_test/headless/test_clipboard_formats_batch.py @@ -0,0 +1,80 @@ +"""Headless tests for clipboard-format classification + diff (pure; Win32 enum skipped).""" +import je_auto_control as ac +from je_auto_control.utils.clipboard_formats import ( + classify_format, classify_formats, diff_formats, +) + + +def test_classify_standard_ids(): + assert classify_format(13) == "text" # CF_UNICODETEXT + assert classify_format(1) == "text" # CF_TEXT + assert classify_format(8) == "image" # CF_DIB + assert classify_format(15) == "files" # CF_HDROP + assert classify_format(99999) == "other" + + +def test_classify_named_takes_priority(): + # registered ids are dynamic (>= 0xC000); the name decides the category + assert classify_format(49161, "Csv") == "csv" + assert classify_format(49383, "HTML Format") == "html" + assert classify_format(50000, "Rich Text Format") == "rtf" + assert classify_format(49500, "PNG") == "image" + + +def test_classify_formats_summary(): + summary = classify_formats([13, {"id": 15, "name": ""}, + {"id": 49383, "name": "HTML Format"}]) + assert summary["categories"] == ["files", "html", "text"] + assert summary["has_text"] is True + assert summary["has_files"] is True + assert summary["has_image"] is False + assert {"id": 15, "name": "", "category": "files"} in summary["formats"] + + +def test_diff_formats_added_removed(): + before = [13, 1] + after = [13, 15, {"id": 49383, "name": "HTML Format"}] + diff = diff_formats(before, after) + assert diff["changed"] is True + added_cats = {f["category"] for f in diff["added"]} + removed_cats = {f["category"] for f in diff["removed"]} + assert added_cats == {"files", "html"} + assert removed_cats == {"text"} # CF_TEXT(1) dropped; CF_UNICODETEXT kept + + +def test_diff_formats_no_change(): + diff = diff_formats([13, 15], [15, 13]) + assert diff == {"added": [], "removed": [], "changed": False} + + +def test_tuple_descriptor_accepted(): + assert classify_formats([(49161, "Csv")])["categories"] == ["csv"] + + +# --- wiring (Win32 enumeration not executed in CI) ------------------------- + +def test_executor_pure_paths(): + from je_auto_control.utils.executor.action_executor import ( + _classify_formats, _diff_formats) + assert _classify_formats("[13, 15]")["has_files"] is True + assert _diff_formats("[13]", "[13, 15]")["changed"] is True + + +def test_wiring(): + known = set(ac.executor.known_commands()) + assert {"AC_clipboard_formats", "AC_classify_formats", + "AC_diff_formats"} <= known + from je_auto_control.utils.mcp_server.tools import build_default_tool_registry + names = {t.name for t in build_default_tool_registry()} + assert {"ac_clipboard_formats", "ac_classify_formats", + "ac_diff_formats"} <= names + from je_auto_control.gui.script_builder.command_schema import _build_specs + specs = {s.command for s in _build_specs()} + assert {"AC_clipboard_formats", "AC_classify_formats", + "AC_diff_formats"} <= specs + + +def test_facade_exports(): + for name in ("classify_format", "classify_formats", "diff_formats", + "list_clipboard_formats", "clipboard_formats"): + assert hasattr(ac, name) and name in ac.__all__ From 97b183ad4b0183600f2428c010ece9296cbf6dab Mon Sep 17 00:00:00 2001 From: JeffreyChen Date: Wed, 24 Jun 2026 14:06:44 +0800 Subject: [PATCH 3/3] Add file_drop: drop files onto a window via WM_DROPFILES clipboard_files stages a file list on the clipboard for Ctrl+V; this actively drops files onto a target window (the completion of a drag-and-drop) by posting WM_DROPFILES. Reuses clipboard_files. build_dropfiles to pack the DROPFILES blob (shared byte layout) and dispatches through an injectable driver seam, so build-and-dispatch is unit-testable with a fake driver; the real GlobalAlloc + PostMessage lives in the default Win32 driver. plan_file_drop is a pure dry-run. --- WHATS_NEW.md | 6 ++ .../doc/new_features/v187_features_doc.rst | 42 +++++++++++ .../Zh/doc/new_features/v187_features_doc.rst | 38 ++++++++++ je_auto_control/__init__.py | 3 + .../gui/script_builder/command_schema.py | 21 ++++++ .../utils/executor/action_executor.py | 34 +++++++++ je_auto_control/utils/file_drop/__init__.py | 4 ++ je_auto_control/utils/file_drop/file_drop.py | 71 ++++++++++++++++++ .../utils/mcp_server/tools/_factories.py | 25 +++++++ .../utils/mcp_server/tools/_handlers.py | 10 +++ .../headless/test_file_drop_batch.py | 72 +++++++++++++++++++ 11 files changed, 326 insertions(+) create mode 100644 docs/source/Eng/doc/new_features/v187_features_doc.rst create mode 100644 docs/source/Zh/doc/new_features/v187_features_doc.rst create mode 100644 je_auto_control/utils/file_drop/__init__.py create mode 100644 je_auto_control/utils/file_drop/file_drop.py create mode 100644 test/unit_test/headless/test_file_drop_batch.py diff --git a/WHATS_NEW.md b/WHATS_NEW.md index e9e0adad..a0be99cc 100644 --- a/WHATS_NEW.md +++ b/WHATS_NEW.md @@ -1,5 +1,11 @@ # What's New — AutoControl +## What's new (2026-06-24) — Drop Files onto a Window (WM_DROPFILES) + +Complete a drag-and-drop programmatically — drop files onto a target window. Full reference: [`docs/source/Eng/doc/new_features/v187_features_doc.rst`](docs/source/Eng/doc/new_features/v187_features_doc.rst). + +- **`plan_file_drop` / `drop_files`** (`AC_plan_file_drop`, `AC_drop_files`): `clipboard_files` *stages* a file list on the clipboard for `Ctrl+V`; this actively **drops** files onto a target window by posting a `WM_DROPFILES` message. It reuses `clipboard_files.build_dropfiles` to pack the `DROPFILES` blob (shared byte layout, not re-implemented) and dispatches through an injectable driver seam, so the build-and-dispatch logic is unit-testable with a fake driver; the real `GlobalAlloc` + `PostMessage` lives in the default Win32 driver. `plan_file_drop` is a pure dry-run returning `{message, paths, point, wide, blob_size}`. No `PySide6`. + ## What's new (2026-06-24) — Clipboard Format Inspection (classify / diff available formats) See which formats are on the clipboard, and detect when its shape changes. Full reference: [`docs/source/Eng/doc/new_features/v186_features_doc.rst`](docs/source/Eng/doc/new_features/v186_features_doc.rst). diff --git a/docs/source/Eng/doc/new_features/v187_features_doc.rst b/docs/source/Eng/doc/new_features/v187_features_doc.rst new file mode 100644 index 00000000..2cdec972 --- /dev/null +++ b/docs/source/Eng/doc/new_features/v187_features_doc.rst @@ -0,0 +1,42 @@ +Drop Files onto a Window (WM_DROPFILES) +======================================= + +``clipboard_files`` *stages* a file-drop list on the clipboard so a user can +``Ctrl+V`` it; ``file_drop`` actively **drops** files onto a target window — the +completion of a drag-and-drop — by posting a ``WM_DROPFILES`` message carrying a +``DROPFILES`` blob. It reuses ``clipboard_files.build_dropfiles`` to pack that +blob (the byte layout is shared, not re-implemented) and dispatches it through an +injectable *driver* seam, so the build-and-dispatch logic is unit-testable on any +platform with a fake driver; the real ``GlobalAlloc`` + ``PostMessage`` lives in +the default Win32 driver. Imports no ``PySide6``. + +Headless API +------------ + +.. code-block:: python + + from je_auto_control import plan_file_drop, drop_files + + # Pure dry-run — inspect the payload without sending: + plan_file_drop(["C:\\a\\one.txt"], point=(10, 20)) + # {"message": 0x233, "paths": [...], "point": [10, 20], "wide": True, + # "blob_size": ...} + + # Real drop onto a window handle (Windows): + drop_files(hwnd, ["C:\\a\\one.txt", "C:\\b\\two.png"], point=(10, 20)) + + # Inject a driver to intercept the send (e.g. in tests): + drop_files(hwnd, ["x.txt"], driver=lambda hwnd, blob, point: True) + +``point`` is the drop coordinate in the window's client area. ``drop_files`` +returns ``bool``; the default driver posts the real ``WM_DROPFILES`` (the +receiving window then owns and frees the memory via ``DragFinish``) and raises +``RuntimeError`` off Windows. + +Executor commands +----------------- + +``AC_drop_files`` (``hwnd`` / ``paths`` / ``point``) performs the drop; +``AC_plan_file_drop`` (``paths`` / ``point``) is the pure dry-run. They are +exposed as the matching ``ac_*`` MCP tools (drop side-effect-only, plan +read-only) and as Script Builder commands under **Window**. diff --git a/docs/source/Zh/doc/new_features/v187_features_doc.rst b/docs/source/Zh/doc/new_features/v187_features_doc.rst new file mode 100644 index 00000000..0bea02d7 --- /dev/null +++ b/docs/source/Zh/doc/new_features/v187_features_doc.rst @@ -0,0 +1,38 @@ +將檔案拖放到視窗(WM_DROPFILES) +============================== + +``clipboard_files`` 只是把檔案拖放清單*放上*剪貼簿,讓使用者可以 ``Ctrl+V``;``file_drop`` 則 +主動把檔案**拖放**到目標視窗——也就是拖放動作的完成——透過送出帶有 ``DROPFILES`` 位元組區塊的 +``WM_DROPFILES`` 訊息達成。它重用 ``clipboard_files.build_dropfiles`` 來打包該區塊(位元組配置 +共用,不重新實作),並透過可注入的 *driver* 接縫分派,因此「打包 + 分派」邏輯可在任何平台以 +假 driver 單元測試;真正的 ``GlobalAlloc`` + ``PostMessage`` 位於預設的 Win32 driver。不匯入 +``PySide6``。 + +無頭 API +-------- + +.. code-block:: python + + from je_auto_control import plan_file_drop, drop_files + + # 純試跑——檢視 payload 但不送出: + plan_file_drop(["C:\\a\\one.txt"], point=(10, 20)) + # {"message": 0x233, "paths": [...], "point": [10, 20], "wide": True, + # "blob_size": ...} + + # 對視窗 handle 真正拖放(Windows): + drop_files(hwnd, ["C:\\a\\one.txt", "C:\\b\\two.png"], point=(10, 20)) + + # 注入 driver 以攔截送出(例如在測試中): + drop_files(hwnd, ["x.txt"], driver=lambda hwnd, blob, point: True) + +``point`` 是視窗工作區(client area)內的拖放座標。``drop_files`` 回傳 ``bool``;預設 driver 送出 +真正的 ``WM_DROPFILES``(接收視窗隨後擁有該記憶體並透過 ``DragFinish`` 釋放),在非 Windows 平台 +拋出 ``RuntimeError``。 + +執行器指令 +---------- + +``AC_drop_files``(``hwnd`` / ``paths`` / ``point``)執行拖放;``AC_plan_file_drop`` +(``paths`` / ``point``)為純試跑。皆以對應的 ``ac_*`` MCP 工具(drop 為僅副作用、plan 為唯讀) +及 Script Builder 指令(位於 **Window** 分類下)形式提供。 diff --git a/je_auto_control/__init__.py b/je_auto_control/__init__.py index e9c9e926..895a182d 100644 --- a/je_auto_control/__init__.py +++ b/je_auto_control/__init__.py @@ -76,6 +76,8 @@ classify_format, classify_formats, clipboard_formats, diff_formats, list_clipboard_formats, ) +# Drop files onto a window (WM_DROPFILES sender) +from je_auto_control.utils.file_drop import drop_files, plan_file_drop # VLM element locator (headless) from je_auto_control.utils.vision import ( VLMNotAvailableError, click_by_description, locate_by_description, @@ -1649,6 +1651,7 @@ def start_autocontrol_gui(*args, **kwargs): "set_clipboard_csv", "get_clipboard_csv", "classify_format", "classify_formats", "diff_formats", "list_clipboard_formats", "clipboard_formats", + "plan_file_drop", "drop_files", # VLM locator "VLMNotAvailableError", "locate_by_description", "click_by_description", "verify_description", diff --git a/je_auto_control/gui/script_builder/command_schema.py b/je_auto_control/gui/script_builder/command_schema.py index dc6d60eb..60844736 100644 --- a/je_auto_control/gui/script_builder/command_schema.py +++ b/je_auto_control/gui/script_builder/command_schema.py @@ -991,6 +991,27 @@ def _add_window_specs(specs: List[CommandSpec]) -> None: "AC_close_window", "Window", "Close Window", fields=(FieldSpec("title_substring", FieldType.STRING),), )) + specs.append(CommandSpec( + "AC_drop_files", "Window", "Drop Files onto Window", + fields=( + FieldSpec("hwnd", FieldType.INT), + FieldSpec("paths", FieldType.STRING, + placeholder='["C:\\\\a\\\\one.txt"]'), + FieldSpec("point", FieldType.STRING, optional=True, + placeholder="[10, 20]"), + ), + description="Drop files onto a window via WM_DROPFILES (Windows).", + )) + specs.append(CommandSpec( + "AC_plan_file_drop", "Window", "Plan File Drop", + fields=( + FieldSpec("paths", FieldType.STRING, + placeholder='["C:\\\\a\\\\one.txt"]'), + FieldSpec("point", FieldType.STRING, optional=True, + placeholder="[10, 20]"), + ), + description="Build the WM_DROPFILES payload without sending (pure).", + )) specs.append(CommandSpec( "AC_snap_window", "Window", "Snap / Tile Window", fields=( diff --git a/je_auto_control/utils/executor/action_executor.py b/je_auto_control/utils/executor/action_executor.py index 150fb5f7..dff38212 100644 --- a/je_auto_control/utils/executor/action_executor.py +++ b/je_auto_control/utils/executor/action_executor.py @@ -4242,6 +4242,38 @@ def _diff_formats(before: Any, after: Any) -> Dict[str, Any]: return diff_formats(before, after) +def _coerce_paths(paths: Any) -> list: + """Normalise a paths argument (JSON list string / single path / list).""" + import json + if isinstance(paths, str): + paths = json.loads(paths) if paths.strip().startswith("[") else [paths] + return [str(p) for p in paths] + + +def _coerce_point(point: Any) -> tuple: + """Normalise a point argument (JSON '[x,y]' / list / default origin).""" + import json + if isinstance(point, str): + point = json.loads(point) if point.strip().startswith("[") else (0, 0) + if not point: + return (0, 0) + return (int(point[0]), int(point[1])) + + +def _plan_file_drop(paths: Any, point: Any = None) -> Dict[str, Any]: + """Adapter: build the WM_DROPFILES payload without sending (pure).""" + from je_auto_control.utils.file_drop import plan_file_drop + return plan_file_drop(_coerce_paths(paths), point=_coerce_point(point)) + + +def _drop_files(hwnd: Any, paths: Any, point: Any = None) -> Dict[str, Any]: + """Adapter: drop files onto a window via WM_DROPFILES (Windows).""" + from je_auto_control.utils.file_drop import drop_files + coerced = _coerce_paths(paths) + dropped = drop_files(int(hwnd), coerced, point=_coerce_point(point)) + return {"dropped": bool(dropped), "count": len(coerced)} + + def _image_histogram(source: Any = None, bins: Any = 32, space: str = "hsv", region: Any = None) -> Dict[str, Any]: """Adapter: per-channel colour histogram of an image / the screen.""" @@ -6462,6 +6494,8 @@ def __init__(self): "AC_clipboard_formats": _clipboard_formats, "AC_classify_formats": _classify_formats, "AC_diff_formats": _diff_formats, + "AC_plan_file_drop": _plan_file_drop, + "AC_drop_files": _drop_files, "AC_image_histogram": _image_histogram, "AC_histogram_changed": _histogram_changed, "AC_changed_regions": _changed_regions, diff --git a/je_auto_control/utils/file_drop/__init__.py b/je_auto_control/utils/file_drop/__init__.py new file mode 100644 index 00000000..e37026c0 --- /dev/null +++ b/je_auto_control/utils/file_drop/__init__.py @@ -0,0 +1,4 @@ +"""Drop files onto a window via WM_DROPFILES (reuses clipboard_files packing).""" +from je_auto_control.utils.file_drop.file_drop import drop_files, plan_file_drop + +__all__ = ["plan_file_drop", "drop_files"] diff --git a/je_auto_control/utils/file_drop/file_drop.py b/je_auto_control/utils/file_drop/file_drop.py new file mode 100644 index 00000000..d8fda491 --- /dev/null +++ b/je_auto_control/utils/file_drop/file_drop.py @@ -0,0 +1,71 @@ +"""Drop files onto a window (Windows ``WM_DROPFILES``). + +``clipboard_files`` *stages* a file-drop list on the clipboard so a user can +``Ctrl+V`` it; this module actively **drops** files onto a target window — the +completion of a drag-and-drop — by posting a ``WM_DROPFILES`` message carrying a +``DROPFILES`` blob. It reuses ``clipboard_files.build_dropfiles`` to pack that +blob (so the byte layout is shared, not re-implemented) and dispatches it through +an injectable *driver* seam, so the build-and-dispatch logic is unit-testable on +any platform by passing a fake driver; the real ``GlobalAlloc`` + ``PostMessage`` +lives in the default Win32 driver. Imports no ``PySide6``. +""" +from typing import Any, Callable, Dict, Optional, Sequence, Tuple + +from je_auto_control.utils.clipboard_files import build_dropfiles + +_WM_DROPFILES = 0x0233 +_GMEM_MOVEABLE = 0x0002 + +# A driver dispatches a packed DROPFILES blob to a window: (hwnd, blob, point) -> bool. +DropDriver = Callable[[int, bytes, Tuple[int, int]], bool] + + +def plan_file_drop(paths: Sequence[str], *, point: Tuple[int, int] = (0, 0), + wide: bool = True) -> Dict[str, Any]: + """Build the ``WM_DROPFILES`` payload for dropping ``paths`` (pure, no send). + + Returns ``{message, paths, point, wide, blob_size}`` — a dry-run description + that reuses the same :func:`build_dropfiles` packing the real drop sends. + """ + blob = build_dropfiles(paths, point=point, wide=wide) + return {"message": _WM_DROPFILES, "paths": [str(p) for p in paths], + "point": [int(point[0]), int(point[1])], "wide": bool(wide), + "blob_size": len(blob)} + + +def _default_driver(hwnd: int, blob: bytes, point: Tuple[int, int]) -> bool: + """Post a real ``WM_DROPFILES`` to ``hwnd`` (Windows only).""" + import sys + if not sys.platform.startswith("win"): + raise RuntimeError("drop_files is only supported on Windows") + import ctypes + from ctypes import wintypes + kernel32, user32 = ctypes.windll.kernel32, ctypes.windll.user32 + kernel32.GlobalAlloc.restype = wintypes.HGLOBAL + kernel32.GlobalLock.restype = ctypes.c_void_p + handle = kernel32.GlobalAlloc(_GMEM_MOVEABLE, len(blob)) + if not handle: + raise RuntimeError("GlobalAlloc failed") + pointer = kernel32.GlobalLock(handle) + ctypes.memmove(pointer, blob, len(blob)) + kernel32.GlobalUnlock(handle) + # The receiving window owns the memory and frees it via DragFinish. + if not user32.PostMessageW(int(hwnd), _WM_DROPFILES, handle, 0): + raise RuntimeError("PostMessage(WM_DROPFILES) failed") + return True + + +def drop_files(hwnd: int, paths: Sequence[str], *, + point: Tuple[int, int] = (0, 0), wide: bool = True, + driver: Optional[DropDriver] = None) -> bool: + """Drop ``paths`` onto window ``hwnd`` via ``WM_DROPFILES``; True on success. + + ``point`` is the drop coordinate in the window's client area. Pass a + ``driver`` ``(hwnd, blob, point) -> bool`` to intercept the send (e.g. in + tests); the default driver posts the real Windows message. + """ + if not paths: + raise ValueError("at least one path is required") + blob = build_dropfiles(paths, point=point, wide=wide) + send = driver if driver is not None else _default_driver + return bool(send(int(hwnd), blob, (int(point[0]), int(point[1])))) diff --git a/je_auto_control/utils/mcp_server/tools/_factories.py b/je_auto_control/utils/mcp_server/tools/_factories.py index f9899647..c3e2401d 100644 --- a/je_auto_control/utils/mcp_server/tools/_factories.py +++ b/je_auto_control/utils/mcp_server/tools/_factories.py @@ -3323,6 +3323,31 @@ def clipboard_files_tools() -> List[MCPTool]: handler=h.diff_formats, annotations=READ_ONLY, ), + MCPTool( + name="ac_plan_file_drop", + description=("Build the WM_DROPFILES payload for dropping 'paths' at " + "'point' without sending it (pure dry-run). Returns " + "{message, paths, point, wide, blob_size}."), + input_schema=schema({ + "paths": {"type": "array", "items": {"type": "string"}}, + "point": {"type": "array", "items": {"type": "integer"}}}, + required=["paths"]), + handler=h.plan_file_drop, + annotations=READ_ONLY, + ), + MCPTool( + name="ac_drop_files", + description=("Drop 'paths' onto window 'hwnd' via WM_DROPFILES — the " + "completion of a drag-and-drop (Windows). 'point' is the " + "client-area drop coordinate. Returns {dropped, count}."), + input_schema=schema({ + "hwnd": {"type": "integer"}, + "paths": {"type": "array", "items": {"type": "string"}}, + "point": {"type": "array", "items": {"type": "integer"}}}, + required=["hwnd", "paths"]), + handler=h.drop_files, + annotations=SIDE_EFFECT_ONLY, + ), ] diff --git a/je_auto_control/utils/mcp_server/tools/_handlers.py b/je_auto_control/utils/mcp_server/tools/_handlers.py index f45dcdc5..2bb64e51 100644 --- a/je_auto_control/utils/mcp_server/tools/_handlers.py +++ b/je_auto_control/utils/mcp_server/tools/_handlers.py @@ -2499,6 +2499,16 @@ def diff_formats(before, after): return _diff_formats(before, after) +def plan_file_drop(paths, point=None): + from je_auto_control.utils.executor.action_executor import _plan_file_drop + return _plan_file_drop(paths, point) + + +def drop_files(hwnd, paths, point=None): + from je_auto_control.utils.executor.action_executor import _drop_files + return _drop_files(hwnd, paths, point) + + def image_histogram(source=None, bins=32, space="hsv", region=None): from je_auto_control.utils.executor.action_executor import _image_histogram return _image_histogram(source, bins, space, region) diff --git a/test/unit_test/headless/test_file_drop_batch.py b/test/unit_test/headless/test_file_drop_batch.py new file mode 100644 index 00000000..a353804f --- /dev/null +++ b/test/unit_test/headless/test_file_drop_batch.py @@ -0,0 +1,72 @@ +"""Headless tests for WM_DROPFILES file drop (injected driver; no Win32).""" +import pytest + +import je_auto_control as ac +from je_auto_control.utils.clipboard_files import parse_dropfiles +from je_auto_control.utils.file_drop import drop_files, plan_file_drop + +_WM_DROPFILES = 0x0233 + + +def test_plan_file_drop_reuses_dropfiles_packing(): + plan = plan_file_drop(["C:\\a\\one.txt", "C:\\b\\two.png"], point=(12, 34)) + assert plan["message"] == _WM_DROPFILES + assert plan["paths"] == ["C:\\a\\one.txt", "C:\\b\\two.png"] + assert plan["point"] == [12, 34] + assert plan["wide"] is True + assert plan["blob_size"] > 20 # header + path list + + +def test_drop_files_dispatches_packed_blob_to_driver(): + captured = {} + + def fake_driver(hwnd, blob, point): + captured["hwnd"] = hwnd + captured["blob"] = blob + captured["point"] = point + return True + + ok = drop_files(0xABCD, ["C:\\docs\\report.pdf"], point=(5, 9), + driver=fake_driver) + assert ok is True + assert captured["hwnd"] == 0xABCD + assert captured["point"] == (5, 9) + # the blob the driver receives is a real DROPFILES the receiver could parse + parsed = parse_dropfiles(captured["blob"]) + assert parsed["paths"] == ["C:\\docs\\report.pdf"] + assert parsed["point"] == [5, 9] + + +def test_drop_files_returns_driver_result(): + assert drop_files(1, ["x"], driver=lambda *_: False) is False + + +def test_empty_paths_raise(): + with pytest.raises(ValueError): + drop_files(1, [], driver=lambda *_: True) + with pytest.raises(ValueError): + plan_file_drop([]) + + +# --- wiring (real Win32 PostMessage not executed in CI) -------------------- + +def test_executor_plan_path_is_pure(): + from je_auto_control.utils.executor.action_executor import _plan_file_drop + plan = _plan_file_drop('["C:\\\\a\\\\one.txt"]', "[3, 4]") + assert plan["point"] == [3, 4] and plan["paths"] == ["C:\\a\\one.txt"] + + +def test_wiring(): + known = set(ac.executor.known_commands()) + assert {"AC_drop_files", "AC_plan_file_drop"} <= known + from je_auto_control.utils.mcp_server.tools import build_default_tool_registry + names = {t.name for t in build_default_tool_registry()} + assert {"ac_drop_files", "ac_plan_file_drop"} <= names + from je_auto_control.gui.script_builder.command_schema import _build_specs + specs = {s.command for s in _build_specs()} + assert {"AC_drop_files", "AC_plan_file_drop"} <= specs + + +def test_facade_exports(): + for name in ("plan_file_drop", "drop_files"): + assert hasattr(ac, name) and name in ac.__all__