hello
") # bytes, valid CF_HTML + assert parse_cf_html(payload) == "hello
" + +``build_cf_html`` returns valid ``CF_HTML`` UTF-8 bytes whose offsets point exactly at +the fragment; ``parse_cf_html`` recovers the fragment from bytes or text (preferring +the comment markers, falling back to the byte offsets). ``set_clipboard_html`` also +seeds plain text via ``fragment_plaintext`` so apps that ignore HTML still paste +something. + +Executor commands +----------------- + +``AC_set_clipboard_html`` (``html`` / ``fragment_plaintext`` → ``{set, length}``) and +``AC_get_clipboard_html`` (→ ``{found, html}``). They are exposed as the MCP tools +``ac_set_clipboard_html`` / ``ac_get_clipboard_html`` and as Script Builder commands +under **Data**. diff --git a/docs/source/Eng/eng_index.rst b/docs/source/Eng/eng_index.rst index 77ff21a5..9a52e556 100644 --- a/docs/source/Eng/eng_index.rst +++ b/docs/source/Eng/eng_index.rst @@ -137,6 +137,36 @@ Comprehensive guides for all AutoControl features. doc/new_features/v112_features_doc doc/new_features/v113_features_doc doc/new_features/v114_features_doc + doc/new_features/v115_features_doc + doc/new_features/v116_features_doc + doc/new_features/v117_features_doc + doc/new_features/v118_features_doc + doc/new_features/v119_features_doc + doc/new_features/v120_features_doc + doc/new_features/v121_features_doc + doc/new_features/v122_features_doc + doc/new_features/v123_features_doc + doc/new_features/v124_features_doc + doc/new_features/v125_features_doc + doc/new_features/v126_features_doc + doc/new_features/v127_features_doc + doc/new_features/v128_features_doc + doc/new_features/v129_features_doc + doc/new_features/v130_features_doc + doc/new_features/v131_features_doc + doc/new_features/v132_features_doc + doc/new_features/v133_features_doc + doc/new_features/v134_features_doc + doc/new_features/v135_features_doc + doc/new_features/v136_features_doc + doc/new_features/v137_features_doc + doc/new_features/v138_features_doc + doc/new_features/v139_features_doc + doc/new_features/v140_features_doc + doc/new_features/v141_features_doc + doc/new_features/v142_features_doc + doc/new_features/v143_features_doc + doc/new_features/v144_features_doc doc/ocr_backends/ocr_backends_doc doc/observability/observability_doc doc/operations_layer/operations_layer_doc diff --git a/docs/source/Zh/doc/new_features/v115_features_doc.rst b/docs/source/Zh/doc/new_features/v115_features_doc.rst new file mode 100644 index 00000000..5a0216b0 --- /dev/null +++ b/docs/source/Zh/doc/new_features/v115_features_doc.rst @@ -0,0 +1,41 @@ +檢查碼演算法 +============ + +``pii_text`` 以正則偵測信用卡與 IBAN 的*形狀*、``data_quality`` 做型別/範圍/正則驗證,但沒有任何功能實際 +計算或驗證*檢查碼*。本功能加入多數真實世界識別碼背後四種方案的共用運算引擎——也是帳號、卡號、IBAN、 +ISBN、EAN 驗證所依據的基本元件。 + +純標準函式庫(整數運算;Verhoeff 與 Damm 表為小型內嵌常數)。每個函式皆為純函式(字串進、bool/str 出), +因此在 CI 中完全具決定性。 + +無頭 API +-------- + +.. code-block:: python + + from je_auto_control import ( + luhn_validate, luhn_check_digit, + verhoeff_validate, verhoeff_check_digit, + damm_validate, damm_check_digit, + mod97_10_validate, mod97_10_check_digits, + ) + + luhn_validate("4111111111111111") # True (信用卡 / IMEI) + luhn_check_digit("7992739871") # '3' -> 79927398713 + verhoeff_validate("2363") # True (可抓出換位錯誤) + damm_check_digit("572") # '4' + mod97_10_validate("3214282912345698765432161182") # True (IBAN 引擎) + +- **Luhn**(mod 10):信用卡、IMEI、多種國民身分碼——可抓出所有單一數字錯誤與多數相鄰換位。 +- **Verhoeff** 與 **Damm**:十進位方案,可抓出*所有*單一數字與相鄰換位錯誤(比 Luhn 更強)。 +- **ISO 7064 MOD 97-10**:IBAN 等使用的雙檢查碼方案。 + +每個方案提供 ``*_validate(number)``(含檢查碼的值是否驗證通過?)與 ``*_check_digit`` / ``*_check_digits`` +(對裸負載應附加哪些檢查碼?)。非數字字元會被忽略,因此含空格/分組的輸入也適用。 + +執行器命令 +---------- + +``AC_checksum_validate`` 接受 ``scheme``(``luhn`` / ``verhoeff`` / ``damm`` / ``mod97``)與 ``number`` 並回傳 +``{valid}``;``AC_checksum_digit`` 對 ``partial`` 回傳 ``{check_digit}``。兩者皆以 MCP 工具 +(``ac_checksum_validate`` / ``ac_checksum_digit``)以及 Script Builder 中 **Data** 分類下的命令提供。 diff --git a/docs/source/Zh/doc/new_features/v116_features_doc.rst b/docs/source/Zh/doc/new_features/v116_features_doc.rst new file mode 100644 index 00000000..bd958ffa --- /dev/null +++ b/docs/source/Zh/doc/new_features/v116_features_doc.rst @@ -0,0 +1,35 @@ +多路徑點滑鼠手勢 +================ + +``humanize.humanized_path`` 與 ``tween_drag`` 只在*單一*起點 → 終點之間插值。真實手勢——簽名、框選 +(marquee / rubber-band)、拖曳經過多個放置目標、形狀手勢——需要任意的路徑點鏈,且可在整段路徑中持續按住按鍵。 + +:func:`plan_path` 為純點運算(重用 ``tween_drag`` 的具名緩動),本身即可單元測試;:func:`move_along_path` 與 +:func:`drag_path` 透過可注入的 ``sink`` 派發,因此手勢可在無真實輸入下測試。不匯入 ``PySide6``。 + +無頭 API +-------- + +.. code-block:: python + + from je_auto_control import ( + plan_path, move_along_path, drag_path, path_easings, + ) + + # 經過每個路徑點的緩動點列(交接點不重複) + plan_path([(100, 100), (400, 150), (400, 500)], per_segment_steps=20) + + move_along_path([(100, 100), (400, 150), (400, 500)]) # 沿折線移動 + drag_path([(50, 50), (300, 50), (300, 300)], button="mouse_left") # L 形拖曳 + +``plan_path`` 以 ``per_segment_steps`` 個緩動步驟對每個相鄰點對插值(``easing`` 為 ``path_easings()`` 中任一 +名稱——``linear`` / ``ease_in_out_quad`` / ``ease_out_cubic`` / ``ease_in_cubic``),且不重複共用的交接點。 +``move_along_path`` 沿路徑發出移動事件;``drag_path`` 在第一個路徑點按下、移動經過整段路徑、在最後一點放開—— +用於多停靠點拖曳。兩者皆可傳入 ``sink`` 以供無頭測試。 + +執行器命令 +---------- + +``AC_move_along_path`` 與 ``AC_drag_path`` 接受 ``waypoints``(JSON ``[[x, y], ...]`` 列表)以及 ``easing`` / +``per_segment_steps``(拖曳另含 ``button``)。兩者皆以 MCP 工具(``ac_move_along_path`` / ``ac_drag_path``) +以及 Script Builder 中 **Mouse** 分類下的命令提供。 diff --git a/docs/source/Zh/doc/new_features/v117_features_doc.rst b/docs/source/Zh/doc/new_features/v117_features_doc.rst new file mode 100644 index 00000000..4c527bda --- /dev/null +++ b/docs/source/Zh/doc/new_features/v117_features_doc.rst @@ -0,0 +1,37 @@ +清空再輸入欄位 +============== + +可靠地設定欄位值,必須先*清空*既有內容再輸入新文字——否則自動化會附加到或破壞既有內容。框架分別有 ``write`` +(可輸入,但對 emoji / CJK / 不在版面表內的字元會拋例外)與 ``set_clipboard`` / ``hotkey``,但沒有單一的 +「聚焦 → 清空 → 設值」基本元件,也沒有可輸入 ``write`` 無法處理之文字的貼上策略。本功能加入 Playwright 的 +``fill`` 慣用法。 + +:func:`plan_field_set` 建立決定性的操作計畫(純函式、可單元測試);:func:`set_field_text` 透過可注入的 ``sink`` +派發,因此可在無真實輸入下測試。不匯入 ``PySide6``。 + +無頭 API +-------- + +.. code-block:: python + + from je_auto_control import set_field_text, plan_field_set + + set_field_text("new value") # 全選、刪除、輸入 + set_field_text("café 🚀", paste=True) # 透過剪貼簿(Unicode 安全) + set_field_text("appended", clear="none") # 不清空,直接輸入 + set_field_text("値", paste=True, modifier="command") # macOS + + plan_field_set("hi") + # [{'op': 'hotkey', 'keys': ['ctrl', 'a']}, + # {'op': 'key', 'key': 'delete'}, + # {'op': 'type', 'text': 'hi'}] + +``clear`` 為 ``"select_all"``(``modifier``+A 再 Delete 的清空)或 ``"none"``。``paste=True`` 透過剪貼簿 +(``modifier``+V)輸入文字——這是 ``write`` 無法輸入之 Unicode / emoji / CJK 的可靠途徑——而非逐鍵輸入。 +``modifier`` 為平台指令鍵(``"ctrl"``;macOS 用 ``"command"``)。未知的 ``clear`` 模式會拋出 ``ValueError``。 + +執行器命令 +---------- + +``AC_set_field_text`` 接受 ``text`` 以及 ``clear`` / ``paste`` / ``modifier``,並回傳 ``{ops, plan}``。它以 +MCP 工具 ``ac_set_field_text`` 以及 Script Builder 中 **Keyboard** 分類下的命令提供。 diff --git a/docs/source/Zh/doc/new_features/v118_features_doc.rst b/docs/source/Zh/doc/new_features/v118_features_doc.rst new file mode 100644 index 00000000..d39d4d59 --- /dev/null +++ b/docs/source/Zh/doc/new_features/v118_features_doc.rst @@ -0,0 +1,35 @@ +等待消失(阻塞式 vanish 等待) +============================== + +``wait_for_image`` / ``wait_for_text`` 會阻塞直到某物*出現*,``observer`` 則以非同步回呼在消失時觸發——但先前 +沒有針對影像或文字的*阻塞式*「等到這個轉圈圈 / toast / 對話框**消失**再繼續」呼叫。``wait_until_window_closed`` +只涵蓋視窗。本功能為 ``smart_waits`` 家族補上缺少的 vanish 等待。 + +通用的 :func:`wait_until_gone` 接受任意述詞,因此其迴圈可在無真實螢幕下做無頭測試;影像 / 文字輔助函式則 +從定位函式建立該述詞。不匯入 ``PySide6``。 + +無頭 API +-------- + +.. code-block:: python + + from je_auto_control import ( + wait_until_gone, wait_until_image_gone, wait_until_text_gone, + ) + + # 通用:等到任意述詞變為 falsey + wait_until_gone(lambda: spinner_is_visible(), timeout_s=15) + + wait_until_image_gone("spinner.png", timeout_s=15) # 影像離開螢幕 + wait_until_text_gone("Loading...", timeout_s=15) # OCR 文字消失 + +每個皆回傳 ``WaitOutcome``(``succeeded`` / ``reason`` / ``elapsed_s`` / ``samples_taken``)——與其他 smart +waits 相同的結果型別。``gone_for_s`` 要求目標需持續缺席該段時間才算成功(可消抖閃爍的元素); +``poll_interval_s`` / ``timeout_s`` 界定迴圈。 + +執行器命令 +---------- + +``AC_wait_image_gone`` 與 ``AC_wait_text_gone`` 接受目標以及 ``timeout_s`` / ``poll_interval_s`` / +``gone_for_s``(影像另含 ``detect_threshold``),並回傳 ``WaitOutcome`` dict。兩者皆以 MCP 工具 +(``ac_wait_image_gone`` / ``ac_wait_text_gone``)以及 Script Builder 中 **Flow** 分類下的命令提供。 diff --git a/docs/source/Zh/doc/new_features/v119_features_doc.rst b/docs/source/Zh/doc/new_features/v119_features_doc.rst new file mode 100644 index 00000000..10df75cf --- /dev/null +++ b/docs/source/Zh/doc/new_features/v119_features_doc.rst @@ -0,0 +1,34 @@ +按住按鍵 / 自動重複 +================== + +``type_keyboard`` 是瞬間的按下+放開,``input_macro.run_sequence`` 雖可手動拼出按下 / 等待 / 放開,但先前沒有 +「按住此鍵 N 秒」(遊戲移動、按住捲動)或「每秒送 R 次」(自動重複)的基本元件。 + +:func:`plan_key_hold` 建立決定性的操作計畫(純函式、可單元測試);:func:`hold_key` 透過可注入的 ``sink`` +與 ``sleep`` 派發,因此可在無真實輸入、無真實等待下測試。不匯入 ``PySide6``。 + +無頭 API +-------- + +.. code-block:: python + + from je_auto_control import hold_key, plan_key_hold + + hold_key("key_d", duration_s=1.5) # 按下、按住 1.5 秒、放開 + hold_key("key_down", duration_s=2.0, rate_hz=20) # 40 個按鍵事件 @ 50ms + + plan_key_hold("space", 1.0) + # [{'op': 'press', 'key': 'space'}, + # {'op': 'wait', 'seconds': 1.0}, + # {'op': 'release', 'key': 'space'}] + +未設定 ``rate_hz`` 時,鍵會被按下、按住 ``duration_s``、再放開。設定 ``rate_hz`` 時,會送出 +``round(duration_s * rate_hz)`` 個相隔 ``1 / rate_hz`` 的離散按鍵事件——用於移動 / 捲動迴圈的模擬自動重複。 +非正數的時長或頻率會拋出 ``ValueError``。``hold_key`` 將 ``wait`` 步驟導向 ``sleep``、按鍵步驟導向 ``sink``, +兩者皆可注入。 + +執行器命令 +---------- + +``AC_hold_key`` 接受 ``key`` 以及 ``duration_s`` 與選用的 ``rate_hz``,並回傳 ``{ops, plan}``。它以 MCP 工具 +``ac_hold_key`` 以及 Script Builder 中 **Keyboard** 分類下的命令提供。 diff --git a/docs/source/Zh/doc/new_features/v120_features_doc.rst b/docs/source/Zh/doc/new_features/v120_features_doc.rst new file mode 100644 index 00000000..95b698f4 --- /dev/null +++ b/docs/source/Zh/doc/new_features/v120_features_doc.rst @@ -0,0 +1,29 @@ +相對滑鼠移動 +============ + +滑鼠 wrapper 只提供絕對的 ``set_mouse_position``——先前沒有「將指標位移 ``(dx, dy)``」(pynput / PyAutoGUI 的 +``moveRel`` 慣用法),而相對指標 / 畫布 / FPS 類應用與漸進式拖曳都需要它。 + +:func:`relative_target` 為純算術(目前位置 + 位移),可單元測試;:func:`move_mouse_relative` 讀取即時位置並 +設定新位置,getter 與 setter 皆可注入,因此可在無真實指標下測試。不匯入 ``PySide6``。 + +無頭 API +-------- + +.. code-block:: python + + from je_auto_control import move_mouse_relative, relative_target + + move_mouse_relative(-40, 12) # 從目前位置往左 40、往下 12 + # {'from': [200, 200], 'to': [160, 212], 'delta': [-40, 12]} + + relative_target((100, 100), 10, -5) # (110, 95) — 純函式、無 I/O + +``move_mouse_relative`` 讀取目前位置(若無法讀取則拋出 ``AutoControlMouseException``)、加上位移、再移動過去。 +``get_position`` / ``set_position`` 預設為真實滑鼠 wrapper,但可注入以供無頭測試。 + +執行器命令 +---------- + +``AC_move_mouse_relative`` 接受 ``dx`` / ``dy`` 並回傳 ``{from, to, delta}``。它以 MCP 工具 +``ac_move_mouse_relative`` 以及 Script Builder 中 **Mouse** 分類下的命令提供。 diff --git a/docs/source/Zh/doc/new_features/v121_features_doc.rst b/docs/source/Zh/doc/new_features/v121_features_doc.rst new file mode 100644 index 00000000..438335ba --- /dev/null +++ b/docs/source/Zh/doc/new_features/v121_features_doc.rst @@ -0,0 +1,35 @@ +等待區域顏色 +============ + +``wait_for_pixel`` 精確比對單一點,``wait_until_pixel_changes`` 偵測單點的*任何*變化——兩者都無法回答 +「等到狀態燈變綠」、「等到進度條大致填滿」或「等到紅色錯誤橫幅消失」。本功能為 ``smart_waits`` 家族加入 +區域顏色等待。 + +像素計數為純函式輔助,:func:`wait_until_color` 接受可注入的 ``sampler``,因此迴圈可在無真實螢幕下測試。 +不匯入 ``PySide6``。 + +無頭 API +-------- + +.. code-block:: python + + from je_auto_control import wait_until_color + + # 等到區域中 ≥ 60% 為(接近)綠色 + wait_until_color(region=[10, 10, 210, 40], target_rgb=[0, 200, 0], + tolerance=15, min_fraction=0.6, timeout_s=20) + + # 等到紅色橫幅消失 + wait_until_color(region=[0, 0, 800, 60], target_rgb=[200, 0, 0], + present=False, timeout_s=10) + +在 ``tolerance``(各通道)內接近 ``target_rgb`` 的像素會被計數。``present=True`` 時,當該比例達到 +``min_fraction`` 即成功;``present=False`` 時,當其低於該值即成功。結果為 ``WaitOutcome`` +(``succeeded`` / ``reason`` / ``elapsed_s`` / ``samples_taken``)。 + +執行器命令 +---------- + +``AC_wait_color`` 接受 ``target_rgb``(與選用的 ``region``)為 JSON 陣列,以及 ``tolerance`` / ``min_fraction`` / +``present`` / ``timeout_s`` / ``poll_interval_s``,並回傳 ``WaitOutcome`` dict。它以 MCP 工具 ``ac_wait_color`` +以及 Script Builder 中 **Flow** 分類下的命令提供。 diff --git a/docs/source/Zh/doc/new_features/v122_features_doc.rst b/docs/source/Zh/doc/new_features/v122_features_doc.rst new file mode 100644 index 00000000..20c9840c --- /dev/null +++ b/docs/source/Zh/doc/new_features/v122_features_doc.rst @@ -0,0 +1,34 @@ +Unicode 文字輸入(emoji / CJK) +============================== + +``write`` 透過平台虛擬鍵表輸入,對任何不在表內的字元會*拋例外*——emoji、CJK、許多重音字母——因此無法以正常 +途徑輸入非 ASCII 文字。可靠的跨平台輸入任意 Unicode 的方法,是將其放上剪貼簿再貼上。 + +:func:`plan_paste` 建立決定性操作計畫,:func:`unicode_code_units` 將文字拆成 UTF-16 碼元(供能做 +``KEYEVENTF_UNICODE`` 的後端使用);兩者皆為純函式、可單元測試。:func:`type_unicode` 透過可注入的 ``sink`` +派發貼上計畫,因此可在不觸碰真實剪貼簿下測試。不匯入 ``PySide6``。 + +無頭 API +-------- + +.. code-block:: python + + from je_auto_control import type_unicode, plan_paste, unicode_code_units + + type_unicode("café 🚀 値") # 設定剪貼簿 + Ctrl+V + type_unicode("値", modifier="command") # macOS + + unicode_code_units("🚀") # [0xD83D, 0xDE80](代理對) + plan_paste("hi") + # [{'op': 'set_clipboard', 'text': 'hi'}, + # {'op': 'hotkey', 'keys': ['ctrl', 'v']}] + +``type_unicode`` 將剪貼簿設為該文字並送出貼上熱鍵(``modifier`` 預設 ``"ctrl"``;macOS 用 ``"command"``), +因此無論鍵盤配置都能輸入*任何*文字——emoji、CJK、RTL、重音。它回傳派發的計畫加上 UTF-16 碼元數。 +``unicode_code_units`` 供想直接注入碼元的後端使用。 + +執行器命令 +---------- + +``AC_type_unicode`` 接受 ``text`` 與選用的 ``modifier``,並回傳 ``{ops, plan, code_units}``。它以 MCP 工具 +``ac_type_unicode`` 以及 Script Builder 中 **Keyboard** 分類下的命令提供。 diff --git a/docs/source/Zh/doc/new_features/v123_features_doc.rst b/docs/source/Zh/doc/new_features/v123_features_doc.rst new file mode 100644 index 00000000..0bb21775 --- /dev/null +++ b/docs/source/Zh/doc/new_features/v123_features_doc.rst @@ -0,0 +1,37 @@ +在動作群組中持續按住修飾鍵 +========================== + +``hotkey`` 按下一組鍵後立即放開——適合一次性的組合鍵,但先前無法在*多個獨立動作之間持續按住* ``ctrl`` +(或 ``shift``)(以 shift 連點做範圍選取、以 ctrl 連點做多選),也無法確保即使其中某個動作拋例外時修飾鍵仍會 +被放開。 + +:func:`plan_with_modifiers` 以 press / release 步驟包覆操作步驟清單,為純函式、可單元測試;:func:`hold_modifiers` +是一個 context manager,進入時按下、離開時(含例外情況)以反向順序放開,並透過可注入的 ``sink`` 派發。 +不匯入 ``PySide6``。 + +無頭 API +-------- + +.. code-block:: python + + from je_auto_control import hold_modifiers, plan_with_modifiers + from je_auto_control import click_mouse + + # 按住 shift 做範圍選取:每次點擊都在 shift 按下狀態進行 + with hold_modifiers(["shift"]): + click_mouse("mouse_left", 100, 100) + click_mouse("mouse_left", 100, 300) + # shift 在此放開——即使某次點擊拋例外也是 + + plan_with_modifiers([{"op": "click"}], ["ctrl", "shift"]) + # 按 ctrl、按 shift、click、放 shift、放 ctrl + +修飾鍵進入時依序按下、離開時在 ``finally`` 區塊以*反向*順序放開,因此卡住的修飾鍵絕不會外洩。 +``plan_with_modifiers`` 是任意操作步驟清單的純計畫。 + +執行器命令 +---------- + +``AC_with_modifiers`` 在按住 ``modifiers``(如 ``["ctrl"]`` 或 ``"ctrl+shift"``)時執行巢狀 JSON 動作清單, +即使某動作失敗也會放開修飾鍵。它以 MCP 工具 ``ac_with_modifiers`` 以及 Script Builder 中 **Keyboard** 分類下 +的命令提供。 diff --git a/docs/source/Zh/doc/new_features/v124_features_doc.rst b/docs/source/Zh/doc/new_features/v124_features_doc.rst new file mode 100644 index 00000000..7ece1c5d --- /dev/null +++ b/docs/source/Zh/doc/new_features/v124_features_doc.rst @@ -0,0 +1,33 @@ +錨點序數與全部定位 +================== + +``anchor_locate`` 依與錨點的空間關係尋找目標,但總是回傳單一*最近*的比對——先前無法選「標題下方的**第 2** 列」 +或列舉每一個符合的列。本功能加入 ``ordinal`` 選擇器與回傳清單的 :func:`anchor_locate_all`。 + +兩者皆建立在共用的排序輔助函式上(純函式:依關係過濾、依距離排序),因此選擇邏輯可藉由注入候選框做單元測試。 +無頭且不依賴 ``PySide6``。 + +無頭 API +-------- + +.. code-block:: python + + from je_auto_control import ( + anchor_locate, anchor_locate_all, ocr_locator, image_locator, + ) + + header = ocr_locator("Name") + row = image_locator("row_handle.png") + + anchor_locate(anchor=header, target=row, relation="below") # 最近 + anchor_locate(anchor=header, target=row, relation="below", ordinal=2) # 第 2 列 + rows = anchor_locate_all(anchor=header, target=row, relation="below") # 所有列 + +``ordinal`` 為 1 起算(``ordinal=1`` 即最近,與先前行為相同,故向後相容);超出範圍的序數回傳未找到的結果。 +``anchor_locate_all`` 回傳依距離排序的 found ``AnchorOutcome`` 清單——表格 / 清單列選取的基礎元件。 + +執行器命令 +---------- + +``AC_anchor_locate`` 新增 ``ordinal`` 參數;``AC_anchor_locate_all`` 回傳 ``{count, matches}``。兩者皆以 MCP +工具(``ac_anchor_locate`` 含 ``ordinal`` / ``ac_anchor_locate_all``)提供。 diff --git a/docs/source/Zh/doc/new_features/v125_features_doc.rst b/docs/source/Zh/doc/new_features/v125_features_doc.rst new file mode 100644 index 00000000..0be94e4c --- /dev/null +++ b/docs/source/Zh/doc/new_features/v125_features_doc.rst @@ -0,0 +1,36 @@ +表格 / 格線儲存格定位 +==================== + +``anchor_locator`` 處理成對的空間關係(目標在錨點*附近* / *下方*),但無法定位二維格線——表格中「第 3 列、 +第 2 欄的儲存格」。給定各儲存格的邊界框(來自影像或 OCR 列舉,例如 ``locate_all_image`` / ``find_text_matches``), +本功能將其分群為列與欄,並回傳所求儲存格的中心。 + +分群與查詢皆為純函式(框進、格線 / 儲存格出),完全可單元測試;框的列舉仍由呼叫端負責,因此此處不需真實螢幕。 +不匯入 ``PySide6``。 + +無頭 API +-------- + +.. code-block:: python + + from je_auto_control import cluster_grid, locate_cell + + boxes = [(10, 100, 20, 10), (110, 100, 20, 10), (210, 100, 20, 10), + (10, 200, 20, 10), (110, 200, 20, 10), (210, 200, 20, 10)] + + locate_cell(boxes, row=1, col=2) + # {'found': True, 'center': [220, 205], 'box': [210, 200, 20, 10], + # 'row': 1, 'col': 2, 'rows': 2, 'cols': 3} + + cluster_grid(boxes) # 列由上到下、儲存格由左到右 + +``cluster_grid`` 依中心 y 排序框,當間距超過 ``row_tolerance`` 時開始新的一列,並將每列的儲存格依中心 x 排序。 +``locate_cell`` 回傳 0 起算 ``(row, col)`` 儲存格的中心(可直接點擊),索引超出範圍時回傳 +``{found: False, reason}``。 + +執行器命令 +---------- + +``AC_grid_cell`` 接受 ``boxes``(JSON ``[[x, y, w, h], ...]`` 清單,例如來自前一個 ``AC_locate_all_image`` 步驟) +以及 ``row`` / ``col`` / ``row_tolerance``,並回傳儲存格 dict。它以 MCP 工具 ``ac_grid_cell`` 以及 Script Builder +中 **Mouse** 分類下的命令提供。 diff --git a/docs/source/Zh/doc/new_features/v126_features_doc.rst b/docs/source/Zh/doc/new_features/v126_features_doc.rst new file mode 100644 index 00000000..9f75b10c --- /dev/null +++ b/docs/source/Zh/doc/new_features/v126_features_doc.rst @@ -0,0 +1,30 @@ +等待視窗標題(正則) +================== + +``wait_for_window`` 以*子字串*比對視窗標題且僅等待其*出現*;``wait_until_window_closed`` 為子字串消失。兩者都 +不支援正則表達式標題或「等到使用中視窗標題符合 P」——例如等待瀏覽器分頁導覽至 ``r".*— Checkout$"``。本功能 +為 ``smart_waits`` 家族加入正則標題等待。 + +標題來源可注入,因此迴圈可在無真實視窗下做無頭測試。不匯入 ``PySide6``。 + +無頭 API +-------- + +.. code-block:: python + + from je_auto_control import wait_until_window_title + + wait_until_window_title(r".*— Checkout$", timeout_s=20) # 分頁已導覽 + wait_until_window_title("Updating", present=False) # 對話框消失 + wait_until_window_title("Checkout", regex=False) # 子字串模式 + +預設 ``pattern`` 為正則表達式(``re.search``);傳入 ``regex=False`` 改用純子字串比對。``present=False`` 等待 +標題*消失*。結果為 ``WaitOutcome``(``succeeded`` / ``reason`` / ``elapsed_s`` / ``samples_taken``); +``title_lister`` 可注入以供測試。 + +執行器命令 +---------- + +``AC_wait_window_title`` 接受 ``pattern`` 以及 ``present`` / ``regex`` / ``timeout_s`` / ``poll_interval_s``, +並回傳 ``WaitOutcome`` dict。它以 MCP 工具 ``ac_wait_window_title`` 以及 Script Builder 中 **Flow** 分類下的命令 +提供。 diff --git a/docs/source/Zh/doc/new_features/v127_features_doc.rst b/docs/source/Zh/doc/new_features/v127_features_doc.rst new file mode 100644 index 00000000..95fbc9a2 --- /dev/null +++ b/docs/source/Zh/doc/new_features/v127_features_doc.rst @@ -0,0 +1,36 @@ +具信心分數的模板比對 +==================== + +本專案的模板比對器(``cv2_utils`` 經由 ``je_open_cv.find_object``)為單一尺度且僅回傳邊界框——其內部計算的 +相關性*分數*被丟棄。因此先前無法為候選排名、設定信心門檻並讀回*比對程度*、在 UI 經 DPI / 縮放時找到控制項, +或列舉*每一個*出現處。本功能加入這些,類似 PyAutoGUI 的 ``confidence`` / ``locateAll`` 與 SikuliX 的 +``similarity`` / ``findAll``。 + +比對接受可注入的 ``haystack`` 影像(ndarray / 路徑 / PIL),因此可在無真實螢幕下對合成陣列做單元測試——僅預設 +(擷取螢幕)為裝置相依。OpenCV + NumPy 透過專案的 ``je_open_cv`` 相依引入。不匯入 ``PySide6``。 + +無頭 API +-------- + +.. code-block:: python + + from je_auto_control import match_template, match_template_all, best_matches + + m = match_template("button.png", min_score=0.85, scales=(0.9, 1.0, 1.1)) + if m: + print(m.score, m.scale, m.center) # 信心 + DPI 尺度 + 點擊點 + + for hit in match_template_all("row_handle.png", min_score=0.8): + click(*hit.center) # 每個出現處,重疊已移除 + +``match_template`` 回傳達到 ``min_score`` 的單一最佳 :class:`Match`(``x`` / ``y`` / ``width`` / ``height`` / +``score`` / ``scale`` / ``center``),並搜尋 ``scales`` 中每個尺度以容忍 DPI / 縮放。``match_template_all`` +回傳所有命中,以非極大值抑制(``nms_iou``)合併重疊偵測並以 ``max_results`` 設上限。``best_matches`` 回傳依 +分數排序的前 N 個(不論門檻,供調校)。 + +執行器命令 +---------- + +``AC_match_template`` 回傳 ``{found, match}``(match dict 帶有分數);``AC_match_template_all`` 回傳 +``{count, matches}``。兩者皆以 MCP 工具(``ac_match_template`` / ``ac_match_template_all``)以及 Script Builder +中 **Image** 分類下的命令提供。 diff --git a/docs/source/Zh/doc/new_features/v128_features_doc.rst b/docs/source/Zh/doc/new_features/v128_features_doc.rst new file mode 100644 index 00000000..df55846d --- /dev/null +++ b/docs/source/Zh/doc/new_features/v128_features_doc.rst @@ -0,0 +1,34 @@ +依顏色定位螢幕區域 +================== + +``color_stats`` 只*描述*區域的主要 / 平均顏色,``assert_pixel`` 檢查單一點及容差——兩者都不*定位*彩色區域。 +當唯一訊號是顏色時(狀態燈、進度條填充、紅色錯誤橫幅),模板比對很脆弱。本功能將接近目標 RGB(在容差內)的 +像素遮罩起來,並回傳相連區塊的邊界框。 + +遮罩與相連元件分析在可注入的 ``haystack`` 影像(ndarray / 路徑 / PIL)上執行,因此可在無真實螢幕下對合成陣列 +做單元測試。OpenCV + NumPy 透過專案的 ``je_open_cv`` 相依引入。不匯入 ``PySide6``。 + +無頭 API +-------- + +.. code-block:: python + + from je_auto_control import find_color_region, find_color_regions + + pill = find_color_region([0, 200, 0], tolerance=25) # 綠色狀態藥丸 + if pill: + click(*pill["center"]) + + for banner in find_color_regions([200, 0, 0], min_area=500): + print(banner["x"], banner["y"], banner["area"]) # 每個紅色區塊 + +``find_color_regions`` 為每個在 ``tolerance``(各通道)內接近 ``rgb`` 且至少 ``min_area`` 像素的區塊回傳 +``{x, y, width, height, area, center}``,由大到小;``find_color_region`` 僅回傳最大的(或 ``None``)。 +``haystack`` 預設為對選用 ``region`` 的螢幕擷取。 + +執行器命令 +---------- + +``AC_find_color_region`` 接受 ``rgb``(JSON ``[r, g, b]`` 陣列)以及 ``tolerance`` / ``min_area`` / ``region``, +並回傳 ``{count, regions, best}``。它以 MCP 工具 ``ac_find_color_region`` 以及 Script Builder 中 **Image** +分類下的命令提供。 diff --git a/docs/source/Zh/doc/new_features/v129_features_doc.rst b/docs/source/Zh/doc/new_features/v129_features_doc.rst new file mode 100644 index 00000000..08afe071 --- /dev/null +++ b/docs/source/Zh/doc/new_features/v129_features_doc.rst @@ -0,0 +1,39 @@ +遮罩模板比對(忽略背景) +======================== + +一般模板比對會計分模板的*每個*像素,因此從某背景裁切出的圖示無法比對到同一圖示在不同背景上的情形—— +工具列圖示在 hover 與閒置按鈕上、游標疊在任意內容上、Logo 在主題化表面上。``match_masked`` 只計算你標記為 +相關的像素:明確的灰階 ``mask``(非零 = 使用),或——若傳入 RGBA 模板——其 alpha 通道。透明 /「不在乎」的 +像素就不會再把分數拉低。 + +它沿用與 :doc:`v127_features_doc` 相同的 ``Match`` 結果(左上角、尺寸、``score``、``center``),並在可注入的 +``haystack``(ndarray / 路徑 / PIL)上執行,因此可對合成陣列做單元測試。比對使用 OpenCV 的遮罩 +``TM_CCORR_NORMED``(唯一能接受遮罩且不產生 NaN 的正規化度量);非有限值會被歸零。OpenCV + NumPy 透過 +``je_open_cv`` 引入;不匯入 ``PySide6``。 + +無頭 API +-------- + +.. code-block:: python + + from je_auto_control import match_masked, match_masked_all + + # 帶透明度的 PNG 圖示——其 alpha 自動作為遮罩。 + hit = match_masked("save_icon.png", min_score=0.9) + if hit: + click(*hit.center) + + # 明確遮罩:只比對 mask.png 的白色像素。 + for hit in match_masked_all("cursor.png", mask="cursor_mask.png", + min_score=0.95): + print(hit.x, hit.y, hit.score) + +``match_masked`` 回傳達到 ``min_score`` 的單一最佳 ``Match``(或 ``None``);``match_masked_all`` 回傳每個 +比對,以非極大值抑制移除重疊,分數由高到低,上限 ``max_results``。遮罩形狀與模板不符會丟出 ``ValueError``。 + +執行器命令 +---------- + +``AC_match_masked`` / ``AC_match_masked_all`` 接受 ``template``(及選用 ``mask``)以及 +``min_score`` / ``region``(*all* 形式另有 ``max_results`` / ``nms_iou``)。它們以 MCP 工具 +``ac_match_masked`` / ``ac_match_masked_all`` 以及 Script Builder 中 **Image** 分類下的命令提供。 diff --git a/docs/source/Zh/doc/new_features/v130_features_doc.rst b/docs/source/Zh/doc/new_features/v130_features_doc.rst new file mode 100644 index 00000000..d511ca50 --- /dev/null +++ b/docs/source/Zh/doc/new_features/v130_features_doc.rst @@ -0,0 +1,38 @@ +結構相似度(SSIM)比較 +======================== + +框架已能以原始像素差(``diff_screenshots``)與顏色直方圖(``detect_drift``)比較畫面——但兩者都不是*結構性*的。 +像素差會因為一像素位移或人眼會忽略的亮度變化而誤報;直方圖則對版面無感(把畫面左右兩半互換,它毫無變化)。 +SSIM 是標準的視覺回歸度量:容忍輕微光照變化,對結構變化(文字被編輯、元素移動或消失)敏感。``ssim_compare`` +回傳單一 0..1 分數,``ssim_changed_regions`` 則回傳*哪裡*真正變了的方框。 + +它是純 NumPy + OpenCV 實作(不需 scikit-image),在可注入的影像配對上執行,因此可在無真實螢幕下對合成陣列做 +單元測試。OpenCV + NumPy 透過 ``je_open_cv`` 引入。不匯入 ``PySide6``。 + +無頭 API +-------- + +.. code-block:: python + + from je_auto_control import ssim_compare, ssim_changed_regions + + # 以黃金截圖把關視覺回歸測試。 + score = ssim_compare("golden.png") # current = 實際螢幕 + assert score > 0.98 + + # 忽略即時時鐘 / 閃爍游標,再列出哪裡移動了。 + for box in ssim_changed_regions("golden.png", ignore=[[0, 0, 120, 30]]): + print(box["x"], box["y"], box["width"], box["height"]) + +``ssim_compare`` 回傳整張影像的平均 SSIM(``1.0`` = 完全相同);``current`` 預設為對選用 ``region`` 的螢幕擷取。 +``ignore`` 是一組從分數與變化偵測中排除的 ``[x, y, w, h]`` 方框。``ssim_changed_regions`` 標記局部不相似度 +``1 - SSIM`` 超過 ``threshold`` 的像素,將相連者(``min_area`` 以上)分群,回傳 ``{x, y, width, height, area, +center}``,由大到小。比較兩張不同尺寸的影像會丟出 ``ValueError``。 + +執行器命令 +---------- + +``AC_ssim_compare``(``reference`` / ``current`` / ``ignore`` / ``region`` → +``{score}``)與 ``AC_ssim_changed_regions``(另有 ``threshold`` / ``min_area`` → +``{count, regions}``)。它們以 MCP 工具 ``ac_ssim_compare`` / ``ac_ssim_changed_regions`` 以及 Script Builder 中 +**Image** 分類下的命令提供。 diff --git a/docs/source/Zh/doc/new_features/v131_features_doc.rst b/docs/source/Zh/doc/new_features/v131_features_doc.rst new file mode 100644 index 00000000..37922b86 --- /dev/null +++ b/docs/source/Zh/doc/new_features/v131_features_doc.rst @@ -0,0 +1,35 @@ +ORB 特徵比對(對旋轉 / 縮放 / 主題穩健) +======================================== + +像素模板比對——``match_template``、``match_masked``——是把模板像素與螢幕做相關運算,因此目標一旦*旋轉*、 +以未列出的倍率縮放、或重新上色(亮 / 暗主題、hover 狀態、不同外觀),就會失效。``feature_match`` 改為比對 +*關鍵點*:以方向不變的二進位描述子(ORB)描述的特徵角點,再用 RANSAC 對一致的點擬合單應矩陣。它能在 +旋轉、縮放與外觀變化下定位元素,並回傳四個投影角點以及做為內建信心訊號的內點數。 + +它在可注入的 ``haystack`` 影像(ndarray / 路徑 / PIL)上執行,因此可在無真實螢幕下對合成陣列做單元測試。 +ORB、暴力比對器與 ``findHomography`` 皆屬於 OpenCV 核心(不需 contrib 模組);OpenCV + NumPy 透過 +``je_open_cv`` 引入。不匯入 ``PySide6``。 + +無頭 API +-------- + +.. code-block:: python + + from je_auto_control import feature_match + + hit = feature_match("logo.png", min_inliers=12) + if hit: + click(*hit.center) # 定位四邊形的中心 + print(hit.corners) # 4 個 [x, y] 點,依模板順序 + print(hit.inliers, hit.score) # 幾何內點數與內點比例 + +``feature_match`` 回傳 ``FeatureMatch``(``corners``、``center``、``inliers``、``matches``、``score``), +或在幾何一致的比對少於 ``min_inliers`` 時回傳 ``None``。``ratio`` 是 Lowe 比例測試的門檻(越低越嚴格); +``max_features`` 限制 ORB 關鍵點預算。ORB 的邊界與 patch 尺寸會針對圖示大小的模板自動縮小,否則 OpenCV +的預設值會將其全數捨棄。 + +執行器命令 +---------- + +``AC_feature_match`` 接受 ``template`` 以及 ``region`` / ``max_features`` / ``ratio`` / ``min_inliers``, +回傳 ``{found, match}``。它以 MCP 工具 ``ac_feature_match`` 以及 Script Builder 中 **Image** 分類下的命令提供。 diff --git a/docs/source/Zh/doc/new_features/v132_features_doc.rst b/docs/source/Zh/doc/new_features/v132_features_doc.rst new file mode 100644 index 00000000..343de33c --- /dev/null +++ b/docs/source/Zh/doc/new_features/v132_features_doc.rst @@ -0,0 +1,38 @@ +以邊緣 / 輪廓定位 UI 元素(免模板) +==================================== + +目前所有定位器都需要一個*尋找對象*:``match_template`` 與 ``feature_match`` 需要參考影像、``find_color_region`` +需要顏色、``locate_text`` 需要文字。它們都無法回答結構性問題「這個畫面上可點擊的方框在哪裡?」。``find_shapes`` +與 ``find_rectangles`` 執行 Canny 邊緣偵測加輪廓擷取,回傳各個形狀的邊界框——讓腳本能在從未見過的畫面上列舉 +卡片、按鈕或輸入框,並對第 N 個操作,完全不需提供樣本。 + +兩者都在可注入的 ``haystack`` 影像(ndarray / 路徑 / PIL)上執行,因此可在無真實螢幕下對合成陣列做單元測試。 +OpenCV + NumPy 透過 ``je_open_cv`` 引入。不匯入 ``PySide6``。 + +無頭 API +-------- + +.. code-block:: python + + from je_auto_control import find_shapes, find_rectangles + + # 每個獨立形狀,由大到小。 + for shape in find_shapes(min_area=500): + print(shape["x"], shape["y"], shape["width"], shape["height"]) + + # 只取寬的按鈕形矩形,點擊第一個。 + buttons = find_rectangles(min_area=800, aspect_range=(1.5, 8.0)) + if buttons: + click(*buttons[0]["center"]) + +``find_shapes`` 為每個輪廓回傳 ``{x, y, width, height, area, center, aspect}``(``area`` 為邊界框面積),由大到小; +``min_area`` / ``max_area`` 去除雜點與整框邊界。``find_rectangles`` 只保留近似凸四邊形的輪廓(``epsilon`` 是 +``approxPolyDP`` 以周長比例表示的容差),並加上選用的 ``aspect_range``(min, max)寬高比過濾——``(1.5, 8)`` 取寬 +按鈕、``(0.8, 1.2)`` 取方形圖示。 + +執行器命令 +---------- + +``AC_find_shapes``(``region`` / ``min_area`` / ``max_area`` → ``{count, shapes}``)與 ``AC_find_rectangles`` +(另有 ``aspect_range`` / ``epsilon`` → ``{count, rectangles}``)。它們以 MCP 工具 ``ac_find_shapes`` / +``ac_find_rectangles`` 以及 Script Builder 中 **Image** 分類下的命令提供。 diff --git a/docs/source/Zh/doc/new_features/v133_features_doc.rst b/docs/source/Zh/doc/new_features/v133_features_doc.rst new file mode 100644 index 00000000..ff3953aa --- /dev/null +++ b/docs/source/Zh/doc/new_features/v133_features_doc.rst @@ -0,0 +1,37 @@ +視窗鋪排 / 版面幾何規劃器 +========================== + +``save_window_layout`` / ``restore_window_layout``擷取並重播使用者已經排好的*精確*位置,``snap_window`` 把*一個* +視窗移到一半或四分之一。沒有任何功能能*計算*出全新的多視窗版面。``tile_rect``、``grid_rects`` 與 ``cascade_rects`` +是純幾何規劃器:給定螢幕工作區,回傳常見鋪排版面的目標矩形——左右半、四分之一、三分之一、R×C 網格、錯位層疊—— +讓腳本能以決定性方式排列應用程式視窗。 + +此規劃器跨平台且無裝置相依,因此完全可單元測試;它回傳的矩形可與任何視窗移動後端組合。不匯入 ``PySide6``。 + +無頭 API +-------- + +.. code-block:: python + + from je_auto_control import tile_rect, grid_rects, cascade_rects + + left = tile_rect((0, 0, 1920, 1080), "left_third", gap=8) + print(left.as_tuple()) # (8, 8, 624, 1064) + + for cell in grid_rects((0, 0, 1920, 1080), rows=2, cols=3): + window_move("Editor", *cell.as_tuple()) # 6 格網格 + + plan = cascade_rects((0, 0, 1920, 1080), count=4, offset=40) + +``tile_rect`` 為具名 ``slot`` 回傳 ``WindowRect``(``x, y, width, height``,含 ``.as_tuple()`` 與 ``.to_dict()``) +——見 :func:`available_slots`(``left``、``top_right``、``center``、``left_third`` …);``gap`` 內縮各邊作為鋪排間距。 +``grid_rects`` 為 ``rows`` × ``cols`` 網格的每格(列優先)回傳一個矩形。``cascade_rects`` 回傳 ``count`` 個錯位、 +重疊且被夾在螢幕內的矩形(``size`` 預設為工作區的 60%)。未知 slot / 非正網格維度會丟出 ``ValueError``。 + +執行器命令 +---------- + +``AC_tile_rect``(``slot`` / ``screen`` / ``gap`` → ``{rect}``)、``AC_grid_rects``(``rows`` / ``cols`` / ``screen`` +/ ``gap`` → ``{count, rects}``)與 ``AC_cascade_rects``(``count`` / ``screen`` / ``offset`` / ``size`` → +``{count, rects}``)。``screen`` 預設為實際主螢幕工作區。它們以 MCP 工具 ``ac_tile_rect`` / ``ac_grid_rects`` / +``ac_cascade_rects`` 以及 Script Builder 中 **Window** 分類下的命令提供。 diff --git a/docs/source/Zh/doc/new_features/v134_features_doc.rst b/docs/source/Zh/doc/new_features/v134_features_doc.rst new file mode 100644 index 00000000..93fd7c88 --- /dev/null +++ b/docs/source/Zh/doc/new_features/v134_features_doc.rst @@ -0,0 +1,36 @@ +排列多個視窗(網格 / 層疊) +============================ + +``snap_window`` 移動*一個*視窗到一半或四分之一,而 :doc:`v133_features_doc` 規劃器只*計算*矩形、並不移動任何東西。 +``arrange_grid`` 與 ``arrange_cascade`` 把這個迴圈補完:給定一組視窗標題,計算版面並實際移動每個符合的視窗—— +把一組應用程式視窗鋪成網格,或以對角線層疊散開,一次呼叫完成。 + +它們以版面規劃器取得幾何,並沿用與 ``snap_window`` 相同的可注入 ``mover`` / ``screen_size`` 接縫,因此排列邏輯 +完全可在無真實視窗下做單元測試。預設 mover 目前為 Win32(其他平台在其後端完成前為 no-op)。不匯入 ``PySide6``。 + +無頭 API +-------- + +.. code-block:: python + + from je_auto_control import arrange_grid, arrange_cascade + + # 把三個編輯器鋪成自動形狀的網格(此處 2x2,使用前 3 格)。 + arrange_grid(["Editor", "Browser", "Terminal"]) + + # 或明確的 1x3 列,含 8px 間距。 + arrange_grid(["Left", "Mid", "Right"], rows=1, cols=3, gap=8) + + # 將視窗以對角線散開。 + arrange_cascade(["Doc 1", "Doc 2", "Doc 3"], offset=40) + +``arrange_grid`` 把 ``titles`` 鋪成 ``rows`` × ``cols`` 網格(預設為依視窗數量的近正方自動形狀),可加 ``gap``; +``arrange_cascade`` 讓每個視窗在前一個的右下方錯位 ``offset`` 像素,尺寸為工作區的 60%。兩者都回傳實際移動的 +視窗數,並對超出網格容量的視窗保持不動。 + +執行器命令 +---------- + +``AC_arrange_grid``(``titles`` JSON 陣列 + ``rows`` / ``cols`` / ``gap``)與 ``AC_arrange_cascade`` +(``titles`` + ``offset``),各回傳 ``{moved, count}``。它們以 MCP 工具 ``ac_arrange_grid`` / ``ac_arrange_cascade`` +(有副作用)以及 Script Builder 中 **Window** 分類下的命令提供。 diff --git a/docs/source/Zh/doc/new_features/v135_features_doc.rst b/docs/source/Zh/doc/new_features/v135_features_doc.rst new file mode 100644 index 00000000..cb5a7e1e --- /dev/null +++ b/docs/source/Zh/doc/new_features/v135_features_doc.rst @@ -0,0 +1,39 @@ +影像前處理(供 OCR / 模板比對) +================================ + +``locate_text`` / ``ocr_read_structure`` 與 ``match_template`` 是把*原始*螢幕擷取直接餵給 OCR 引擎或比對器。 +小字、暗色主題、低對比與略為旋轉的截圖會嚴重影響兩者——而框架先前完全沒有前處理接縫。本功能加入標準前處理 +流程——灰階 → 放大 → 二值化 → 去歪斜 → 去噪 → CLAHE 對比——以倍增你已在使用的 OCR 與比對功能的準確度。 + +每個函式都在可注入的 ``haystack`` 影像(ndarray / 路徑 / PIL,預設為對 ``region`` 擷取螢幕)上執行並回傳 +NumPy ndarray,因此可對合成陣列做單元測試。OpenCV + NumPy 透過 ``je_open_cv`` 引入。不匯入 ``PySide6``。 + +無頭 API +-------- + +.. code-block:: python + + from je_auto_control import preprocess_image, binarize, deskew, upscale + + # 一次性流程,再對清理後的影像做 OCR。 + clean = preprocess_image("receipt.png", steps=("grayscale", "upscale", + "deskew", "binarize"), scale=2.0) + + # 或個別組合各步驟。 + bw = binarize("panel.png", method="adaptive_gaussian", block_size=41) + straight = deskew("scan.png", max_angle=10.0) + big = upscale("tiny_label.png", scale=3.0, interp="lanczos") + +基礎元件有 ``to_grayscale``、``upscale``(``scale`` / ``interp``)、``binarize``(``method`` = +``otsu`` / ``adaptive_mean`` / ``adaptive_gaussian``)、``denoise``、``enhance_contrast``(CLAHE)、``deskew`` +以及 ``detect_skew_angle``(回傳量測到的文字歪斜角度,夾在 ``±max_angle``)。``preprocess_image`` 依序串接任意 +具名 ``steps``——``grayscale``、``upscale``、``binarize``、``denoise``、``deskew``、``contrast``;未知步驟名稱 +會丟出 ``ValueError``。 + +執行器命令 +---------- + +``AC_preprocess_image`` 執行流程並把結果*寫入* ``output_path``(因此可從 JSON / MCP / builder 使用): +``source`` 為影像路徑(預設為對 ``region`` 擷取螢幕)、``steps`` 為有序清單(或逗號字串),另有 ``scale`` / +``block_size`` / ``c``;回傳 ``{path, width, height}``。它以 MCP 工具 ``ac_preprocess_image`` 以及 Script +Builder 中 **Image** 分類下的命令提供。 diff --git a/docs/source/Zh/doc/new_features/v136_features_doc.rst b/docs/source/Zh/doc/new_features/v136_features_doc.rst new file mode 100644 index 00000000..ebdb1a6d --- /dev/null +++ b/docs/source/Zh/doc/new_features/v136_features_doc.rst @@ -0,0 +1,39 @@ +多螢幕 / 虛擬桌面幾何 +====================== + +``snap_window``、``arrange_grid`` 與版面規劃器都只取單一主螢幕 ``(width, height)``——它們對多螢幕無感: +無法在第二台顯示器上鋪排、也無法處理負原點的虛擬桌面,而 ``coordinate_space`` 只縮放模型網格。本功能補上缺少 +的實體層:列舉各螢幕、計算聯集虛擬邊界、查詢某點或某視窗位於哪台螢幕、在虛擬座標與各螢幕區域座標間轉換, +並把某點重映射到另一台螢幕上的等效位置。 + +幾何運算皆是對純 ``Monitor`` dataclass 的算術,因此完全可單元測試;只有 ``enumerate_monitors`` 的預設 provider +會碰到 OS(透過 ``mss``),且可注入。不匯入 ``PySide6``。 + +無頭 API +-------- + +.. code-block:: python + + from je_auto_control import (enumerate_monitors, monitor_at_point, + virtual_bounds, to_local, remap_point) + + monitors = enumerate_monitors() + print(virtual_bounds(monitors)) # 涵蓋所有顯示器的 (x, y, w, h) + + here = monitor_at_point(monitors, x, y) # 此點屬於哪台螢幕 + idx, lx, ly = to_local(monitors, x, y) # 虛擬 -> (螢幕, 區域 x, 區域 y) + + # 把某點移到另一台螢幕上的等效相對位置。 + second = remap_point(monitors[0], monitors[1], 960, 540) + +``Monitor`` 帶有 ``index, x, y, width, height, scale, primary`` 與 ``work`` 區域(``.bounds`` / +``.contains(x, y)`` / ``.to_dict()``)。``virtual_bounds`` 回傳聯集框(原點可能為負);``primary_monitor`` 取主螢幕; +``monitor_for_window(rect, monitors)`` 回傳視窗主要佔據的顯示器(最大重疊);``to_virtual`` 是 ``to_local`` 的反向; +``remap_point`` 保留分數位置,因此可跨不同解析度與 DPI 運作。 + +執行器命令 +---------- + +``AC_enumerate_monitors`` → ``{count, monitors, virtual_bounds}`` 與 ``AC_monitor_at_point``(``x`` / ``y``)→ +``{found, monitor}``。它們以 MCP 工具 ``ac_enumerate_monitors`` / ``ac_monitor_at_point`` 以及 Script Builder 中 +**Window** 分類下的命令提供。 diff --git a/docs/source/Zh/doc/new_features/v137_features_doc.rst b/docs/source/Zh/doc/new_features/v137_features_doc.rst new file mode 100644 index 00000000..bb40e254 --- /dev/null +++ b/docs/source/Zh/doc/new_features/v137_features_doc.rst @@ -0,0 +1,42 @@ +可操作性閘門——在操作前先等待就緒 +================================== + +現代 UI 框架(Playwright、Cypress、WebdriverIO)在每次點擊前都會執行*可操作性*檢查:目標必須存在、已停止移動、 +為啟用狀態,且確實能接收事件(未被遮蓋)。AutoControl 先前沒有對應功能——``self_heal_click`` 定位後立即點擊, +``wait_until_screen_stable`` 只觀察*整個*畫面。``wait_actionable`` 把這四項檢查合成單一閘門,讓點擊落在真正就緒的 +按鈕上,而非動畫中、停用、或被對話框擋住的狀態。 + +每個訊號都是可注入的 callable——``bbox_provider``(定位目標)、``region_sampler``(像素穩定 token)、 +``enabled_probe``、``hit_tester``——再加上透過 :class:`GateConfig` 注入的 ``clock`` / ``sleep``,因此閘門完全 +決定性且可無頭測試。不匯入 ``PySide6``。 + +無頭 API +-------- + +.. code-block:: python + + from je_auto_control import wait_actionable, act_when_ready, GateConfig + + report = wait_actionable( + bbox_provider=lambda: locate_button(), # () -> (x, y, w, h) 或 None + enabled_probe=lambda: not is_greyed_out(), + config=GateConfig(timeout_s=8.0, stable_for_s=0.4)) + if report.actionable: + click(*report.point) + else: + print("blocked:", report.reason) # not visible / not stable / … + + # 或一次完成「等待 + 操作」(若始終未就緒則丟例外): + act_when_ready(lambda point: click(*point), bbox_provider=locate_button) + +``wait_actionable`` 回傳 :class:`ActionabilityReport`,含 ``actionable`` 以及各項檢查布林值 +(``visible`` / ``stable`` / ``enabled`` / ``receives_events``)、目標 ``point``、``waited_s`` 與 ``reason`` +(第一個失敗的檢查)。``act_when_ready`` 等待後呼叫 ``action(center_point)``,逾時則丟出 +``AutoControlActionException``。 + +執行器命令 +---------- + +``AC_wait_actionable`` 將閘門綁定到 ``template`` 影像(每次輪詢時定位),並取樣該區域像素以判斷穩定: +``timeout_s`` / ``stable_for_s`` / ``min_score`` / ``region`` → 回傳報告字典。它以 MCP 工具 +``ac_wait_actionable`` 以及 Script Builder 中 **Flow** 分類下的命令提供。 diff --git a/docs/source/Zh/doc/new_features/v138_features_doc.rst b/docs/source/Zh/doc/new_features/v138_features_doc.rst new file mode 100644 index 00000000..2e913482 --- /dev/null +++ b/docs/source/Zh/doc/new_features/v138_features_doc.rst @@ -0,0 +1,38 @@ +融合並排序螢幕元素框 +==================== + +``set_of_marks.mark_elements`` 為單一、已乾淨的元素清單編號——但沒有任何功能*產生*那份清單。真實的畫面解析會 +產出三個彼此重疊的來源(OCR 文字框、圖示 / 形狀框、無障礙樹框),有大量重複且無一致順序。本模組是定位器 +(``locate_text``、``find_shapes``、a11y 樹)與 ``set_of_marks`` 之間缺少的連接組織:依重疊去重、聯集各來源並 +保留最可信的框、再排成閱讀順序並給予穩定索引。 + +每個框都是帶 ``x, y, width, height`` 的純 ``dict``(可附帶 ``text`` / ``source`` / ``score`` 等鍵),因此整個模組 +皆為純標準函式庫且完全可單元測試。不匯入 ``PySide6``。 + +無頭 API +-------- + +.. code-block:: python + + from je_auto_control import iou, merge_boxes, fuse_elements, reading_order + + iou(box_a, box_b) # 兩框的重疊度,0..1 + deduped = merge_boxes(raw_boxes, iou_threshold=0.9) + + # 聯集三個偵測器輸出;重疊時 a11y 框勝出,其次 OCR,再其次 icon。 + elements = fuse_elements(ocr_boxes=ocr, icon_boxes=icons, a11y_boxes=tree) + + # 由上到下、由左到右排序並為每個元素加上 "index"。 + for el in reading_order(elements): + print(el["index"], el.get("text"), el["x"], el["y"]) + +``iou`` 回傳兩框的交集除以聯集。``merge_boxes`` 在任一群重疊超過 ``iou_threshold`` 時保留最大者。``fuse_elements`` +為每個輸入標記 ``source``,再依 ``source_priority``(預設 ``a11y`` > ``ocr`` > ``icon``,其後較大面積)丟棄跨來源 +重疊。``reading_order`` 將相距 ``row_tol`` 像素內的元素歸為同列、列內依 ``x`` 排序,並回傳帶有遞增 ``index`` 的新字典。 + +執行器命令 +---------- + +``AC_fuse_elements``(``ocr`` / ``icon`` / ``a11y`` JSON 陣列 + ``iou_threshold`` → ``{count, elements}``)與 +``AC_reading_order``(``elements`` + ``row_tol`` → ``{count, elements}``)。它們以 MCP 工具 ``ac_fuse_elements`` / +``ac_reading_order`` 以及 Script Builder 中 **Image** 分類下的命令提供。 diff --git a/docs/source/Zh/doc/new_features/v139_features_doc.rst b/docs/source/Zh/doc/new_features/v139_features_doc.rst new file mode 100644 index 00000000..cc0d9003 --- /dev/null +++ b/docs/source/Zh/doc/new_features/v139_features_doc.rst @@ -0,0 +1,36 @@ +HSV 色彩空間分割 +================ + +``find_color_region`` 在 RGB 以各通道 ± 容差框遮罩,這在經典情境會失效:狀態燈、強調色或主題色調是「同一個顏色」 +但*亮度*不同。HSV 把色相與飽和度 / 明度分離,因此「色相帶 + 飽和度 / 明度下限」可在不同光照下捕捉某顏色的所有色階。 +本功能加入 HSV 遮罩與區塊框,沿用共用的連通元件輔助函式,並正確處理紅色的色相環繞(跨越 0/180 邊界)。 + +在可注入的 ``haystack``(ndarray / 路徑 / PIL,RGB)上執行,因此可對合成陣列做無頭測試。OpenCV + NumPy 透過 +``je_open_cv`` 引入。不匯入 ``PySide6``。 + +無頭 API +-------- + +.. code-block:: python + + from je_auto_control import dominant_hue_regions, segment_hsv, color_mask + + # 每個紅色區域——不論明暗、不論光照(已處理紅色環繞)。 + for r in dominant_hue_regions(hue=0, hue_tol=10, sat_min=80, val_min=80): + click(*r["center"]) + + # 或明確的 HSV 帶(H 0-179、S/V 0-255)。 + greens = segment_hsv(lower_hsv=[40, 80, 80], upper_hsv=[80, 255, 255]) + mask = color_mask(lower_hsv=[40, 80, 80], upper_hsv=[80, 255, 255]) + +``dominant_hue_regions`` 只限制色相(± ``hue_tol``)再加 ``sat_min`` / ``val_min`` 下限以略過灰階,為每個區塊回傳 +``{x, y, width, height, area, center}``,由大到小——因此能在任何亮度下找到某顏色,不像 RGB 框。``segment_hsv`` 接受 +明確的 ``lower_hsv`` / ``upper_hsv`` 帶;``color_mask`` 回傳原始 uint8 遮罩。 + +執行器命令 +---------- + +``AC_segment_hsv``(``lower_hsv`` / ``upper_hsv`` / ``min_area`` / ``region``)與 +``AC_dominant_hue_regions``(``hue`` / ``hue_tol`` / ``sat_min`` / ``val_min`` / ``min_area`` / ``region``), +皆回傳 ``{count, regions, best}``。它們以 MCP 工具 ``ac_segment_hsv`` / ``ac_dominant_hue_regions`` 以及 Script +Builder 中 **Image** 分類下的命令提供。 diff --git a/docs/source/Zh/doc/new_features/v140_features_doc.rst b/docs/source/Zh/doc/new_features/v140_features_doc.rst new file mode 100644 index 00000000..dfc136bc --- /dev/null +++ b/docs/source/Zh/doc/new_features/v140_features_doc.rst @@ -0,0 +1,37 @@ +免模型文字區偵測(MSER) +======================== + +``shape_locator`` 找的是矩形輪廓(按鈕 / 卡片,不是文字),``locate_text`` 需要 Tesseract / Paddle 引擎*以及*要 +搜尋的確切字串。兩者都無法在不跑 OCR、也不知道內容的情況下回答「畫面上哪裡有*任何*文字?」。``find_text_regions`` +與 ``find_text_lines`` 以 MSER(最大穩定極值區域)找出字元 / 詞 / 行的區塊,讓腳本能裁切候選文字框餵給 OCR +(比全畫面 OCR 更快更準),或在未安裝 OCR 相依時單純偵測「某標籤出現了」。 + +在可注入的 ``haystack``(ndarray / 路徑 / PIL)上執行,因此可對合成陣列做無頭測試。``cv2.MSER_create`` 屬於 +OpenCV 核心(不需 contrib);OpenCV + NumPy 透過 ``je_open_cv`` 引入。不匯入 ``PySide6``。 + +無頭 API +-------- + +.. code-block:: python + + from je_auto_control import find_text_regions, find_text_lines + + # 裁切每一行文字,只對該長條做 OCR。 + for line in find_text_lines(y_tolerance=8): + print(line["x"], line["y"], line["width"], line["height"]) + + # 或逐字元 / 逐詞的區域。 + for box in find_text_regions(min_area=80): + highlight(box["x"], box["y"], box["width"], box["height"]) + +``find_text_regions`` 為每個區域回傳 ``{x, y, width, height, area, center}``,由大到小;``merge`` 會聯集 MSER +逐字元的巢狀偵測,``min_area`` / ``max_area`` 去除雜點與整頁大小的區塊,``max_aspect`` 排除長條狀的分隔線。 +``find_text_lines`` 將垂直中心在 ``y_tolerance`` 像素內的字元框歸為同一行,每行一個框、由上到下。空白畫面回傳空 +清單(整框極值區域已被濾除)。 + +執行器命令 +---------- + +``AC_find_text_regions``(``min_area`` / ``max_area`` / ``merge`` / ``max_aspect`` / ``region`` → +``{count, regions}``)與 ``AC_find_text_lines``(``y_tolerance`` / ``region`` → ``{count, lines}``)。它們以 +MCP 工具 ``ac_find_text_regions`` / ``ac_find_text_lines`` 以及 Script Builder 中 **Image** 分類下的命令提供。 diff --git a/docs/source/Zh/doc/new_features/v141_features_doc.rst b/docs/source/Zh/doc/new_features/v141_features_doc.rst new file mode 100644 index 00000000..92bad09b --- /dev/null +++ b/docs/source/Zh/doc/new_features/v141_features_doc.rst @@ -0,0 +1,38 @@ +線條 / 網格 / 分隔線偵測(Hough) +================================== + +``grid_locator`` 把*已找到的*元素框分群成網格;它無法從原始像素找出表格 / 試算表的格線或 UI 分隔線,而 +``shape_locator`` 只找封閉矩形。``find_lines``、``find_grid`` 與 ``find_separators`` 以 Canny + 機率 Hough 轉換 +偵測直線段、分類為水平 / 垂直 / 斜向、還原表格的列與欄座標(及儲存格),並回傳長分隔線的位置——讓腳本能在無模板下 +定址「第 3 列、第 2 欄」或在分隔處切分面板。 + +在可注入的 ``haystack``(ndarray / 路徑 / PIL)上執行,因此可對合成陣列做無頭測試。``cv2.HoughLinesP`` 屬於 +OpenCV 核心;OpenCV + NumPy 透過 ``je_open_cv`` 引入。不匯入 ``PySide6``。 + +無頭 API +-------- + +.. code-block:: python + + from je_auto_control import find_lines, find_grid, find_separators + + for seg in find_lines(min_length=80, orientation="vertical"): + print(seg["x1"], seg["y1"], seg["x2"], seg["y2"], seg["length"]) + + grid = find_grid(min_length=120) + cell = grid["cells"][0] # 第 0 列、第 0 欄的 {x, y, width, height} + click(cell["x"] + cell["width"] // 2, cell["y"] + cell["height"] // 2) + + dividers = find_separators(axis="horizontal") # 各格線的 [y0, y1, ...] + +``find_lines`` 為每段回傳 ``{x1, y1, x2, y2, angle, length, orientation}``,最長者優先;傳入非 ``any`` 的 +``orientation`` 只保留該類。``find_grid`` 將水平格線分群為列座標、垂直格線分群為欄,回傳 ``{rows, cols, cells}`` +(儲存格為相鄰格線之間的矩形)。``find_separators`` 回傳沿 ``axis`` 的長分隔線合併後座標。空白畫面不產生線條 / 儲存格。 + +執行器命令 +---------- + +``AC_find_lines``(``min_length`` / ``max_gap`` / ``orientation`` / ``region`` → ``{count, lines}``)、 +``AC_find_grid``(``min_length`` / ``tol`` / ``region`` → ``{rows, cols, cells}``)與 ``AC_find_separators`` +(``axis`` / ``min_length`` / ``tol`` / ``region`` → ``{count, axis, coordinates}``)。它們以 MCP 工具 +``ac_find_lines`` / ``ac_find_grid`` / ``ac_find_separators`` 以及 Script Builder 中 **Image** 分類下的命令提供。 diff --git a/docs/source/Zh/doc/new_features/v142_features_doc.rst b/docs/source/Zh/doc/new_features/v142_features_doc.rst new file mode 100644 index 00000000..211814b5 --- /dev/null +++ b/docs/source/Zh/doc/new_features/v142_features_doc.rst @@ -0,0 +1,41 @@ +重試式數值斷言(expect.poll) +============================== + +``assert_eventually`` 只能輪詢框架固定的字典規格分派表(文字 / 影像 / 像素 / 視窗 / 剪貼簿 / 行程 / 檔案 / http), +無法重試*任意*值——OCR 出的總額等於 ``"$42.00"``、列數穩定下來、或自訂判斷式。``expect_poll`` 接受任何零參數 +``getter`` 與任何 ``matcher`` 判斷式,輪詢直到通過或逾時,並可注入 ``clock`` / ``sleep`` 讓測試具決定性(既有 +輔助函式呼叫真實 ``time.sleep``)。它對應 Playwright 的 ``expect.poll`` / web-first 重試式斷言。 + +純標準函式庫,不匯入 ``PySide6``。 + +無頭 API +-------- + +.. code-block:: python + + from je_auto_control import (expect_poll, assert_poll, to_equal, to_contain, + to_be_greater_than, to_match_regex, to_be_stable) + + # 輪詢任意 getter 直到符合。 + result = expect_poll(lambda: read_cart_total(), to_equal("$42.00"), + timeout_s=8.0, interval_s=0.5) + if result.ok: + print("settled after", result.attempts, "tries") + + # 失敗時拋例外(斷言風格)。 + assert_poll(lambda: row_count(), to_be_greater_than(0)) + + # 等待某值不再變動。 + expect_poll(lambda: ocr_value(), to_be_stable(3)) + +``expect_poll`` 回傳 ``PollResult``(``ok``、``value``、``attempts``、``waited_s``、``description``); +``assert_poll`` 在始終不符時丟出 ``AutoControlActionException``。matcher 工廠有 ``to_equal``、``to_contain``、 +``to_be_greater_than``、``to_match_regex``、``to_be_truthy`` 與 ``to_be_stable(n)``(值重複 ``n`` 次後符合)。 + +執行器命令 +---------- + +``AC_expect_poll`` 重複執行巢狀 ``action``(例如 ``["AC_get_clipboard"]``),直到其結果的 ``key`` 以 ``op`` +(``truthy`` / ``equals`` / ``contains`` / ``gt`` / ``regex``)對 ``expected`` 符合,或 ``timeout_s`` 逾時—— +回傳 ``{ok, value, attempts, waited_s}``。它以 MCP 工具 ``ac_expect_poll`` 以及 Script Builder 中 **Flow** +分類下的命令提供。 diff --git a/docs/source/Zh/doc/new_features/v143_features_doc.rst b/docs/source/Zh/doc/new_features/v143_features_doc.rst new file mode 100644 index 00000000..973b4a9f --- /dev/null +++ b/docs/source/Zh/doc/new_features/v143_features_doc.rst @@ -0,0 +1,37 @@ +可串接 / 可過濾的候選定位器 +============================ + +``anchor_locator`` 解析單一錨點→目標關係,``grid_locator`` 定址網格儲存格;兩者都不支援對候選集合做*可組合的細化* +——``.within(panel).filter(has_text="Delete").nth(2)``——這是 Selenium-4 / Playwright 的串接並過濾定位慣用法。 +目前細化得重新查詢後端。本功能是對來自*任何*來源(模板比對、OCR、a11y 樹、:doc:`v138_features_doc`)的框做純 +後置過濾。 + +``Candidates`` 包裝一串 ``{x, y, width, height, …}`` 框;每個方法都回傳*新的* ``Candidates``,因此鏈式呼叫無副作用 +且完全可單元測試。純標準函式庫,不匯入 ``PySide6``。 + +無頭 API +-------- + +.. code-block:: python + + from je_auto_control import from_boxes + + target = (from_boxes(boxes) # 來自任何定位器的框 + .within((0, 0, 1920, 120)) # 只取工具列 + .filter(has_text="Delete") # 只取刪除按鈕 + .sort_reading() # 由左到右 + .nth(1)) # 第二個 + if target.center(): + click(*target.center()) + +``within(region)`` 保留中心落在矩形內的框;``filter`` 保留符合所有條件的框(``has_text`` 子字串、``near`` +``(x, y, max_dist)`` 鄰近、``min_area`` / ``max_area``,或任意 ``predicate``);``sort_reading`` 排序; +``nth`` / ``first`` / ``last`` 選取;``resolve()`` 回傳存活清單、``center()`` 回傳第一個框的中心。鏈式呼叫永不 +變動原集合。 + +執行器命令 +---------- + +``AC_locate_chain`` 對 ``boxes`` 陣列依序套用一串 JSON ``ops``——``{op:"within",region:[…]}``、 +``{op:"filter",has_text:…}``、``{op:"reading"}``、``{op:"nth",index:…}``、``{op:"first"}``、``{op:"last"}``—— +回傳 ``{count, boxes, center}``。它以 MCP 工具 ``ac_locate_chain`` 以及 Script Builder 中 **Image** 分類下的命令提供。 diff --git a/docs/source/Zh/doc/new_features/v144_features_doc.rst b/docs/source/Zh/doc/new_features/v144_features_doc.rst new file mode 100644 index 00000000..a2da9610 --- /dev/null +++ b/docs/source/Zh/doc/new_features/v144_features_doc.rst @@ -0,0 +1,38 @@ +豐富剪貼簿——HTML(CF_HTML) +============================= + +基礎 ``clipboard`` 模組只處理純文字(``CF_UNICODETEXT``)與影像(``CF_DIB``)。將*格式化*內容貼進 Word / +Outlook / 富文字編輯器需要 ``CF_HTML`` 格式,其 ``Version / StartHTML / EndHTML / StartFragment / +EndFragment`` **位元組偏移**標頭以手寫極易出錯。``build_cf_html`` / ``parse_cf_html`` 以純 Python 計算與還原 +該標頭(完整單元測試的往返,且在多位元組 UTF-8 下正確),而 ``set_clipboard_html`` / ``get_clipboard_html`` +將其包裝於 Win32 剪貼簿之上。 + +位元組偏移運算與平台無關且可無頭測試;只有實際的剪貼簿 I/O 為 Windows(在其他平台丟出 ``RuntimeError``, +與基礎模組一致)。不匯入 ``PySide6``。 + +無頭 API +-------- + +.. code-block:: python + + from je_auto_control import (build_cf_html, parse_cf_html, + set_clipboard_html, get_clipboard_html) + + set_clipboard_html("Bold and italic", + fragment_plaintext="Bold and italic") # Windows + html = get_clipboard_html() # Windows + + # 純函式在任何平台皆可用(例如預先建立 payload): + payload = build_cf_html("hello
") # bytes,有效 CF_HTML + assert parse_cf_html(payload) == "hello
" + +``build_cf_html`` 回傳偏移精確指向片段的有效 ``CF_HTML`` UTF-8 位元組;``parse_cf_html`` 從 bytes 或文字還原片段 +(優先用註解標記,退而用位元組偏移)。``set_clipboard_html`` 也會以 ``fragment_plaintext`` 設定純文字,讓忽略 +HTML 的程式仍能貼上內容。 + +執行器命令 +---------- + +``AC_set_clipboard_html``(``html`` / ``fragment_plaintext`` → ``{set, length}``)與 ``AC_get_clipboard_html`` +(→ ``{found, html}``)。它們以 MCP 工具 ``ac_set_clipboard_html`` / ``ac_get_clipboard_html`` 以及 Script +Builder 中 **Data** 分類下的命令提供。 diff --git a/docs/source/Zh/zh_index.rst b/docs/source/Zh/zh_index.rst index 6f85b45d..d35b4ac7 100644 --- a/docs/source/Zh/zh_index.rst +++ b/docs/source/Zh/zh_index.rst @@ -137,6 +137,36 @@ AutoControl 所有功能的完整使用指南。 doc/new_features/v112_features_doc doc/new_features/v113_features_doc doc/new_features/v114_features_doc + doc/new_features/v115_features_doc + doc/new_features/v116_features_doc + doc/new_features/v117_features_doc + doc/new_features/v118_features_doc + doc/new_features/v119_features_doc + doc/new_features/v120_features_doc + doc/new_features/v121_features_doc + doc/new_features/v122_features_doc + doc/new_features/v123_features_doc + doc/new_features/v124_features_doc + doc/new_features/v125_features_doc + doc/new_features/v126_features_doc + doc/new_features/v127_features_doc + doc/new_features/v128_features_doc + doc/new_features/v129_features_doc + doc/new_features/v130_features_doc + doc/new_features/v131_features_doc + doc/new_features/v132_features_doc + doc/new_features/v133_features_doc + doc/new_features/v134_features_doc + doc/new_features/v135_features_doc + doc/new_features/v136_features_doc + doc/new_features/v137_features_doc + doc/new_features/v138_features_doc + doc/new_features/v139_features_doc + doc/new_features/v140_features_doc + doc/new_features/v141_features_doc + doc/new_features/v142_features_doc + doc/new_features/v143_features_doc + doc/new_features/v144_features_doc doc/ocr_backends/ocr_backends_doc doc/observability/observability_doc doc/operations_layer/operations_layer_doc diff --git a/je_auto_control/__init__.py b/je_auto_control/__init__.py index 61aff45b..a4d59a66 100644 --- a/je_auto_control/__init__.py +++ b/je_auto_control/__init__.py @@ -74,8 +74,8 @@ from je_auto_control.utils.color_stats import ColorStats, region_color_stats # Per-window capture, window-layout save / restore, snap/tile. from je_auto_control.utils.window_capture import ( - capture_window, get_window_geometry, restore_window_layout, - save_window_layout, snap_window, + arrange_cascade, arrange_grid, capture_window, get_window_geometry, + restore_window_layout, save_window_layout, snap_window, ) # Scroll until a target image / text is visible. from je_auto_control.utils.scroll_find import scroll_until_visible @@ -245,6 +245,100 @@ from je_auto_control.utils.gettext_catalog import ( GettextCatalog, parse_po, parse_po_file, read_mo, read_mo_file, ) +# Check-digit algorithms (Luhn / Verhoeff / Damm / ISO 7064 MOD 97-10) +from je_auto_control.utils.checksum import ( + damm_check_digit, damm_validate, luhn_check_digit, luhn_validate, + mod97_10_check_digits, mod97_10_validate, verhoeff_check_digit, + verhoeff_validate, +) +# Multi-waypoint mouse gestures (move / drag through a polyline of points) +from je_auto_control.utils.mouse_path import ( + drag_path, move_along_path, path_easings, plan_path, +) +# Clear-then-type a text field (Playwright `fill` idiom; paste for Unicode) +from je_auto_control.utils.field_entry import plan_field_set, set_field_text +# Hold a key for a duration / auto-repeat at a fixed rate +from je_auto_control.utils.key_hold import hold_key, plan_key_hold +# Relative mouse movement (move by a delta from the current position) +from je_auto_control.utils.mouse_relative import ( + move_mouse_relative, relative_target, +) +# Type arbitrary Unicode (emoji / CJK) via the clipboard +from je_auto_control.utils.text_unicode import ( + plan_paste, type_unicode, unicode_code_units, +) +# Hold modifier keys across a group of actions (release-on-error) +from je_auto_control.utils.modifier_state import ( + hold_modifiers, plan_with_modifiers, +) +# Address a table / grid cell by (row, column) from bounding boxes +from je_auto_control.utils.grid_locator import cluster_grid, locate_cell +# Confidence-returning template matching (score / multi-scale / find-all + NMS) +from je_auto_control.utils.visual_match import ( + best_matches, match_masked, match_masked_all, match_template, + match_template_all, +) +from je_auto_control.utils.visual_match import Match as TemplateMatch +# Locate on-screen regions by colour (mask + connected components) +from je_auto_control.utils.color_region import ( + find_color_region, find_color_regions, +) +# Structural-similarity comparison (perceptual score + changed regions) +from je_auto_control.utils.ssim import ( + ssim_changed_regions, ssim_compare, +) +# ORB feature matching (rotation / scale / theme-robust template location) +from je_auto_control.utils.feature_match import feature_match +from je_auto_control.utils.feature_match import FeatureMatch +# Locate UI elements by edge/contour detection (rectangles / shapes, no template) +from je_auto_control.utils.shape_locator import ( + find_rectangles, find_shapes, +) +# Window tiling/layout geometry planner (halves, quadrants, grids, cascade) +from je_auto_control.utils.window_layout import ( + WindowRect, available_slots, cascade_rects, grid_rects, tile_rect, +) +# Image pre-processing for OCR / template matching (grayscale, binarize, deskew, …) +from je_auto_control.utils.preprocess import ( + binarize, denoise, deskew, detect_skew_angle, enhance_contrast, + preprocess_image, to_grayscale, upscale, +) +# Multi-monitor / virtual-desktop geometry (which monitor, where, remapping) +from je_auto_control.utils.monitor_layout import ( + Monitor, enumerate_monitors, monitor_at_point, monitor_for_window, + primary_monitor, remap_point, to_local, to_virtual, virtual_bounds, +) +# Pre-action readiness gate (visible + stable + enabled + not-occluded) +from je_auto_control.utils.actionability import ( + ActionabilityReport, GateConfig, act_when_ready, wait_actionable, +) +# Fuse and order on-screen element boxes (IoU, merge, fuse sources, reading order) +from je_auto_control.utils.element_parse import ( + fuse_elements, iou, merge_boxes, reading_order, +) +# HSV colour-space segmentation (lighting-robust colour masking + blob boxes) +from je_auto_control.utils.hsv_segment import ( + color_mask, dominant_hue_regions, segment_hsv, +) +# Model-free on-screen text-region detection (MSER): regions and lines +from je_auto_control.utils.text_regions import ( + find_text_lines, find_text_regions, +) +# Line / grid / separator detection on raw pixels (Hough transform) +from je_auto_control.utils.edge_lines import ( + find_grid, find_lines, find_separators, +) +# Retry an arbitrary value until it matches (Playwright-style expect.poll) +from je_auto_control.utils.expect_poll import ( + PollResult, assert_poll, expect_poll, to_be_greater_than, to_be_stable, + to_be_truthy, to_contain, to_equal, to_match_regex, +) +# Composable / filtered candidate locators (chained-locator idiom) +from je_auto_control.utils.locator_chain import Candidates, from_boxes +# Rich clipboard formats — HTML (CF_HTML) build / parse / get / set +from je_auto_control.utils.rich_clipboard import ( + build_cf_html, get_clipboard_html, parse_cf_html, set_clipboard_html, +) # CI workflow annotations (GitHub Actions) from je_auto_control.utils.ci_annotations import ( emit_annotations, format_annotation, @@ -581,8 +675,8 @@ # Anchor-based locators (spatial composition of locator backends) from je_auto_control.utils.anchor_locator import ( AnchorLocatorError, AnchorOutcome, Locator as AnchorLocator, - a11y_locator, anchor_locate, image_locator, ocr_locator, - vlm_locator, + a11y_locator, anchor_locate, anchor_locate_all, image_locator, + ocr_locator, vlm_locator, ) # Structured OCR (rows / tables / form fields) from je_auto_control.utils.ocr.structure import ( @@ -592,9 +686,11 @@ ) # Smart waits (frame-diff replacements for time.sleep) from je_auto_control.utils.smart_waits import ( - WaitOutcome, wait_until_clipboard_changes, wait_until_file, + WaitOutcome, wait_until_clipboard_changes, wait_until_color, + wait_until_file, wait_until_gone, wait_until_image_gone, wait_until_pixel_changes, wait_until_port, wait_until_process, - wait_until_region_idle, wait_until_screen_stable, wait_until_window_closed, + wait_until_region_idle, wait_until_screen_stable, wait_until_text_gone, + wait_until_window_closed, wait_until_window_title, ) # Visual regression (golden-image comparison) from je_auto_control.utils.visual_regression import ( @@ -1007,6 +1103,98 @@ def start_autocontrol_gui(*args, **kwargs): "parse_po_file", "read_mo", "read_mo_file", + "luhn_validate", + "luhn_check_digit", + "verhoeff_validate", + "verhoeff_check_digit", + "damm_validate", + "damm_check_digit", + "mod97_10_validate", + "mod97_10_check_digits", + "plan_path", + "move_along_path", + "drag_path", + "path_easings", + "plan_field_set", + "set_field_text", + "plan_key_hold", + "hold_key", + "move_mouse_relative", + "relative_target", + "type_unicode", + "plan_paste", + "unicode_code_units", + "hold_modifiers", + "plan_with_modifiers", + "cluster_grid", + "locate_cell", + "TemplateMatch", + "match_template", + "match_template_all", + "match_masked", + "match_masked_all", + "best_matches", + "find_color_region", + "find_color_regions", + "ssim_compare", + "ssim_changed_regions", + "feature_match", + "FeatureMatch", + "find_shapes", + "find_rectangles", + "WindowRect", + "available_slots", + "tile_rect", + "grid_rects", + "cascade_rects", + "preprocess_image", + "to_grayscale", + "binarize", + "upscale", + "denoise", + "deskew", + "detect_skew_angle", + "enhance_contrast", + "Monitor", + "enumerate_monitors", + "monitor_at_point", + "monitor_for_window", + "primary_monitor", + "remap_point", + "to_local", + "to_virtual", + "virtual_bounds", + "wait_actionable", + "act_when_ready", + "ActionabilityReport", + "GateConfig", + "iou", + "merge_boxes", + "fuse_elements", + "reading_order", + "segment_hsv", + "color_mask", + "dominant_hue_regions", + "find_text_regions", + "find_text_lines", + "find_lines", + "find_grid", + "find_separators", + "expect_poll", + "assert_poll", + "PollResult", + "to_equal", + "to_contain", + "to_be_greater_than", + "to_match_regex", + "to_be_truthy", + "to_be_stable", + "Candidates", + "from_boxes", + "build_cf_html", + "parse_cf_html", + "get_clipboard_html", + "set_clipboard_html", "emit_annotations", "format_annotation", "ClipboardHistory", "default_clipboard_history", "analyze_heal_log", "heal_stats", "scan_secrets", @@ -1194,7 +1382,8 @@ def start_autocontrol_gui(*args, **kwargs): "register_chatops_default_commands", # Anchor-based locator "AnchorLocator", "AnchorLocatorError", "AnchorOutcome", - "a11y_locator", "anchor_locate", "image_locator", "ocr_locator", + "a11y_locator", "anchor_locate", "anchor_locate_all", "image_locator", + "ocr_locator", "vlm_locator", # Structured OCR "OCRField", "OCRRow", "OCRTable", "StructuredOCR", @@ -1204,6 +1393,8 @@ def start_autocontrol_gui(*args, **kwargs): "wait_until_region_idle", "wait_until_screen_stable", "wait_until_clipboard_changes", "wait_until_window_closed", "wait_until_file", "wait_until_port", "wait_until_process", + "wait_until_gone", "wait_until_image_gone", "wait_until_text_gone", + "wait_until_color", "wait_until_window_title", # Visual regression + state machine "take_golden", "compare_to_golden", "image_difference", "DiffResult", "MaskRegion", @@ -1263,6 +1454,7 @@ def start_autocontrol_gui(*args, **kwargs): # Per-window capture + window-layout save / restore + snap "capture_window", "get_window_geometry", "save_window_layout", "restore_window_layout", "snap_window", + "arrange_grid", "arrange_cascade", # Scroll-to-find "scroll_until_visible", # Recoverable deletion (recycle bin) diff --git a/je_auto_control/gui/script_builder/command_schema.py b/je_auto_control/gui/script_builder/command_schema.py index e2a66b3d..5bd7d1ff 100644 --- a/je_auto_control/gui/script_builder/command_schema.py +++ b/je_auto_control/gui/script_builder/command_schema.py @@ -44,6 +44,7 @@ class CommandSpec: _MOUSE_BUTTONS = ("mouse_left", "mouse_right", "mouse_middle") +_REGION_PLACEHOLDER = "[left, top, right, bottom]" def _build_specs() -> List[CommandSpec]: @@ -162,6 +163,47 @@ def _add_keyboard_specs(specs: List[CommandSpec]) -> None: ), description="Type text with randomized per-key delays.", )) + specs.append(CommandSpec( + "AC_set_field_text", "Keyboard", "Set Field Text", + fields=( + FieldSpec("text", FieldType.STRING, placeholder="new value"), + FieldSpec("clear", FieldType.ENUM, choices=("select_all", "none"), + optional=True, default="select_all"), + FieldSpec("paste", FieldType.BOOL, optional=True, default=False), + FieldSpec("modifier", FieldType.STRING, optional=True, + default="ctrl", placeholder="ctrl | command"), + ), + description="Clear the focused field then enter text (paste for Unicode).", + )) + specs.append(CommandSpec( + "AC_hold_key", "Keyboard", "Hold Key", + fields=( + FieldSpec("key", FieldType.STRING, placeholder="e.g. key_d, space"), + FieldSpec("duration_s", FieldType.FLOAT, default=1.0, + min_value=0.01), + FieldSpec("rate_hz", FieldType.FLOAT, optional=True, + placeholder="auto-repeat presses/sec (blank = hold)"), + ), + description="Hold a key for a duration, or auto-repeat it at rate_hz.", + )) + specs.append(CommandSpec( + "AC_type_unicode", "Keyboard", "Type Unicode (emoji / CJK)", + fields=( + FieldSpec("text", FieldType.STRING, placeholder="café 🚀 値"), + FieldSpec("modifier", FieldType.STRING, optional=True, + default="ctrl", placeholder="ctrl | command"), + ), + description="Enter any Unicode text via clipboard paste (write can't).", + )) + specs.append(CommandSpec( + "AC_with_modifiers", "Keyboard", "With Modifiers Held", + fields=( + FieldSpec("modifiers", FieldType.STRING, placeholder="ctrl+shift"), + FieldSpec("actions", FieldType.STRING, + placeholder='[["AC_click_mouse", {...}], ...]'), + ), + description="Run nested actions while modifiers are held (release-safe).", + )) specs.append(CommandSpec( "AC_hotkey", "Keyboard", "Hotkey", fields=( @@ -212,6 +254,254 @@ def _add_image_specs(specs: List[CommandSpec]) -> None: default=0.8, min_value=0.0, max_value=1.0), ), )) + specs.append(CommandSpec( + "AC_match_template", "Image", "Match Template (scored)", + fields=( + FieldSpec("template", FieldType.FILE_PATH), + FieldSpec("min_score", FieldType.FLOAT, optional=True, default=0.8, + min_value=0.0, max_value=1.0), + FieldSpec("scales", FieldType.STRING, optional=True, + placeholder="[0.9, 1.0, 1.1]"), + FieldSpec("region", FieldType.STRING, optional=True, + placeholder=_REGION_PLACEHOLDER), + ), + description="Locate a template and return its confidence score + scale.", + )) + specs.append(CommandSpec( + "AC_match_template_all", "Image", "Match Template All (scored)", + fields=( + FieldSpec("template", FieldType.FILE_PATH), + FieldSpec("min_score", FieldType.FLOAT, optional=True, default=0.8, + min_value=0.0, max_value=1.0), + FieldSpec("max_results", FieldType.INT, optional=True, default=20), + FieldSpec("nms_iou", FieldType.FLOAT, optional=True, default=0.3, + min_value=0.0, max_value=1.0), + ), + description="Find every occurrence of a template (scored, NMS-deduped).", + )) + specs.append(CommandSpec( + "AC_match_masked", "Image", "Match Masked Template", + fields=( + FieldSpec("template", FieldType.FILE_PATH), + FieldSpec("mask", FieldType.FILE_PATH, optional=True), + FieldSpec("min_score", FieldType.FLOAT, optional=True, default=0.9, + min_value=0.0, max_value=1.0), + FieldSpec("region", FieldType.STRING, optional=True, + placeholder=_REGION_PLACEHOLDER), + ), + description="Match counting only opaque/masked pixels (alpha or mask).", + )) + specs.append(CommandSpec( + "AC_match_masked_all", "Image", "Match Masked Template All", + fields=( + FieldSpec("template", FieldType.FILE_PATH), + FieldSpec("mask", FieldType.FILE_PATH, optional=True), + FieldSpec("min_score", FieldType.FLOAT, optional=True, default=0.9, + min_value=0.0, max_value=1.0), + FieldSpec("max_results", FieldType.INT, optional=True, default=20), + FieldSpec("nms_iou", FieldType.FLOAT, optional=True, default=0.3, + min_value=0.0, max_value=1.0), + ), + description="Find every masked match of a template (NMS-deduped).", + )) + specs.append(CommandSpec( + "AC_find_color_region", "Image", "Find Colour Region", + fields=( + FieldSpec("rgb", FieldType.STRING, placeholder="[0, 200, 0]"), + FieldSpec("tolerance", FieldType.INT, optional=True, default=20), + FieldSpec("min_area", FieldType.INT, optional=True, default=50), + FieldSpec("region", FieldType.STRING, optional=True, + placeholder=_REGION_PLACEHOLDER), + ), + description="Locate regions by colour (status light / banner / fill).", + )) + specs.append(CommandSpec( + "AC_ssim_compare", "Image", "SSIM Compare", + fields=( + FieldSpec("reference", FieldType.FILE_PATH), + FieldSpec("current", FieldType.FILE_PATH, optional=True), + FieldSpec("ignore", FieldType.STRING, optional=True, + placeholder="[[x, y, w, h], ...]"), + FieldSpec("region", FieldType.STRING, optional=True, + placeholder=_REGION_PLACEHOLDER), + ), + description="Structural-similarity score (0..1) vs reference / screen.", + )) + specs.append(CommandSpec( + "AC_ssim_changed_regions", "Image", "SSIM Changed Regions", + fields=( + FieldSpec("reference", FieldType.FILE_PATH), + FieldSpec("current", FieldType.FILE_PATH, optional=True), + FieldSpec("ignore", FieldType.STRING, optional=True, + placeholder="[[x, y, w, h], ...]"), + FieldSpec("threshold", FieldType.FLOAT, optional=True, default=0.35, + min_value=0.0, max_value=1.0), + FieldSpec("min_area", FieldType.INT, optional=True, default=50), + FieldSpec("region", FieldType.STRING, optional=True, + placeholder=_REGION_PLACEHOLDER), + ), + description="Boxes of the regions that structurally changed.", + )) + specs.append(CommandSpec( + "AC_feature_match", "Image", "Feature Match (ORB)", + fields=( + FieldSpec("template", FieldType.FILE_PATH), + FieldSpec("region", FieldType.STRING, optional=True, + placeholder=_REGION_PLACEHOLDER), + FieldSpec("max_features", FieldType.INT, optional=True, default=500), + FieldSpec("ratio", FieldType.FLOAT, optional=True, default=0.75, + min_value=0.0, max_value=1.0), + FieldSpec("min_inliers", FieldType.INT, optional=True, default=10), + ), + description="Locate a template under rotation / scale / theme change.", + )) + specs.append(CommandSpec( + "AC_find_shapes", "Image", "Find Shapes", + fields=( + FieldSpec("region", FieldType.STRING, optional=True, + placeholder=_REGION_PLACEHOLDER), + FieldSpec("min_area", FieldType.INT, optional=True, default=400), + FieldSpec("max_area", FieldType.INT, optional=True), + ), + description="Locate distinct shapes by edge/contour detection (no template).", + )) + specs.append(CommandSpec( + "AC_find_rectangles", "Image", "Find Rectangles", + fields=( + FieldSpec("region", FieldType.STRING, optional=True, + placeholder=_REGION_PLACEHOLDER), + FieldSpec("min_area", FieldType.INT, optional=True, default=400), + FieldSpec("max_area", FieldType.INT, optional=True), + FieldSpec("aspect_range", FieldType.STRING, optional=True, + placeholder="[1.5, 8.0]"), + FieldSpec("epsilon", FieldType.FLOAT, optional=True, default=0.04, + min_value=0.0, max_value=1.0), + ), + description="Locate rectangular regions (buttons / cards / input fields).", + )) + specs.append(CommandSpec( + "AC_preprocess_image", "Image", "Preprocess Image (OCR/match)", + fields=( + FieldSpec("output_path", FieldType.FILE_PATH), + FieldSpec("source", FieldType.FILE_PATH, optional=True), + FieldSpec("steps", FieldType.STRING, optional=True, + placeholder="grayscale,upscale,binarize"), + FieldSpec("scale", FieldType.FLOAT, optional=True, default=2.0), + FieldSpec("region", FieldType.STRING, optional=True, + placeholder=_REGION_PLACEHOLDER), + ), + description="Clean up an image for OCR / matching (grayscale/binarize/…).", + )) + specs.append(CommandSpec( + "AC_segment_hsv", "Image", "Segment by HSV", + fields=( + FieldSpec("lower_hsv", FieldType.STRING, placeholder="[40, 80, 80]"), + FieldSpec("upper_hsv", FieldType.STRING, placeholder="[80, 255, 255]"), + FieldSpec("min_area", FieldType.INT, optional=True, default=50), + FieldSpec("region", FieldType.STRING, optional=True, + placeholder=_REGION_PLACEHOLDER), + ), + description="Locate regions inside an HSV band (lighting-robust).", + )) + specs.append(CommandSpec( + "AC_dominant_hue_regions", "Image", "Find Hue Regions", + fields=( + FieldSpec("hue", FieldType.INT, placeholder="0=red, 60=green, 120=blue"), + FieldSpec("hue_tol", FieldType.INT, optional=True, default=10), + FieldSpec("sat_min", FieldType.INT, optional=True, default=80), + FieldSpec("val_min", FieldType.INT, optional=True, default=80), + FieldSpec("min_area", FieldType.INT, optional=True, default=50), + FieldSpec("region", FieldType.STRING, optional=True, + placeholder=_REGION_PLACEHOLDER), + ), + description="Locate any shade of a hue, any brightness (handles red wrap).", + )) + specs.append(CommandSpec( + "AC_find_text_regions", "Image", "Find Text Regions (MSER)", + fields=( + FieldSpec("min_area", FieldType.INT, optional=True, default=60), + FieldSpec("max_area", FieldType.INT, optional=True), + FieldSpec("merge", FieldType.BOOL, optional=True, default=True), + FieldSpec("max_aspect", FieldType.FLOAT, optional=True, default=12.0), + FieldSpec("region", FieldType.STRING, optional=True, + placeholder=_REGION_PLACEHOLDER), + ), + description="Locate text regions without OCR (crop to feed an OCR engine).", + )) + specs.append(CommandSpec( + "AC_find_text_lines", "Image", "Find Text Lines (MSER)", + fields=( + FieldSpec("y_tolerance", FieldType.INT, optional=True, default=8), + FieldSpec("region", FieldType.STRING, optional=True, + placeholder=_REGION_PLACEHOLDER), + ), + description="Locate horizontal text lines without OCR.", + )) + specs.append(CommandSpec( + "AC_find_lines", "Image", "Find Lines (Hough)", + fields=( + FieldSpec("min_length", FieldType.INT, optional=True, default=80), + FieldSpec("max_gap", FieldType.INT, optional=True, default=10), + FieldSpec("orientation", FieldType.ENUM, optional=True, default="any", + choices=("any", "horizontal", "vertical", "diagonal")), + FieldSpec("region", FieldType.STRING, optional=True, + placeholder=_REGION_PLACEHOLDER), + ), + description="Detect straight line segments on raw pixels.", + )) + specs.append(CommandSpec( + "AC_find_grid", "Image", "Find Table Grid", + fields=( + FieldSpec("min_length", FieldType.INT, optional=True, default=120), + FieldSpec("tol", FieldType.INT, optional=True, default=10), + FieldSpec("region", FieldType.STRING, optional=True, + placeholder=_REGION_PLACEHOLDER), + ), + description="Recover a table's rows / columns / cells from its lines.", + )) + specs.append(CommandSpec( + "AC_find_separators", "Image", "Find Separator Lines", + fields=( + FieldSpec("axis", FieldType.ENUM, optional=True, default="horizontal", + choices=("horizontal", "vertical")), + FieldSpec("min_length", FieldType.INT, optional=True, default=120), + FieldSpec("tol", FieldType.INT, optional=True, default=10), + FieldSpec("region", FieldType.STRING, optional=True, + placeholder=_REGION_PLACEHOLDER), + ), + description="Coordinates of long divider lines along an axis.", + )) + specs.append(CommandSpec( + "AC_fuse_elements", "Image", "Fuse Element Boxes", + fields=( + FieldSpec("ocr", FieldType.STRING, optional=True, + placeholder='[{"x":..,"y":..,"width":..,"height":..}]'), + FieldSpec("icon", FieldType.STRING, optional=True), + FieldSpec("a11y", FieldType.STRING, optional=True), + FieldSpec("iou_threshold", FieldType.FLOAT, optional=True, default=0.9, + min_value=0.0, max_value=1.0), + ), + description="Union OCR/icon/a11y boxes, dropping cross-source duplicates.", + )) + specs.append(CommandSpec( + "AC_reading_order", "Image", "Reading Order", + fields=( + FieldSpec("elements", FieldType.STRING, + placeholder='[{"x":..,"y":..,"width":..,"height":..}]'), + FieldSpec("row_tol", FieldType.INT, optional=True, default=12), + ), + description="Order element boxes top-to-bottom, left-to-right (+ index).", + )) + specs.append(CommandSpec( + "AC_locate_chain", "Image", "Locate Chain (refine boxes)", + fields=( + FieldSpec("boxes", FieldType.STRING, + placeholder='[{"x":..,"y":..,"width":..,"height":..}]'), + FieldSpec("ops", FieldType.STRING, + placeholder='[{"op":"filter","has_text":"OK"},{"op":"first"}]'), + ), + description="Refine candidate boxes (within / filter / reading / nth / …).", + )) def _add_ocr_specs(specs: List[CommandSpec]) -> None: @@ -315,6 +605,26 @@ def _add_window_specs(specs: List[CommandSpec]) -> None: ), description="Move a window to a screen half / quarter / maximize.", )) + specs.append(CommandSpec( + "AC_arrange_grid", "Window", "Arrange Windows in Grid", + fields=( + FieldSpec("titles", FieldType.STRING, + placeholder='["Editor", "Browser", "Terminal"]'), + FieldSpec("rows", FieldType.INT, optional=True), + FieldSpec("cols", FieldType.INT, optional=True), + FieldSpec("gap", FieldType.INT, optional=True, default=0), + ), + description="Tile a list of windows into a grid (auto-shape if unset).", + )) + specs.append(CommandSpec( + "AC_arrange_cascade", "Window", "Arrange Windows in Cascade", + fields=( + FieldSpec("titles", FieldType.STRING, + placeholder='["Editor", "Browser", "Terminal"]'), + FieldSpec("offset", FieldType.INT, optional=True, default=30), + ), + description="Cascade a list of windows diagonally.", + )) specs.append(CommandSpec( "AC_wait_window_closed", "Window", "Wait for Window to Close", fields=( @@ -386,6 +696,55 @@ def _add_window_specs(specs: List[CommandSpec]) -> None: fields=(FieldSpec("layout", FieldType.FILE_PATH),), description="Move windows back to a saved layout file.", )) + specs.append(CommandSpec( + "AC_tile_rect", "Window", "Tile Rect (compute)", + fields=( + FieldSpec("slot", FieldType.ENUM, + choices=("full", "left", "right", "top", "bottom", + "top_left", "top_right", "bottom_left", + "bottom_right", "center", "left_third", + "center_third", "right_third"), default="left"), + FieldSpec("screen", FieldType.STRING, optional=True, + placeholder="[x, y, width, height]"), + FieldSpec("gap", FieldType.INT, optional=True, default=0), + ), + description="Compute the rectangle for a tiling slot of the screen.", + )) + specs.append(CommandSpec( + "AC_grid_rects", "Window", "Grid Rects (compute)", + fields=( + FieldSpec("rows", FieldType.INT, default=2), + FieldSpec("cols", FieldType.INT, default=2), + FieldSpec("screen", FieldType.STRING, optional=True, + placeholder="[x, y, width, height]"), + FieldSpec("gap", FieldType.INT, optional=True, default=0), + ), + description="Compute the cell rectangles of an R×C screen grid.", + )) + specs.append(CommandSpec( + "AC_cascade_rects", "Window", "Cascade Rects (compute)", + fields=( + FieldSpec("count", FieldType.INT, default=3), + FieldSpec("screen", FieldType.STRING, optional=True, + placeholder="[x, y, width, height]"), + FieldSpec("offset", FieldType.INT, optional=True, default=30), + FieldSpec("size", FieldType.STRING, optional=True, + placeholder="[width, height]"), + ), + description="Compute staggered, overlapping window rectangles.", + )) + specs.append(CommandSpec( + "AC_enumerate_monitors", "Window", "Enumerate Monitors", + description="List monitors + virtual bounds (multi-display geometry).", + )) + specs.append(CommandSpec( + "AC_monitor_at_point", "Window", "Monitor at Point", + fields=( + FieldSpec("x", FieldType.INT), + FieldSpec("y", FieldType.INT), + ), + description="Report which monitor contains a virtual point.", + )) def _add_flow_specs(specs: List[CommandSpec]) -> None: @@ -406,6 +765,34 @@ def _add_flow_specs(specs: List[CommandSpec]) -> None: min_value=0.01), ), )) + specs.append(CommandSpec( + "AC_wait_actionable", "Flow", "Wait Until Actionable", + fields=( + FieldSpec("template", FieldType.FILE_PATH), + FieldSpec("timeout_s", FieldType.FLOAT, optional=True, default=5.0), + FieldSpec("stable_for_s", FieldType.FLOAT, optional=True, default=0.3), + FieldSpec("min_score", FieldType.FLOAT, optional=True, default=0.8, + min_value=0.0, max_value=1.0), + FieldSpec("region", FieldType.STRING, optional=True, + placeholder=_REGION_PLACEHOLDER), + ), + description="Wait until a target is visible + stable before acting.", + )) + specs.append(CommandSpec( + "AC_expect_poll", "Flow", "Expect (poll until match)", + fields=( + FieldSpec("action", FieldType.STRING, + placeholder='["AC_get_clipboard"]'), + FieldSpec("key", FieldType.STRING, optional=True, + placeholder="result dict key, e.g. text"), + FieldSpec("op", FieldType.ENUM, optional=True, default="truthy", + choices=("truthy", "equals", "contains", "gt", "regex")), + FieldSpec("expected", FieldType.STRING, optional=True), + FieldSpec("timeout_s", FieldType.FLOAT, optional=True, default=5.0), + FieldSpec("interval_s", FieldType.FLOAT, optional=True, default=0.25), + ), + description="Re-run an action until a key of its result matches.", + )) specs.append(CommandSpec( "AC_wait_pixel", "Flow", "Wait for Pixel", fields=( @@ -431,6 +818,58 @@ def _add_flow_specs(specs: List[CommandSpec]) -> None: ), description="Wait until the clipboard changes or matches target.", )) + specs.append(CommandSpec( + "AC_wait_image_gone", "Flow", "Wait for Image to Vanish", + fields=( + FieldSpec("image", FieldType.STRING, placeholder="path/to/spinner.png"), + FieldSpec("detect_threshold", FieldType.FLOAT, optional=True, + default=1.0), + FieldSpec("timeout_s", FieldType.FLOAT, optional=True, default=10.0), + FieldSpec("poll_interval_s", FieldType.FLOAT, optional=True, + default=0.2, min_value=0.01), + FieldSpec("gone_for_s", FieldType.FLOAT, optional=True, default=0.0), + ), + description="Block until an image (spinner/toast) leaves the screen.", + )) + specs.append(CommandSpec( + "AC_wait_text_gone", "Flow", "Wait for Text to Vanish", + fields=( + FieldSpec("text", FieldType.STRING, placeholder="Loading..."), + FieldSpec("timeout_s", FieldType.FLOAT, optional=True, default=10.0), + FieldSpec("poll_interval_s", FieldType.FLOAT, optional=True, + default=0.2, min_value=0.01), + FieldSpec("gone_for_s", FieldType.FLOAT, optional=True, default=0.0), + ), + description="Block until on-screen text (OCR) disappears.", + )) + specs.append(CommandSpec( + "AC_wait_color", "Flow", "Wait for Region Colour", + fields=( + FieldSpec("target_rgb", FieldType.STRING, placeholder="[0, 200, 0]"), + FieldSpec("region", FieldType.STRING, optional=True, + placeholder=_REGION_PLACEHOLDER), + FieldSpec("tolerance", FieldType.INT, optional=True, default=10), + FieldSpec("min_fraction", FieldType.FLOAT, optional=True, + default=0.5, min_value=0.0, max_value=1.0), + FieldSpec("present", FieldType.BOOL, optional=True, default=True), + FieldSpec("timeout_s", FieldType.FLOAT, optional=True, default=10.0), + FieldSpec("poll_interval_s", FieldType.FLOAT, optional=True, + default=0.2, min_value=0.01), + ), + description="Block until a colour fills (or leaves) a screen region.", + )) + specs.append(CommandSpec( + "AC_wait_window_title", "Flow", "Wait for Window Title", + fields=( + FieldSpec("pattern", FieldType.STRING, placeholder="Checkout$"), + FieldSpec("present", FieldType.BOOL, optional=True, default=True), + FieldSpec("regex", FieldType.BOOL, optional=True, default=True), + FieldSpec("timeout_s", FieldType.FLOAT, optional=True, default=10.0), + FieldSpec("poll_interval_s", FieldType.FLOAT, optional=True, + default=0.2, min_value=0.01), + ), + description="Block until a window title matches a regex (or vanishes).", + )) specs.append(CommandSpec( "AC_loop", "Flow", "Loop (N times)", fields=( @@ -605,6 +1044,19 @@ def _add_native_control_specs(specs: List[CommandSpec]) -> None: def _add_misc_specs(specs: List[CommandSpec]) -> None: _add_native_control_specs(specs) + specs.append(CommandSpec( + "AC_set_clipboard_html", "Data", "Set Clipboard HTML", + fields=( + FieldSpec("html", FieldType.STRING, + placeholder="Bold rich text"), + FieldSpec("fragment_plaintext", FieldType.STRING, optional=True), + ), + description="Put rich HTML on the clipboard (CF_HTML, Windows).", + )) + specs.append(CommandSpec( + "AC_get_clipboard_html", "Data", "Get Clipboard HTML", + description="Read the clipboard's HTML fragment (CF_HTML, Windows).", + )) specs.append(CommandSpec( "AC_watchdog_add", "Flow", "Watchdog: Add Popup Rule", fields=( @@ -681,6 +1133,55 @@ def _add_misc_specs(specs: List[CommandSpec]) -> None: description="Drag along an eased path; 'start'/'end' [x,y] via JSON " "view.", )) + specs.append(CommandSpec( + "AC_move_along_path", "Mouse", "Move Along Path", + fields=( + FieldSpec("waypoints", FieldType.STRING, + placeholder="[[100,100],[400,150],[400,500]]"), + FieldSpec("easing", FieldType.ENUM, + choices=("linear", "ease_in_out_quad", "ease_out_cubic", + "ease_in_cubic"), + optional=True, default="linear"), + FieldSpec("per_segment_steps", FieldType.INT, optional=True, + default=20), + ), + description="Move the pointer through a polyline of waypoints.", + )) + specs.append(CommandSpec( + "AC_drag_path", "Mouse", "Drag Along Path", + fields=( + FieldSpec("waypoints", FieldType.STRING, + placeholder="[[50,50],[300,50],[300,300]]"), + FieldSpec("button", FieldType.ENUM, choices=_MOUSE_BUTTONS, + optional=True, default="mouse_left"), + FieldSpec("easing", FieldType.ENUM, + choices=("linear", "ease_in_out_quad", "ease_out_cubic", + "ease_in_cubic"), + optional=True, default="linear"), + FieldSpec("per_segment_steps", FieldType.INT, optional=True, + default=20), + ), + description="Press, drag through a polyline of waypoints, release.", + )) + specs.append(CommandSpec( + "AC_move_mouse_relative", "Mouse", "Move Relative", + fields=( + FieldSpec("dx", FieldType.INT, placeholder="-40"), + FieldSpec("dy", FieldType.INT, placeholder="12"), + ), + description="Move the pointer by (dx, dy) from its current position.", + )) + specs.append(CommandSpec( + "AC_grid_cell", "Mouse", "Grid Cell (row / col)", + fields=( + FieldSpec("boxes", FieldType.STRING, + placeholder="[[10,100,20,10],[110,100,20,10], ...]"), + FieldSpec("row", FieldType.INT, placeholder="0"), + FieldSpec("col", FieldType.INT, placeholder="0"), + FieldSpec("row_tolerance", FieldType.INT, optional=True, default=10), + ), + description="Resolve a table cell centre by row/col from cell boxes.", + )) specs.append(CommandSpec( "AC_list_plugins", "Tools", "List Plugin Commands", fields=(FieldSpec("group", FieldType.STRING, optional=True, @@ -2170,6 +2671,24 @@ def _add_resilience_specs(specs: List[CommandSpec]) -> None: ), description="Pick the plural-correct translation for count n.", )) + specs.append(CommandSpec( + "AC_checksum_validate", "Data", "Checksum: Validate", + fields=( + FieldSpec("scheme", FieldType.STRING, + placeholder="luhn | verhoeff | damm | mod97"), + FieldSpec("number", FieldType.STRING, placeholder="4111111111111111"), + ), + description="Validate a number's check digit (Luhn/Verhoeff/Damm/mod97).", + )) + specs.append(CommandSpec( + "AC_checksum_digit", "Data", "Checksum: Check Digit", + fields=( + FieldSpec("scheme", FieldType.STRING, + placeholder="luhn | verhoeff | damm | mod97"), + FieldSpec("partial", FieldType.STRING, placeholder="799273987"), + ), + description="Compute the check digit(s) to append to a value.", + )) specs.append(CommandSpec( "AC_diff_rows", "Data", "Dataset Diff: Rows by Key", fields=( diff --git a/je_auto_control/utils/actionability/__init__.py b/je_auto_control/utils/actionability/__init__.py new file mode 100644 index 00000000..453ca1ff --- /dev/null +++ b/je_auto_control/utils/actionability/__init__.py @@ -0,0 +1,7 @@ +"""Pre-action readiness gate (visible + stable + enabled + not-occluded).""" +from je_auto_control.utils.actionability.actionability import ( + ActionabilityReport, GateConfig, act_when_ready, wait_actionable, +) + +__all__ = ["ActionabilityReport", "GateConfig", "act_when_ready", + "wait_actionable"] diff --git a/je_auto_control/utils/actionability/actionability.py b/je_auto_control/utils/actionability/actionability.py new file mode 100644 index 00000000..3389d8ac --- /dev/null +++ b/je_auto_control/utils/actionability/actionability.py @@ -0,0 +1,156 @@ +"""Pre-action readiness gate — visible + stable + enabled + not-occluded before acting. + +Modern UI frameworks (Playwright, Cypress, WebdriverIO) auto-run an *actionability* +check before every click: the target must be present, have stopped moving, be enabled, +and actually receive the event (not be covered). AutoControl had none — ``self_heal_click`` +locates and clicks immediately, and ``wait_until_screen_stable`` only watches the *whole* +frame. This composes the four checks into one gate. + +Every signal is an injectable callable — ``bbox_provider`` (locate the target), +``region_sampler`` (pixel-stability token), ``enabled_probe`` (is it enabled?), +``hit_tester`` (is the click point on top?) — plus an injectable ``clock`` / ``sleep``, +so the gate is fully deterministic and headless-testable. Imports no ``PySide6``. +""" +import time +from dataclasses import dataclass, field +from typing import Any, Callable, Dict, List, Optional, Tuple + +from je_auto_control.utils.exception.exceptions import AutoControlActionException + +Bbox = Tuple[int, int, int, int] +BboxProvider = Callable[[], Optional[Bbox]] + + +@dataclass +class GateConfig: + """Timing knobs and test seams for the actionability gate.""" + + timeout_s: float = 5.0 + stable_for_s: float = 0.3 + poll_interval_s: float = 0.1 + clock: Callable[[], float] = time.monotonic + sleep: Callable[[float], None] = time.sleep + + +@dataclass(frozen=True) +class ActionabilityReport: + """The outcome of a gate: per-check booleans, the target point, and timing.""" + + actionable: bool + visible: bool + stable: bool + enabled: bool + receives_events: bool + waited_s: float + reason: str + point: Optional[List[int]] = None + checks: Dict[str, bool] = field(default_factory=dict) + + def to_dict(self) -> Dict[str, Any]: + """Return the report as a plain dict.""" + return {"actionable": self.actionable, "visible": self.visible, + "stable": self.stable, "enabled": self.enabled, + "receives_events": self.receives_events, "waited_s": self.waited_s, + "reason": self.reason, "point": self.point, "checks": self.checks} + + +def _center(bbox: Bbox) -> List[int]: + return [bbox[0] + bbox[2] // 2, bbox[1] + bbox[3] // 2] + + +def _reason(visible: bool, stable: bool, enabled: bool, receives: bool) -> str: + if not visible: + return "not visible" + if not stable: + return "not stable" + if not enabled: + return "disabled" + if not receives: + return "occluded" + return "actionable" + + +class _StabilityTracker: + """Tracks how long the bbox (and optional pixel sample) has stayed unchanged.""" + + def __init__(self, sampler, stable_for_s: float): + self._sampler = sampler + self._stable_for_s = stable_for_s + self._prev: Any = object() + self._since: Optional[float] = None + + def update(self, bbox: Optional[Bbox], now: float) -> bool: + if bbox is None: + self._since = None + self._prev = object() + return False + token = (bbox, self._sampler(bbox) if self._sampler else None) + if token != self._prev: + self._prev = token + self._since = now + return False + return (now - self._since) >= self._stable_for_s + + +def _evaluate(bbox, tracker, now, enabled_probe, hit_tester): + """Return ``(visible, stable, enabled, receives, point)`` for one poll.""" + visible = bbox is not None + stable = tracker.update(bbox, now) + enabled = enabled_probe is None or bool(enabled_probe()) + point = _center(bbox) if visible else None + receives = hit_tester is None or point is None or bool(hit_tester(point)) + return visible, stable, enabled, receives, point + + +def _report(visible, stable, enabled, receives, point, waited): + actionable = visible and stable and enabled and receives + return ActionabilityReport( + actionable, visible, stable, enabled, receives, round(waited, 4), + _reason(visible, stable, enabled, receives), point, + {"visible": visible, "stable": stable, "enabled": enabled, + "receives_events": receives}) + + +def wait_actionable(bbox_provider: BboxProvider, *, + region_sampler: Optional[Callable[[Bbox], Any]] = None, + enabled_probe: Optional[Callable[[], Optional[bool]]] = None, + hit_tester: Optional[Callable[[List[int]], bool]] = None, + config: Optional[GateConfig] = None) -> ActionabilityReport: + """Poll until the target is visible + stable + enabled + unoccluded, or timeout. + + ``bbox_provider`` returns the target ``(x, y, w, h)`` or ``None``. The optional + ``region_sampler`` returns a pixel token over the bbox (movement/animation + settling); ``enabled_probe`` reports enabled state; ``hit_tester`` reports + whether the centre point is on top. Returns an :class:`ActionabilityReport`. + """ + cfg = config or GateConfig() + tracker = _StabilityTracker(region_sampler, cfg.stable_for_s) + start = cfg.clock() + deadline = start + cfg.timeout_s + while True: + now = cfg.clock() + signals = _evaluate(bbox_provider(), tracker, now, enabled_probe, + hit_tester) + report = _report(*signals, now - start) + if report.actionable or now >= deadline: + return report + cfg.sleep(cfg.poll_interval_s) + + +def act_when_ready(action: Callable[[List[int]], Any], bbox_provider: BboxProvider, + *, region_sampler: Optional[Callable[[Bbox], Any]] = None, + enabled_probe: Optional[Callable[[], Optional[bool]]] = None, + hit_tester: Optional[Callable[[List[int]], bool]] = None, + config: Optional[GateConfig] = None) -> Any: + """Wait for the target to be actionable, then call ``action(center_point)``. + + Raises ``AutoControlActionException`` (with the failing check) if the gate times + out before the target becomes actionable. + """ + report = wait_actionable(bbox_provider, region_sampler=region_sampler, + enabled_probe=enabled_probe, hit_tester=hit_tester, + config=config) + if not report.actionable: + raise AutoControlActionException( + f"target not actionable ({report.reason}) after {report.waited_s}s") + return action(report.point) diff --git a/je_auto_control/utils/anchor_locator/__init__.py b/je_auto_control/utils/anchor_locator/__init__.py index 9668da02..a6d281d5 100644 --- a/je_auto_control/utils/anchor_locator/__init__.py +++ b/je_auto_control/utils/anchor_locator/__init__.py @@ -22,8 +22,8 @@ AnchorLocatorError, AnchorOutcome, KIND_A11Y, KIND_IMAGE, KIND_OCR, KIND_VLM, Locator, REL_ABOVE, REL_BELOW, REL_LEFT_OF, REL_NEAR, REL_RIGHT_OF, - a11y_locator, anchor_locate, image_locator, ocr_locator, - vlm_locator, + a11y_locator, anchor_locate, anchor_locate_all, image_locator, + ocr_locator, vlm_locator, ) @@ -31,6 +31,6 @@ "AnchorLocatorError", "AnchorOutcome", "KIND_A11Y", "KIND_IMAGE", "KIND_OCR", "KIND_VLM", "Locator", "REL_ABOVE", "REL_BELOW", "REL_LEFT_OF", "REL_NEAR", "REL_RIGHT_OF", - "a11y_locator", "anchor_locate", "image_locator", "ocr_locator", - "vlm_locator", + "a11y_locator", "anchor_locate", "anchor_locate_all", "image_locator", + "ocr_locator", "vlm_locator", ] diff --git a/je_auto_control/utils/anchor_locator/locator.py b/je_auto_control/utils/anchor_locator/locator.py index 3190108e..feb156f8 100644 --- a/je_auto_control/utils/anchor_locator/locator.py +++ b/je_auto_control/utils/anchor_locator/locator.py @@ -137,7 +137,8 @@ def to_dict(self) -> Dict[str, Any]: def anchor_locate(*, anchor: Locator, target: Locator, relation: str = REL_NEAR, - max_distance_px: float = 200.0) -> AnchorOutcome: + max_distance_px: float = 200.0, + ordinal: int = 1) -> AnchorOutcome: """Find ``target`` near / above / below / beside ``anchor``. Strategy: @@ -147,8 +148,10 @@ def anchor_locate(*, anchor: Locator, target: Locator, 2. Resolve the target to a list of candidate bboxes. Image and OCR can enumerate; VLM / a11y always return one point so only that one candidate is considered. - 3. Filter candidates by the spatial relation; tie-break by the - smallest centre-to-centre distance. + 3. Filter candidates by the spatial relation and sort by the + centre-to-centre distance; return the ``ordinal``-th (1-based, + so ``ordinal=1`` is the nearest — "the 2nd row below the header" + is ``ordinal=2``). """ relation_norm = _normalise_relation(relation) anchor_point = _resolve_single(anchor) @@ -160,7 +163,6 @@ def anchor_locate(*, anchor: Locator, target: Locator, error="anchor not found", ) candidates = _resolve_candidates(target) - candidates_considered = len(candidates) if not candidates: return AnchorOutcome( found=False, target_coords=None, anchor_coords=anchor_point, @@ -168,26 +170,54 @@ def anchor_locate(*, anchor: Locator, target: Locator, target_kind=target.kind, anchor_kind=anchor.kind, candidates_considered=0, error="target not found", ) - chosen = _pick_best( - anchor_point, candidates, relation_norm, float(max_distance_px), - ) - if chosen is None: + ranked = _ranked(anchor_point, candidates, relation_norm, + float(max_distance_px)) + index = int(ordinal) - 1 + if index < 0 or index >= len(ranked): return AnchorOutcome( found=False, target_coords=None, anchor_coords=anchor_point, distance_px=None, relation=relation_norm, target_kind=target.kind, anchor_kind=anchor.kind, - candidates_considered=candidates_considered, - error=f"no candidate satisfies relation {relation_norm!r}", + candidates_considered=len(candidates), + error=f"no candidate at ordinal {ordinal} for relation " + f"{relation_norm!r}", ) - coords, distance = chosen + coords, distance = ranked[index] return AnchorOutcome( found=True, target_coords=coords, anchor_coords=anchor_point, distance_px=round(distance, 2), relation=relation_norm, target_kind=target.kind, anchor_kind=anchor.kind, - candidates_considered=candidates_considered, + candidates_considered=len(candidates), ) +def anchor_locate_all(*, anchor: Locator, target: Locator, + relation: str = REL_NEAR, + max_distance_px: float = 200.0) -> List[AnchorOutcome]: + """Return every target matching the relation, nearest-first. + + The building block for table / list-row selection: each result is a found + :class:`AnchorOutcome` ordered by distance from the anchor. Returns an empty + list when the anchor or all targets are missing. + """ + relation_norm = _normalise_relation(relation) + anchor_point = _resolve_single(anchor) + if anchor_point is None: + return [] + candidates = _resolve_candidates(target) + ranked = _ranked(anchor_point, candidates, relation_norm, + float(max_distance_px)) + return [ + AnchorOutcome( + found=True, target_coords=coords, anchor_coords=anchor_point, + distance_px=round(distance, 2), relation=relation_norm, + target_kind=target.kind, anchor_kind=anchor.kind, + candidates_considered=len(candidates), + ) + for coords, distance in ranked + ] + + def _normalise_relation(relation: str) -> str: normalised = (relation or "").strip().lower() if normalised not in _VALID_RELATIONS: @@ -221,21 +251,22 @@ def _resolve_candidates(locator: Locator) -> List[_Bbox]: return [] -def _pick_best(anchor_point: Tuple[int, int], - candidates: List[_Bbox], - relation: str, - max_distance: float, - ) -> Optional[Tuple[Tuple[int, int], float]]: - best: Optional[Tuple[Tuple[int, int], float]] = None +def _ranked(anchor_point: Tuple[int, int], + candidates: List[_Bbox], + relation: str, + max_distance: float, + ) -> List[Tuple[Tuple[int, int], float]]: + """Return (centre, distance) of matching candidates, nearest first.""" + matches: List[Tuple[Tuple[int, int], float]] = [] for bbox in candidates: if not _matches_relation(anchor_point, bbox, relation): continue distance = _euclid(anchor_point, bbox.center) if relation == REL_NEAR and distance > max_distance: continue - if best is None or distance < best[1]: - best = (bbox.center, distance) - return best + matches.append((bbox.center, distance)) + matches.sort(key=lambda item: item[1]) + return matches def _matches_relation(anchor_point: Tuple[int, int], @@ -352,5 +383,6 @@ def _a11y_point(locator: Locator) -> Optional[Tuple[int, int]]: "AnchorLocatorError", "AnchorOutcome", "KIND_A11Y", "KIND_IMAGE", "KIND_OCR", "KIND_VLM", "Locator", "REL_ABOVE", "REL_BELOW", "REL_LEFT_OF", "REL_NEAR", "REL_RIGHT_OF", "a11y_locator", - "anchor_locate", "image_locator", "ocr_locator", "vlm_locator", + "anchor_locate", "anchor_locate_all", "image_locator", "ocr_locator", + "vlm_locator", ] diff --git a/je_auto_control/utils/checksum/__init__.py b/je_auto_control/utils/checksum/__init__.py new file mode 100644 index 00000000..4edddc3c --- /dev/null +++ b/je_auto_control/utils/checksum/__init__.py @@ -0,0 +1,12 @@ +"""Check-digit algorithms: Luhn, Verhoeff, Damm, ISO 7064 MOD 97-10.""" +from je_auto_control.utils.checksum.checksum import ( + damm_check_digit, damm_validate, luhn_check_digit, luhn_validate, + mod97_10_check_digits, mod97_10_validate, verhoeff_check_digit, + verhoeff_validate, +) + +__all__ = [ + "damm_check_digit", "damm_validate", "luhn_check_digit", "luhn_validate", + "mod97_10_check_digits", "mod97_10_validate", "verhoeff_check_digit", + "verhoeff_validate", +] diff --git a/je_auto_control/utils/checksum/checksum.py b/je_auto_control/utils/checksum/checksum.py new file mode 100644 index 00000000..aa162b2b --- /dev/null +++ b/je_auto_control/utils/checksum/checksum.py @@ -0,0 +1,120 @@ +"""Check-digit algorithms: Luhn, Verhoeff, Damm, ISO 7064 MOD 97-10. + +``pii_text`` detects credit-card and IBAN *shapes* by regex and ``data_quality`` +does type/range/regex validation, but nothing in the project actually computes or +verifies a *check digit*. This is the shared arithmetic engine that catches the +single-digit and adjacent-transposition typos those formats are designed to +detect, and the primitive that ``identifier_validate`` (IBAN / ISBN / EAN / card) +builds on. + +Pure standard library (integer arithmetic only; the Verhoeff and Damm tables are +small embedded constants). Every function is pure (string in, bool/str out), so it +is fully deterministic in CI. +""" +from typing import List + +# Verhoeff dihedral-group multiplication, permutation and inverse tables. +_VERHOEFF_D = ( + (0, 1, 2, 3, 4, 5, 6, 7, 8, 9), (1, 2, 3, 4, 0, 6, 7, 8, 9, 5), + (2, 3, 4, 0, 1, 7, 8, 9, 5, 6), (3, 4, 0, 1, 2, 8, 9, 5, 6, 7), + (4, 0, 1, 2, 3, 9, 5, 6, 7, 8), (5, 9, 8, 7, 6, 0, 4, 3, 2, 1), + (6, 5, 9, 8, 7, 1, 0, 4, 3, 2), (7, 6, 5, 9, 8, 2, 1, 0, 4, 3), + (8, 7, 6, 5, 9, 3, 2, 1, 0, 4), (9, 8, 7, 6, 5, 4, 3, 2, 1, 0), +) +_VERHOEFF_P = ( + (0, 1, 2, 3, 4, 5, 6, 7, 8, 9), (1, 5, 7, 6, 2, 8, 3, 0, 9, 4), + (5, 8, 0, 3, 7, 9, 6, 1, 4, 2), (8, 9, 1, 6, 0, 4, 3, 5, 2, 7), + (9, 4, 5, 3, 1, 2, 6, 8, 7, 0), (4, 2, 8, 6, 5, 7, 3, 9, 0, 1), + (2, 7, 9, 3, 8, 0, 6, 4, 1, 5), (7, 0, 4, 6, 9, 1, 3, 2, 5, 8), +) +_VERHOEFF_INV = (0, 4, 3, 2, 1, 5, 6, 7, 8, 9) + +# Damm quasigroup table (totally anti-symmetric). +_DAMM = ( + (0, 3, 1, 7, 5, 9, 8, 6, 4, 2), (7, 0, 9, 2, 1, 5, 4, 8, 6, 3), + (4, 2, 0, 6, 8, 7, 1, 3, 5, 9), (1, 7, 5, 0, 9, 8, 3, 4, 2, 6), + (6, 1, 2, 3, 0, 4, 5, 9, 7, 8), (3, 6, 7, 4, 2, 0, 9, 5, 8, 1), + (5, 8, 6, 9, 7, 2, 0, 1, 3, 4), (8, 9, 4, 5, 3, 6, 2, 0, 1, 7), + (9, 4, 3, 8, 6, 1, 7, 2, 0, 5), (2, 5, 8, 1, 4, 3, 6, 7, 9, 0), +) + + +def _digits(value: object) -> List[int]: + """Extract the decimal digits of a value as a list of ints.""" + return [int(ch) for ch in str(value) if ch.isdigit()] + + +# --- Luhn (mod 10) -------------------------------------------------------- + +def _luhn_sum(digits: List[int]) -> int: + total = 0 + for index, digit in enumerate(reversed(digits)): + if index % 2 == 1: + digit *= 2 + if digit > 9: + digit -= 9 + total += digit + return total + + +def luhn_validate(number: object) -> bool: + """Whether ``number`` (incl. its trailing check digit) passes Luhn.""" + digits = _digits(number) + return bool(digits) and _luhn_sum(digits) % 10 == 0 + + +def luhn_check_digit(partial: object) -> str: + """Return the Luhn check digit to append to ``partial`` (no check digit).""" + total = _luhn_sum(_digits(partial) + [0]) + return str((10 - total % 10) % 10) + + +# --- Verhoeff ------------------------------------------------------------- + +def verhoeff_validate(number: object) -> bool: + """Whether ``number`` (incl. check digit) passes the Verhoeff scheme.""" + check = 0 + for index, digit in enumerate(reversed(_digits(number))): + check = _VERHOEFF_D[check][_VERHOEFF_P[index % 8][digit]] + return check == 0 + + +def verhoeff_check_digit(partial: object) -> str: + """Return the Verhoeff check digit to append to ``partial``.""" + check = 0 + for index, digit in enumerate(reversed(_digits(partial))): + check = _VERHOEFF_D[check][_VERHOEFF_P[(index + 1) % 8][digit]] + return str(_VERHOEFF_INV[check]) + + +# --- Damm ----------------------------------------------------------------- + +def damm_validate(number: object) -> bool: + """Whether ``number`` (incl. check digit) passes the Damm scheme.""" + interim = 0 + for digit in _digits(number): + interim = _DAMM[interim][digit] + return interim == 0 + + +def damm_check_digit(partial: object) -> str: + """Return the Damm check digit to append to ``partial``.""" + interim = 0 + for digit in _digits(partial): + interim = _DAMM[interim][digit] + return str(interim) + + +# --- ISO 7064 MOD 97-10 (the IBAN engine) --------------------------------- + +def mod97_10_validate(number: object) -> bool: + """Whether the numeric string ``number`` satisfies ISO 7064 MOD 97-10.""" + digits = "".join(ch for ch in str(number) if ch.isdigit()) + return bool(digits) and int(digits) % 97 == 1 + + +def mod97_10_check_digits(partial: object) -> str: + """Return the two MOD 97-10 check digits to append to ``partial``.""" + digits = "".join(ch for ch in str(partial) if ch.isdigit()) + value = int(digits) if digits else 0 + return f"{(1 - value * 100) % 97:02d}" diff --git a/je_auto_control/utils/color_region/__init__.py b/je_auto_control/utils/color_region/__init__.py new file mode 100644 index 00000000..c59b80a5 --- /dev/null +++ b/je_auto_control/utils/color_region/__init__.py @@ -0,0 +1,6 @@ +"""Locate on-screen regions by colour (mask + connected components).""" +from je_auto_control.utils.color_region.color_region import ( + find_color_region, find_color_regions, +) + +__all__ = ["find_color_region", "find_color_regions"] diff --git a/je_auto_control/utils/color_region/color_region.py b/je_auto_control/utils/color_region/color_region.py new file mode 100644 index 00000000..7adb3c1c --- /dev/null +++ b/je_auto_control/utils/color_region/color_region.py @@ -0,0 +1,73 @@ +"""Locate on-screen regions by colour — find the green pill, the red banner. + +``color_stats`` only *describes* a region's dominant / average colour and +``assert_pixel`` checks a single point with a tolerance; neither *locates* a +coloured region. Template matching is brittle when only the colour is the signal +(a status light, a progress fill, an error banner). This masks pixels within a +tolerance of a target RGB and returns the bounding boxes of the connected blobs. + +The masking + connected-components run on an injectable ``haystack`` image +(ndarray / path / PIL), so it is unit-testable on synthetic arrays without a real +screen. OpenCV + NumPy come in via the project's ``je_open_cv`` dependency and are +imported lazily. Imports no ``PySide6``. +""" +from typing import Any, Dict, List, Optional, Sequence + +ImageSource = Any + + +def _to_rgb(source: ImageSource): + """Load a path / ndarray / PIL image as an RGB ndarray.""" + import cv2 + import numpy as np + if hasattr(source, "shape"): + return np.asarray(source) + if isinstance(source, (str, bytes)) or hasattr(source, "__fspath__"): + bgr = cv2.imread(str(source), cv2.IMREAD_COLOR) + if bgr is None: + raise ValueError(f"could not read image: {source!r}") + return cv2.cvtColor(bgr, cv2.COLOR_BGR2RGB) + return np.asarray(source) + + +def _grab_rgb(region: Optional[Sequence[int]]): + import numpy as np + from je_auto_control.utils.cv2_utils.screenshot import pil_screenshot + image = pil_screenshot(screen_region=list(region) if region else None) + return np.asarray(image.convert("RGB")) + + +def find_color_regions(rgb: Sequence[int], *, + haystack: Optional[ImageSource] = None, + region: Optional[Sequence[int]] = None, + tolerance: int = 20, + min_area: int = 50) -> List[Dict[str, Any]]: + """Return bounding boxes of blobs within ``tolerance`` of ``rgb``, largest first. + + Each result is ``{x, y, width, height, area, center}``. ``tolerance`` is the + per-channel band around ``rgb``; ``min_area`` drops specks. ``haystack`` is an + RGB ndarray / path / PIL image (default: grab the screen / ``region``). + """ + import cv2 + import numpy as np + from je_auto_control.utils.cv2_utils.blobs import connected_boxes + image = _to_rgb(haystack) if haystack is not None else _grab_rgb(region) + red, green, blue = (int(channel) for channel in rgb[:3]) + tol = int(tolerance) + lower = np.array([max(0, red - tol), max(0, green - tol), + max(0, blue - tol)], dtype=np.uint8) + upper = np.array([min(255, red + tol), min(255, green + tol), + min(255, blue + tol)], dtype=np.uint8) + mask = cv2.inRange(image, lower, upper) + return connected_boxes(mask, int(min_area)) + + +def find_color_region(rgb: Sequence[int], *, + haystack: Optional[ImageSource] = None, + region: Optional[Sequence[int]] = None, + tolerance: int = 20, + min_area: int = 50) -> Optional[Dict[str, Any]]: + """Return the largest blob within ``tolerance`` of ``rgb`` (or ``None``).""" + regions = find_color_regions(rgb, haystack=haystack, region=region, + tolerance=tolerance, min_area=min_area) + return regions[0] if regions else None diff --git a/je_auto_control/utils/cv2_utils/blobs.py b/je_auto_control/utils/cv2_utils/blobs.py new file mode 100644 index 00000000..a9f262e5 --- /dev/null +++ b/je_auto_control/utils/cv2_utils/blobs.py @@ -0,0 +1,29 @@ +"""Connected-component bounding boxes from a binary mask. + +Shared by the colour-region locator and the SSIM change detector: both threshold +an image into a binary mask and then need the bounding boxes of the connected +blobs. OpenCV + NumPy come in via ``je_open_cv`` and are imported lazily. Imports +no ``PySide6``. +""" +from typing import Any, Dict, List + + +def connected_boxes(mask, min_area: int = 1) -> List[Dict[str, Any]]: + """Return ``{x, y, width, height, area, center}`` per blob, largest first. + + ``mask`` is a uint8 binary image (non-zero = foreground). Components whose + area is below ``min_area`` are dropped; ``center`` is the component centroid + (truncated to int). Uses 8-connectivity; the background label (0) is skipped. + """ + import cv2 + count, _labels, stats, centroids = cv2.connectedComponentsWithStats( + mask, connectivity=8) + boxes: List[Dict[str, Any]] = [] + for index in range(1, count): # 0 is the background + x, y, width, height, area = (int(v) for v in stats[index]) + if area >= int(min_area): + cx, cy = centroids[index] + boxes.append({"x": x, "y": y, "width": width, "height": height, + "area": area, "center": [int(cx), int(cy)]}) + boxes.sort(key=lambda item: item["area"], reverse=True) + return boxes diff --git a/je_auto_control/utils/edge_lines/__init__.py b/je_auto_control/utils/edge_lines/__init__.py new file mode 100644 index 00000000..1bfb56a6 --- /dev/null +++ b/je_auto_control/utils/edge_lines/__init__.py @@ -0,0 +1,6 @@ +"""Line / grid / separator detection on raw pixels (Hough transform).""" +from je_auto_control.utils.edge_lines.edge_lines import ( + find_grid, find_lines, find_separators, +) + +__all__ = ["find_grid", "find_lines", "find_separators"] diff --git a/je_auto_control/utils/edge_lines/edge_lines.py b/je_auto_control/utils/edge_lines/edge_lines.py new file mode 100644 index 00000000..0682cc31 --- /dev/null +++ b/je_auto_control/utils/edge_lines/edge_lines.py @@ -0,0 +1,113 @@ +"""Line, grid and separator detection on raw pixels (Hough) — tables and dividers. + +``grid_locator`` clusters *already-found* element boxes into a grid; it cannot find +the ruling lines of a table / spreadsheet or a UI divider from raw pixels, and +``shape_locator`` only finds closed rectangles. This detects straight line segments +via Canny + the probabilistic Hough transform, classifies them horizontal / vertical / +diagonal, recovers a table's row/column coordinates and cells, and returns the +positions of long divider lines — so a script can address "row 3, column 2" or split a +panel at its separators without templates. + +Runs on an injectable ``haystack`` (ndarray / path / PIL), so it is headless-testable +on synthetic arrays. ``cv2.HoughLinesP`` is base OpenCV; OpenCV + NumPy come in via +``je_open_cv``. Imports no ``PySide6``. +""" +import math +from typing import Any, Dict, List, Optional, Sequence + +from je_auto_control.utils.visual_match.visual_match import _haystack_gray + +ImageSource = Any +_CANNY_LOW = 50 +_CANNY_HIGH = 150 + + +def _orientation(angle: float) -> str: + a = abs(angle) % 180 + if a < 10 or a > 170: + return "horizontal" + if 80 < a < 100: + return "vertical" + return "diagonal" + + +def _segments(gray, min_length: int, max_gap: int): + import cv2 + import numpy as np + edges = cv2.Canny(gray, _CANNY_LOW, _CANNY_HIGH) + return cv2.HoughLinesP(edges, 1, np.pi / 180, threshold=50, + minLineLength=int(min_length), maxLineGap=int(max_gap)) + + +def find_lines(haystack: Optional[ImageSource] = None, *, + region: Optional[Sequence[int]] = None, min_length: int = 80, + max_gap: int = 10, orientation: str = "any" + ) -> List[Dict[str, Any]]: + """Return straight line segments on screen, longest first. + + Each is ``{x1, y1, x2, y2, angle, length, orientation}`` where ``orientation`` is + ``horizontal`` / ``vertical`` / ``diagonal``. Pass ``orientation`` other than + ``any`` to keep only that kind. ``min_length`` / ``max_gap`` tune the Hough probe. + """ + segments = _segments(_haystack_gray(haystack, region), int(min_length), + int(max_gap)) + out: List[Dict[str, Any]] = [] + if segments is None: + return out + for x1, y1, x2, y2 in segments[:, 0]: + angle = math.degrees(math.atan2(int(y2) - int(y1), int(x2) - int(x1))) + kind = _orientation(angle) + if orientation not in ("any", kind): + continue + out.append({"x1": int(x1), "y1": int(y1), "x2": int(x2), "y2": int(y2), + "angle": round(angle, 1), + "length": round(math.hypot(x2 - x1, y2 - y1), 1), + "orientation": kind}) + out.sort(key=lambda seg: seg["length"], reverse=True) + return out + + +def _cluster(values: Sequence[int], tol: int) -> List[int]: + """Group near-equal coordinates and return each cluster's mean (sorted).""" + groups: List[List[int]] = [] + for value in sorted(values): + if groups and value - groups[-1][-1] <= tol: + groups[-1].append(value) + else: + groups.append([value]) + return [int(round(sum(group) / len(group))) for group in groups] + + +def find_separators(haystack: Optional[ImageSource] = None, *, + region: Optional[Sequence[int]] = None, + axis: str = "horizontal", min_length: int = 120, + tol: int = 10) -> List[int]: + """Return the coordinates of long divider lines along ``axis``. + + ``axis="horizontal"`` returns the ``y`` of each horizontal rule (top-to-bottom); + ``axis="vertical"`` returns the ``x`` of each vertical rule. Near-equal lines are + merged within ``tol`` pixels. + """ + lines = find_lines(haystack, region=region, min_length=int(min_length), + orientation=axis) + coords = [seg["y1"] if axis == "horizontal" else seg["x1"] for seg in lines] + return _cluster(coords, int(tol)) + + +def find_grid(haystack: Optional[ImageSource] = None, *, + region: Optional[Sequence[int]] = None, min_length: int = 120, + tol: int = 10) -> Dict[str, Any]: + """Recover a table's grid: ``{rows: [y…], cols: [x…], cells: [{x,y,width,height}]}``. + + Horizontal rules give the row coordinates, vertical rules the columns; the cells + are the rectangles between consecutive rules. ``min_length`` filters short edges. + """ + lines = find_lines(haystack, region=region, min_length=int(min_length)) + rows = _cluster([seg["y1"] for seg in lines + if seg["orientation"] == "horizontal"], int(tol)) + cols = _cluster([seg["x1"] for seg in lines + if seg["orientation"] == "vertical"], int(tol)) + cells = [{"x": cols[i], "y": rows[j], "width": cols[i + 1] - cols[i], + "height": rows[j + 1] - rows[j]} + for j in range(len(rows) - 1) for i in range(len(cols) - 1)] + return {"rows": rows, "cols": cols, "cells": cells} diff --git a/je_auto_control/utils/element_parse/__init__.py b/je_auto_control/utils/element_parse/__init__.py new file mode 100644 index 00000000..38caada2 --- /dev/null +++ b/je_auto_control/utils/element_parse/__init__.py @@ -0,0 +1,6 @@ +"""Fuse and order on-screen element boxes (IoU, merge, fuse sources, reading order).""" +from je_auto_control.utils.element_parse.element_parse import ( + fuse_elements, iou, merge_boxes, reading_order, +) + +__all__ = ["fuse_elements", "iou", "merge_boxes", "reading_order"] diff --git a/je_auto_control/utils/element_parse/element_parse.py b/je_auto_control/utils/element_parse/element_parse.py new file mode 100644 index 00000000..fee457db --- /dev/null +++ b/je_auto_control/utils/element_parse/element_parse.py @@ -0,0 +1,100 @@ +"""Fuse and order on-screen element boxes — the step before set-of-marks numbering. + +``set_of_marks.mark_elements`` assumes a single, already-deduplicated element list +and just numbers it; nothing produces that list. A real screen parse yields *three* +overlapping sources — OCR text boxes, icon/shape boxes, accessibility boxes — with +heavy overlap and no consistent order. This module supplies the missing connective +tissue: ``iou`` / ``merge_boxes`` drop near-duplicates, ``fuse_elements`` unions the +three sources keeping the most trustworthy one on overlap, and ``reading_order`` +sorts the survivors top-to-bottom, left-to-right and assigns a stable index. + +Every box is a plain ``dict`` with ``x, y, width, height`` (plus any extra keys such +as ``text`` / ``source`` / ``score``), so this is pure-stdlib and fully unit-testable. +Imports no ``PySide6``. +""" +from typing import Any, Dict, List, Optional, Sequence, Tuple + +Box = Dict[str, Any] +_DEFAULT_PRIORITY = ("a11y", "ocr", "icon") + + +def _xywh(box: Box) -> Tuple[int, int, int, int]: + return (int(box["x"]), int(box["y"]), int(box["width"]), int(box["height"])) + + +def _area(box: Box) -> int: + _x, _y, width, height = _xywh(box) + return width * height + + +def iou(box_a: Box, box_b: Box) -> float: + """Return the intersection-over-union (0..1) of two ``{x,y,width,height}`` boxes.""" + ax, ay, aw, ah = _xywh(box_a) + bx, by, bw, bh = _xywh(box_b) + left, top = max(ax, bx), max(ay, by) + right, bottom = min(ax + aw, bx + bw), min(ay + ah, by + bh) + inter = max(0, right - left) * max(0, bottom - top) + if inter == 0: + return 0.0 + union = aw * ah + bw * bh - inter + return inter / union if union else 0.0 + + +def _dedup(boxes: Sequence[Box], iou_threshold: float) -> List[Box]: + kept: List[Box] = [] + for box in boxes: + if all(iou(box, other) <= iou_threshold for other in kept): + kept.append(box) + return kept + + +def merge_boxes(boxes: Sequence[Box], *, iou_threshold: float = 0.9) -> List[Box]: + """Drop near-duplicate boxes (IoU above ``iou_threshold``), largest kept first.""" + ordered = sorted(boxes, key=_area, reverse=True) + return _dedup(ordered, float(iou_threshold)) + + +def fuse_elements(ocr_boxes: Optional[Sequence[Box]] = None, + icon_boxes: Optional[Sequence[Box]] = None, + a11y_boxes: Optional[Sequence[Box]] = None, *, + iou_threshold: float = 0.9, + source_priority: Sequence[str] = _DEFAULT_PRIORITY + ) -> List[Box]: + """Union OCR + icon + accessibility boxes, dropping cross-source duplicates. + + On overlap (IoU above ``iou_threshold``) the box from the higher-priority source + wins (``source_priority`` default a11y > ocr > icon, then larger area). Each box + is tagged with its ``source``. + """ + rank = {name: index for index, name in enumerate(source_priority)} + tagged: List[Box] = [] + for source, boxes in (("a11y", a11y_boxes), ("ocr", ocr_boxes), + ("icon", icon_boxes)): + for box in boxes or (): + item = dict(box) + item.setdefault("source", source) + tagged.append(item) + tagged.sort(key=lambda box: (rank.get(box.get("source"), len(rank)), + -_area(box))) + return _dedup(tagged, float(iou_threshold)) + + +def reading_order(elements: Sequence[Box], *, row_tol: int = 12) -> List[Box]: + """Return the elements sorted top-to-bottom, left-to-right, with an ``index`` key. + + Elements whose tops are within ``row_tol`` pixels are treated as the same row and + ordered by ``x`` within it. + """ + rows: List[Dict[str, Any]] = [] + for element in sorted(elements, key=lambda box: _xywh(box)[1]): + top = _xywh(element)[1] + row = next((candidate for candidate in rows + if abs(top - candidate["top"]) <= int(row_tol)), None) + if row is None: + rows.append({"top": top, "items": [element]}) + else: + row["items"].append(element) + ordered: List[Box] = [] + for row in rows: + ordered.extend(sorted(row["items"], key=lambda box: _xywh(box)[0])) + return [dict(element, index=index) for index, element in enumerate(ordered)] diff --git a/je_auto_control/utils/executor/action_executor.py b/je_auto_control/utils/executor/action_executor.py index 65539211..874f1477 100644 --- a/je_auto_control/utils/executor/action_executor.py +++ b/je_auto_control/utils/executor/action_executor.py @@ -361,6 +361,58 @@ def _wait_clipboard_change(baseline: Optional[str] = None, ).to_dict() +def _wait_image_gone(image: Any, detect_threshold: float = 1.0, + timeout_s: float = 10.0, poll_interval_s: float = 0.2, + gone_for_s: float = 0.0) -> Dict[str, Any]: + """Executor adapter: wait until an image is no longer on screen.""" + from je_auto_control.utils.smart_waits import wait_until_image_gone + return wait_until_image_gone( + image, detect_threshold=float(detect_threshold), + timeout_s=float(timeout_s), poll_interval_s=float(poll_interval_s), + gone_for_s=float(gone_for_s), + ).to_dict() + + +def _wait_text_gone(text: str, timeout_s: float = 10.0, + poll_interval_s: float = 0.2, + gone_for_s: float = 0.0) -> Dict[str, Any]: + """Executor adapter: wait until text is no longer on screen (OCR).""" + from je_auto_control.utils.smart_waits import wait_until_text_gone + return wait_until_text_gone( + text, timeout_s=float(timeout_s), + poll_interval_s=float(poll_interval_s), gone_for_s=float(gone_for_s), + ).to_dict() + + +def _wait_window_title(pattern: str, present: bool = True, regex: bool = True, + timeout_s: float = 10.0, + poll_interval_s: float = 0.2) -> Dict[str, Any]: + """Executor adapter: wait for a window title (regex) to appear / vanish.""" + from je_auto_control.utils.smart_waits import wait_until_window_title + return wait_until_window_title( + pattern, present=bool(present), regex=bool(regex), + timeout_s=float(timeout_s), poll_interval_s=float(poll_interval_s), + ).to_dict() + + +def _wait_color(target_rgb: Any, region: Any = None, + tolerance: int = 10, min_fraction: float = 0.5, + present: bool = True, timeout_s: float = 10.0, + poll_interval_s: float = 0.2) -> Dict[str, Any]: + """Executor adapter: wait until a colour fills/leaves a region.""" + import json + from je_auto_control.utils.smart_waits import wait_until_color + if isinstance(target_rgb, str): + target_rgb = json.loads(target_rgb) + if isinstance(region, str): + region = json.loads(region) if region.strip() else None + return wait_until_color( + region=region, target_rgb=target_rgb, tolerance=int(tolerance), + min_fraction=float(min_fraction), present=bool(present), + timeout_s=float(timeout_s), poll_interval_s=float(poll_interval_s), + ).to_dict() + + def _wait_window_closed(title: str, case_sensitive: bool = False, timeout_s: float = 10.0, poll_interval_s: float = 0.2) -> Dict[str, Any]: @@ -437,8 +489,9 @@ def _ocr_read_structure(region: Optional[List[int]] = None, def _anchor_locate(anchor: Dict[str, Any], target: Dict[str, Any], relation: str = "near", - max_distance_px: float = 200.0) -> Dict[str, Any]: - """Executor adapter: anchor-based spatial locator.""" + max_distance_px: float = 200.0, + ordinal: Any = 1) -> Dict[str, Any]: + """Executor adapter: anchor-based spatial locator (Nth match via ordinal).""" from je_auto_control.utils.anchor_locator import ( Locator, anchor_locate, ) @@ -447,10 +500,23 @@ def _anchor_locate(anchor: Dict[str, Any], target: Dict[str, Any], outcome = anchor_locate( anchor=anchor_loc, target=target_loc, relation=relation, max_distance_px=float(max_distance_px), + ordinal=int(ordinal), ) return outcome.to_dict() +def _anchor_locate_all(anchor: Dict[str, Any], target: Dict[str, Any], + relation: str = "near", + max_distance_px: float = 200.0) -> Dict[str, Any]: + """Executor adapter: every anchor-relative match, nearest-first.""" + from je_auto_control.utils.anchor_locator import Locator, anchor_locate_all + outcomes = anchor_locate_all( + anchor=Locator(**anchor), target=Locator(**target), + relation=relation, max_distance_px=float(max_distance_px), + ) + return {"count": len(outcomes), "matches": [o.to_dict() for o in outcomes]} + + def _anchor_click(anchor: Dict[str, Any], target: Dict[str, Any], mouse_keycode: str = "mouse_left", relation: str = "near", @@ -2112,6 +2178,31 @@ def _snap_window(title: str, position: str = "left") -> Dict[str, Any]: return {"moved": snap_window(title, position)} +def _arrange_grid(titles: Any, rows: Any = None, cols: Any = None, + gap: Any = 0) -> Dict[str, Any]: + """Executor adapter: tile a list of window titles into a grid.""" + import json + from je_auto_control.utils.window_capture import arrange_grid + if isinstance(titles, str): + titles = json.loads(titles) + moved = arrange_grid(list(titles), + rows=int(rows) if rows is not None else None, + cols=int(cols) if cols is not None else None, + gap=int(gap)) + return {"moved": moved, "count": len(list(titles))} + + +def _arrange_cascade(titles: Any, offset: Any = 30) -> Dict[str, Any]: + """Executor adapter: cascade a list of window titles diagonally.""" + import json + from je_auto_control.utils.window_capture import arrange_cascade + if isinstance(titles, str): + titles = json.loads(titles) + titles = list(titles) + return {"moved": arrange_cascade(titles, offset=int(offset)), + "count": len(titles)} + + def _save_window_layout(path: Optional[str] = None) -> Dict[str, Any]: """Executor adapter: snapshot every window's geometry (optionally to file).""" from je_auto_control.utils.window_capture import save_window_layout @@ -3047,6 +3138,558 @@ def _gettext_ngettext(po: str, msgid: str, msgid_plural: str, return {"text": catalog.ngettext(msgid, msgid_plural, int(n))} +def _checksum_validate(scheme: str, number: str) -> Dict[str, Any]: + """Adapter: validate a number's check digit under a named scheme.""" + from je_auto_control.utils import checksum as cs + validators = {"luhn": cs.luhn_validate, "verhoeff": cs.verhoeff_validate, + "damm": cs.damm_validate, "mod97": cs.mod97_10_validate} + func = validators.get(scheme) + if func is None: + raise AutoControlActionException(f"unknown checksum scheme: {scheme!r}") + return {"valid": func(number)} + + +def _checksum_digit(scheme: str, partial: str) -> Dict[str, Any]: + """Adapter: compute the check digit(s) for a value under a named scheme.""" + from je_auto_control.utils import checksum as cs + digits = {"luhn": cs.luhn_check_digit, "verhoeff": cs.verhoeff_check_digit, + "damm": cs.damm_check_digit, "mod97": cs.mod97_10_check_digits} + func = digits.get(scheme) + if func is None: + raise AutoControlActionException(f"unknown checksum scheme: {scheme!r}") + return {"check_digit": func(partial)} + + +def _waypoints(value: Any) -> Any: + """Coerce a JSON string of waypoints into a list.""" + import json + return json.loads(value) if isinstance(value, str) else value + + +def _move_along_path(waypoints: Any, easing: str = "linear", + per_segment_steps: Any = 20) -> Dict[str, Any]: + """Adapter: move the pointer through a polyline of waypoints.""" + from je_auto_control.utils.mouse_path import move_along_path + return move_along_path(_waypoints(waypoints), easing=easing, + per_segment_steps=int(per_segment_steps)) + + +def _drag_path(waypoints: Any, button: str = "mouse_left", + easing: str = "linear", + per_segment_steps: Any = 20) -> Dict[str, Any]: + """Adapter: press, drag through a polyline of waypoints, release.""" + from je_auto_control.utils.mouse_path import drag_path + return drag_path(_waypoints(waypoints), button=button, easing=easing, + per_segment_steps=int(per_segment_steps)) + + +def _set_field_text(text: str, clear: str = "select_all", paste: Any = False, + modifier: str = "ctrl") -> Dict[str, Any]: + """Adapter: clear the focused field and enter text.""" + from je_auto_control.utils.field_entry import set_field_text + return set_field_text(text, clear=clear, paste=bool(paste), + modifier=modifier) + + +def _hold_key(key: str, duration_s: Any = 1.0, + rate_hz: Any = None) -> Dict[str, Any]: + """Adapter: hold a key for a duration (or auto-repeat at rate_hz).""" + from je_auto_control.utils.key_hold import hold_key + rate = float(rate_hz) if rate_hz not in (None, "") else None + return hold_key(key, float(duration_s), rate_hz=rate) + + +def _move_mouse_relative(dx: Any, dy: Any) -> Dict[str, Any]: + """Adapter: move the pointer by a delta from its current position.""" + from je_auto_control.utils.mouse_relative import move_mouse_relative + return move_mouse_relative(int(dx), int(dy)) + + +def _type_unicode(text: str, modifier: str = "ctrl") -> Dict[str, Any]: + """Adapter: enter arbitrary Unicode text via clipboard paste.""" + from je_auto_control.utils.text_unicode import type_unicode + return type_unicode(text, modifier=modifier) + + +def _grid_cell(boxes: Any, row: Any, col: Any, + row_tolerance: Any = 10) -> Dict[str, Any]: + """Adapter: address a grid cell by (row, col) from a JSON list of boxes.""" + import json + from je_auto_control.utils.grid_locator import locate_cell + if isinstance(boxes, str): + boxes = json.loads(boxes) + return locate_cell(list(boxes), int(row), int(col), + row_tolerance=int(row_tolerance)) + + +def _match_template(template: str, min_score: Any = 0.8, scales: Any = None, + region: Any = None, + method: str = "ccoeff_normed") -> Dict[str, Any]: + """Adapter: best confidence-scored template match on the screen.""" + import json + from je_auto_control.utils.visual_match import match_template + if isinstance(scales, str): + scales = json.loads(scales) if scales.strip() else None + if isinstance(region, str): + region = json.loads(region) if region.strip() else None + match = match_template(template, region=region, + scales=tuple(scales) if scales else (1.0,), + min_score=float(min_score), method=method) + return {"found": match is not None, + "match": match.to_dict() if match else None} + + +def _match_template_all(template: str, min_score: Any = 0.8, + max_results: Any = 20, nms_iou: Any = 0.3, + region: Any = None) -> Dict[str, Any]: + """Adapter: every confidence-scored template match on the screen (NMS).""" + import json + from je_auto_control.utils.visual_match import match_template_all + if isinstance(region, str): + region = json.loads(region) if region.strip() else None + matches = match_template_all(template, region=region, + min_score=float(min_score), + max_results=int(max_results), + nms_iou=float(nms_iou)) + return {"count": len(matches), "matches": [m.to_dict() for m in matches]} + + +def _match_masked(template: str, mask: Any = None, min_score: Any = 0.9, + region: Any = None) -> Dict[str, Any]: + """Adapter: best masked template match (alpha / mask ignores background).""" + import json + from je_auto_control.utils.visual_match import match_masked + if isinstance(region, str): + region = json.loads(region) if region.strip() else None + match = match_masked(template, mask=mask, region=region, + min_score=float(min_score)) + return {"found": match is not None, + "match": match.to_dict() if match else None} + + +def _match_masked_all(template: str, mask: Any = None, min_score: Any = 0.9, + max_results: Any = 20, nms_iou: Any = 0.3, + region: Any = None) -> Dict[str, Any]: + """Adapter: every masked template match on the screen (NMS).""" + import json + from je_auto_control.utils.visual_match import match_masked_all + if isinstance(region, str): + region = json.loads(region) if region.strip() else None + matches = match_masked_all(template, mask=mask, region=region, + min_score=float(min_score), + max_results=int(max_results), + nms_iou=float(nms_iou)) + return {"count": len(matches), "matches": [m.to_dict() for m in matches]} + + +def _find_color_region(rgb: Any, tolerance: Any = 20, min_area: Any = 50, + region: Any = None) -> Dict[str, Any]: + """Adapter: locate coloured regions on the screen, largest first.""" + import json + from je_auto_control.utils.color_region import find_color_regions + if isinstance(rgb, str): + rgb = json.loads(rgb) + if isinstance(region, str): + region = json.loads(region) if region.strip() else None + regions = find_color_regions(list(rgb), region=region, + tolerance=int(tolerance), + min_area=int(min_area)) + return {"count": len(regions), "regions": regions, + "best": regions[0] if regions else None} + + +def _ssim_compare(reference: str, current: Any = None, ignore: Any = None, + region: Any = None) -> Dict[str, Any]: + """Adapter: structural-similarity score between reference and current/screen.""" + import json + from je_auto_control.utils.ssim import ssim_compare + if isinstance(ignore, str): + ignore = json.loads(ignore) if ignore.strip() else None + if isinstance(region, str): + region = json.loads(region) if region.strip() else None + score = ssim_compare(reference, current, ignore=ignore, region=region) + return {"score": score} + + +def _ssim_changed_regions(reference: str, current: Any = None, ignore: Any = None, + threshold: Any = 0.35, min_area: Any = 50, + region: Any = None) -> Dict[str, Any]: + """Adapter: boxes of the regions that structurally changed, largest first.""" + import json + from je_auto_control.utils.ssim import ssim_changed_regions + if isinstance(ignore, str): + ignore = json.loads(ignore) if ignore.strip() else None + if isinstance(region, str): + region = json.loads(region) if region.strip() else None + regions = ssim_changed_regions(reference, current, ignore=ignore, + threshold=float(threshold), + min_area=int(min_area), region=region) + return {"count": len(regions), "regions": regions} + + +def _feature_match(template: str, region: Any = None, max_features: Any = 500, + ratio: Any = 0.75, min_inliers: Any = 10) -> Dict[str, Any]: + """Adapter: locate a template by ORB keypoints (rotation/scale/theme robust).""" + import json + from je_auto_control.utils.feature_match import feature_match + if isinstance(region, str): + region = json.loads(region) if region.strip() else None + match = feature_match(template, region=region, max_features=int(max_features), + ratio=float(ratio), min_inliers=int(min_inliers)) + return {"found": match is not None, + "match": match.to_dict() if match else None} + + +def _find_shapes(region: Any = None, min_area: Any = 400, + max_area: Any = None) -> Dict[str, Any]: + """Adapter: bounding boxes of all distinct on-screen shapes, largest first.""" + import json + from je_auto_control.utils.shape_locator import find_shapes + if isinstance(region, str): + region = json.loads(region) if region.strip() else None + shapes = find_shapes(region=region, min_area=int(min_area), + max_area=int(max_area) if max_area is not None else None) + return {"count": len(shapes), "shapes": shapes} + + +def _find_rectangles(region: Any = None, min_area: Any = 400, max_area: Any = None, + aspect_range: Any = None, epsilon: Any = 0.04 + ) -> Dict[str, Any]: + """Adapter: boxes of the ~rectangular shapes (buttons / cards), largest first.""" + import json + from je_auto_control.utils.shape_locator import find_rectangles + if isinstance(region, str): + region = json.loads(region) if region.strip() else None + if isinstance(aspect_range, str): + aspect_range = json.loads(aspect_range) if aspect_range.strip() else None + rects = find_rectangles( + region=region, min_area=int(min_area), + max_area=int(max_area) if max_area is not None else None, + aspect_range=tuple(aspect_range) if aspect_range else None, + epsilon=float(epsilon)) + return {"count": len(rects), "rectangles": rects} + + +def _resolve_screen(screen: Any) -> list: + """Parse a JSON screen rect, or default to the live primary screen work area.""" + import json + if isinstance(screen, str): + screen = json.loads(screen) if screen.strip() else None + if screen: + return list(screen) + from je_auto_control.wrapper.auto_control_screen import screen_size + width, height = screen_size() + return [0, 0, int(width), int(height)] + + +def _tile_rect(slot: str, screen: Any = None, gap: Any = 0) -> Dict[str, Any]: + """Adapter: rectangle for a named tiling slot of the screen work area.""" + from je_auto_control.utils.window_layout import tile_rect + rect = tile_rect(_resolve_screen(screen), str(slot), gap=int(gap)) + return {"rect": rect.to_dict()} + + +def _grid_rects(rows: Any, cols: Any, screen: Any = None, + gap: Any = 0) -> Dict[str, Any]: + """Adapter: one rectangle per cell of an rows x cols grid over the screen.""" + from je_auto_control.utils.window_layout import grid_rects + rects = grid_rects(_resolve_screen(screen), int(rows), int(cols), gap=int(gap)) + return {"count": len(rects), "rects": [rect.to_dict() for rect in rects]} + + +def _cascade_rects(count: Any, screen: Any = None, offset: Any = 30, + size: Any = None) -> Dict[str, Any]: + """Adapter: count staggered, overlapping window rectangles (a cascade).""" + import json + from je_auto_control.utils.window_layout import cascade_rects + if isinstance(size, str): + size = json.loads(size) if size.strip() else None + rects = cascade_rects(_resolve_screen(screen), int(count), offset=int(offset), + size=tuple(size) if size else None) + return {"count": len(rects), "rects": [rect.to_dict() for rect in rects]} + + +def _preprocess_image(output_path: str, source: Any = None, steps: Any = None, + scale: Any = 2.0, region: Any = None, block_size: Any = 31, + c: Any = 11) -> Dict[str, Any]: + """Adapter: run the preprocessing pipeline and write the result to a file.""" + import json + import cv2 + from je_auto_control.utils.preprocess import preprocess_image + if isinstance(steps, str): + steps = (json.loads(steps) if steps.strip().startswith("[") + else [part.strip() for part in steps.split(",") if part.strip()]) + if isinstance(region, str): + region = json.loads(region) if region.strip() else None + result = preprocess_image( + source, region=region, + steps=tuple(steps) if steps else ("grayscale", "upscale", "binarize"), + scale=float(scale), block_size=int(block_size), c=int(c)) + if not cv2.imwrite(str(output_path), result): + raise AutoControlActionException(f"could not write image: {output_path!r}") + return {"path": str(output_path), "width": int(result.shape[1]), + "height": int(result.shape[0])} + + +def _enumerate_monitors() -> Dict[str, Any]: + """Adapter: list connected monitors with virtual-desktop geometry.""" + from je_auto_control.utils.monitor_layout import ( + enumerate_monitors, virtual_bounds) + monitors = enumerate_monitors() + bounds = virtual_bounds(monitors) if monitors else (0, 0, 0, 0) + return {"count": len(monitors), + "monitors": [monitor.to_dict() for monitor in monitors], + "virtual_bounds": list(bounds)} + + +def _monitor_at_point(x: Any, y: Any) -> Dict[str, Any]: + """Adapter: report which monitor contains a virtual point.""" + from je_auto_control.utils.monitor_layout import ( + enumerate_monitors, monitor_at_point) + monitor = monitor_at_point(enumerate_monitors(), int(x), int(y)) + return {"found": monitor is not None, + "monitor": monitor.to_dict() if monitor else None} + + +def _region_pixel_token(bbox): + """Stability token: a hash of the bbox region's pixels (changes on movement).""" + from je_auto_control.utils.cv2_utils.screenshot import pil_screenshot + left, top, width, height = bbox + image = pil_screenshot(screen_region=[left, top, left + width, top + height]) + return hash(image.tobytes()) + + +def _wait_actionable(template: str, timeout_s: Any = 5.0, stable_for_s: Any = 0.3, + min_score: Any = 0.8, region: Any = None) -> Dict[str, Any]: + """Adapter: wait until a template is visible + stable before acting.""" + import json + from je_auto_control.utils.actionability import GateConfig, wait_actionable + from je_auto_control.utils.visual_match import match_template + if isinstance(region, str): + region = json.loads(region) if region.strip() else None + + def locate(): + match = match_template(template, region=region, min_score=float(min_score)) + return (match.x, match.y, match.width, match.height) if match else None + + report = wait_actionable( + locate, region_sampler=_region_pixel_token, + config=GateConfig(timeout_s=float(timeout_s), + stable_for_s=float(stable_for_s))) + return report.to_dict() + + +def _fuse_elements(ocr: Any = None, icon: Any = None, a11y: Any = None, + iou_threshold: Any = 0.9) -> Dict[str, Any]: + """Adapter: union OCR / icon / a11y element boxes, dropping duplicates.""" + import json + from je_auto_control.utils.element_parse import fuse_elements + + def parse(value: Any) -> list: + if isinstance(value, str): + return json.loads(value) if value.strip() else [] + return list(value) if value else [] + + elements = fuse_elements(parse(ocr), parse(icon), parse(a11y), + iou_threshold=float(iou_threshold)) + return {"count": len(elements), "elements": elements} + + +def _reading_order(elements: Any, row_tol: Any = 12) -> Dict[str, Any]: + """Adapter: order element boxes top-to-bottom, left-to-right, with an index.""" + import json + from je_auto_control.utils.element_parse import reading_order + if isinstance(elements, str): + elements = json.loads(elements) + ordered = reading_order(list(elements), row_tol=int(row_tol)) + return {"count": len(ordered), "elements": ordered} + + +def _segment_hsv(lower_hsv: Any, upper_hsv: Any, min_area: Any = 50, + region: Any = None) -> Dict[str, Any]: + """Adapter: locate blobs inside an explicit HSV band on the screen.""" + import json + from je_auto_control.utils.hsv_segment import segment_hsv + if isinstance(lower_hsv, str): + lower_hsv = json.loads(lower_hsv) + if isinstance(upper_hsv, str): + upper_hsv = json.loads(upper_hsv) + if isinstance(region, str): + region = json.loads(region) if region.strip() else None + boxes = segment_hsv(region=region, lower_hsv=list(lower_hsv), + upper_hsv=list(upper_hsv), min_area=int(min_area)) + return {"count": len(boxes), "regions": boxes, + "best": boxes[0] if boxes else None} + + +def _dominant_hue_regions(hue: Any, hue_tol: Any = 10, sat_min: Any = 80, + val_min: Any = 80, min_area: Any = 50, + region: Any = None) -> Dict[str, Any]: + """Adapter: locate any-brightness regions near a hue on the screen.""" + import json + from je_auto_control.utils.hsv_segment import dominant_hue_regions + if isinstance(region, str): + region = json.loads(region) if region.strip() else None + boxes = dominant_hue_regions(region=region, hue=int(hue), hue_tol=int(hue_tol), + sat_min=int(sat_min), val_min=int(val_min), + min_area=int(min_area)) + return {"count": len(boxes), "regions": boxes, + "best": boxes[0] if boxes else None} + + +def _find_text_regions(min_area: Any = 60, max_area: Any = None, merge: Any = True, + max_aspect: Any = 12.0, region: Any = None) -> Dict[str, Any]: + """Adapter: locate text/glyph regions on screen via MSER (no OCR).""" + import json + from je_auto_control.utils.text_regions import find_text_regions + if isinstance(region, str): + region = json.loads(region) if region.strip() else None + regions = find_text_regions( + region=region, min_area=int(min_area), + max_area=int(max_area) if max_area is not None else None, + merge=bool(merge), max_aspect=float(max_aspect)) + return {"count": len(regions), "regions": regions} + + +def _find_text_lines(y_tolerance: Any = 8, region: Any = None) -> Dict[str, Any]: + """Adapter: locate horizontal text lines on screen via MSER (no OCR).""" + import json + from je_auto_control.utils.text_regions import find_text_lines + if isinstance(region, str): + region = json.loads(region) if region.strip() else None + lines = find_text_lines(region=region, y_tolerance=int(y_tolerance)) + return {"count": len(lines), "lines": lines} + + +def _find_lines(min_length: Any = 80, max_gap: Any = 10, orientation: str = "any", + region: Any = None) -> Dict[str, Any]: + """Adapter: detect straight line segments on screen (Hough).""" + import json + from je_auto_control.utils.edge_lines import find_lines + if isinstance(region, str): + region = json.loads(region) if region.strip() else None + lines = find_lines(region=region, min_length=int(min_length), + max_gap=int(max_gap), orientation=str(orientation)) + return {"count": len(lines), "lines": lines} + + +def _find_grid(min_length: Any = 120, tol: Any = 10, + region: Any = None) -> Dict[str, Any]: + """Adapter: recover a table grid (rows / cols / cells) from screen lines.""" + import json + from je_auto_control.utils.edge_lines import find_grid + if isinstance(region, str): + region = json.loads(region) if region.strip() else None + return find_grid(region=region, min_length=int(min_length), tol=int(tol)) + + +def _find_separators(axis: str = "horizontal", min_length: Any = 120, tol: Any = 10, + region: Any = None) -> Dict[str, Any]: + """Adapter: coordinates of long divider lines along an axis.""" + import json + from je_auto_control.utils.edge_lines import find_separators + if isinstance(region, str): + region = json.loads(region) if region.strip() else None + coords = find_separators(region=region, axis=str(axis), + min_length=int(min_length), tol=int(tol)) + return {"count": len(coords), "axis": str(axis), "coordinates": coords} + + +def _expect_poll(action: Any, key: Any = None, op: str = "truthy", + expected: Any = None, timeout_s: Any = 5.0, + interval_s: Any = 0.25) -> Dict[str, Any]: + """Adapter: re-run a nested action until a key of its result matches.""" + import json + from je_auto_control.utils.expect_poll import ( + expect_poll, to_be_greater_than, to_be_truthy, to_contain, to_equal, + to_match_regex) + if isinstance(action, str): + action = json.loads(action) + builders = {"equals": lambda: to_equal(expected), + "contains": lambda: to_contain(expected), + "gt": lambda: to_be_greater_than(expected), + "regex": lambda: to_match_regex(str(expected)), + "truthy": to_be_truthy} + matcher = builders.get(str(op), to_be_truthy)() + + def getter(): + record = executor.execute_action([list(action)]) + value = next(iter(record.values()), None) + if key is not None and isinstance(value, dict): + return value.get(key) + return value + + result = expect_poll(getter, matcher, timeout_s=float(timeout_s), + interval_s=float(interval_s)) + return {"ok": result.ok, "value": result.value, "attempts": result.attempts, + "waited_s": result.waited_s} + + +def _apply_locate_op(candidates, op: Dict[str, Any]): + """Apply one locate-chain op spec to a Candidates set.""" + name = op.get("op") + if name == "within": + return candidates.within(op["region"]) + if name == "filter": + return candidates.filter(has_text=op.get("has_text"), near=op.get("near"), + min_area=op.get("min_area"), + max_area=op.get("max_area")) + if name == "reading": + return candidates.sort_reading(row_tol=int(op.get("row_tol", 12))) + if name == "nth": + return candidates.nth(int(op["index"])) + if name == "first": + return candidates.first() + if name == "last": + return candidates.last() + raise AutoControlActionException(f"unknown locate-chain op: {name!r}") + + +def _locate_chain(boxes: Any, ops: Any = None) -> Dict[str, Any]: + """Adapter: apply a chain of refinement ops to a set of element boxes.""" + import json + from je_auto_control.utils.locator_chain import from_boxes + if isinstance(boxes, str): + boxes = json.loads(boxes) + if isinstance(ops, str): + ops = json.loads(ops) if ops.strip() else [] + candidates = from_boxes(list(boxes)) + for op in ops or (): + candidates = _apply_locate_op(candidates, op) + resolved = candidates.resolve() + return {"count": len(resolved), "boxes": resolved, + "center": candidates.center()} + + +def _set_clipboard_html(html: str, fragment_plaintext: Any = None + ) -> Dict[str, Any]: + """Adapter: put an HTML fragment on the clipboard as CF_HTML (Windows).""" + from je_auto_control.utils.rich_clipboard import set_clipboard_html + set_clipboard_html(str(html), fragment_plaintext=fragment_plaintext) + return {"set": True, "length": len(str(html))} + + +def _get_clipboard_html() -> Dict[str, Any]: + """Adapter: read the clipboard's HTML fragment (Windows).""" + from je_auto_control.utils.rich_clipboard import get_clipboard_html + html = get_clipboard_html() + return {"found": html is not None, "html": html} + + +def _with_modifiers(modifiers: Any, actions: Any) -> Dict[str, Any]: + """Adapter: run nested actions while modifier keys are held down.""" + import json + from je_auto_control.utils.modifier_state import hold_modifiers + if isinstance(modifiers, str): + modifiers = (json.loads(modifiers) if modifiers.strip().startswith("[") + else [part.strip() for part in modifiers.split("+")]) + if isinstance(actions, str): + actions = json.loads(actions) + with hold_modifiers(list(modifiers)): + record = executor.execute_action(list(actions), raise_on_error=True) + return {"modifiers": list(modifiers), "record": record} + + def _cas_put(name: str, key: str, value: Any, expected_version: Any = None) -> Dict[str, Any]: """Adapter: optimistic put into a named versioned store.""" @@ -4740,6 +5383,46 @@ def __init__(self): "AC_format_message": _format_message, "AC_gettext_translate": _gettext_translate, "AC_gettext_ngettext": _gettext_ngettext, + "AC_checksum_validate": _checksum_validate, + "AC_checksum_digit": _checksum_digit, + "AC_move_along_path": _move_along_path, + "AC_drag_path": _drag_path, + "AC_set_field_text": _set_field_text, + "AC_hold_key": _hold_key, + "AC_move_mouse_relative": _move_mouse_relative, + "AC_type_unicode": _type_unicode, + "AC_with_modifiers": _with_modifiers, + "AC_grid_cell": _grid_cell, + "AC_match_template": _match_template, + "AC_match_template_all": _match_template_all, + "AC_match_masked": _match_masked, + "AC_match_masked_all": _match_masked_all, + "AC_ssim_compare": _ssim_compare, + "AC_ssim_changed_regions": _ssim_changed_regions, + "AC_feature_match": _feature_match, + "AC_find_shapes": _find_shapes, + "AC_find_rectangles": _find_rectangles, + "AC_preprocess_image": _preprocess_image, + "AC_enumerate_monitors": _enumerate_monitors, + "AC_monitor_at_point": _monitor_at_point, + "AC_wait_actionable": _wait_actionable, + "AC_fuse_elements": _fuse_elements, + "AC_reading_order": _reading_order, + "AC_segment_hsv": _segment_hsv, + "AC_dominant_hue_regions": _dominant_hue_regions, + "AC_find_text_regions": _find_text_regions, + "AC_find_text_lines": _find_text_lines, + "AC_find_lines": _find_lines, + "AC_find_grid": _find_grid, + "AC_find_separators": _find_separators, + "AC_expect_poll": _expect_poll, + "AC_locate_chain": _locate_chain, + "AC_set_clipboard_html": _set_clipboard_html, + "AC_get_clipboard_html": _get_clipboard_html, + "AC_tile_rect": _tile_rect, + "AC_grid_rects": _grid_rects, + "AC_cascade_rects": _cascade_rects, + "AC_find_color_region": _find_color_region, "AC_detect_drift": _detect_drift, "AC_categorical_drift": _categorical_drift, "AC_diff_rows": _diff_rows, @@ -4900,6 +5583,7 @@ def __init__(self): # Anchor-based locator (spatial composition of locator backends) "AC_anchor_locate": _anchor_locate, + "AC_anchor_locate_all": _anchor_locate_all, "AC_anchor_click": _anchor_click, # Structured OCR (rows / tables / form fields) @@ -4913,6 +5597,10 @@ def __init__(self): "AC_wait_for_port": _wait_for_port, "AC_wait_for_process": _wait_for_process, "AC_wait_clipboard_change": _wait_clipboard_change, + "AC_wait_image_gone": _wait_image_gone, + "AC_wait_text_gone": _wait_text_gone, + "AC_wait_color": _wait_color, + "AC_wait_window_title": _wait_window_title, "AC_wait_window_closed": _wait_window_closed, # Cost telemetry (LLM token + USD tracking) @@ -5101,6 +5789,8 @@ def __init__(self): "AC_save_window_layout": _save_window_layout, "AC_restore_window_layout": _restore_window_layout, "AC_snap_window": _snap_window, + "AC_arrange_grid": _arrange_grid, + "AC_arrange_cascade": _arrange_cascade, } def known_commands(self) -> set: diff --git a/je_auto_control/utils/expect_poll/__init__.py b/je_auto_control/utils/expect_poll/__init__.py new file mode 100644 index 00000000..b72e6ac7 --- /dev/null +++ b/je_auto_control/utils/expect_poll/__init__.py @@ -0,0 +1,9 @@ +"""Retry an arbitrary value until it matches (Playwright-style expect.poll).""" +from je_auto_control.utils.expect_poll.expect_poll import ( + PollResult, assert_poll, expect_poll, to_be_greater_than, to_be_stable, + to_be_truthy, to_contain, to_equal, to_match_regex, +) + +__all__ = ["PollResult", "assert_poll", "expect_poll", "to_be_greater_than", + "to_be_stable", "to_be_truthy", "to_contain", "to_equal", + "to_match_regex"] diff --git a/je_auto_control/utils/expect_poll/expect_poll.py b/je_auto_control/utils/expect_poll/expect_poll.py new file mode 100644 index 00000000..d442b2a0 --- /dev/null +++ b/je_auto_control/utils/expect_poll/expect_poll.py @@ -0,0 +1,110 @@ +"""Retry an arbitrary value until it matches — Playwright-style ``expect.poll``. + +``assert_eventually`` can only poll the framework's fixed dict-spec dispatch table +(text / image / pixel / window / clipboard / process / file / http). It cannot retry an +*arbitrary* value: an OCR'd total equalling ``"$42.00"``, a row count stabilising, a +custom predicate. ``expect_poll`` takes any zero-arg ``getter`` and any ``matcher`` +predicate and polls until it passes or the timeout elapses, with injectable +``clock`` / ``sleep`` so it is deterministic in tests (the existing helper calls real +``time.sleep``). Ships a small library of matcher factories. + +Pure-stdlib, imports no ``PySide6``. +""" +import time +from dataclasses import dataclass +from typing import Any, Callable + +from je_auto_control.utils.exception.exceptions import AutoControlActionException + +Matcher = Callable[[Any], bool] + + +@dataclass(frozen=True) +class PollResult: + """The outcome of a poll: whether it matched, the last value, and timing.""" + + ok: bool + value: Any + attempts: int + waited_s: float + description: str + + +def to_equal(expected: Any) -> Matcher: + """Match when the value equals ``expected``.""" + return lambda value: value == expected + + +def to_contain(item: Any) -> Matcher: + """Match when ``item`` is contained in the value.""" + return lambda value: item in value + + +def to_be_greater_than(threshold: Any) -> Matcher: + """Match when the value is greater than ``threshold``.""" + return lambda value: value > threshold + + +def to_match_regex(pattern: str) -> Matcher: + """Match when ``str(value)`` contains a match for ``pattern``.""" + import re + regex = re.compile(pattern) + return lambda value: bool(regex.search(str(value))) + + +def to_be_truthy() -> Matcher: + """Match when the value is truthy.""" + return bool + + +def to_be_stable(times: int = 3) -> Matcher: + """Match once the value has been equal across ``times`` consecutive polls.""" + state = {"last": object(), "count": 0} + + def matcher(value: Any) -> bool: + if value == state["last"]: + state["count"] += 1 + else: + state["last"], state["count"] = value, 1 + return state["count"] >= int(times) + + return matcher + + +def expect_poll(getter: Callable[[], Any], matcher: Matcher, *, + timeout_s: float = 5.0, interval_s: float = 0.25, + describe: Callable[[Any], str] = repr, + clock: Callable[[], float] = time.monotonic, + sleep: Callable[[float], None] = time.sleep) -> PollResult: + """Poll ``getter`` until ``matcher`` passes or ``timeout_s`` elapses. + + Returns a :class:`PollResult` with the final value, attempt count and elapsed + time. ``clock`` / ``sleep`` are injectable for deterministic tests. + """ + start = clock() + deadline = start + float(timeout_s) + attempts = 0 + value: Any = None + while True: + value = getter() + attempts += 1 + if matcher(value): + return PollResult(True, value, attempts, round(clock() - start, 4), + describe(value)) + if clock() >= deadline: + return PollResult(False, value, attempts, round(clock() - start, 4), + describe(value)) + sleep(float(interval_s)) + + +def assert_poll(getter: Callable[[], Any], matcher: Matcher, *, + timeout_s: float = 5.0, interval_s: float = 0.25, + describe: Callable[[Any], str] = repr) -> PollResult: + """Like :func:`expect_poll` but raise ``AutoControlActionException`` on failure.""" + result = expect_poll(getter, matcher, timeout_s=timeout_s, + interval_s=interval_s, describe=describe) + if not result.ok: + raise AutoControlActionException( + f"expect_poll failed after {result.attempts} attempt(s): " + f"{result.description}") + return result diff --git a/je_auto_control/utils/feature_match/__init__.py b/je_auto_control/utils/feature_match/__init__.py new file mode 100644 index 00000000..696321b6 --- /dev/null +++ b/je_auto_control/utils/feature_match/__init__.py @@ -0,0 +1,6 @@ +"""ORB feature matching: locate templates under rotation / scale / theme change.""" +from je_auto_control.utils.feature_match.feature_match import ( + FeatureMatch, feature_match, +) + +__all__ = ["FeatureMatch", "feature_match"] diff --git a/je_auto_control/utils/feature_match/feature_match.py b/je_auto_control/utils/feature_match/feature_match.py new file mode 100644 index 00000000..ca44560b --- /dev/null +++ b/je_auto_control/utils/feature_match/feature_match.py @@ -0,0 +1,123 @@ +"""ORB feature matching: locate a template that is rotated, rescaled or re-themed. + +Template matching (``match_template`` / ``match_masked``) correlates pixels, so it +fails the moment the target is rotated, scaled by a non-listed factor, or its +colours change (light vs dark theme, hover state). ORB matches *keypoints* — +distinctive corners described by orientation-invariant binary descriptors — and +fits a homography through them, so it finds the element under rotation, scale and +appearance change, and reports the four projected corners plus the inlier count +as a confidence signal. + +It runs on an injectable ``haystack`` image (ndarray / path / PIL), so it is +unit-testable on synthetic arrays without a real screen. OpenCV + NumPy come in +via the project's ``je_open_cv`` dependency and are imported lazily. Imports no +``PySide6``. ORB, the brute-force matcher and ``findHomography`` are all in core +OpenCV (no contrib modules). +""" +from dataclasses import asdict, dataclass +from typing import Any, Dict, List, Optional, Sequence + +from je_auto_control.utils.visual_match.visual_match import ( + _haystack_gray, _to_gray, +) + +ImageSource = Any + + +@dataclass(frozen=True) +class FeatureMatch: + """A homography-located match: projected corners, centre, inliers, score.""" + + corners: List[List[int]] + center: List[int] + inliers: int + matches: int + score: float + + def to_dict(self) -> Dict[str, Any]: + """Return the match as a plain dict.""" + return asdict(self) + + +def _ratio_filter(knn_matches, ratio: float) -> list: + """Lowe's ratio test: keep matches clearly better than their runner-up.""" + good = [] + for pair in knn_matches: + if len(pair) == 2 and pair[0].distance < ratio * pair[1].distance: + good.append(pair[0]) + return good + + +def _make_orb(template_gray, max_features: int): + """Build an ORB detector with border/patch sizes that suit small templates. + + OpenCV's default ``edgeThreshold``/``patchSize`` (31) reject every keypoint + in an icon-sized template, so they are scaled down to the template. + """ + import cv2 + smaller = min(template_gray.shape[:2]) + patch = max(7, min(31, smaller // 3)) + edge = max(2, patch // 3) + return cv2.ORB_create(nfeatures=int(max_features), edgeThreshold=edge, + patchSize=patch) + + +def _keypoint_matches(template_gray, scene_gray, max_features: int, ratio: float): + """Return (kp_template, kp_scene, good_matches) or None if too few features.""" + import cv2 + orb = _make_orb(template_gray, max_features) + kp1, des1 = orb.detectAndCompute(template_gray, None) + kp2, des2 = orb.detectAndCompute(scene_gray, None) + if des1 is None or des2 is None or len(des1) < 2 or len(des2) < 2: + return None + matcher = cv2.BFMatcher(cv2.NORM_HAMMING) + good = _ratio_filter(matcher.knnMatch(des1, des2, k=2), float(ratio)) + return kp1, kp2, good + + +def _locate(template_shape, kp1, kp2, good, min_inliers: int + ) -> Optional[FeatureMatch]: + """Fit a RANSAC homography from ``good`` matches and project the template box.""" + import cv2 + import numpy as np + src = np.float32([kp1[m.queryIdx].pt for m in good]).reshape(-1, 1, 2) + dst = np.float32([kp2[m.trainIdx].pt for m in good]).reshape(-1, 1, 2) + homography, mask = cv2.findHomography(src, dst, cv2.RANSAC, 5.0) + if homography is None: + return None + inliers = int(mask.sum()) + if inliers < int(min_inliers): + return None + height, width = template_shape[:2] + box = np.float32([[0, 0], [width, 0], [width, height], + [0, height]]).reshape(-1, 1, 2) + projected = cv2.perspectiveTransform(box, homography).reshape(-1, 2) + corners = [[int(round(px)), int(round(py))] for px, py in projected] + center = [int(round(float(projected[:, 0].mean()))), + int(round(float(projected[:, 1].mean())))] + return FeatureMatch(corners, center, inliers, len(good), + round(inliers / len(good), 4)) + + +def feature_match(template: ImageSource, *, + haystack: Optional[ImageSource] = None, + region: Optional[Sequence[int]] = None, + max_features: int = 500, ratio: float = 0.75, + min_inliers: int = 10) -> Optional[FeatureMatch]: + """Locate ``template`` in the scene by ORB keypoints + a RANSAC homography. + + Robust to rotation, scale and appearance change. ``haystack`` is an ndarray / + path / PIL image (default: grab the screen / ``region``). ``ratio`` is Lowe's + ratio-test cutoff; at least ``min_inliers`` geometrically consistent matches + are required. Returns a :class:`FeatureMatch` (projected ``corners``, + ``center``, ``inliers``, ``score``) or ``None`` when the target is not found. + """ + template_gray = _to_gray(template) + scene_gray = _haystack_gray(haystack, region) + found = _keypoint_matches(template_gray, scene_gray, max_features, ratio) + if found is None: + return None + kp1, kp2, good = found + if len(good) < int(min_inliers): + return None + return _locate(template_gray.shape, kp1, kp2, good, min_inliers) diff --git a/je_auto_control/utils/field_entry/__init__.py b/je_auto_control/utils/field_entry/__init__.py new file mode 100644 index 00000000..be31039e --- /dev/null +++ b/je_auto_control/utils/field_entry/__init__.py @@ -0,0 +1,6 @@ +"""Clear-then-type a text field (the Playwright ``fill`` idiom).""" +from je_auto_control.utils.field_entry.field_entry import ( + plan_field_set, set_field_text, +) + +__all__ = ["plan_field_set", "set_field_text"] diff --git a/je_auto_control/utils/field_entry/field_entry.py b/je_auto_control/utils/field_entry/field_entry.py new file mode 100644 index 00000000..991c30d4 --- /dev/null +++ b/je_auto_control/utils/field_entry/field_entry.py @@ -0,0 +1,70 @@ +"""Clear-then-type a text field (the Playwright ``fill`` idiom). + +Setting a field's value reliably means *clearing* whatever is there first, then +entering the new text — otherwise automation appends to or corrupts the existing +content. The framework has ``write`` (types, but raises on emoji / CJK / chars +outside the layout table) and ``set_clipboard`` / ``hotkey`` separately, but no +single "focus → clear → set value" primitive, and no paste strategy for text +``write`` cannot type. + +:func:`plan_field_set` builds the deterministic op-plan (pure, unit-testable); +:func:`set_field_text` dispatches it through an injectable ``sink`` so it is +tested without real input. Imports no ``PySide6``. +""" +from typing import Any, Callable, Dict, List, Optional + +_CLEAR_MODES = ("select_all", "none") +Sink = Callable[[Dict[str, Any]], None] + + +def plan_field_set(text: str, *, clear: str = "select_all", paste: bool = False, + modifier: str = "ctrl") -> List[Dict[str, Any]]: + """Return the op-plan to set a focused field to ``text``. + + ``clear`` is ``"select_all"`` (``modifier``+A then Delete) or ``"none"``. + ``paste=True`` enters the text via the clipboard (``modifier``+V) — the + reliable path for Unicode / emoji / CJK that ``write`` cannot type — instead + of typing it key by key. ``modifier`` is the platform command key + (``"ctrl"``; use ``"command"`` on macOS). Raises ``ValueError`` on an unknown + ``clear`` mode. + """ + if clear not in _CLEAR_MODES: + raise ValueError(f"unknown clear mode: {clear!r}") + plan: List[Dict[str, Any]] = [] + if clear == "select_all": + plan.append({"op": "hotkey", "keys": [modifier, "a"]}) + plan.append({"op": "key", "key": "delete"}) + if paste: + plan.append({"op": "set_clipboard", "text": text}) + plan.append({"op": "hotkey", "keys": [modifier, "v"]}) + else: + plan.append({"op": "type", "text": text}) + return plan + + +def _default_sink(event: Dict[str, Any]) -> None: + """Default dispatch: drive the real keyboard / clipboard backend.""" + op = event["op"] + if op == "hotkey": + from je_auto_control.wrapper.auto_control_keyboard import hotkey + hotkey(list(event["keys"])) + elif op == "key": + from je_auto_control.wrapper.auto_control_keyboard import type_keyboard + type_keyboard(event["key"]) + elif op == "type": + from je_auto_control.wrapper.auto_control_keyboard import write + write(event["text"]) + elif op == "set_clipboard": + from je_auto_control.utils.clipboard.clipboard import set_clipboard + set_clipboard(event["text"]) + + +def set_field_text(text: str, *, clear: str = "select_all", paste: bool = False, + modifier: str = "ctrl", + sink: Optional[Sink] = None) -> Dict[str, Any]: + """Clear the focused field and enter ``text``; return the dispatched plan.""" + plan = plan_field_set(text, clear=clear, paste=paste, modifier=modifier) + dispatch = sink or _default_sink + for event in plan: + dispatch(event) + return {"ops": len(plan), "plan": plan} diff --git a/je_auto_control/utils/grid_locator/__init__.py b/je_auto_control/utils/grid_locator/__init__.py new file mode 100644 index 00000000..1bca3635 --- /dev/null +++ b/je_auto_control/utils/grid_locator/__init__.py @@ -0,0 +1,6 @@ +"""Address a table / grid cell by (row, column) from bounding boxes.""" +from je_auto_control.utils.grid_locator.grid_locator import ( + cluster_grid, locate_cell, +) + +__all__ = ["cluster_grid", "locate_cell"] diff --git a/je_auto_control/utils/grid_locator/grid_locator.py b/je_auto_control/utils/grid_locator/grid_locator.py new file mode 100644 index 00000000..8542ff17 --- /dev/null +++ b/je_auto_control/utils/grid_locator/grid_locator.py @@ -0,0 +1,65 @@ +"""Address a table / grid cell by (row, column) from a set of bounding boxes. + +``anchor_locator`` does pairwise spatial relations (target *near* / *below* an +anchor) but nothing addresses a 2-D grid — "the cell at row 3, column 2" of a +table. Given the bounding boxes of the cells (from an image or OCR enumeration — +e.g. ``locate_all_image`` / ``find_text_matches``), this clusters them into rows +and columns and returns the requested cell's centre. + +The clustering and lookup are pure (boxes in, grid / cell out) and fully +unit-testable; the box enumeration stays the caller's job, so nothing here needs a +real screen. Imports no ``PySide6``. +""" +from typing import Any, Dict, List, Sequence, Tuple + +Box = Sequence[int] + + +def _center(box: Box) -> Tuple[int, int]: + """Return the integer centre ``(x, y)`` of an ``(x, y, w, h)`` box.""" + x, y, width, height = (int(value) for value in box[:4]) + return x + width // 2, y + height // 2 + + +def cluster_grid(boxes: Sequence[Box], *, + row_tolerance: int = 10) -> List[List[List[int]]]: + """Cluster ``(x, y, w, h)`` boxes into rows (top-down), cells left-to-right. + + Boxes whose centre-y values are within ``row_tolerance`` of the previous + box (after sorting by y) share a row; within a row the cells are ordered by + centre-x. Returns a list of rows, each a list of ``[x, y, w, h]`` boxes. + """ + items = sorted((list(map(int, box[:4])) for box in boxes), + key=lambda box: _center(box)[1]) + rows: List[List[List[int]]] = [] + current: List[List[int]] = [] + last_cy = None + for box in items: + center_y = _center(box)[1] + if last_cy is not None and abs(center_y - last_cy) > int(row_tolerance): + rows.append(current) + current = [] + current.append(box) + last_cy = center_y + if current: + rows.append(current) + for row in rows: + row.sort(key=lambda box: _center(box)[0]) + return rows + + +def locate_cell(boxes: Sequence[Box], row: int, col: int, *, + row_tolerance: int = 10) -> Dict[str, Any]: + """Return the cell at ``(row, col)`` (both 0-based) of the clustered grid.""" + grid = cluster_grid(boxes, row_tolerance=row_tolerance) + if not 0 <= row < len(grid): + return {"found": False, "reason": "row out of range", + "rows": len(grid), "cols": 0} + line = grid[row] + if not 0 <= col < len(line): + return {"found": False, "reason": "col out of range", + "rows": len(grid), "cols": len(line)} + box = line[col] + center = _center(box) + return {"found": True, "center": [center[0], center[1]], "box": list(box), + "row": row, "col": col, "rows": len(grid), "cols": len(line)} diff --git a/je_auto_control/utils/hsv_segment/__init__.py b/je_auto_control/utils/hsv_segment/__init__.py new file mode 100644 index 00000000..0ee58aa7 --- /dev/null +++ b/je_auto_control/utils/hsv_segment/__init__.py @@ -0,0 +1,6 @@ +"""HSV colour-space segmentation (lighting-robust colour masking + blob boxes).""" +from je_auto_control.utils.hsv_segment.hsv_segment import ( + color_mask, dominant_hue_regions, segment_hsv, +) + +__all__ = ["color_mask", "dominant_hue_regions", "segment_hsv"] diff --git a/je_auto_control/utils/hsv_segment/hsv_segment.py b/je_auto_control/utils/hsv_segment/hsv_segment.py new file mode 100644 index 00000000..8dc7029f --- /dev/null +++ b/je_auto_control/utils/hsv_segment/hsv_segment.py @@ -0,0 +1,85 @@ +"""HSV colour-space segmentation — find "any shade of red" regardless of lighting. + +``color_region`` masks in RGB with a per-channel ± tolerance box, which fails the +canonical case: a status light, highlight or theme tint that is "the same colour" but +at a different brightness. HSV separates hue from saturation/value, so a hue band with +a saturation/value floor catches every shade of a colour across lighting. This adds +HSV masking + blob boxes, reusing the shared connected-component helper, with correct +hue-wrap handling for red (which straddles the 0/180 boundary). + +Runs on an injectable ``haystack`` (ndarray / path / PIL, RGB), so it is headless- +testable on synthetic arrays. OpenCV + NumPy come in via ``je_open_cv``. Imports no +``PySide6``. +""" +from typing import Any, Dict, List, Optional, Sequence + +# Reuse the RGB loader / screen grab from color_region (single source of truth). +from je_auto_control.utils.color_region.color_region import _grab_rgb, _to_rgb + +ImageSource = Any + + +def _hsv(haystack: Optional[ImageSource], region: Optional[Sequence[int]]): + import cv2 + rgb = _to_rgb(haystack) if haystack is not None else _grab_rgb(region) + return cv2.cvtColor(rgb, cv2.COLOR_RGB2HSV) + + +def color_mask(haystack: Optional[ImageSource] = None, *, + region: Optional[Sequence[int]] = None, + lower_hsv: Sequence[int], upper_hsv: Sequence[int]): + """Return a uint8 mask of pixels inside the ``lower_hsv``..``upper_hsv`` band. + + HSV ranges are OpenCV's: H in 0..179, S and V in 0..255. + """ + import cv2 + import numpy as np + hsv = _hsv(haystack, region) + lower = np.array([int(value) for value in lower_hsv], dtype=np.uint8) + upper = np.array([int(value) for value in upper_hsv], dtype=np.uint8) + return cv2.inRange(hsv, lower, upper) + + +def segment_hsv(haystack: Optional[ImageSource] = None, *, + region: Optional[Sequence[int]] = None, + lower_hsv: Sequence[int], upper_hsv: Sequence[int], + min_area: int = 50) -> List[Dict[str, Any]]: + """Return blob boxes for pixels inside an explicit HSV band, largest first.""" + from je_auto_control.utils.cv2_utils.blobs import connected_boxes + mask = color_mask(haystack, region=region, lower_hsv=lower_hsv, + upper_hsv=upper_hsv) + return connected_boxes(mask, int(min_area)) + + +def _hue_mask(hsv, low_h: int, high_h: int, sat_min: int, val_min: int): + """Build an inRange mask for a hue band, OR-ing the two parts when it wraps 0/180.""" + import cv2 + import numpy as np + floor = [int(sat_min), int(val_min)] + top = [255, 255] + + def band(start: int, end: int): + return cv2.inRange(hsv, np.array([start, *floor], dtype=np.uint8), + np.array([end, *top], dtype=np.uint8)) + + if low_h < 0: + return cv2.bitwise_or(band(180 + low_h, 179), band(0, high_h)) + if high_h > 179: + return cv2.bitwise_or(band(low_h, 179), band(0, high_h - 180)) + return band(low_h, high_h) + + +def dominant_hue_regions(haystack: Optional[ImageSource] = None, *, + region: Optional[Sequence[int]] = None, hue: int, + hue_tol: int = 10, sat_min: int = 80, val_min: int = 80, + min_area: int = 50) -> List[Dict[str, Any]]: + """Return blob boxes for any pixel near ``hue`` (± ``hue_tol``), any brightness. + + Only the hue is constrained (plus a ``sat_min`` / ``val_min`` floor to skip greys), + so this catches every shade of a colour across lighting — unlike ``color_region``'s + RGB box. Red's 0/180 hue wrap is handled automatically. + """ + from je_auto_control.utils.cv2_utils.blobs import connected_boxes + mask = _hue_mask(_hsv(haystack, region), int(hue) - int(hue_tol), + int(hue) + int(hue_tol), int(sat_min), int(val_min)) + return connected_boxes(mask, int(min_area)) diff --git a/je_auto_control/utils/key_hold/__init__.py b/je_auto_control/utils/key_hold/__init__.py new file mode 100644 index 00000000..4fb6c7be --- /dev/null +++ b/je_auto_control/utils/key_hold/__init__.py @@ -0,0 +1,4 @@ +"""Hold a key for a duration, or auto-repeat it at a fixed rate.""" +from je_auto_control.utils.key_hold.key_hold import hold_key, plan_key_hold + +__all__ = ["hold_key", "plan_key_hold"] diff --git a/je_auto_control/utils/key_hold/key_hold.py b/je_auto_control/utils/key_hold/key_hold.py new file mode 100644 index 00000000..cdc79102 --- /dev/null +++ b/je_auto_control/utils/key_hold/key_hold.py @@ -0,0 +1,73 @@ +"""Hold a key for a duration, or auto-repeat it at a fixed rate. + +``type_keyboard`` is an instant down+up and ``input_macro.run_sequence`` can hand- +roll a press / wait / release, but there is no primitive for "hold this key for N +seconds" (game movement, hold-to-scroll) or "send it at R presses per second" +(auto-repeat). :func:`plan_key_hold` builds the deterministic op-plan (pure, +unit-testable); :func:`hold_key` dispatches it through an injectable ``sink`` and +``sleep`` so it is tested without real input or real waiting. Imports no +``PySide6``. +""" +import time +from typing import Any, Callable, Dict, List, Optional + +Sink = Callable[[Dict[str, Any]], None] + + +def plan_key_hold(key: str, duration_s: float, *, + rate_hz: Optional[float] = None) -> List[Dict[str, Any]]: + """Return the op-plan to hold (or auto-repeat) ``key`` for ``duration_s``. + + With ``rate_hz`` unset the key is pressed, held for ``duration_s``, then + released. With ``rate_hz`` set it is sent as ``round(duration_s * rate_hz)`` + discrete key events spaced ``1 / rate_hz`` apart (simulated auto-repeat). + Raises ``ValueError`` on a non-positive duration or rate. + """ + if duration_s <= 0: + raise ValueError("duration_s must be positive") + if rate_hz is None: + return [{"op": "press", "key": key}, + {"op": "wait", "seconds": float(duration_s)}, + {"op": "release", "key": key}] + if rate_hz <= 0: + raise ValueError("rate_hz must be positive") + interval = 1.0 / float(rate_hz) + count = max(1, round(float(duration_s) * float(rate_hz))) + plan: List[Dict[str, Any]] = [] + for index in range(count): + plan.append({"op": "key", "key": key}) + if index != count - 1: + plan.append({"op": "wait", "seconds": interval}) + return plan + + +def _default_sink(event: Dict[str, Any]) -> None: + """Default dispatch: drive the real keyboard backend.""" + from je_auto_control.wrapper.auto_control_keyboard import ( + press_keyboard_key, release_keyboard_key, type_keyboard) + op = event["op"] + if op == "press": + press_keyboard_key(event["key"]) + elif op == "release": + release_keyboard_key(event["key"]) + elif op == "key": + type_keyboard(event["key"]) + + +def hold_key(key: str, duration_s: float, *, rate_hz: Optional[float] = None, + sink: Optional[Sink] = None, + sleep: Optional[Callable[[float], None]] = None) -> Dict[str, Any]: + """Hold or auto-repeat ``key`` for ``duration_s``; return the dispatched plan. + + ``wait`` ops go to ``sleep`` (default :func:`time.sleep`); key ops go to + ``sink`` (default: the real keyboard backend). + """ + plan = plan_key_hold(key, duration_s, rate_hz=rate_hz) + dispatch = sink or _default_sink + pause = sleep or time.sleep + for event in plan: + if event["op"] == "wait": + pause(event["seconds"]) + else: + dispatch(event) + return {"ops": len(plan), "plan": plan} diff --git a/je_auto_control/utils/locator_chain/__init__.py b/je_auto_control/utils/locator_chain/__init__.py new file mode 100644 index 00000000..efa8a78b --- /dev/null +++ b/je_auto_control/utils/locator_chain/__init__.py @@ -0,0 +1,6 @@ +"""Composable / filtered candidate locators (chained-locator idiom).""" +from je_auto_control.utils.locator_chain.locator_chain import ( + Candidates, from_boxes, +) + +__all__ = ["Candidates", "from_boxes"] diff --git a/je_auto_control/utils/locator_chain/locator_chain.py b/je_auto_control/utils/locator_chain/locator_chain.py new file mode 100644 index 00000000..b4131579 --- /dev/null +++ b/je_auto_control/utils/locator_chain/locator_chain.py @@ -0,0 +1,106 @@ +"""Composable / filtered candidate locators — Playwright-style chained locators. + +``anchor_locator`` resolves a single anchor→target relation and ``grid_locator`` +addresses grid cells; neither supports *composable refinement* of a candidate set — +``.within(panel).filter(has_text="Delete").nth(2)`` — the Selenium-4 / Playwright +chained-and-filtered locator idiom. Today refining means re-querying a backend; this is +a pure post-filter over boxes from *any* source (template / OCR / a11y). + +A ``Candidates`` wraps a list of ``{x, y, width, height, ...}`` boxes; every method +returns a new ``Candidates`` so chains are side-effect-free and fully unit-testable. +Pure-stdlib, imports no ``PySide6``. +""" +import math +from typing import Any, Callable, Dict, List, Optional, Sequence, Tuple + +Box = Dict[str, Any] + + +def _center(box: Box) -> List[int]: + return [int(box["x"]) + int(box["width"]) // 2, + int(box["y"]) + int(box["height"]) // 2] + + +def _area(box: Box) -> int: + return int(box["width"]) * int(box["height"]) + + +class Candidates: + """An immutable, chainable set of element boxes.""" + + def __init__(self, boxes: Sequence[Box]): + self._boxes: List[Box] = [dict(box) for box in boxes] + + def __iter__(self): + return iter(self._boxes) + + def __len__(self) -> int: + return len(self._boxes) + + def resolve(self) -> List[Box]: + """Return the surviving boxes as a list.""" + return list(self._boxes) + + def center(self) -> Optional[List[int]]: + """Return the centre of the first surviving box (or ``None``).""" + return _center(self._boxes[0]) if self._boxes else None + + def within(self, region: Sequence[int]) -> "Candidates": + """Keep boxes whose centre falls inside ``region`` ``(x, y, w, h)``.""" + rx, ry, rw, rh = (int(value) for value in region[:4]) + kept = [box for box in self._boxes + if rx <= _center(box)[0] <= rx + rw + and ry <= _center(box)[1] <= ry + rh] + return Candidates(kept) + + def filter(self, *, has_text: Optional[str] = None, + near: Optional[Tuple[int, int, float]] = None, + min_area: Optional[int] = None, max_area: Optional[int] = None, + predicate: Optional[Callable[[Box], bool]] = None) -> "Candidates": + """Keep boxes matching every supplied criterion. + + ``has_text`` substring (case-insensitive) of the box ``text``; ``near`` + ``(x, y, max_dist)`` centre proximity; ``min_area`` / ``max_area`` size; + ``predicate`` an arbitrary callable. + """ + checks: List[Callable[[Box], bool]] = [] + if has_text is not None: + needle = str(has_text).lower() + checks.append(lambda b: needle in str(b.get("text", "")).lower()) + if near is not None: + nx, ny, dist = near + checks.append( + lambda b: math.hypot(_center(b)[0] - nx, _center(b)[1] - ny) <= dist) + if min_area is not None: + low = int(min_area) + checks.append(lambda b: _area(b) >= low) + if max_area is not None: + high = int(max_area) + checks.append(lambda b: _area(b) <= high) + if predicate is not None: + checks.append(predicate) + return Candidates([b for b in self._boxes if all(c(b) for c in checks)]) + + def sort_reading(self, row_tol: int = 12) -> "Candidates": + """Order the boxes top-to-bottom, left-to-right (reuses element_parse).""" + from je_auto_control.utils.element_parse import reading_order + return Candidates(reading_order(self._boxes, row_tol=int(row_tol))) + + def nth(self, index: int) -> "Candidates": + """Keep only the box at ``index`` (negative allowed), else empty.""" + if -len(self._boxes) <= index < len(self._boxes): + return Candidates([self._boxes[index]]) + return Candidates([]) + + def first(self) -> "Candidates": + """Keep only the first box.""" + return self.nth(0) + + def last(self) -> "Candidates": + """Keep only the last box.""" + return self.nth(-1) + + +def from_boxes(boxes: Sequence[Box]) -> Candidates: + """Wrap a list of element boxes in a :class:`Candidates` for chaining.""" + return Candidates(boxes) diff --git a/je_auto_control/utils/mcp_server/tools/_factories.py b/je_auto_control/utils/mcp_server/tools/_factories.py index be267fbc..116fd9d7 100644 --- a/je_auto_control/utils/mcp_server/tools/_factories.py +++ b/je_auto_control/utils/mcp_server/tools/_factories.py @@ -1277,6 +1277,67 @@ def cost_telemetry_tools() -> List[MCPTool]: def smart_wait_tools() -> List[MCPTool]: return [ + MCPTool( + name="ac_wait_image_gone", + description=("Block until 'image' is no longer found on screen " + "(spinner/toast/dialog vanished). 'detect_threshold', " + "'timeout_s', 'poll_interval_s', 'gone_for_s'."), + input_schema=schema({ + "image": {"type": "string"}, + "detect_threshold": {"type": "number"}, + "timeout_s": {"type": "number"}, + "poll_interval_s": {"type": "number"}, + "gone_for_s": {"type": "number"}}, + required=["image"]), + handler=h.wait_image_gone, + annotations=READ_ONLY, + ), + MCPTool( + name="ac_wait_text_gone", + description=("Block until 'text' is no longer found on screen (OCR). " + "'timeout_s', 'poll_interval_s', 'gone_for_s'."), + input_schema=schema({ + "text": {"type": "string"}, + "timeout_s": {"type": "number"}, + "poll_interval_s": {"type": "number"}, + "gone_for_s": {"type": "number"}}, + required=["text"]), + handler=h.wait_text_gone, + annotations=READ_ONLY, + ), + MCPTool( + name="ac_wait_color", + description=("Block until 'target_rgb' [r,g,b] covers >= " + "'min_fraction' of 'region' [l,t,r,b] within " + "'tolerance' (present=True), or drops below it " + "(present=False). A status-light / progress-bar wait."), + input_schema=schema({ + "target_rgb": {"type": "array", "items": {"type": "integer"}}, + "region": {"type": "array", "items": {"type": "integer"}}, + "tolerance": {"type": "integer"}, + "min_fraction": {"type": "number"}, + "present": {"type": "boolean"}, + "timeout_s": {"type": "number"}, + "poll_interval_s": {"type": "number"}}, + required=["target_rgb"]), + handler=h.wait_color, + annotations=READ_ONLY, + ), + MCPTool( + name="ac_wait_window_title", + description=("Block until a window title matches 'pattern' (a regex " + "by default; regex=False for substring) — or vanishes " + "with present=False. e.g. r'.*— Checkout$'."), + input_schema=schema({ + "pattern": {"type": "string"}, + "present": {"type": "boolean"}, + "regex": {"type": "boolean"}, + "timeout_s": {"type": "number"}, + "poll_interval_s": {"type": "number"}}, + required=["pattern"]), + handler=h.wait_window_title, + annotations=READ_ONLY, + ), MCPTool( name="ac_wait_screen_stable", description=("Block until the screen stops moving (consecutive " @@ -1426,10 +1487,27 @@ def anchor_locator_tools() -> List[MCPTool]: "enum": ["above", "below", "left_of", "right_of", "near"]}, "max_distance_px": {"type": "number"}, + "ordinal": {"type": "integer"}, }, required=["anchor", "target"]), handler=h.anchor_locate, annotations=READ_ONLY, ), + MCPTool( + name="ac_anchor_locate_all", + description=("Every target matching the spatial relation to the " + "anchor, nearest-first (for table / list-row " + "selection). Returns {count, matches}."), + input_schema=schema({ + "anchor": locator_schema, + "target": locator_schema, + "relation": {"type": "string", + "enum": ["above", "below", "left_of", + "right_of", "near"]}, + "max_distance_px": {"type": "number"}, + }, required=["anchor", "target"]), + handler=h.anchor_locate_all, + annotations=READ_ONLY, + ), MCPTool( name="ac_anchor_click", description="Anchor-locate then click the resolved target point.", @@ -2550,6 +2628,717 @@ def tween_drag_tools() -> List[MCPTool]: ] +def color_region_tools() -> List[MCPTool]: + return [ + MCPTool( + name="ac_find_color_region", + description=("Locate on-screen regions matching 'rgb' [r,g,b] within " + "'tolerance' (per channel), blobs >= 'min_area'. " + "Returns {count, regions, best} (largest first). " + "For status lights / progress bars / coloured banners."), + input_schema=schema({ + "rgb": {"type": "array", "items": {"type": "integer"}}, + "tolerance": {"type": "integer"}, + "min_area": {"type": "integer"}, + "region": {"type": "array", "items": {"type": "integer"}}}, + required=["rgb"]), + handler=h.find_color_region, + annotations=READ_ONLY, + ), + ] + + +def feature_match_tools() -> List[MCPTool]: + return [ + MCPTool( + name="ac_feature_match", + description=("Locate 'template' (image path) on screen by ORB " + "keypoints + a RANSAC homography — robust to ROTATION, " + "SCALE and theme/colour change, where pixel template " + "matching fails. Returns {found, match:{corners (4 " + "points), center, inliers, matches, score}}. 'min_inliers' " + "is the confidence floor; 'ratio' the match cutoff."), + input_schema=schema({ + "template": {"type": "string"}, + "region": {"type": "array", "items": {"type": "integer"}}, + "max_features": {"type": "integer"}, + "ratio": {"type": "number"}, + "min_inliers": {"type": "integer"}}, + required=["template"]), + handler=h.feature_match, + annotations=READ_ONLY, + ), + ] + + +def shape_locator_tools() -> List[MCPTool]: + return [ + MCPTool( + name="ac_find_shapes", + description=("Locate distinct on-screen shapes by edge/contour " + "detection — NO template/colour/text needed. Returns " + "{count, shapes:[{x,y,width,height,area,center,aspect}]} " + "(largest first). 'min_area'/'max_area' filter by " + "bounding-box area."), + input_schema=schema({ + "region": {"type": "array", "items": {"type": "integer"}}, + "min_area": {"type": "integer"}, + "max_area": {"type": "integer"}}, + required=[]), + handler=h.find_shapes, + annotations=READ_ONLY, + ), + MCPTool( + name="ac_find_rectangles", + description=("Locate ~rectangular regions (buttons / cards / input " + "fields) by contour approximation. Returns {count, " + "rectangles:[{x,y,width,height,area,center,aspect}]} " + "(largest first). 'aspect_range' [min,max] filters w/h " + "(e.g. [1.5,8] wide buttons); 'epsilon' the approx " + "tolerance."), + input_schema=schema({ + "region": {"type": "array", "items": {"type": "integer"}}, + "min_area": {"type": "integer"}, + "max_area": {"type": "integer"}, + "aspect_range": {"type": "array", "items": {"type": "number"}}, + "epsilon": {"type": "number"}}, + required=[]), + handler=h.find_rectangles, + annotations=READ_ONLY, + ), + ] + + +def window_layout_tools() -> List[MCPTool]: + return [ + MCPTool( + name="ac_tile_rect", + description=("Compute the rectangle for a named tiling 'slot' of the " + "screen work area: full/left/right/top/bottom/top_left/" + "top_right/bottom_left/bottom_right/center/left_third/" + "center_third/right_third. 'screen' [x,y,w,h] defaults to " + "the live primary screen; 'gap' insets all sides. Returns " + "{rect:{x,y,width,height}} — feed to a window-move."), + input_schema=schema({ + "slot": {"type": "string"}, + "screen": {"type": "array", "items": {"type": "integer"}}, + "gap": {"type": "integer"}}, + required=["slot"]), + handler=h.tile_rect, + annotations=READ_ONLY, + ), + MCPTool( + name="ac_grid_rects", + description=("Split the screen work area into an 'rows' x 'cols' grid; " + "return {count, rects:[{x,y,width,height}]} row-major. " + "'screen' [x,y,w,h] defaults to the live screen; 'gap' " + "insets each cell."), + input_schema=schema({ + "rows": {"type": "integer"}, + "cols": {"type": "integer"}, + "screen": {"type": "array", "items": {"type": "integer"}}, + "gap": {"type": "integer"}}, + required=["rows", "cols"]), + handler=h.grid_rects, + annotations=READ_ONLY, + ), + MCPTool( + name="ac_cascade_rects", + description=("Compute 'count' staggered, overlapping window rectangles " + "(a cascade), each 'offset' px down-right of the previous, " + "clamped to the screen. 'size' [w,h] defaults to 60% of " + "the work area. Returns {count, rects}."), + input_schema=schema({ + "count": {"type": "integer"}, + "screen": {"type": "array", "items": {"type": "integer"}}, + "offset": {"type": "integer"}, + "size": {"type": "array", "items": {"type": "integer"}}}, + required=["count"]), + handler=h.cascade_rects, + annotations=READ_ONLY, + ), + ] + + +def window_arrange_tools() -> List[MCPTool]: + return [ + MCPTool( + name="ac_arrange_grid", + description=("Tile the given window 'titles' into a grid and MOVE " + "them. 'rows'/'cols' default to a near-square auto-shape; " + "'gap' spaces cells. Returns {moved, count}."), + input_schema=schema({ + "titles": {"type": "array", "items": {"type": "string"}}, + "rows": {"type": "integer"}, + "cols": {"type": "integer"}, + "gap": {"type": "integer"}}, + required=["titles"]), + handler=h.arrange_grid, + annotations=SIDE_EFFECT_ONLY, + ), + MCPTool( + name="ac_arrange_cascade", + description=("Cascade the given window 'titles' diagonally and MOVE " + "them, each 'offset' px down-right of the previous. " + "Returns {moved, count}."), + input_schema=schema({ + "titles": {"type": "array", "items": {"type": "string"}}, + "offset": {"type": "integer"}}, + required=["titles"]), + handler=h.arrange_cascade, + annotations=SIDE_EFFECT_ONLY, + ), + ] + + +def preprocess_tools() -> List[MCPTool]: + return [ + MCPTool( + name="ac_preprocess_image", + description=("Pre-process an image for OCR / template matching and " + "WRITE the result to 'output_path'. 'source' is an image " + "path (default: screen grab of 'region'). 'steps' is an " + "ordered list from grayscale/upscale/binarize/denoise/" + "deskew/contrast (default grayscale,upscale,binarize); " + "'scale' for upscale. Returns {path, width, height}."), + input_schema=schema({ + "output_path": {"type": "string"}, + "source": {"type": "string"}, + "steps": {"type": "array", "items": {"type": "string"}}, + "scale": {"type": "number"}, + "region": {"type": "array", "items": {"type": "integer"}}, + "block_size": {"type": "integer"}, + "c": {"type": "integer"}}, + required=["output_path"]), + handler=h.preprocess_image, + annotations=SIDE_EFFECT_ONLY, + ), + ] + + +def monitor_layout_tools() -> List[MCPTool]: + return [ + MCPTool( + name="ac_enumerate_monitors", + description=("List connected monitors with virtual-desktop geometry: " + "{count, monitors:[{index,x,y,width,height,scale,primary," + "work}], virtual_bounds:[x,y,w,h]}. Unlike a single " + "screen_size, this exposes per-monitor origins (which may " + "be negative) for multi-display placement."), + input_schema=schema({}, required=[]), + handler=h.enumerate_monitors, + annotations=READ_ONLY, + ), + MCPTool( + name="ac_monitor_at_point", + description=("Report which monitor contains virtual point (x, y): " + "{found, monitor}. Returns found=false when the point is " + "off every display."), + input_schema=schema({ + "x": {"type": "integer"}, + "y": {"type": "integer"}}, + required=["x", "y"]), + handler=h.monitor_at_point, + annotations=READ_ONLY, + ), + ] + + +def actionability_tools() -> List[MCPTool]: + return [ + MCPTool( + name="ac_wait_actionable", + description=("Wait until 'template' (image path) is VISIBLE and STABLE " + "(stopped moving / animating) on screen before you act — " + "the Playwright-style actionability gate. Returns {actionable, " + "visible, stable, enabled, receives_events, point, reason, " + "waited_s}. 'timeout_s', 'stable_for_s', 'min_score', " + "'region'."), + input_schema=schema({ + "template": {"type": "string"}, + "timeout_s": {"type": "number"}, + "stable_for_s": {"type": "number"}, + "min_score": {"type": "number"}, + "region": {"type": "array", "items": {"type": "integer"}}}, + required=["template"]), + handler=h.wait_actionable, + annotations=READ_ONLY, + ), + ] + + +def element_parse_tools() -> List[MCPTool]: + return [ + MCPTool( + name="ac_fuse_elements", + description=("Union OCR + icon + accessibility element boxes into one " + "de-duplicated list (drop cross-source overlaps > " + "'iou_threshold'; higher-priority source wins). Each box " + "is {x,y,width,height,...}. Returns {count, elements}."), + input_schema=schema({ + "ocr": {"type": "array", "items": {"type": "object"}}, + "icon": {"type": "array", "items": {"type": "object"}}, + "a11y": {"type": "array", "items": {"type": "object"}}, + "iou_threshold": {"type": "number"}}, + required=[]), + handler=h.fuse_elements, + annotations=READ_ONLY, + ), + MCPTool( + name="ac_reading_order", + description=("Sort element boxes top-to-bottom, left-to-right and add a " + "stable 'index' to each (elements within 'row_tol' px of " + "each other count as one row). Returns {count, elements}."), + input_schema=schema({ + "elements": {"type": "array", "items": {"type": "object"}}, + "row_tol": {"type": "integer"}}, + required=["elements"]), + handler=h.reading_order, + annotations=READ_ONLY, + ), + ] + + +def hsv_segment_tools() -> List[MCPTool]: + return [ + MCPTool( + name="ac_segment_hsv", + description=("Locate on-screen blobs inside an explicit HSV band " + "(H 0-179, S/V 0-255): 'lower_hsv' [h,s,v] .. 'upper_hsv' " + "[h,s,v], blobs >= 'min_area'. Returns {count, regions, " + "best}. HSV is lighting-robust where RGB tolerance is not."), + input_schema=schema({ + "lower_hsv": {"type": "array", "items": {"type": "integer"}}, + "upper_hsv": {"type": "array", "items": {"type": "integer"}}, + "min_area": {"type": "integer"}, + "region": {"type": "array", "items": {"type": "integer"}}}, + required=["lower_hsv", "upper_hsv"]), + handler=h.segment_hsv, + annotations=READ_ONLY, + ), + MCPTool( + name="ac_dominant_hue_regions", + description=("Locate regions of ANY shade of a colour near 'hue' " + "(0-179, ± 'hue_tol'), any brightness, with a 'sat_min' / " + "'val_min' floor. Red's 0/180 wrap is handled. Returns " + "{count, regions, best}."), + input_schema=schema({ + "hue": {"type": "integer"}, + "hue_tol": {"type": "integer"}, + "sat_min": {"type": "integer"}, + "val_min": {"type": "integer"}, + "min_area": {"type": "integer"}, + "region": {"type": "array", "items": {"type": "integer"}}}, + required=["hue"]), + handler=h.dominant_hue_regions, + annotations=READ_ONLY, + ), + ] + + +def text_regions_tools() -> List[MCPTool]: + return [ + MCPTool( + name="ac_find_text_regions", + description=("Locate text/glyph regions on screen via MSER — NO OCR " + "engine or known string needed. Returns {count, regions:" + "[{x,y,width,height,area,center}]} (largest first). " + "'merge' unions nested glyph boxes; 'min_area'/'max_area'/" + "'max_aspect' filter. Crop these to feed OCR."), + input_schema=schema({ + "min_area": {"type": "integer"}, + "max_area": {"type": "integer"}, + "merge": {"type": "boolean"}, + "max_aspect": {"type": "number"}, + "region": {"type": "array", "items": {"type": "integer"}}}, + required=[]), + handler=h.find_text_regions, + annotations=READ_ONLY, + ), + MCPTool( + name="ac_find_text_lines", + description=("Locate horizontal lines of text on screen via MSER: one " + "box per line (glyphs within 'y_tolerance' px grouped). " + "Returns {count, lines}. No OCR needed."), + input_schema=schema({ + "y_tolerance": {"type": "integer"}, + "region": {"type": "array", "items": {"type": "integer"}}}, + required=[]), + handler=h.find_text_lines, + annotations=READ_ONLY, + ), + ] + + +def edge_lines_tools() -> List[MCPTool]: + return [ + MCPTool( + name="ac_find_lines", + description=("Detect straight line segments on screen (Hough): " + "{count, lines:[{x1,y1,x2,y2,angle,length,orientation}]} " + "longest first. 'orientation' horizontal/vertical/diagonal/" + "any filters; 'min_length'/'max_gap' tune the probe."), + input_schema=schema({ + "min_length": {"type": "integer"}, + "max_gap": {"type": "integer"}, + "orientation": {"type": "string"}, + "region": {"type": "array", "items": {"type": "integer"}}}, + required=[]), + handler=h.find_lines, + annotations=READ_ONLY, + ), + MCPTool( + name="ac_find_grid", + description=("Recover a table's grid from screen lines: {rows:[y...], " + "cols:[x...], cells:[{x,y,width,height}]}. Address 'row 3, " + "col 2' without a template. 'min_length' filters edges."), + input_schema=schema({ + "min_length": {"type": "integer"}, + "tol": {"type": "integer"}, + "region": {"type": "array", "items": {"type": "integer"}}}, + required=[]), + handler=h.find_grid, + annotations=READ_ONLY, + ), + MCPTool( + name="ac_find_separators", + description=("Coordinates of long divider lines along 'axis' " + "(horizontal -> y of each rule, vertical -> x). Returns " + "{count, axis, coordinates}. Split a panel at its dividers."), + input_schema=schema({ + "axis": {"type": "string"}, + "min_length": {"type": "integer"}, + "tol": {"type": "integer"}, + "region": {"type": "array", "items": {"type": "integer"}}}, + required=[]), + handler=h.find_separators, + annotations=READ_ONLY, + ), + ] + + +def expect_poll_tools() -> List[MCPTool]: + return [ + MCPTool( + name="ac_expect_poll", + description=("Re-run a nested 'action' (e.g. [\"AC_get_clipboard\"]) " + "until a 'key' of its result dict matches 'op' " + "(truthy/equals/contains/gt/regex) vs 'expected', or " + "'timeout_s' elapses. Retries an ARBITRARY value, unlike " + "assert_eventually's fixed checks. Returns {ok, value, " + "attempts, waited_s}."), + input_schema=schema({ + "action": {"type": "array"}, + "key": {"type": "string"}, + "op": {"type": "string"}, + "expected": {}, + "timeout_s": {"type": "number"}, + "interval_s": {"type": "number"}}, + required=["action"]), + handler=h.expect_poll, + annotations=NON_DESTRUCTIVE, + ), + ] + + +def locator_chain_tools() -> List[MCPTool]: + return [ + MCPTool( + name="ac_locate_chain", + description=("Refine a set of element 'boxes' with a chain of 'ops' " + "applied in order: {op:'within',region:[x,y,w,h]}, " + "{op:'filter',has_text/near/min_area/max_area}, " + "{op:'reading'}, {op:'nth',index}, {op:'first'}, " + "{op:'last'}. Returns {count, boxes, center}."), + input_schema=schema({ + "boxes": {"type": "array", "items": {"type": "object"}}, + "ops": {"type": "array", "items": {"type": "object"}}}, + required=["boxes"]), + handler=h.locate_chain, + annotations=READ_ONLY, + ), + ] + + +def rich_clipboard_tools() -> List[MCPTool]: + return [ + MCPTool( + name="ac_set_clipboard_html", + description=("Put an HTML fragment on the clipboard as CF_HTML for rich " + "paste into Word / Outlook / rich editors (Windows). " + "'fragment_plaintext' is also set as plain text. Returns " + "{set, length}."), + input_schema=schema({ + "html": {"type": "string"}, + "fragment_plaintext": {"type": "string"}}, + required=["html"]), + handler=h.set_clipboard_html, + annotations=SIDE_EFFECT_ONLY, + ), + MCPTool( + name="ac_get_clipboard_html", + description=("Read the clipboard's HTML fragment (CF_HTML, Windows). " + "Returns {found, html}."), + input_schema=schema({}, required=[]), + handler=h.get_clipboard_html, + annotations=READ_ONLY, + ), + ] + + +def ssim_tools() -> List[MCPTool]: + return [ + MCPTool( + name="ac_ssim_compare", + description=("Structural-similarity (SSIM) score 0..1 between " + "'reference' (image path) and 'current' (path; default: " + "screen grab of 'region'). 1.0 = identical. 'ignore' is " + "a list of [x,y,w,h] boxes to exclude (clocks/cursors). " + "Returns {score}. Perceptual, unlike pixel diff."), + input_schema=schema({ + "reference": {"type": "string"}, + "current": {"type": "string"}, + "ignore": {"type": "array", + "items": {"type": "array", + "items": {"type": "integer"}}}, + "region": {"type": "array", "items": {"type": "integer"}}}, + required=["reference"]), + handler=h.ssim_compare, + annotations=READ_ONLY, + ), + MCPTool( + name="ac_ssim_changed_regions", + description=("Boxes of the regions that STRUCTURALLY changed between " + "'reference' and 'current' (default: screen). A pixel " + "changed where 1-SSIM > 'threshold'; blobs >= 'min_area'. " + "'ignore' [x,y,w,h] boxes suppressed. Returns " + "{count, regions} (largest first)."), + input_schema=schema({ + "reference": {"type": "string"}, + "current": {"type": "string"}, + "ignore": {"type": "array", + "items": {"type": "array", + "items": {"type": "integer"}}}, + "threshold": {"type": "number"}, + "min_area": {"type": "integer"}, + "region": {"type": "array", "items": {"type": "integer"}}}, + required=["reference"]), + handler=h.ssim_changed_regions, + annotations=READ_ONLY, + ), + ] + + +def visual_match_tools() -> List[MCPTool]: + return [ + MCPTool( + name="ac_match_template", + description=("Find 'template' (image path) on screen and return the " + "match WITH its score: {found, match:{x,y,width,height," + "score,scale,center}}. 'scales' [..] for DPI/zoom, " + "'min_score', 'region', 'method'."), + input_schema=schema({ + "template": {"type": "string"}, + "min_score": {"type": "number"}, + "scales": {"type": "array", "items": {"type": "number"}}, + "region": {"type": "array", "items": {"type": "integer"}}, + "method": {"type": "string"}}, + required=["template"]), + handler=h.match_template, + annotations=READ_ONLY, + ), + MCPTool( + name="ac_match_template_all", + description=("Find EVERY occurrence of 'template' on screen " + ">= 'min_score', overlaps removed by NMS. " + "Returns {count, matches}."), + input_schema=schema({ + "template": {"type": "string"}, + "min_score": {"type": "number"}, + "max_results": {"type": "integer"}, + "nms_iou": {"type": "number"}, + "region": {"type": "array", "items": {"type": "integer"}}}, + required=["template"]), + handler=h.match_template_all, + annotations=READ_ONLY, + ), + MCPTool( + name="ac_match_masked", + description=("Find 'template' on screen counting only masked/opaque " + "pixels: a grayscale 'mask' (non-zero = use) or, if " + "omitted, the template's RGBA alpha. For glyphs/icons " + "over a transparent or varying background. Returns " + "{found, match}. 'min_score', 'region'."), + input_schema=schema({ + "template": {"type": "string"}, + "mask": {"type": "string"}, + "min_score": {"type": "number"}, + "region": {"type": "array", "items": {"type": "integer"}}}, + required=["template"]), + handler=h.match_masked, + annotations=READ_ONLY, + ), + MCPTool( + name="ac_match_masked_all", + description=("Find EVERY masked match of 'template' >= 'min_score', " + "overlaps removed by NMS. Returns {count, matches}."), + input_schema=schema({ + "template": {"type": "string"}, + "mask": {"type": "string"}, + "min_score": {"type": "number"}, + "max_results": {"type": "integer"}, + "nms_iou": {"type": "number"}, + "region": {"type": "array", "items": {"type": "integer"}}}, + required=["template"]), + handler=h.match_masked_all, + annotations=READ_ONLY, + ), + ] + + +def grid_locator_tools() -> List[MCPTool]: + return [ + MCPTool( + name="ac_grid_cell", + description=("Address a table cell: cluster 'boxes' ([[x,y,w,h],...] " + "from an image/OCR enumeration) into a grid and return " + "the centre of cell ('row','col') (0-based). " + "Returns {found, center, row, col, rows, cols}."), + input_schema=schema({ + "boxes": {"type": "array", + "items": {"type": "array", + "items": {"type": "integer"}}}, + "row": {"type": "integer"}, "col": {"type": "integer"}, + "row_tolerance": {"type": "integer"}}, + required=["boxes", "row", "col"]), + handler=h.grid_cell, + annotations=READ_ONLY, + ), + ] + + +def modifier_state_tools() -> List[MCPTool]: + return [ + MCPTool( + name="ac_with_modifiers", + description=("Run 'actions' (a JSON action list) while 'modifiers' " + "(e.g. ['ctrl'] or 'ctrl+shift') are held down; the " + "modifiers are released even if an action fails. " + "For shift-click range / ctrl-click multi-select."), + input_schema=schema({ + "modifiers": {"type": "array", "items": {"type": "string"}}, + "actions": {"type": "array"}}, + required=["modifiers", "actions"]), + handler=h.with_modifiers, + annotations=SIDE_EFFECT_ONLY, + ), + ] + + +def text_unicode_tools() -> List[MCPTool]: + return [ + MCPTool( + name="ac_type_unicode", + description=("Enter arbitrary Unicode 'text' (emoji / CJK / accented " + "that the normal type cannot) via clipboard paste. " + "'modifier' is the paste key (ctrl / command). " + "Returns {ops, plan, code_units}."), + input_schema=schema( + {"text": {"type": "string"}, "modifier": {"type": "string"}}, + required=["text"]), + handler=h.type_unicode, + annotations=SIDE_EFFECT_ONLY, + ), + ] + + +def mouse_relative_tools() -> List[MCPTool]: + return [ + MCPTool( + name="ac_move_mouse_relative", + description=("Move the pointer by ('dx', 'dy') relative to its " + "current position. Returns {from, to, delta}."), + input_schema=schema( + {"dx": {"type": "integer"}, "dy": {"type": "integer"}}, + required=["dx", "dy"]), + handler=h.move_mouse_relative, + annotations=SIDE_EFFECT_ONLY, + ), + ] + + +def key_hold_tools() -> List[MCPTool]: + return [ + MCPTool( + name="ac_hold_key", + description=("Hold 'key' for 'duration_s' seconds, or set 'rate_hz' " + "to auto-repeat it at that many presses/second. " + "Returns {ops, plan}."), + input_schema=schema({ + "key": {"type": "string"}, + "duration_s": {"type": "number"}, + "rate_hz": {"type": "number"}}, + required=["key", "duration_s"]), + handler=h.hold_key, + annotations=SIDE_EFFECT_ONLY, + ), + ] + + +def field_entry_tools() -> List[MCPTool]: + return [ + MCPTool( + name="ac_set_field_text", + description=("Clear the focused field and enter 'text' (Playwright " + "fill). 'clear' select_all|none; 'paste' True for " + "Unicode/emoji via clipboard; 'modifier' ctrl|command. " + "Returns {ops, plan}."), + input_schema=schema({ + "text": {"type": "string"}, "clear": {"type": "string"}, + "paste": {"type": "boolean"}, "modifier": {"type": "string"}}, + required=["text"]), + handler=h.set_field_text, + annotations=SIDE_EFFECT_ONLY, + ), + ] + + +def mouse_path_tools() -> List[MCPTool]: + return [ + MCPTool( + name="ac_move_along_path", + description=("Move the pointer through 'waypoints' ([[x,y],...]) as " + "an eased polyline. 'per_segment_steps' + 'easing' " + "(linear / ease_*). Returns {points, path}."), + input_schema=schema({ + "waypoints": {"type": "array", + "items": {"type": "array", + "items": {"type": "integer"}}}, + "easing": {"type": "string"}, + "per_segment_steps": {"type": "integer"}}, + required=["waypoints"]), + handler=h.move_along_path, + annotations=SIDE_EFFECT_ONLY, + ), + MCPTool( + name="ac_drag_path", + description=("Press at the first of 'waypoints' ([[x,y],...]), drag " + "through them, release at the last. 'button', 'easing', " + "'per_segment_steps'. Returns {points, path}."), + input_schema=schema({ + "waypoints": {"type": "array", + "items": {"type": "array", + "items": {"type": "integer"}}}, + "button": {"type": "string"}, + "easing": {"type": "string"}, + "per_segment_steps": {"type": "integer"}}, + required=["waypoints"]), + handler=h.drag_path, + annotations=SIDE_EFFECT_ONLY, + ), + ] + + def plugin_sdk_tools() -> List[MCPTool]: _G = {"group": {"type": "string"}} return [ @@ -3709,6 +4498,32 @@ def gettext_catalog_tools() -> List[MCPTool]: ] +def checksum_tools() -> List[MCPTool]: + return [ + MCPTool( + name="ac_checksum_validate", + description=("Validate a number's check digit. 'scheme' is " + "luhn|verhoeff|damm|mod97. Returns {valid}."), + input_schema=schema( + {"scheme": {"type": "string"}, "number": {"type": "string"}}, + ["scheme", "number"]), + handler=h.checksum_validate, + annotations=READ_ONLY, + ), + MCPTool( + name="ac_checksum_digit", + description=("Compute the check digit(s) to append to 'partial'. " + "'scheme' is luhn|verhoeff|damm|mod97. " + "Returns {check_digit}."), + input_schema=schema( + {"scheme": {"type": "string"}, "partial": {"type": "string"}}, + ["scheme", "partial"]), + handler=h.checksum_digit, + annotations=READ_ONLY, + ), + ] + + def message_format_tools() -> List[MCPTool]: return [ MCPTool( @@ -5769,7 +6584,14 @@ def media_assert_tools() -> List[MCPTool]: checkpoint_tools, set_of_marks_tools, screen_state_tools, input_macro_tools, resilience_tools, ci_annotation_tools, clipboard_history_tools, audit_analysis_tools, - process_doc_tools, tween_drag_tools, plugin_sdk_tools, governance_tools, + process_doc_tools, tween_drag_tools, mouse_path_tools, field_entry_tools, + key_hold_tools, mouse_relative_tools, text_unicode_tools, + modifier_state_tools, grid_locator_tools, visual_match_tools, + color_region_tools, ssim_tools, feature_match_tools, shape_locator_tools, + window_layout_tools, window_arrange_tools, preprocess_tools, + monitor_layout_tools, actionability_tools, element_parse_tools, + hsv_segment_tools, text_regions_tools, edge_lines_tools, expect_poll_tools, + locator_chain_tools, rich_clipboard_tools, plugin_sdk_tools, governance_tools, credential_lease_tools, egress_tools, approval_testing_tools, trajectory_eval_tools, compliance_tools, agent_trace_tools, video_report_tools, fuzzy_tools, artifact_store_tools, image_dedup_tools, @@ -5789,7 +6611,7 @@ def media_assert_tools() -> List[MCPTool]: dedup_window_tools, sequence_gap_tools, optimistic_tools, outbox_tools, locale_collation_tools, confusables_tools, readability_tools, bidi_check_tools, list_format_tools, message_format_tools, - gettext_catalog_tools, + gettext_catalog_tools, checksum_tools, dataset_diff_tools, referential_tools, link_header_tools, multipart_tools, http_content_tools, cookie_jar_tools, http_conditional_tools, saga_tools, decision_table_tools, locator_repair_tools, diff --git a/je_auto_control/utils/mcp_server/tools/_handlers.py b/je_auto_control/utils/mcp_server/tools/_handlers.py index 882db17b..b6378854 100644 --- a/je_auto_control/utils/mcp_server/tools/_handlers.py +++ b/je_auto_control/utils/mcp_server/tools/_handlers.py @@ -2017,6 +2017,236 @@ def gettext_ngettext(po, msgid, msgid_plural, n): return _gettext_ngettext(po, msgid, msgid_plural, n) +def checksum_validate(scheme, number): + from je_auto_control.utils.executor.action_executor import _checksum_validate + return _checksum_validate(scheme, number) + + +def checksum_digit(scheme, partial): + from je_auto_control.utils.executor.action_executor import _checksum_digit + return _checksum_digit(scheme, partial) + + +def move_along_path(waypoints, easing="linear", per_segment_steps=20): + from je_auto_control.utils.executor.action_executor import _move_along_path + return _move_along_path(waypoints, easing, per_segment_steps) + + +def drag_path(waypoints, button="mouse_left", easing="linear", + per_segment_steps=20): + from je_auto_control.utils.executor.action_executor import _drag_path + return _drag_path(waypoints, button, easing, per_segment_steps) + + +def set_field_text(text, clear="select_all", paste=False, modifier="ctrl"): + from je_auto_control.utils.executor.action_executor import _set_field_text + return _set_field_text(text, clear, paste, modifier) + + +def hold_key(key, duration_s=1.0, rate_hz=None): + from je_auto_control.utils.executor.action_executor import _hold_key + return _hold_key(key, duration_s, rate_hz) + + +def move_mouse_relative(dx, dy): + from je_auto_control.utils.executor.action_executor import _move_mouse_relative + return _move_mouse_relative(dx, dy) + + +def type_unicode(text, modifier="ctrl"): + from je_auto_control.utils.executor.action_executor import _type_unicode + return _type_unicode(text, modifier) + + +def with_modifiers(modifiers, actions): + from je_auto_control.utils.executor.action_executor import _with_modifiers + return _with_modifiers(modifiers, actions) + + +def grid_cell(boxes, row, col, row_tolerance=10): + from je_auto_control.utils.executor.action_executor import _grid_cell + return _grid_cell(boxes, row, col, row_tolerance) + + +def match_template(template, min_score=0.8, scales=None, region=None, + method="ccoeff_normed"): + from je_auto_control.utils.executor.action_executor import _match_template + return _match_template(template, min_score, scales, region, method) + + +def match_template_all(template, min_score=0.8, max_results=20, nms_iou=0.3, + region=None): + from je_auto_control.utils.executor.action_executor import ( + _match_template_all) + return _match_template_all(template, min_score, max_results, nms_iou, region) + + +def match_masked(template, mask=None, min_score=0.9, region=None): + from je_auto_control.utils.executor.action_executor import _match_masked + return _match_masked(template, mask, min_score, region) + + +def match_masked_all(template, mask=None, min_score=0.9, max_results=20, + nms_iou=0.3, region=None): + from je_auto_control.utils.executor.action_executor import ( + _match_masked_all) + return _match_masked_all(template, mask, min_score, max_results, nms_iou, + region) + + +def find_color_region(rgb, tolerance=20, min_area=50, region=None): + from je_auto_control.utils.executor.action_executor import ( + _find_color_region) + return _find_color_region(rgb, tolerance, min_area, region) + + +def ssim_compare(reference, current=None, ignore=None, region=None): + from je_auto_control.utils.executor.action_executor import _ssim_compare + return _ssim_compare(reference, current, ignore, region) + + +def ssim_changed_regions(reference, current=None, ignore=None, threshold=0.35, + min_area=50, region=None): + from je_auto_control.utils.executor.action_executor import ( + _ssim_changed_regions) + return _ssim_changed_regions(reference, current, ignore, threshold, min_area, + region) + + +def feature_match(template, region=None, max_features=500, ratio=0.75, + min_inliers=10): + from je_auto_control.utils.executor.action_executor import _feature_match + return _feature_match(template, region, max_features, ratio, min_inliers) + + +def find_shapes(region=None, min_area=400, max_area=None): + from je_auto_control.utils.executor.action_executor import _find_shapes + return _find_shapes(region, min_area, max_area) + + +def find_rectangles(region=None, min_area=400, max_area=None, aspect_range=None, + epsilon=0.04): + from je_auto_control.utils.executor.action_executor import _find_rectangles + return _find_rectangles(region, min_area, max_area, aspect_range, epsilon) + + +def tile_rect(slot, screen=None, gap=0): + from je_auto_control.utils.executor.action_executor import _tile_rect + return _tile_rect(slot, screen, gap) + + +def grid_rects(rows, cols, screen=None, gap=0): + from je_auto_control.utils.executor.action_executor import _grid_rects + return _grid_rects(rows, cols, screen, gap) + + +def cascade_rects(count, screen=None, offset=30, size=None): + from je_auto_control.utils.executor.action_executor import _cascade_rects + return _cascade_rects(count, screen, offset, size) + + +def arrange_grid(titles, rows=None, cols=None, gap=0): + from je_auto_control.utils.executor.action_executor import _arrange_grid + return _arrange_grid(titles, rows, cols, gap) + + +def arrange_cascade(titles, offset=30): + from je_auto_control.utils.executor.action_executor import _arrange_cascade + return _arrange_cascade(titles, offset) + + +def preprocess_image(output_path, source=None, steps=None, scale=2.0, region=None, + block_size=31, c=11): + from je_auto_control.utils.executor.action_executor import _preprocess_image + return _preprocess_image(output_path, source, steps, scale, region, + block_size, c) + + +def enumerate_monitors(): + from je_auto_control.utils.executor.action_executor import _enumerate_monitors + return _enumerate_monitors() + + +def monitor_at_point(x, y): + from je_auto_control.utils.executor.action_executor import _monitor_at_point + return _monitor_at_point(x, y) + + +def wait_actionable(template, timeout_s=5.0, stable_for_s=0.3, min_score=0.8, + region=None): + from je_auto_control.utils.executor.action_executor import _wait_actionable + return _wait_actionable(template, timeout_s, stable_for_s, min_score, region) + + +def fuse_elements(ocr=None, icon=None, a11y=None, iou_threshold=0.9): + from je_auto_control.utils.executor.action_executor import _fuse_elements + return _fuse_elements(ocr, icon, a11y, iou_threshold) + + +def reading_order(elements, row_tol=12): + from je_auto_control.utils.executor.action_executor import _reading_order + return _reading_order(elements, row_tol) + + +def segment_hsv(lower_hsv, upper_hsv, min_area=50, region=None): + from je_auto_control.utils.executor.action_executor import _segment_hsv + return _segment_hsv(lower_hsv, upper_hsv, min_area, region) + + +def dominant_hue_regions(hue, hue_tol=10, sat_min=80, val_min=80, min_area=50, + region=None): + from je_auto_control.utils.executor.action_executor import ( + _dominant_hue_regions) + return _dominant_hue_regions(hue, hue_tol, sat_min, val_min, min_area, region) + + +def find_text_regions(min_area=60, max_area=None, merge=True, max_aspect=12.0, + region=None): + from je_auto_control.utils.executor.action_executor import _find_text_regions + return _find_text_regions(min_area, max_area, merge, max_aspect, region) + + +def find_text_lines(y_tolerance=8, region=None): + from je_auto_control.utils.executor.action_executor import _find_text_lines + return _find_text_lines(y_tolerance, region) + + +def find_lines(min_length=80, max_gap=10, orientation="any", region=None): + from je_auto_control.utils.executor.action_executor import _find_lines + return _find_lines(min_length, max_gap, orientation, region) + + +def find_grid(min_length=120, tol=10, region=None): + from je_auto_control.utils.executor.action_executor import _find_grid + return _find_grid(min_length, tol, region) + + +def find_separators(axis="horizontal", min_length=120, tol=10, region=None): + from je_auto_control.utils.executor.action_executor import _find_separators + return _find_separators(axis, min_length, tol, region) + + +def expect_poll(action, key=None, op="truthy", expected=None, timeout_s=5.0, + interval_s=0.25): + from je_auto_control.utils.executor.action_executor import _expect_poll + return _expect_poll(action, key, op, expected, timeout_s, interval_s) + + +def locate_chain(boxes, ops=None): + from je_auto_control.utils.executor.action_executor import _locate_chain + return _locate_chain(boxes, ops) + + +def set_clipboard_html(html, fragment_plaintext=None): + from je_auto_control.utils.executor.action_executor import _set_clipboard_html + return _set_clipboard_html(html, fragment_plaintext) + + +def get_clipboard_html(): + from je_auto_control.utils.executor.action_executor import _get_clipboard_html + return _get_clipboard_html() + + def detect_drift(reference, current, threshold=0.25, bins=10): from je_auto_control.utils.executor.action_executor import _detect_drift return _detect_drift(reference, current, threshold, bins) @@ -2392,6 +2622,35 @@ def wait_screen_stable(region: Optional[List[int]] = None, ).to_dict() +def wait_image_gone(image, detect_threshold: float = 1.0, + timeout_s: float = 10.0, poll_interval_s: float = 0.2, + gone_for_s: float = 0.0) -> Dict[str, Any]: + from je_auto_control.utils.executor.action_executor import _wait_image_gone + return _wait_image_gone(image, detect_threshold, timeout_s, + poll_interval_s, gone_for_s) + + +def wait_text_gone(text: str, timeout_s: float = 10.0, + poll_interval_s: float = 0.2, + gone_for_s: float = 0.0) -> Dict[str, Any]: + from je_auto_control.utils.executor.action_executor import _wait_text_gone + return _wait_text_gone(text, timeout_s, poll_interval_s, gone_for_s) + + +def wait_color(target_rgb, region=None, tolerance=10, min_fraction=0.5, + present=True, timeout_s=10.0, poll_interval_s=0.2): + from je_auto_control.utils.executor.action_executor import _wait_color + return _wait_color(target_rgb, region, tolerance, min_fraction, + present, timeout_s, poll_interval_s) + + +def wait_window_title(pattern, present=True, regex=True, timeout_s=10.0, + poll_interval_s=0.2): + from je_auto_control.utils.executor.action_executor import _wait_window_title + return _wait_window_title(pattern, present, regex, timeout_s, + poll_interval_s) + + def wait_for_file(path: str, timeout_s: float = 30.0, poll_interval_s: float = 0.25, stable_for_s: float = 1.0, @@ -2464,14 +2723,17 @@ def ocr_read_structure(region: Optional[List[int]] = None, def anchor_locate(anchor: Dict[str, Any], target: Dict[str, Any], relation: str = "near", - max_distance_px: float = 200.0) -> Dict[str, Any]: - from je_auto_control.utils.anchor_locator import ( - Locator, anchor_locate as _impl, - ) - return _impl( - anchor=Locator(**anchor), target=Locator(**target), - relation=relation, max_distance_px=float(max_distance_px), - ).to_dict() + max_distance_px: float = 200.0, + ordinal: int = 1) -> Dict[str, Any]: + from je_auto_control.utils.executor.action_executor import _anchor_locate + return _anchor_locate(anchor, target, relation, max_distance_px, ordinal) + + +def anchor_locate_all(anchor: Dict[str, Any], target: Dict[str, Any], + relation: str = "near", + max_distance_px: float = 200.0) -> Dict[str, Any]: + from je_auto_control.utils.executor.action_executor import _anchor_locate_all + return _anchor_locate_all(anchor, target, relation, max_distance_px) def anchor_click(anchor: Dict[str, Any], target: Dict[str, Any], diff --git a/je_auto_control/utils/modifier_state/__init__.py b/je_auto_control/utils/modifier_state/__init__.py new file mode 100644 index 00000000..b277e9a0 --- /dev/null +++ b/je_auto_control/utils/modifier_state/__init__.py @@ -0,0 +1,6 @@ +"""Hold modifier keys across a group of actions, releasing them safely.""" +from je_auto_control.utils.modifier_state.modifier_state import ( + hold_modifiers, plan_with_modifiers, +) + +__all__ = ["hold_modifiers", "plan_with_modifiers"] diff --git a/je_auto_control/utils/modifier_state/modifier_state.py b/je_auto_control/utils/modifier_state/modifier_state.py new file mode 100644 index 00000000..21e058b4 --- /dev/null +++ b/je_auto_control/utils/modifier_state/modifier_state.py @@ -0,0 +1,57 @@ +"""Hold modifier keys across a group of actions, releasing them safely. + +``hotkey`` presses a set of keys and releases them immediately — fine for a +one-shot chord, but there is no way to hold ``ctrl`` (or ``shift``) *down across +several independent actions* (range-select with shift-clicks, ctrl-clicks to +multi-select) and be sure the modifiers are released even if one of those actions +raises. + +:func:`plan_with_modifiers` wraps an op-step list with press / release steps and +is pure / unit-testable; :func:`hold_modifiers` is a context manager that presses +on enter and releases (in reverse) on exit — including on exception — dispatching +through an injectable ``sink``. Imports no ``PySide6``. +""" +import contextlib +from typing import Any, Callable, Dict, Iterator, List, Optional, Sequence + +Sink = Callable[[Dict[str, Any]], None] + + +def plan_with_modifiers(steps: Sequence[Dict[str, Any]], + modifiers: Sequence[str]) -> List[Dict[str, Any]]: + """Wrap ``steps`` with press-modifiers (in order) … release-modifiers (reversed).""" + mods = list(modifiers) + plan: List[Dict[str, Any]] = [{"op": "press", "key": mod} for mod in mods] + plan.extend(dict(step) for step in steps) + plan.extend({"op": "release", "key": mod} for mod in reversed(mods)) + return plan + + +def _default_sink(event: Dict[str, Any]) -> None: + """Default dispatch: drive the real keyboard backend.""" + from je_auto_control.wrapper.auto_control_keyboard import ( + press_keyboard_key, release_keyboard_key) + if event["op"] == "press": + press_keyboard_key(event["key"]) + elif event["op"] == "release": + release_keyboard_key(event["key"]) + + +@contextlib.contextmanager +def hold_modifiers(modifiers: Sequence[str], *, + sink: Optional[Sink] = None) -> Iterator[List[str]]: + """Hold ``modifiers`` down for the ``with`` block; release them on exit. + + Modifiers are pressed in order on entry and released in reverse order on + exit — even if the body raises — so a stuck modifier can never leak. Dispatch + goes through ``sink`` (default: the real keyboard backend). + """ + dispatch = sink or _default_sink + mods = list(modifiers) + for mod in mods: + dispatch({"op": "press", "key": mod}) + try: + yield mods + finally: + for mod in reversed(mods): + dispatch({"op": "release", "key": mod}) diff --git a/je_auto_control/utils/monitor_layout/__init__.py b/je_auto_control/utils/monitor_layout/__init__.py new file mode 100644 index 00000000..0dcdd9a0 --- /dev/null +++ b/je_auto_control/utils/monitor_layout/__init__.py @@ -0,0 +1,9 @@ +"""Multi-monitor / virtual-desktop geometry (which monitor, where, remapping).""" +from je_auto_control.utils.monitor_layout.monitor_layout import ( + Monitor, enumerate_monitors, monitor_at_point, monitor_for_window, + primary_monitor, remap_point, to_local, to_virtual, virtual_bounds, +) + +__all__ = ["Monitor", "enumerate_monitors", "monitor_at_point", + "monitor_for_window", "primary_monitor", "remap_point", "to_local", + "to_virtual", "virtual_bounds"] diff --git a/je_auto_control/utils/monitor_layout/monitor_layout.py b/je_auto_control/utils/monitor_layout/monitor_layout.py new file mode 100644 index 00000000..2fb46f32 --- /dev/null +++ b/je_auto_control/utils/monitor_layout/monitor_layout.py @@ -0,0 +1,158 @@ +"""Multi-monitor / virtual-desktop geometry — which monitor, where, and remapping. + +``snap_window`` / ``arrange_grid`` / the layout planner all take a single primary +``(width, height)`` — they are monitor-blind and cannot tile on the second display +or handle a negative-origin virtual desktop, and ``coordinate_space`` only rescales +a model grid. This adds the missing physical layer: enumerate the monitors, find the +union virtual bounds, ask which monitor contains a point or a window, convert between +virtual and per-monitor-local coordinates, and remap a point to the equivalent spot +on another monitor. + +The geometry is pure arithmetic over plain ``Monitor`` dataclasses, so it is fully +unit-testable; only ``enumerate_monitors``' default provider touches the OS (and it +is injectable). Imports no ``PySide6``. +""" +from dataclasses import asdict, dataclass +from typing import Any, Callable, Dict, List, Optional, Sequence, Tuple + +Rect = Tuple[int, int, int, int] +MonitorRows = Callable[[], List[Dict[str, Any]]] + + +@dataclass(frozen=True) +class Monitor: + """One display: virtual-desktop origin ``(x, y)``, size, DPI ``scale``, primary.""" + + index: int + x: int + y: int + width: int + height: int + scale: float = 1.0 + primary: bool = False + work_area: Optional[Rect] = None + + @property + def bounds(self) -> Rect: + """Full monitor rectangle ``(x, y, width, height)`` in virtual coordinates.""" + return (self.x, self.y, self.width, self.height) + + @property + def work(self) -> Rect: + """Usable work area (minus taskbar/dock), or the full bounds if unknown.""" + return self.work_area if self.work_area is not None else self.bounds + + def contains(self, x: int, y: int) -> bool: + """True when virtual point ``(x, y)`` falls inside this monitor.""" + return (self.x <= x < self.x + self.width + and self.y <= y < self.y + self.height) + + def to_dict(self) -> Dict[str, Any]: + """Return the monitor as a plain dict (``work_area`` as a list).""" + data = asdict(self) + data["work"] = list(self.work) + return data + + +def virtual_bounds(monitors: Sequence[Monitor]) -> Rect: + """Return the union bounding box of all monitors (origin may be negative).""" + if not monitors: + raise ValueError("no monitors") + left = min(monitor.x for monitor in monitors) + top = min(monitor.y for monitor in monitors) + right = max(monitor.x + monitor.width for monitor in monitors) + bottom = max(monitor.y + monitor.height for monitor in monitors) + return (left, top, right - left, bottom - top) + + +def primary_monitor(monitors: Sequence[Monitor]) -> Optional[Monitor]: + """Return the monitor flagged ``primary`` (or the first one, or ``None``).""" + for monitor in monitors: + if monitor.primary: + return monitor + return monitors[0] if monitors else None + + +def monitor_at_point(monitors: Sequence[Monitor], x: int, + y: int) -> Optional[Monitor]: + """Return the monitor containing virtual point ``(x, y)`` (or ``None``).""" + for monitor in monitors: + if monitor.contains(int(x), int(y)): + return monitor + return None + + +def _overlap(rect: Rect, monitor: Monitor) -> int: + rx, ry, rw, rh = rect + left = max(rx, monitor.x) + top = max(ry, monitor.y) + right = min(rx + rw, monitor.x + monitor.width) + bottom = min(ry + rh, monitor.y + monitor.height) + return max(0, right - left) * max(0, bottom - top) + + +def monitor_for_window(rect: Rect, + monitors: Sequence[Monitor]) -> Optional[Monitor]: + """Return the monitor a window ``rect`` mostly occupies (max overlap area).""" + best: Optional[Monitor] = None + best_area = 0 + for monitor in monitors: + area = _overlap(rect, monitor) + if area > best_area: + best_area, best = area, monitor + return best + + +def to_local(monitors: Sequence[Monitor], x: int, + y: int) -> Optional[Tuple[int, int, int]]: + """Map a virtual point to ``(monitor_index, local_x, local_y)`` (or ``None``).""" + monitor = monitor_at_point(monitors, x, y) + if monitor is None: + return None + return (monitor.index, int(x) - monitor.x, int(y) - monitor.y) + + +def to_virtual(monitor: Monitor, local_x: int, local_y: int) -> Tuple[int, int]: + """Map a monitor-local point back to virtual-desktop coordinates.""" + return (monitor.x + int(local_x), monitor.y + int(local_y)) + + +def remap_point(src: Monitor, dst: Monitor, local_x: int, + local_y: int) -> Tuple[int, int]: + """Remap a ``src``-local point to the equivalent relative spot on ``dst`` (local). + + Preserves the fractional position within the monitor, so it works across + differing resolutions and physical sizes. + """ + frac_x = (local_x / src.width) if src.width else 0.0 + frac_y = (local_y / src.height) if src.height else 0.0 + return (round(frac_x * dst.width), round(frac_y * dst.height)) + + +def _mss_rows() -> List[Dict[str, Any]]: + import mss + with mss.mss() as screen: + monitors = screen.monitors + return [{"x": int(m["left"]), "y": int(m["top"]), + "width": int(m["width"]), "height": int(m["height"])} + for m in monitors[1:]] # [0] is the combined desktop + + +def enumerate_monitors(provider: Optional[MonitorRows] = None) -> List[Monitor]: + """Return the connected monitors as ``Monitor`` objects, primary-first index 0. + + ``provider`` yields raw ``{x, y, width, height[, scale, primary, work_area]}`` + rows; the default reads them from ``mss``. Injecting a provider makes the whole + module headless-testable. + """ + rows = (provider or _mss_rows)() + monitors: List[Monitor] = [] + for index, row in enumerate(rows): + work = row.get("work_area") + monitors.append(Monitor( + index=index, x=int(row["x"]), y=int(row["y"]), + width=int(row["width"]), height=int(row["height"]), + scale=float(row.get("scale", 1.0)), + primary=bool(row.get("primary", index == 0)), + work_area=tuple(work) if work else None)) + return monitors diff --git a/je_auto_control/utils/mouse_path/__init__.py b/je_auto_control/utils/mouse_path/__init__.py new file mode 100644 index 00000000..369cc9a1 --- /dev/null +++ b/je_auto_control/utils/mouse_path/__init__.py @@ -0,0 +1,6 @@ +"""Multi-waypoint mouse gestures (move or drag through a polyline of points).""" +from je_auto_control.utils.mouse_path.mouse_path import ( + drag_path, move_along_path, path_easings, plan_path, +) + +__all__ = ["drag_path", "move_along_path", "path_easings", "plan_path"] diff --git a/je_auto_control/utils/mouse_path/mouse_path.py b/je_auto_control/utils/mouse_path/mouse_path.py new file mode 100644 index 00000000..121ca93e --- /dev/null +++ b/je_auto_control/utils/mouse_path/mouse_path.py @@ -0,0 +1,86 @@ +"""Multi-waypoint mouse gestures: move or drag through a polyline of points. + +``humanize.humanized_path`` and ``tween_drag`` only interpolate a *single* +start -> end hop. Real gestures — signatures, marquee/rubber-band selections, +drag-through-multiple-drop-targets, shape gestures — need an arbitrary chain of +waypoints with the button optionally held down across the whole path. + +:func:`plan_path` is pure point math (reusing the named easings from +``tween_drag``) and is unit-testable on its own; :func:`move_along_path` and +:func:`drag_path` dispatch through an injectable ``sink`` so the gesture is +tested without real input. Imports no ``PySide6``. +""" +from typing import Any, Callable, Dict, List, Optional, Sequence + +from je_auto_control.utils.tween_drag.tween_drag import easing_names, tween_points + +Point = Sequence[int] +Sink = Callable[[Dict[str, Any]], None] + + +def plan_path(waypoints: Sequence[Point], *, easing: str = "linear", + per_segment_steps: int = 20) -> List[List[int]]: + """Return the eased point list passing through every waypoint in order. + + Each consecutive pair is interpolated with ``per_segment_steps`` eased steps + (named easings from ``tween_drag``); shared junction points are not + duplicated. Fewer than two waypoints yields the points unchanged. + """ + points: List[List[int]] = [] + if not waypoints: + return points + if len(waypoints) == 1: + first = waypoints[0] + return [[int(first[0]), int(first[1])]] + for index in range(len(waypoints) - 1): + segment = tween_points(waypoints[index], waypoints[index + 1], + per_segment_steps, easing) + points.extend(segment[1:] if index else segment) + return points + + +def _default_sink(event: Dict[str, Any]) -> None: + """Default dispatch: drive the real mouse backend.""" + from je_auto_control.wrapper.auto_control_mouse import ( + press_mouse, release_mouse, set_mouse_position) + x, y = int(event["x"]), int(event["y"]) + set_mouse_position(x, y) + op = event["op"] + if op == "press": + press_mouse(event.get("button", "mouse_left"), x, y) + elif op == "release": + release_mouse(event.get("button", "mouse_left"), x, y) + + +def move_along_path(waypoints: Sequence[Point], *, easing: str = "linear", + per_segment_steps: int = 20, + sink: Optional[Sink] = None) -> Dict[str, Any]: + """Move the pointer through ``waypoints`` (no button held).""" + points = plan_path(waypoints, easing=easing, + per_segment_steps=per_segment_steps) + dispatch = sink or _default_sink + for x, y in points: + dispatch({"op": "move", "x": x, "y": y}) + return {"points": len(points), "path": points} + + +def drag_path(waypoints: Sequence[Point], *, button: str = "mouse_left", + easing: str = "linear", per_segment_steps: int = 20, + sink: Optional[Sink] = None) -> Dict[str, Any]: + """Press at the first waypoint, move through the path, release at the last.""" + points = plan_path(waypoints, easing=easing, + per_segment_steps=per_segment_steps) + if not points: + return {"points": 0, "path": points} + dispatch = sink or _default_sink + first, last = points[0], points[-1] + dispatch({"op": "press", "button": button, "x": first[0], "y": first[1]}) + for x, y in points: + dispatch({"op": "move", "x": x, "y": y}) + dispatch({"op": "release", "button": button, "x": last[0], "y": last[1]}) + return {"points": len(points), "path": points} + + +def path_easings() -> List[str]: + """Return the available easing names (shared with ``tween_drag``).""" + return easing_names() diff --git a/je_auto_control/utils/mouse_relative/__init__.py b/je_auto_control/utils/mouse_relative/__init__.py new file mode 100644 index 00000000..ffdd7a7c --- /dev/null +++ b/je_auto_control/utils/mouse_relative/__init__.py @@ -0,0 +1,6 @@ +"""Relative mouse movement — move by a delta from the current position.""" +from je_auto_control.utils.mouse_relative.mouse_relative import ( + move_mouse_relative, relative_target, +) + +__all__ = ["move_mouse_relative", "relative_target"] diff --git a/je_auto_control/utils/mouse_relative/mouse_relative.py b/je_auto_control/utils/mouse_relative/mouse_relative.py new file mode 100644 index 00000000..088f36df --- /dev/null +++ b/je_auto_control/utils/mouse_relative/mouse_relative.py @@ -0,0 +1,53 @@ +"""Relative mouse movement — move by a delta from the current position. + +The mouse wrapper exposes only absolute ``set_mouse_position``; there is no +"nudge the pointer by (dx, dy)" (pynput / PyAutoGUI ``moveRel`` staple), which is +what relative-pointer / canvas / FPS-style apps and incremental drags need. + +:func:`relative_target` is the pure arithmetic (current + delta) and is +unit-testable; :func:`move_mouse_relative` reads the live position and sets the +new one, with both the getter and setter injectable so it is tested without a +real pointer. Imports no ``PySide6``. +""" +from typing import Any, Callable, Dict, Optional, Tuple + +from je_auto_control.utils.exception.exceptions import AutoControlMouseException + +PositionGetter = Callable[[], Optional[Tuple[int, int]]] +PositionSetter = Callable[[int, int], Any] + + +def relative_target(current: Tuple[int, int], dx: int, dy: int) -> Tuple[int, int]: + """Return ``current`` offset by ``(dx, dy)`` as an integer ``(x, y)``.""" + return int(current[0]) + int(dx), int(current[1]) + int(dy) + + +def move_mouse_relative(dx: int, dy: int, *, + get_position: Optional[PositionGetter] = None, + set_position: Optional[PositionSetter] = None, + ) -> Dict[str, Any]: + """Move the pointer by ``(dx, dy)`` relative to where it is now. + + ``get_position`` / ``set_position`` default to the real mouse wrapper but are + injectable for headless tests. Raises :class:`AutoControlMouseException` if + the current position cannot be read. + """ + getter = get_position or _default_get_position + setter = set_position or _default_set_position + current = getter() + if current is None: + raise AutoControlMouseException("could not read the current mouse position") + target = relative_target((int(current[0]), int(current[1])), dx, dy) + setter(target[0], target[1]) + return {"from": [int(current[0]), int(current[1])], + "to": [target[0], target[1]], "delta": [int(dx), int(dy)]} + + +def _default_get_position() -> Optional[Tuple[int, int]]: + from je_auto_control.wrapper.auto_control_mouse import get_mouse_position + return get_mouse_position() + + +def _default_set_position(x: int, y: int) -> Any: + from je_auto_control.wrapper.auto_control_mouse import set_mouse_position + return set_mouse_position(x, y) diff --git a/je_auto_control/utils/preprocess/__init__.py b/je_auto_control/utils/preprocess/__init__.py new file mode 100644 index 00000000..ad8aaebb --- /dev/null +++ b/je_auto_control/utils/preprocess/__init__.py @@ -0,0 +1,8 @@ +"""Image pre-processing for OCR / template matching (grayscale, binarize, deskew, …).""" +from je_auto_control.utils.preprocess.preprocess import ( + binarize, denoise, deskew, detect_skew_angle, enhance_contrast, + preprocess_image, to_grayscale, upscale, +) + +__all__ = ["binarize", "denoise", "deskew", "detect_skew_angle", + "enhance_contrast", "preprocess_image", "to_grayscale", "upscale"] diff --git a/je_auto_control/utils/preprocess/preprocess.py b/je_auto_control/utils/preprocess/preprocess.py new file mode 100644 index 00000000..0f62e63e --- /dev/null +++ b/je_auto_control/utils/preprocess/preprocess.py @@ -0,0 +1,177 @@ +"""Image pre-processing for OCR / template matching — grayscale, binarize, deskew, upscale. + +``locate_text`` / ``ocr_read_structure`` and ``match_template`` feed the *raw* capture +to the OCR engine / matcher; small UI text, dark themes and low contrast wreck both. +This is the standard pre-step pipeline — grayscale → upscale → binarize → deskew → +denoise → CLAHE contrast — that multiplies their accuracy, with no preprocessing seam +anywhere in the framework today. + +Every function runs on an injectable ``haystack`` image (ndarray / path / PIL, default: +grab the screen / ``region``) and returns a NumPy ndarray you can pass straight to an +OCR / match call or save. OpenCV + NumPy come in via the project's ``je_open_cv`` +dependency and are imported lazily. Imports no ``PySide6``. +""" +from typing import Any, Optional, Sequence + +ImageSource = Any +_INTERP = ("nearest", "linear", "cubic", "lanczos") + + +def _to_array(source: ImageSource): + """Load a path / ndarray / PIL image as a uint8 ndarray (as stored).""" + import cv2 + import numpy as np + if hasattr(source, "shape"): + return np.asarray(source) + if isinstance(source, (str, bytes)) or hasattr(source, "__fspath__"): + array = cv2.imread(str(source), cv2.IMREAD_UNCHANGED) + if array is None: + raise ValueError(f"could not read image: {source!r}") + return array + return np.asarray(source) + + +def _resolve(haystack: Optional[ImageSource], region: Optional[Sequence[int]]): + import numpy as np + if haystack is not None: + return _to_array(haystack) + from je_auto_control.utils.cv2_utils.screenshot import pil_screenshot + image = pil_screenshot(screen_region=list(region) if region else None) + return np.asarray(image.convert("RGB")) + + +def _gray(array): + import cv2 + if array.ndim == 2: + return array + code = cv2.COLOR_BGRA2GRAY if array.shape[2] == 4 else cv2.COLOR_BGR2GRAY + return cv2.cvtColor(array, code) + + +def to_grayscale(haystack: Optional[ImageSource] = None, *, + region: Optional[Sequence[int]] = None): + """Return the image as a single-channel grayscale ndarray.""" + return _gray(_resolve(haystack, region)) + + +def upscale(haystack: Optional[ImageSource] = None, *, + region: Optional[Sequence[int]] = None, scale: float = 2.0, + interp: str = "cubic"): + """Return the image resized by ``scale`` — enlarge small UI text before OCR.""" + import cv2 + if interp not in _INTERP: + raise ValueError(f"unknown interp: {interp!r}") + table = {"nearest": cv2.INTER_NEAREST, "linear": cv2.INTER_LINEAR, + "cubic": cv2.INTER_CUBIC, "lanczos": cv2.INTER_LANCZOS4} + array = _resolve(haystack, region) + height, width = array.shape[:2] + size = (max(1, round(width * float(scale))), max(1, round(height * float(scale)))) + return cv2.resize(array, size, interpolation=table[interp]) + + +def binarize(haystack: Optional[ImageSource] = None, *, + region: Optional[Sequence[int]] = None, method: str = "otsu", + block_size: int = 31, c: int = 11): + """Return a black/white image. ``method``: otsu / adaptive_mean / adaptive_gaussian.""" + import cv2 + gray = _gray(_resolve(haystack, region)) + if method == "otsu": + _, result = cv2.threshold(gray, 0, 255, + cv2.THRESH_BINARY + cv2.THRESH_OTSU) + return result + table = {"adaptive_mean": cv2.ADAPTIVE_THRESH_MEAN_C, + "adaptive_gaussian": cv2.ADAPTIVE_THRESH_GAUSSIAN_C} + if method not in table: + raise ValueError(f"unknown method: {method!r}") + block = int(block_size) | 1 # adaptiveThreshold needs odd + return cv2.adaptiveThreshold(gray, 255, table[method], cv2.THRESH_BINARY, + block, int(c)) + + +def denoise(haystack: Optional[ImageSource] = None, *, + region: Optional[Sequence[int]] = None, strength: int = 7): + """Return a denoised grayscale image (non-local means).""" + import cv2 + return cv2.fastNlMeansDenoising(_gray(_resolve(haystack, region)), None, + float(strength), 7, 21) + + +def enhance_contrast(haystack: Optional[ImageSource] = None, *, + region: Optional[Sequence[int]] = None, clip: float = 2.0, + grid: int = 8): + """Return a CLAHE contrast-enhanced grayscale image (rescues dark/low-contrast UI).""" + import cv2 + clahe = cv2.createCLAHE(clipLimit=float(clip), + tileGridSize=(int(grid), int(grid))) + return clahe.apply(_gray(_resolve(haystack, region))) + + +def detect_skew_angle(haystack: Optional[ImageSource] = None, *, + region: Optional[Sequence[int]] = None, + max_angle: float = 15.0) -> float: + """Return the text skew angle in degrees within ``[-max_angle, max_angle]`` (else 0).""" + import cv2 + gray = _gray(_resolve(haystack, region)) + _, mask = cv2.threshold(gray, 0, 255, + cv2.THRESH_BINARY_INV + cv2.THRESH_OTSU) + coords = cv2.findNonZero(mask) + if coords is None: + return 0.0 + angle = cv2.minAreaRect(coords)[-1] % 90 # version-robust normalisation + if angle > 45: + angle -= 90 + return round(float(angle), 3) if abs(angle) <= float(max_angle) else 0.0 + + +def deskew(haystack: Optional[ImageSource] = None, *, + region: Optional[Sequence[int]] = None, max_angle: float = 15.0): + """Return the image rotated to remove text skew (no-op when none is detected).""" + import cv2 + array = _resolve(haystack, region) + angle = detect_skew_angle(array, max_angle=max_angle) + if abs(angle) < 1e-9: + return array + height, width = array.shape[:2] + matrix = cv2.getRotationMatrix2D((width / 2.0, height / 2.0), angle, 1.0) + return cv2.warpAffine(array, matrix, (width, height), + flags=cv2.INTER_CUBIC, borderMode=cv2.BORDER_REPLICATE) + + +def _step_grayscale(array, **_kwargs): + return to_grayscale(array) + + +def _step_upscale(array, *, scale: float, **_kwargs): + return upscale(array, scale=scale) + + +def _step_binarize(array, *, block_size: int, c: int, **_kwargs): + return binarize(array, block_size=block_size, c=c) + + +_STEPS = { + "grayscale": _step_grayscale, + "upscale": _step_upscale, + "binarize": _step_binarize, + "denoise": lambda array, **_kwargs: denoise(array), + "deskew": lambda array, **_kwargs: deskew(array), + "contrast": lambda array, **_kwargs: enhance_contrast(array), +} + + +def preprocess_image(haystack: Optional[ImageSource] = None, *, + region: Optional[Sequence[int]] = None, + steps: Sequence[str] = ("grayscale", "upscale", "binarize"), + scale: float = 2.0, block_size: int = 31, c: int = 11): + """Apply a pipeline of named preprocessing ``steps`` in order, returning the result. + + Steps: ``grayscale``, ``upscale`` (by ``scale``), ``binarize`` (otsu, tuned by + ``block_size`` / ``c`` only for the adaptive variants), ``denoise``, ``deskew``, + ``contrast`` (CLAHE). Unknown step names raise ``ValueError``. + """ + array = _resolve(haystack, region) + for step in steps: + if step not in _STEPS: + raise ValueError(f"unknown step: {step!r}") + array = _STEPS[step](array, scale=scale, block_size=block_size, c=c) + return array diff --git a/je_auto_control/utils/rich_clipboard/__init__.py b/je_auto_control/utils/rich_clipboard/__init__.py new file mode 100644 index 00000000..09f3548d --- /dev/null +++ b/je_auto_control/utils/rich_clipboard/__init__.py @@ -0,0 +1,7 @@ +"""Rich clipboard formats — HTML (CF_HTML) build / parse / get / set.""" +from je_auto_control.utils.rich_clipboard.rich_clipboard import ( + build_cf_html, get_clipboard_html, parse_cf_html, set_clipboard_html, +) + +__all__ = ["build_cf_html", "get_clipboard_html", "parse_cf_html", + "set_clipboard_html"] diff --git a/je_auto_control/utils/rich_clipboard/rich_clipboard.py b/je_auto_control/utils/rich_clipboard/rich_clipboard.py new file mode 100644 index 00000000..0719d486 --- /dev/null +++ b/je_auto_control/utils/rich_clipboard/rich_clipboard.py @@ -0,0 +1,142 @@ +"""Rich clipboard formats — HTML (CF_HTML) get / set, for rich paste into Office. + +The base ``clipboard`` module handles plain text (CF_UNICODETEXT) and image (CF_DIB) +only; pasting formatted content into Word / Outlook / a rich editor needs the +``CF_HTML`` format, whose ``Version / StartHTML / EndHTML / StartFragment / +EndFragment`` *byte-offset* header is notoriously error-prone to build. This module's +``build_cf_html`` / ``parse_cf_html`` compute and recover that header in pure Python — +fully unit-testable round-trip — and the Windows ``get_clipboard_html`` / +``set_clipboard_html`` wrap them over the Win32 clipboard. + +The byte-offset math is platform-independent and headless-testable; only the actual +clipboard I/O is Win32 (raising ``RuntimeError`` elsewhere, like the base module). +Imports no ``PySide6``. +""" +import re +import sys +from typing import Optional + +_HEADER = ("Version:0.9\r\n" + "StartHTML:{:010d}\r\n" + "EndHTML:{:010d}\r\n" + "StartFragment:{:010d}\r\n" + "EndFragment:{:010d}\r\n") +_PRE = "" +_POST = "" +_START_MARK = "" +_END_MARK = "" +_HTML_FORMAT_NAME = "HTML Format" + + +def build_cf_html(html: str) -> bytes: + """Wrap an HTML fragment in a valid ``CF_HTML`` clipboard payload (UTF-8 bytes). + + The fixed-width 10-digit offsets make the header length constant, so the byte + offsets can be computed in a single pass. + """ + if not isinstance(html, str): + raise TypeError("build_cf_html expects a str") + header_len = len(_HEADER.format(0, 0, 0, 0).encode("utf-8")) + start_html = header_len + start_fragment = start_html + len(_PRE.encode("utf-8")) + end_fragment = start_fragment + len(html.encode("utf-8")) + end_html = end_fragment + len(_POST.encode("utf-8")) + header = _HEADER.format(start_html, end_html, start_fragment, end_fragment) + return (header + _PRE + html + _POST).encode("utf-8") + + +def _header_offset(text: str, name: str) -> Optional[int]: + match = re.search(rf"{name}:(\d+)", text) + return int(match.group(1)) if match else None + + +def parse_cf_html(blob) -> str: + """Extract the HTML fragment from a ``CF_HTML`` payload (bytes or str). + + Prefers the ``StartFragment`` / ``EndFragment`` comment markers, falling back to + the header's byte offsets. + """ + raw = bytes(blob) if isinstance(blob, (bytes, bytearray)) else None + text = (raw.decode("utf-8", "replace") if raw is not None else str(blob)) + start = text.find(_START_MARK) + end = text.find(_END_MARK) + if start != -1 and end != -1: + return text[start + len(_START_MARK):end] + start_offset = _header_offset(text, "StartFragment") + end_offset = _header_offset(text, "EndFragment") + if raw is not None and start_offset is not None and end_offset is not None: + return raw[start_offset:end_offset].decode("utf-8", "replace") + return text + + +def set_clipboard_html(html: str, *, fragment_plaintext: Optional[str] = None) -> None: + """Put an HTML fragment on the clipboard as ``CF_HTML`` (Windows only). + + ``fragment_plaintext`` is also placed as plain text so apps that ignore HTML still + paste something. Raises ``RuntimeError`` on non-Windows platforms. + """ + if not isinstance(html, str): + raise TypeError("set_clipboard_html expects a str") + if not sys.platform.startswith("win"): + raise RuntimeError("set_clipboard_html is only supported on Windows") + _win_set_html(build_cf_html(html), fragment_plaintext) + + +def get_clipboard_html() -> Optional[str]: + """Return the clipboard's HTML fragment, or ``None`` (Windows only).""" + if not sys.platform.startswith("win"): + raise RuntimeError("get_clipboard_html is only supported on Windows") + blob = _win_get_html() + return parse_cf_html(blob) if blob is not None else None + + +def _html_format_id(): + import ctypes + return ctypes.windll.user32.RegisterClipboardFormatW(_HTML_FORMAT_NAME) + + +def _win_set_html(cf_html: bytes, fragment_plaintext: Optional[str]) -> None: + import ctypes + from ctypes import wintypes + from je_auto_control.utils.clipboard.clipboard import set_clipboard + user32, kernel32 = ctypes.windll.user32, ctypes.windll.kernel32 + kernel32.GlobalAlloc.restype = wintypes.HGLOBAL + kernel32.GlobalLock.restype = ctypes.c_void_p + if fragment_plaintext is not None: + set_clipboard(fragment_plaintext) # seeds CF_UNICODETEXT first + if not user32.OpenClipboard(None): + raise RuntimeError("OpenClipboard failed") + try: + if fragment_plaintext is None: + user32.EmptyClipboard() + handle = kernel32.GlobalAlloc(0x0002, len(cf_html) + 1) + if not handle: + raise RuntimeError("GlobalAlloc failed") + pointer = kernel32.GlobalLock(handle) + ctypes.memmove(pointer, cf_html + b"\x00", len(cf_html) + 1) + kernel32.GlobalUnlock(handle) + if not user32.SetClipboardData(_html_format_id(), handle): + raise RuntimeError("SetClipboardData(CF_HTML) failed") + finally: + user32.CloseClipboard() + + +def _win_get_html() -> Optional[bytes]: + import ctypes + from ctypes import wintypes + user32, kernel32 = ctypes.windll.user32, ctypes.windll.kernel32 + user32.GetClipboardData.restype = wintypes.HANDLE + kernel32.GlobalLock.restype = ctypes.c_void_p + if not user32.OpenClipboard(None): + raise RuntimeError("OpenClipboard failed") + try: + handle = user32.GetClipboardData(_html_format_id()) + if not handle: + return None + pointer = kernel32.GlobalLock(handle) + size = kernel32.GlobalSize(handle) + data = ctypes.string_at(pointer, size) + kernel32.GlobalUnlock(handle) + return data.split(b"\x00", 1)[0] + finally: + user32.CloseClipboard() diff --git a/je_auto_control/utils/shape_locator/__init__.py b/je_auto_control/utils/shape_locator/__init__.py new file mode 100644 index 00000000..971fffad --- /dev/null +++ b/je_auto_control/utils/shape_locator/__init__.py @@ -0,0 +1,6 @@ +"""Locate UI elements by edge/contour detection (rectangles / shapes, no template).""" +from je_auto_control.utils.shape_locator.shape_locator import ( + find_rectangles, find_shapes, +) + +__all__ = ["find_rectangles", "find_shapes"] diff --git a/je_auto_control/utils/shape_locator/shape_locator.py b/je_auto_control/utils/shape_locator/shape_locator.py new file mode 100644 index 00000000..f1a01f6d --- /dev/null +++ b/je_auto_control/utils/shape_locator/shape_locator.py @@ -0,0 +1,99 @@ +"""Locate UI elements by edge/contour detection — buttons, cards, fields, no template. + +Template matching needs a reference image, colour-region needs a known colour, OCR +needs text — none of them answers "where are the rectangular, clickable regions on +this screen?". This runs Canny edge detection plus contour extraction and returns +the bounding boxes of the distinct shapes, optionally filtered to rectangles, so a +script can enumerate cards / buttons / inputs structurally and act on the Nth one +without ever supplying a sample image. + +It runs on an injectable ``haystack`` image (ndarray / path / PIL), so it is +unit-testable on synthetic arrays without a real screen. OpenCV + NumPy come in via +the project's ``je_open_cv`` dependency and are imported lazily. Imports no +``PySide6``. +""" +from typing import Any, Dict, List, Optional, Sequence, Tuple + +from je_auto_control.utils.visual_match.visual_match import _haystack_gray + +ImageSource = Any +AspectRange = Optional[Tuple[float, float]] +_CANNY_LOW = 50 +_CANNY_HIGH = 150 + + +def _contours(gray): + """Canny edges (dilated to close gaps) -> external contours, version-agnostic.""" + import cv2 + edges = cv2.Canny(gray, _CANNY_LOW, _CANNY_HIGH) + kernel = cv2.getStructuringElement(cv2.MORPH_RECT, (3, 3)) + edges = cv2.dilate(edges, kernel, iterations=1) + found = cv2.findContours(edges, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE) + return found[0] if len(found) == 2 else found[1] + + +def _box(contour) -> Dict[str, Any]: + """Bounding box of one contour as ``{x,y,width,height,area,center,aspect}``.""" + import cv2 + x, y, width, height = cv2.boundingRect(contour) + return {"x": int(x), "y": int(y), "width": int(width), "height": int(height), + "area": int(width * height), + "center": [int(x + width // 2), int(y + height // 2)], + "aspect": round(width / height, 3) if height else 0.0} + + +def _passes(box: Dict[str, Any], min_area: int, max_area: Optional[int], + aspect_range: AspectRange) -> bool: + """Apply the size / aspect-ratio filters to one box.""" + if box["area"] < int(min_area): + return False + if max_area is not None and box["area"] > int(max_area): + return False + if aspect_range is not None and not ( + aspect_range[0] <= box["aspect"] <= aspect_range[1]): + return False + return True + + +def _is_rectangle(contour, epsilon: float) -> bool: + """True when the contour approximates to a convex 4-gon (a rectangle).""" + import cv2 + perimeter = cv2.arcLength(contour, True) + approx = cv2.approxPolyDP(contour, epsilon * perimeter, True) + return len(approx) == 4 and cv2.isContourConvex(approx) + + +def find_shapes(haystack: Optional[ImageSource] = None, *, + region: Optional[Sequence[int]] = None, min_area: int = 400, + max_area: Optional[int] = None) -> List[Dict[str, Any]]: + """Return bounding boxes of all distinct shapes on screen, largest first. + + Each result is ``{x, y, width, height, area, center, aspect}`` where ``area`` + is the bounding-box area. ``haystack`` is an ndarray / path / PIL image + (default: grab the screen / ``region``); ``min_area`` / ``max_area`` drop + specks and full-frame borders. + """ + boxes = [_box(contour) for contour in _contours(_haystack_gray(haystack, region))] + boxes = [box for box in boxes if _passes(box, min_area, max_area, None)] + boxes.sort(key=lambda box: box["area"], reverse=True) + return boxes + + +def find_rectangles(haystack: Optional[ImageSource] = None, *, + region: Optional[Sequence[int]] = None, min_area: int = 400, + max_area: Optional[int] = None, + aspect_range: AspectRange = None, + epsilon: float = 0.04) -> List[Dict[str, Any]]: + """Return boxes of the ~rectangular shapes (buttons / cards / fields), largest first. + + Keeps only contours that approximate to a convex quadrilateral (``epsilon`` is + the ``approxPolyDP`` tolerance as a fraction of the perimeter). ``aspect_range`` + is an optional ``(min, max)`` width/height filter — e.g. ``(1.5, 8)`` for wide + buttons, ``(0.8, 1.2)`` for square icons. + """ + gray = _haystack_gray(haystack, region) + boxes = [_box(contour) for contour in _contours(gray) + if _is_rectangle(contour, float(epsilon))] + boxes = [box for box in boxes if _passes(box, min_area, max_area, aspect_range)] + boxes.sort(key=lambda box: box["area"], reverse=True) + return boxes diff --git a/je_auto_control/utils/smart_waits/__init__.py b/je_auto_control/utils/smart_waits/__init__.py index e71bb355..8c3315ae 100644 --- a/je_auto_control/utils/smart_waits/__init__.py +++ b/je_auto_control/utils/smart_waits/__init__.py @@ -9,18 +9,21 @@ """ from je_auto_control.utils.smart_waits.waits import ( ClipboardReader, FileStatReader, Frame, PortConnector, ProcessLister, - ScreenSampler, WaitOutcome, WindowFinder, wait_until_clipboard_changes, - wait_until_file, wait_until_pixel_changes, wait_until_port, - wait_until_process, wait_until_region_idle, wait_until_screen_stable, - wait_until_window_closed, + ScreenSampler, WaitOutcome, WindowFinder, WindowTitleLister, + wait_until_clipboard_changes, wait_until_color, wait_until_file, + wait_until_gone, wait_until_image_gone, wait_until_pixel_changes, + wait_until_port, wait_until_process, wait_until_region_idle, + wait_until_screen_stable, wait_until_text_gone, wait_until_window_closed, + wait_until_window_title, ) __all__ = [ "ClipboardReader", "FileStatReader", "Frame", "PortConnector", "ProcessLister", "ScreenSampler", "WaitOutcome", "WindowFinder", - "wait_until_clipboard_changes", "wait_until_file", + "WindowTitleLister", "wait_until_clipboard_changes", "wait_until_color", + "wait_until_file", "wait_until_gone", "wait_until_image_gone", "wait_until_pixel_changes", "wait_until_port", "wait_until_process", - "wait_until_region_idle", "wait_until_screen_stable", - "wait_until_window_closed", + "wait_until_region_idle", "wait_until_screen_stable", "wait_until_text_gone", + "wait_until_window_closed", "wait_until_window_title", ] diff --git a/je_auto_control/utils/smart_waits/waits.py b/je_auto_control/utils/smart_waits/waits.py index 44d89808..fd597096 100644 --- a/je_auto_control/utils/smart_waits/waits.py +++ b/je_auto_control/utils/smart_waits/waits.py @@ -18,6 +18,7 @@ """ from __future__ import annotations +import re import time from dataclasses import asdict, dataclass from typing import Any, Callable, Dict, List, Optional, Sequence, Tuple @@ -238,6 +239,52 @@ def _default_window_finder(title: str, case_sensitive: bool) -> bool: return find_window(title, case_sensitive=case_sensitive) is not None +WindowTitleLister = Callable[[], List[str]] + + +def _title_matches(titles: List[str], pattern: str, + compiled: Optional["re.Pattern"]) -> bool: + if compiled is not None: + return any(compiled.search(title) for title in titles) + return any(pattern in title for title in titles) + + +def wait_until_window_title(pattern: str, *, present: bool = True, + regex: bool = True, timeout_s: float = 10.0, + poll_interval_s: float = 0.2, + title_lister: Optional[WindowTitleLister] = None + ) -> WaitOutcome: + """Wait until a window whose title matches ``pattern`` appears (or vanishes). + + Unlike ``wait_for_window`` (substring, appear only) this matches a regular + expression by default (``regex=False`` falls back to a substring test) and + can wait for the title to *vanish* with ``present=False`` — e.g. wait for a + browser tab to navigate to ``r".*— Checkout$"``. ``title_lister() -> [titles]`` + is injectable for tests. + """ + if timeout_s <= 0: + raise ValueError(_TIMEOUT_POSITIVE) + if poll_interval_s <= 0: + raise ValueError(_POLL_POSITIVE) + titles_of = title_lister or _default_title_lister + compiled = re.compile(pattern) if regex else None + started = time.monotonic() + deadline = started + float(timeout_s) + samples = 0 + while time.monotonic() < deadline: + samples += 1 + if _title_matches(titles_of(), pattern, compiled) == bool(present): + return _finish(True, "window title condition met", started, samples) + time.sleep(float(poll_interval_s)) + return _finish(False, "timeout while waiting for window title", + started, samples) + + +def _default_title_lister() -> List[str]: + from je_auto_control.wrapper.auto_control_window import list_windows + return [title for _id, title in list_windows()] + + FileStatReader = Callable[[str], Optional[int]] @@ -378,6 +425,123 @@ def wait_until_process(name: str, *, present: bool = True, started, samples) +def wait_until_gone(present: Callable[[], bool], *, + timeout_s: float = 10.0, poll_interval_s: float = 0.2, + gone_for_s: float = 0.0) -> WaitOutcome: + """Return once ``present()`` has been falsey for ``gone_for_s`` seconds. + + The blocking complement of ``wait_for_image`` / ``wait_for_text``: wait for a + spinner / toast / dialog to *disappear*. ``present`` is any predicate (e.g. + "is this image still on screen"); it is polled every ``poll_interval_s`` up to + ``timeout_s``. Injecting ``present`` keeps the loop headless-testable. + """ + if timeout_s <= 0: + raise ValueError(_TIMEOUT_POSITIVE) + if poll_interval_s <= 0: + raise ValueError(_POLL_POSITIVE) + if gone_for_s < 0: + raise ValueError("gone_for_s must be >= 0") + started = time.monotonic() + deadline = started + float(timeout_s) + samples = 0 + gone_since: Optional[float] = None + while time.monotonic() < deadline: + samples += 1 + if present(): + gone_since = None + else: + if gone_since is None: + gone_since = time.monotonic() + if time.monotonic() - gone_since >= float(gone_for_s): + return _finish(True, "target gone", started, samples) + time.sleep(float(poll_interval_s)) + return _finish(False, "timeout while waiting for target to vanish", + started, samples) + + +def _image_present(image: Any, detect_threshold: float) -> bool: + """Whether ``image`` is currently locatable on screen.""" + from je_auto_control.utils.exception.exceptions import ImageNotFoundException + from je_auto_control.wrapper.auto_control_image import locate_image_center + try: + locate_image_center(image, detect_threshold=detect_threshold) + return True + except ImageNotFoundException: + return False + + +def wait_until_image_gone(image: Any, *, detect_threshold: float = 1.0, + timeout_s: float = 10.0, poll_interval_s: float = 0.2, + gone_for_s: float = 0.0) -> WaitOutcome: + """Wait until ``image`` is no longer found on screen.""" + return wait_until_gone(lambda: _image_present(image, detect_threshold), + timeout_s=timeout_s, poll_interval_s=poll_interval_s, + gone_for_s=gone_for_s) + + +def _text_present(text: str) -> bool: + """Whether ``text`` is currently found on screen via OCR.""" + from je_auto_control.utils.ocr.ocr_engine import find_text_matches + return bool(find_text_matches(text)) + + +def wait_until_text_gone(text: str, *, timeout_s: float = 10.0, + poll_interval_s: float = 0.2, + gone_for_s: float = 0.0) -> WaitOutcome: + """Wait until ``text`` is no longer found on screen (OCR).""" + return wait_until_gone(lambda: _text_present(text), timeout_s=timeout_s, + poll_interval_s=poll_interval_s, gone_for_s=gone_for_s) + + +def _color_fraction(frame: "Frame", target: Sequence[int], + tolerance: int) -> float: + """Fraction of the frame's pixels within ``tolerance`` of ``target`` RGB.""" + pixels = frame.pixels + total = len(pixels) // 3 + if total == 0: + return 0.0 + red, green, blue = int(target[0]), int(target[1]), int(target[2]) + tol = int(tolerance) + matched = 0 + for offset in range(0, total * 3, 3): + if (abs(pixels[offset] - red) <= tol + and abs(pixels[offset + 1] - green) <= tol + and abs(pixels[offset + 2] - blue) <= tol): + matched += 1 + return matched / total + + +def wait_until_color(*, region: Optional[Sequence[int]] = None, + target_rgb: Sequence[int], tolerance: int = 10, + min_fraction: float = 0.5, present: bool = True, + timeout_s: float = 10.0, poll_interval_s: float = 0.2, + sampler: Optional[ScreenSampler] = None) -> WaitOutcome: + """Wait until ``target_rgb`` covers (or stops covering) a fraction of ``region``. + + Counts pixels within ``tolerance`` (per channel) of ``target_rgb``. With + ``present=True`` the wait succeeds once that fraction reaches + ``min_fraction`` (a status light turns green, a progress bar fills); with + ``present=False`` it succeeds once the fraction drops below it (the colour + disappears). ``sampler`` is injectable for headless tests. + """ + if timeout_s <= 0: + raise ValueError(_TIMEOUT_POSITIVE) + if poll_interval_s <= 0: + raise ValueError(_POLL_POSITIVE) + grab = sampler or _default_sampler + started = time.monotonic() + deadline = started + float(timeout_s) + samples = 0 + while time.monotonic() < deadline: + samples += 1 + fraction = _color_fraction(grab(region), target_rgb, tolerance) + reached = fraction >= float(min_fraction) + if reached == bool(present): + return _finish(True, "colour condition met", started, samples) + time.sleep(float(poll_interval_s)) + return _finish(False, "timeout while waiting for colour", started, samples) + + def _default_process_lister(name: str) -> List[str]: """List running process names matching ``name`` (requires psutil).""" from je_auto_control.utils.assertion.assertions import _running_process_names diff --git a/je_auto_control/utils/ssim/__init__.py b/je_auto_control/utils/ssim/__init__.py new file mode 100644 index 00000000..4297fdf0 --- /dev/null +++ b/je_auto_control/utils/ssim/__init__.py @@ -0,0 +1,4 @@ +"""Structural-similarity (SSIM) comparison: perceptual score + changed regions.""" +from je_auto_control.utils.ssim.ssim import ssim_changed_regions, ssim_compare + +__all__ = ["ssim_changed_regions", "ssim_compare"] diff --git a/je_auto_control/utils/ssim/ssim.py b/je_auto_control/utils/ssim/ssim.py new file mode 100644 index 00000000..c882a941 --- /dev/null +++ b/je_auto_control/utils/ssim/ssim.py @@ -0,0 +1,120 @@ +"""Structural-similarity (SSIM) comparison: perceptual score + changed regions. + +The framework already has pixel diff (``diff_screenshots``) and histogram drift +(``detect_drift``); neither is *structural*. SSIM is the standard visual-regression +metric — tolerant of small illumination shifts, sensitive to structural change +(text edits, moved or missing elements) — and yields a 0..1 similarity plus the +boxes of the regions that actually changed, so a test can both gate on a score and +point at *what* moved. + +It is a pure NumPy + OpenCV implementation (no scikit-image, which is not a +dependency) over an injectable image pair, so it is unit-testable on synthetic +arrays without a real screen. OpenCV + NumPy come in via ``je_open_cv`` and are +imported lazily. Imports no ``PySide6``. +""" +from typing import Any, Dict, List, Optional, Sequence + +ImageSource = Any +IgnoreBoxes = Optional[Sequence[Sequence[int]]] +_WINDOW = (11, 11) +_SIGMA = 1.5 +_C1 = (0.01 * 255) ** 2 +_C2 = (0.03 * 255) ** 2 + + +def _to_gray_f(source: ImageSource): + """Load a path / ndarray / PIL image as a 2-D float64 grayscale image.""" + import cv2 + import numpy as np + if hasattr(source, "shape"): + array = np.asarray(source) + elif isinstance(source, (str, bytes)) or hasattr(source, "__fspath__"): + array = cv2.imread(str(source), cv2.IMREAD_COLOR) + if array is None: + raise ValueError(f"could not read image: {source!r}") + else: + array = np.asarray(source) + if array.ndim == 3: + code = cv2.COLOR_BGRA2GRAY if array.shape[2] == 4 else cv2.COLOR_BGR2GRAY + array = cv2.cvtColor(array, code) + return array.astype(np.float64) + + +def _grab_gray_f(region: Optional[Sequence[int]]): + from je_auto_control.utils.cv2_utils.screenshot import pil_screenshot + image = pil_screenshot(screen_region=list(region) if region else None) + return _to_gray_f(image) + + +def _resolve_pair(reference: ImageSource, current: Optional[ImageSource], + region: Optional[Sequence[int]]): + reference_gray = _to_gray_f(reference) + current_gray = (_to_gray_f(current) if current is not None + else _grab_gray_f(region)) + if reference_gray.shape != current_gray.shape: + raise ValueError(f"reference {reference_gray.shape} and current " + f"{current_gray.shape} must be the same size") + return reference_gray, current_gray + + +def _ssim_map(reference, current): + """Per-pixel SSIM map via an 11x11 Gaussian window (sigma 1.5).""" + import cv2 + mu_ref = cv2.GaussianBlur(reference, _WINDOW, _SIGMA) + mu_cur = cv2.GaussianBlur(current, _WINDOW, _SIGMA) + mu_ref2, mu_cur2, mu_cross = mu_ref * mu_ref, mu_cur * mu_cur, mu_ref * mu_cur + var_ref = cv2.GaussianBlur(reference * reference, _WINDOW, _SIGMA) - mu_ref2 + var_cur = cv2.GaussianBlur(current * current, _WINDOW, _SIGMA) - mu_cur2 + cov = cv2.GaussianBlur(reference * current, _WINDOW, _SIGMA) - mu_cross + numerator = (2 * mu_cross + _C1) * (2 * cov + _C2) + denominator = (mu_ref2 + mu_cur2 + _C1) * (var_ref + var_cur + _C2) + return numerator / denominator + + +def _keep_mask(shape, ignore: IgnoreBoxes): + """Boolean keep-mask (True = counted); ``ignore`` boxes [x,y,w,h] set False.""" + import numpy as np + keep = np.ones(shape, dtype=bool) + for box in ignore or (): + x, y, width, height = (int(value) for value in box[:4]) + keep[y:y + height, x:x + width] = False + return keep + + +def ssim_compare(reference: ImageSource, current: Optional[ImageSource] = None, + *, ignore: IgnoreBoxes = None, + region: Optional[Sequence[int]] = None) -> float: + """Return the mean SSIM (0..1) between ``reference`` and ``current``. + + ``current`` defaults to a screen grab of the optional ``region``. ``ignore`` + is a list of ``[x, y, w, h]`` boxes excluded from the score (dynamic clocks, + blinking cursors). ``1.0`` means structurally identical; lower means more + change. Raises ``ValueError`` if the two images differ in size. + """ + import numpy as np + reference_gray, current_gray = _resolve_pair(reference, current, region) + smap = _ssim_map(reference_gray, current_gray) + keep = _keep_mask(smap.shape, ignore) + return round(float(np.mean(smap[keep])), 4) if keep.any() else 1.0 + + +def ssim_changed_regions(reference: ImageSource, + current: Optional[ImageSource] = None, *, + ignore: IgnoreBoxes = None, threshold: float = 0.35, + min_area: int = 50, + region: Optional[Sequence[int]] = None + ) -> List[Dict[str, Any]]: + """Return boxes of the regions that structurally changed, largest first. + + A pixel counts as changed where local dissimilarity ``1 - SSIM`` exceeds + ``threshold``; connected changed pixels covering at least ``min_area`` are + returned as ``{x, y, width, height, area, center}``. ``ignore`` boxes are + suppressed before detection. + """ + import numpy as np + from je_auto_control.utils.cv2_utils.blobs import connected_boxes + reference_gray, current_gray = _resolve_pair(reference, current, region) + smap = _ssim_map(reference_gray, current_gray) + changed = (1.0 - smap) > float(threshold) + changed &= _keep_mask(smap.shape, ignore) + return connected_boxes(changed.astype(np.uint8), int(min_area)) diff --git a/je_auto_control/utils/text_regions/__init__.py b/je_auto_control/utils/text_regions/__init__.py new file mode 100644 index 00000000..a27f316c --- /dev/null +++ b/je_auto_control/utils/text_regions/__init__.py @@ -0,0 +1,6 @@ +"""Model-free on-screen text-region detection (MSER): regions and lines.""" +from je_auto_control.utils.text_regions.text_regions import ( + find_text_lines, find_text_regions, +) + +__all__ = ["find_text_lines", "find_text_regions"] diff --git a/je_auto_control/utils/text_regions/text_regions.py b/je_auto_control/utils/text_regions/text_regions.py new file mode 100644 index 00000000..fe78af2e --- /dev/null +++ b/je_auto_control/utils/text_regions/text_regions.py @@ -0,0 +1,132 @@ +"""Model-free on-screen text-region detection (MSER) — where is text, without OCR. + +``shape_locator`` finds rectangular contours (buttons / cards, not text) and +``locate_text`` needs a Tesseract / Paddle engine *and* the exact string to search +for. Neither answers "where is there *any* text on screen" without running OCR or +knowing the words. This uses MSER (Maximally Stable Extremal Regions) to find the +glyph/word blobs, so a script can crop candidate text boxes to feed OCR (much faster +and more accurate than full-frame OCR) or detect that a label appeared with no OCR +dependency installed. + +Runs on an injectable ``haystack`` (ndarray / path / PIL), so it is headless-testable +on synthetic arrays. OpenCV + NumPy come in via ``je_open_cv`` (``cv2.MSER_create`` is +base OpenCV, no contrib). Imports no ``PySide6``. +""" +from typing import Any, Dict, List, Optional, Sequence, Tuple + +from je_auto_control.utils.visual_match.visual_match import _haystack_gray + +ImageSource = Any +Rect = Tuple[int, int, int, int] + + +def _box_dict(rect: Rect) -> Dict[str, Any]: + x, y, width, height = rect + return {"x": x, "y": y, "width": width, "height": height, + "area": width * height, "center": [x + width // 2, y + height // 2]} + + +def _accept(rect: Rect, shape, min_area: int, max_area: Optional[int], + max_aspect: float) -> bool: + """Whether an MSER box looks like text (size / aspect, not the whole frame).""" + _x, _y, width, height = rect + frame_h, frame_w = shape[:2] + if width >= 0.95 * frame_w and height >= 0.95 * frame_h: + return False # whole-frame extremal region + area = width * height + if area < min_area or (max_area is not None and area > max_area): + return False + aspect = width / height if height else 0.0 + return aspect <= max_aspect + + +def _filtered_boxes(gray, min_area: int, max_area: Optional[int], + max_aspect: float) -> List[Rect]: + """Return de-duplicated MSER bounding boxes passing the size / aspect filters.""" + import cv2 + regions, _bboxes = cv2.MSER_create().detectRegions(gray) + out: List[Rect] = [] + seen = set() + for points in regions: + rect = cv2.boundingRect(points.reshape(-1, 1, 2)) + if rect not in seen and _accept(rect, gray.shape, min_area, max_area, + max_aspect): + out.append(rect) + seen.add(rect) + return out + + +def _overlaps(a: Rect, b: Rect) -> bool: + return (a[0] < b[0] + b[2] and b[0] < a[0] + a[2] + and a[1] < b[1] + b[3] and b[1] < a[1] + a[3]) + + +def _union(a: Rect, b: Rect) -> Rect: + x, y = min(a[0], b[0]), min(a[1], b[1]) + return (x, y, max(a[0] + a[2], b[0] + b[2]) - x, + max(a[1] + a[3], b[1] + b[3]) - y) + + +def _merge_pass(rects: Sequence[Rect]) -> Tuple[bool, List[Rect]]: + out: List[Rect] = [] + changed = False + for rect in rects: + index = next((i for i, kept in enumerate(out) if _overlaps(rect, kept)), + None) + if index is None: + out.append(rect) + else: + out[index] = _union(rect, out[index]) + changed = True + return changed, out + + +def _merge_overlapping(rects: Sequence[Rect]) -> List[Rect]: + """Union all overlapping rectangles (collapses MSER's nested glyph regions).""" + result = list(rects) + changed = True + while changed: + changed, result = _merge_pass(result) + return result + + +def find_text_regions(haystack: Optional[ImageSource] = None, *, + region: Optional[Sequence[int]] = None, min_area: int = 60, + max_area: Optional[int] = None, merge: bool = True, + max_aspect: float = 12.0) -> List[Dict[str, Any]]: + """Return boxes of text/glyph regions on screen, largest first (no OCR needed). + + ``merge`` unions overlapping detections (MSER reports nested regions per glyph). + ``min_area`` / ``max_area`` drop specks and page-sized blobs; ``max_aspect`` + rejects long thin lines that are rules rather than text. Each result is + ``{x, y, width, height, area, center}``. + """ + rects = _filtered_boxes(_haystack_gray(haystack, region), int(min_area), + int(max_area) if max_area is not None else None, + float(max_aspect)) + if merge: + rects = _merge_overlapping(rects) + rects.sort(key=lambda rect: rect[2] * rect[3], reverse=True) + return [_box_dict(rect) for rect in rects] + + +def find_text_lines(haystack: Optional[ImageSource] = None, *, + region: Optional[Sequence[int]] = None, + y_tolerance: int = 8) -> List[Dict[str, Any]]: + """Return one box per horizontal line of text, top-to-bottom. + + Glyph boxes whose vertical centres are within ``y_tolerance`` pixels are grouped + into a line spanning their combined extent — ready to crop and feed to OCR. + """ + boxes = _filtered_boxes(_haystack_gray(haystack, region), 40, None, 20.0) + rows: List[Dict[str, Any]] = [] + for rect in sorted(boxes, key=lambda item: item[1]): + center_y = rect[1] + rect[3] // 2 + row = next((candidate for candidate in rows + if abs(center_y - candidate["cy"]) <= int(y_tolerance)), None) + if row is None: + rows.append({"cy": center_y, "rect": rect}) + else: + row["rect"] = _union(row["rect"], rect) + return [_box_dict(row["rect"]) + for row in sorted(rows, key=lambda item: item["cy"])] diff --git a/je_auto_control/utils/text_unicode/__init__.py b/je_auto_control/utils/text_unicode/__init__.py new file mode 100644 index 00000000..e94ea2ce --- /dev/null +++ b/je_auto_control/utils/text_unicode/__init__.py @@ -0,0 +1,6 @@ +"""Type arbitrary Unicode (emoji / CJK / accented) via the clipboard.""" +from je_auto_control.utils.text_unicode.text_unicode import ( + plan_paste, type_unicode, unicode_code_units, +) + +__all__ = ["plan_paste", "type_unicode", "unicode_code_units"] diff --git a/je_auto_control/utils/text_unicode/text_unicode.py b/je_auto_control/utils/text_unicode/text_unicode.py new file mode 100644 index 00000000..de7edf2a --- /dev/null +++ b/je_auto_control/utils/text_unicode/text_unicode.py @@ -0,0 +1,62 @@ +"""Type arbitrary Unicode (emoji / CJK / accented) via the clipboard. + +``write`` types through the platform virtual-key table and *raises* on any +character outside it — emoji, CJK, many accented letters — so non-ASCII text +entry is impossible through the normal path. The reliable, cross-platform way to +enter arbitrary Unicode is to put it on the clipboard and paste it. + +:func:`plan_paste` builds the deterministic op-plan and :func:`unicode_code_units` +splits text into UTF-16 code units (for a backend that can do +``KEYEVENTF_UNICODE``); both are pure and unit-testable. :func:`type_unicode` +dispatches the paste plan through an injectable ``sink`` so it is tested without +touching the real clipboard. Imports no ``PySide6``. +""" +from typing import Any, Callable, Dict, List, Optional + +Sink = Callable[[Dict[str, Any]], None] + + +def unicode_code_units(text: str) -> List[int]: + """Return the UTF-16 code units of ``text`` (surrogate pairs for > U+FFFF).""" + units: List[int] = [] + for char in text or "": + code = ord(char) + if code > 0xFFFF: + code -= 0x10000 + units.append(0xD800 + (code >> 10)) + units.append(0xDC00 + (code & 0x3FF)) + else: + units.append(code) + return units + + +def plan_paste(text: str, *, modifier: str = "ctrl") -> List[Dict[str, Any]]: + """Return the op-plan to enter ``text`` via clipboard paste.""" + return [{"op": "set_clipboard", "text": text}, + {"op": "hotkey", "keys": [modifier, "v"]}] + + +def _default_sink(event: Dict[str, Any]) -> None: + """Default dispatch: drive the real clipboard / keyboard backend.""" + op = event["op"] + if op == "set_clipboard": + from je_auto_control.utils.clipboard.clipboard import set_clipboard + set_clipboard(event["text"]) + elif op == "hotkey": + from je_auto_control.wrapper.auto_control_keyboard import hotkey + hotkey(list(event["keys"])) + + +def type_unicode(text: str, *, modifier: str = "ctrl", + sink: Optional[Sink] = None) -> Dict[str, Any]: + """Enter ``text`` (any Unicode) by setting the clipboard then pasting. + + ``modifier`` is the platform paste key (``"ctrl"``; use ``"command"`` on + macOS). Returns the dispatched plan plus the UTF-16 code-unit count. + """ + plan = plan_paste(text, modifier=modifier) + dispatch = sink or _default_sink + for event in plan: + dispatch(event) + return {"ops": len(plan), "plan": plan, + "code_units": len(unicode_code_units(text))} diff --git a/je_auto_control/utils/visual_match/__init__.py b/je_auto_control/utils/visual_match/__init__.py new file mode 100644 index 00000000..2e5959dc --- /dev/null +++ b/je_auto_control/utils/visual_match/__init__.py @@ -0,0 +1,8 @@ +"""Confidence-returning template matching (score, multi-scale, find-all + NMS).""" +from je_auto_control.utils.visual_match.visual_match import ( + Match, best_matches, match_masked, match_masked_all, match_template, + match_template_all, +) + +__all__ = ["Match", "best_matches", "match_masked", "match_masked_all", + "match_template", "match_template_all"] diff --git a/je_auto_control/utils/visual_match/visual_match.py b/je_auto_control/utils/visual_match/visual_match.py new file mode 100644 index 00000000..a4919c75 --- /dev/null +++ b/je_auto_control/utils/visual_match/visual_match.py @@ -0,0 +1,261 @@ +"""Confidence-returning template matching: score, multi-scale, find-all + NMS. + +The project's template matcher (``je_open_cv.find_object`` via ``cv2_utils``) is +single-scale and returns only bounding boxes — the correlation *score* it computes +internally is discarded, so there is no way to rank candidates, set a confidence +threshold and read back how well it matched, find a button when the UI is +DPI/zoom-scaled, or enumerate *every* occurrence. This adds those, in the style of +PyAutoGUI ``confidence`` / ``locateAll`` and SikuliX ``similarity`` / ``findAll``. + +The matching takes an injectable ``haystack`` image (ndarray / path / PIL), so it +is unit-testable on synthetic arrays without a real screen; only the default +(grab the screen) is device-bound. OpenCV + NumPy come in via the project's +``je_open_cv`` dependency and are imported lazily. Imports no ``PySide6``. +""" +from dataclasses import asdict, dataclass +from typing import Any, Dict, List, Optional, Sequence + +# cv2 method name -> the OpenCV constant is resolved lazily in _method(). +_METHOD_NAMES = ("ccoeff_normed", "ccorr_normed", "sqdiff_normed") +ImageSource = Any + + +@dataclass(frozen=True) +class Match: + """One template match: top-left (x, y), size, correlation score, scale.""" + + x: int + y: int + width: int + height: int + score: float + scale: float + + @property + def center(self) -> List[int]: + """The match's centre point ``[x, y]`` (ready to click).""" + return [self.x + self.width // 2, self.y + self.height // 2] + + def to_dict(self) -> Dict[str, Any]: + """Return the match as a plain dict including the centre point.""" + data = asdict(self) + data["center"] = self.center + return data + + +def _method(name: str) -> int: + import cv2 + table = {"ccoeff_normed": cv2.TM_CCOEFF_NORMED, + "ccorr_normed": cv2.TM_CCORR_NORMED, + "sqdiff_normed": cv2.TM_SQDIFF_NORMED} + if name not in table: + raise ValueError(f"unknown method: {name!r}") + return table[name] + + +def _to_gray(source: ImageSource): + """Load a path / ndarray / PIL image as a 2-D grayscale ndarray.""" + import cv2 + import numpy as np + if hasattr(source, "shape"): + array = np.asarray(source) + elif isinstance(source, (str, bytes)) or hasattr(source, "__fspath__"): + array = cv2.imread(str(source), cv2.IMREAD_COLOR) + if array is None: + raise ValueError(f"could not read image: {source!r}") + else: + array = np.asarray(source) + if array.ndim == 2: + return array + channels = array.shape[2] + code = cv2.COLOR_BGRA2GRAY if channels == 4 else cv2.COLOR_BGR2GRAY + return cv2.cvtColor(array, code) + + +def _grab_gray(region: Optional[Sequence[int]]): + from je_auto_control.utils.cv2_utils.screenshot import pil_screenshot + image = pil_screenshot(screen_region=list(region) if region else None) + return _to_gray(image) + + +def _haystack_gray(haystack: Optional[ImageSource], + region: Optional[Sequence[int]]): + return _to_gray(haystack) if haystack is not None else _grab_gray(region) + + +def _resize(template, scale: float): + import cv2 + if abs(scale - 1.0) < 1e-9: + return template + height, width = template.shape[:2] + new_size = (max(1, round(width * scale)), max(1, round(height * scale))) + return cv2.resize(template, new_size) + + +def match_template(template: ImageSource, *, haystack: Optional[ImageSource] = None, + region: Optional[Sequence[int]] = None, + scales: Sequence[float] = (1.0,), min_score: float = 0.8, + method: str = "ccoeff_normed") -> Optional[Match]: + """Return the single best match of ``template`` at or above ``min_score``. + + Searches each scale in ``scales`` (e.g. ``(0.9, 1.0, 1.1)`` for DPI / zoom + tolerance) and keeps the highest-scoring hit, or ``None`` if none clear the + threshold. + """ + import cv2 + tmpl = _to_gray(template) + hay = _haystack_gray(haystack, region) + metric = _method(method) + best: Optional[Match] = None + for scale in scales: + scaled = _resize(tmpl, float(scale)) + if scaled.shape[0] > hay.shape[0] or scaled.shape[1] > hay.shape[1]: + continue + _, max_val, _, max_loc = cv2.minMaxLoc(cv2.matchTemplate(hay, scaled, + metric)) + if max_val >= min_score and (best is None or max_val > best.score): + best = Match(int(max_loc[0]), int(max_loc[1]), scaled.shape[1], + scaled.shape[0], round(float(max_val), 4), float(scale)) + return best + + +def _iou(a: Match, b: Match) -> float: + left = max(a.x, b.x) + top = max(a.y, b.y) + right = min(a.x + a.width, b.x + b.width) + bottom = min(a.y + a.height, b.y + b.height) + inter = max(0, right - left) * max(0, bottom - top) + if inter == 0: + return 0.0 + union = a.width * a.height + b.width * b.height - inter + return inter / union + + +def _nms(matches: List[Match], iou_threshold: float) -> List[Match]: + kept: List[Match] = [] + for candidate in sorted(matches, key=lambda m: m.score, reverse=True): + if all(_iou(candidate, k) <= iou_threshold for k in kept): + kept.append(candidate) + return kept + + +def match_template_all(template: ImageSource, *, + haystack: Optional[ImageSource] = None, + region: Optional[Sequence[int]] = None, + min_score: float = 0.8, max_results: int = 20, + nms_iou: float = 0.3) -> List[Match]: + """Return every match of ``template`` >= ``min_score``, overlaps removed. + + Overlapping detections (the matcher fires on neighbouring pixels) are merged + by non-maximum suppression on the intersection-over-union, highest score + kept. Results are ordered by score, capped at ``max_results``. + """ + import cv2 + import numpy as np + tmpl = _to_gray(template) + hay = _haystack_gray(haystack, region) + height, width = tmpl.shape[:2] + result = cv2.matchTemplate(hay, tmpl, cv2.TM_CCOEFF_NORMED) + ys, xs = np.nonzero(result >= float(min_score)) + candidates = [Match(int(x), int(y), width, height, + round(float(result[y, x]), 4), 1.0) + for y, x in zip(ys, xs)] + return _nms(candidates, float(nms_iou))[:int(max_results)] + + +def best_matches(template: ImageSource, *, + haystack: Optional[ImageSource] = None, + region: Optional[Sequence[int]] = None, + top_n: int = 5) -> List[Match]: + """Return the top ``top_n`` matches by score (any score), nearest-best first.""" + return match_template_all(template, haystack=haystack, region=region, + min_score=-1.0, max_results=int(top_n)) + + +def _load_unchanged(source: ImageSource): + """Load a path / ndarray / PIL image keeping an alpha channel if present.""" + import cv2 + import numpy as np + if hasattr(source, "shape"): + return np.asarray(source) + if isinstance(source, (str, bytes)) or hasattr(source, "__fspath__"): + array = cv2.imread(str(source), cv2.IMREAD_UNCHANGED) + if array is None: + raise ValueError(f"could not read image: {source!r}") + return array + return np.asarray(source) + + +def _template_and_mask(template: ImageSource, mask: Optional[ImageSource]): + """Return (gray_template, uint8_mask_or_None); alpha is the implicit mask.""" + import cv2 + import numpy as np + array = _load_unchanged(template) + if array.ndim == 3 and array.shape[2] == 4: + gray = cv2.cvtColor(array, cv2.COLOR_BGRA2GRAY) + implicit = array[:, :, 3] + else: + gray = _to_gray(array) + implicit = None + chosen = _to_gray(mask) if mask is not None else implicit + if chosen is None: + return gray, None + chosen = np.ascontiguousarray(chosen, dtype=np.uint8) + if chosen.shape[:2] != gray.shape[:2]: + raise ValueError("mask shape must match template shape") + return gray, chosen + + +def _masked_scores(template: ImageSource, mask: Optional[ImageSource], + haystack: Optional[ImageSource], + region: Optional[Sequence[int]]): + """Return (score_map, gray_template) for masked correlation, NaNs zeroed.""" + import cv2 + import numpy as np + tmpl, msk = _template_and_mask(template, mask) + hay = _haystack_gray(haystack, region) + if tmpl.shape[0] > hay.shape[0] or tmpl.shape[1] > hay.shape[1]: + return None, tmpl + result = cv2.matchTemplate(hay, tmpl, cv2.TM_CCORR_NORMED, mask=msk) + return np.nan_to_num(result, nan=0.0, posinf=0.0, neginf=0.0), tmpl + + +def match_masked(template: ImageSource, *, mask: Optional[ImageSource] = None, + haystack: Optional[ImageSource] = None, + region: Optional[Sequence[int]] = None, + min_score: float = 0.9) -> Optional[Match]: + """Return the best match counting only masked (opaque) template pixels. + + ``mask`` is a grayscale image where non-zero pixels participate; if omitted + and ``template`` is RGBA, its alpha channel is the mask. This finds icons / + buttons whose background is transparent or varies (a glyph over any colour) + where a plain template would be dragged down by the irrelevant pixels. + Returns ``None`` when nothing clears ``min_score``. + """ + import cv2 + scores, tmpl = _masked_scores(template, mask, haystack, region) + if scores is None: + return None + _, max_val, _, max_loc = cv2.minMaxLoc(scores) + if max_val < min_score: + return None + return Match(int(max_loc[0]), int(max_loc[1]), tmpl.shape[1], tmpl.shape[0], + round(float(max_val), 4), 1.0) + + +def match_masked_all(template: ImageSource, *, mask: Optional[ImageSource] = None, + haystack: Optional[ImageSource] = None, + region: Optional[Sequence[int]] = None, + min_score: float = 0.9, max_results: int = 20, + nms_iou: float = 0.3) -> List[Match]: + """Return every masked match >= ``min_score`` with overlaps removed (NMS).""" + import numpy as np + scores, tmpl = _masked_scores(template, mask, haystack, region) + if scores is None: + return [] + height, width = tmpl.shape[:2] + ys, xs = np.nonzero(scores >= float(min_score)) + candidates = [Match(int(x), int(y), width, height, + round(float(scores[y, x]), 4), 1.0) + for y, x in zip(ys, xs)] + return _nms(candidates, float(nms_iou))[:int(max_results)] diff --git a/je_auto_control/utils/window_capture/__init__.py b/je_auto_control/utils/window_capture/__init__.py index 6975b12d..9892aa6f 100644 --- a/je_auto_control/utils/window_capture/__init__.py +++ b/je_auto_control/utils/window_capture/__init__.py @@ -1,10 +1,12 @@ -"""Per-window capture, window-layout save / restore, and snap/tile.""" +"""Per-window capture, window-layout save / restore, snap/tile, and arrange.""" from je_auto_control.utils.window_capture.window_capture import ( - capture_window, get_window_geometry, restore_window_layout, - save_window_layout, snap_window, + arrange_cascade, arrange_grid, capture_window, get_window_geometry, + restore_window_layout, save_window_layout, snap_window, ) __all__ = [ + "arrange_cascade", + "arrange_grid", "capture_window", "get_window_geometry", "restore_window_layout", diff --git a/je_auto_control/utils/window_capture/window_capture.py b/je_auto_control/utils/window_capture/window_capture.py index aa3dd82f..56053bb2 100644 --- a/je_auto_control/utils/window_capture/window_capture.py +++ b/je_auto_control/utils/window_capture/window_capture.py @@ -11,6 +11,7 @@ logic is fully unit-testable without real windows. GUI-free. """ import json +import math import sys from pathlib import Path from typing import Any, Callable, Dict, List, Optional, Tuple, Union @@ -168,3 +169,64 @@ def snap_window(title: str, position: str = "left", *, width, height = (screen_size or _default_screen_size)() x, y, w, h = _snap_rect(position, int(width), int(height)) return (mover or _default_mover)(title, x, y, w, h) + + +def _move_into(titles: List[str], rects, move: WindowMover) -> int: + """Move each title to the matching rectangle; return the number moved.""" + moved = 0 + for title, rect in zip(titles, rects): + if move(title, rect.x, rect.y, rect.width, rect.height): + moved += 1 + return moved + + +def _grid_shape(count: int, rows: Optional[int], + cols: Optional[int]) -> Tuple[int, int]: + """Resolve the (rows, cols) grid shape, auto-sizing to near-square when unset.""" + if rows and cols: + return int(rows), int(cols) + if cols: + return math.ceil(count / int(cols)), int(cols) + if rows: + return int(rows), math.ceil(count / int(rows)) + side = math.ceil(math.sqrt(count)) + return math.ceil(count / side), side + + +def arrange_grid(titles: List[str], *, rows: Optional[int] = None, + cols: Optional[int] = None, gap: int = 0, + mover: Optional[WindowMover] = None, + screen_size: Optional[SizeProvider] = None) -> int: + """Tile the given window ``titles`` into a grid; return the count moved. + + ``rows`` / ``cols`` default to a near-square auto-shape for the number of + windows; ``gap`` spaces the cells. The mover and size provider are injectable + for tests. Windows beyond the grid capacity are left untouched. + """ + from je_auto_control.utils.window_layout import grid_rects + titles = list(titles) + if not titles: + return 0 + width, height = (screen_size or _default_screen_size)() + grid_rows, grid_cols = _grid_shape(len(titles), rows, cols) + rects = grid_rects((0, 0, int(width), int(height)), grid_rows, grid_cols, + gap=int(gap)) + return _move_into(titles, rects, mover or _default_mover) + + +def arrange_cascade(titles: List[str], *, offset: int = 30, + mover: Optional[WindowMover] = None, + screen_size: Optional[SizeProvider] = None) -> int: + """Cascade the given window ``titles`` diagonally; return the count moved. + + Each window is ``offset`` pixels down-right of the previous, sized to 60% of + the work area and clamped on-screen. The mover and size provider are injectable. + """ + from je_auto_control.utils.window_layout import cascade_rects + titles = list(titles) + if not titles: + return 0 + width, height = (screen_size or _default_screen_size)() + rects = cascade_rects((0, 0, int(width), int(height)), len(titles), + offset=int(offset)) + return _move_into(titles, rects, mover or _default_mover) diff --git a/je_auto_control/utils/window_layout/__init__.py b/je_auto_control/utils/window_layout/__init__.py new file mode 100644 index 00000000..68a71b22 --- /dev/null +++ b/je_auto_control/utils/window_layout/__init__.py @@ -0,0 +1,7 @@ +"""Window tiling/layout geometry planner (halves, quadrants, grids, cascade).""" +from je_auto_control.utils.window_layout.window_layout import ( + WindowRect, available_slots, cascade_rects, grid_rects, tile_rect, +) + +__all__ = ["WindowRect", "available_slots", "cascade_rects", "grid_rects", + "tile_rect"] diff --git a/je_auto_control/utils/window_layout/window_layout.py b/je_auto_control/utils/window_layout/window_layout.py new file mode 100644 index 00000000..bd538977 --- /dev/null +++ b/je_auto_control/utils/window_layout/window_layout.py @@ -0,0 +1,127 @@ +"""Window tiling/layout geometry planner — halves, quadrants, grids, cascade. + +``save_window_layout`` / ``restore_window_layout`` capture and replay the *exact* +positions a user already arranged; nothing *computes* new ones. This is a pure- +geometry planner: given a screen work area it returns the target rectangles for the +common tiling layouts (left/right/top/bottom halves, quadrants, thirds, an R×C grid, +a staggered cascade) so a script can arrange application windows deterministically. + +The result is plain geometry that composes with any window-move backend; the planner +itself is cross-platform and has no device dependency, so it is fully unit-testable. +Imports no ``PySide6``. +""" +from dataclasses import asdict, dataclass +from typing import Dict, List, Optional, Sequence, Tuple + +Screen = Sequence[int] +Size = Optional[Sequence[int]] + +# slot name -> (x_fraction, y_fraction, width_fraction, height_fraction) +_THIRD = 1.0 / 3.0 +_SLOTS: Dict[str, Tuple[float, float, float, float]] = { + "full": (0.0, 0.0, 1.0, 1.0), + "left": (0.0, 0.0, 0.5, 1.0), + "right": (0.5, 0.0, 0.5, 1.0), + "top": (0.0, 0.0, 1.0, 0.5), + "bottom": (0.0, 0.5, 1.0, 0.5), + "top_left": (0.0, 0.0, 0.5, 0.5), + "top_right": (0.5, 0.0, 0.5, 0.5), + "bottom_left": (0.0, 0.5, 0.5, 0.5), + "bottom_right": (0.5, 0.5, 0.5, 0.5), + "center": (0.25, 0.25, 0.5, 0.5), + "left_third": (0.0, 0.0, _THIRD, 1.0), + "center_third": (_THIRD, 0.0, _THIRD, 1.0), + "right_third": (2.0 * _THIRD, 0.0, _THIRD, 1.0), +} + + +@dataclass(frozen=True) +class WindowRect: + """A target window geometry: top-left ``(x, y)`` and ``width`` / ``height``.""" + + x: int + y: int + width: int + height: int + + def as_tuple(self) -> Tuple[int, int, int, int]: + """Return ``(x, y, width, height)`` — ready for a window-move call.""" + return (self.x, self.y, self.width, self.height) + + def to_dict(self) -> Dict[str, int]: + """Return the rectangle as a plain dict.""" + return asdict(self) + + +def available_slots() -> List[str]: + """Return the names of the supported single-window slots.""" + return list(_SLOTS) + + +def _apply_gap(x: int, y: int, width: int, height: int, + gap: int) -> Tuple[int, int, int, int]: + """Inset a rectangle by ``gap`` on every side (min size 1).""" + return (x + gap, y + gap, max(1, width - 2 * gap), max(1, height - 2 * gap)) + + +def tile_rect(screen: Screen, slot: str, *, gap: int = 0) -> WindowRect: + """Return the rectangle for a named ``slot`` of the ``screen`` work area. + + ``screen`` is ``(x, y, width, height)``. ``slot`` is one of + :func:`available_slots` (``left``, ``top_right``, ``center``, ``left_third`` …). + ``gap`` insets the result on all sides for a margin between tiled windows. + """ + if slot not in _SLOTS: + raise ValueError(f"unknown slot: {slot!r}") + screen_x, screen_y, screen_w, screen_h = (int(value) for value in screen[:4]) + fx, fy, fw, fh = _SLOTS[slot] + rect = _apply_gap(screen_x + round(fx * screen_w), + screen_y + round(fy * screen_h), + round(fw * screen_w), round(fh * screen_h), int(gap)) + return WindowRect(*rect) + + +def grid_rects(screen: Screen, rows: int, cols: int, *, + gap: int = 0) -> List[WindowRect]: + """Return one rectangle per cell of an ``rows`` × ``cols`` grid, row-major. + + Splits the ``screen`` work area into equal cells; ``gap`` insets each cell for + uniform spacing. Raises ``ValueError`` if ``rows`` or ``cols`` is below 1. + """ + rows, cols = int(rows), int(cols) + if rows < 1 or cols < 1: + raise ValueError("rows and cols must be >= 1") + screen_x, screen_y, screen_w, screen_h = (int(value) for value in screen[:4]) + cell_w, cell_h = screen_w / cols, screen_h / rows + rects: List[WindowRect] = [] + for row in range(rows): + for col in range(cols): + rect = _apply_gap(screen_x + round(col * cell_w), + screen_y + round(row * cell_h), + round(cell_w), round(cell_h), int(gap)) + rects.append(WindowRect(*rect)) + return rects + + +def cascade_rects(screen: Screen, count: int, *, offset: int = 30, + size: Size = None) -> List[WindowRect]: + """Return ``count`` staggered, overlapping rectangles (a window cascade). + + Each window is ``offset`` pixels down-right of the previous one, clamped to stay + within the ``screen`` work area; ``size`` defaults to 60% of the work area. + """ + count = int(count) + if count < 1: + raise ValueError("count must be >= 1") + screen_x, screen_y, screen_w, screen_h = (int(value) for value in screen[:4]) + if size is not None: + win_w, win_h = int(size[0]), int(size[1]) + else: + win_w, win_h = round(screen_w * 0.6), round(screen_h * 0.6) + max_x, max_y = screen_x + screen_w - win_w, screen_y + screen_h - win_h + rects: List[WindowRect] = [] + for index in range(count): + pos_x = min(screen_x + index * int(offset), max_x) + pos_y = min(screen_y + index * int(offset), max_y) + rects.append(WindowRect(int(pos_x), int(pos_y), int(win_w), int(win_h))) + return rects diff --git a/test/unit_test/headless/test_actionability_batch.py b/test/unit_test/headless/test_actionability_batch.py new file mode 100644 index 00000000..177699ec --- /dev/null +++ b/test/unit_test/headless/test_actionability_batch.py @@ -0,0 +1,109 @@ +"""Headless tests for the pre-action actionability gate. No Qt; clock is injected.""" +import pytest + +import je_auto_control as ac +from je_auto_control.utils.actionability import ( + GateConfig, act_when_ready, wait_actionable, +) + + +class _Clock: + """A fake clock that only advances when the gate sleeps (deterministic).""" + + def __init__(self): + self.t = 0.0 + + def now(self): + return self.t + + def sleep(self, seconds): + self.t += seconds + + +def _config(): + clock = _Clock() + return GateConfig(timeout_s=5.0, stable_for_s=0.3, poll_interval_s=0.1, + clock=clock.now, sleep=clock.sleep) + + +def test_becomes_actionable_once_visible_and_stable(): + calls = {"n": 0} + + def bbox(): + calls["n"] += 1 + return (10, 10, 40, 20) if calls["n"] >= 2 else None + + report = wait_actionable(bbox, config=_config()) + assert report.actionable is True and report.reason == "actionable" + assert report.point == [30, 20] + + +def test_never_visible_times_out(): + report = wait_actionable(lambda: None, config=_config()) + assert report.actionable is False and report.reason == "not visible" + + +def test_moving_target_is_not_stable(): + counter = {"n": 0} + + def moving(): + counter["n"] += 1 + return (counter["n"], 10, 40, 20) # x shifts every poll + + report = wait_actionable(moving, config=_config()) + assert report.actionable is False and report.reason == "not stable" + + +def test_disabled_blocks(): + report = wait_actionable(lambda: (0, 0, 10, 10), enabled_probe=lambda: False, + config=_config()) + assert report.actionable is False and report.reason == "disabled" + + +def test_occluded_blocks(): + report = wait_actionable(lambda: (0, 0, 10, 10), hit_tester=lambda point: False, + config=_config()) + assert report.actionable is False and report.reason == "occluded" + + +def test_region_sampler_must_settle(): + samples = {"n": 0} + + def churning(_bbox): + samples["n"] += 1 + return samples["n"] # pixel token keeps changing + + report = wait_actionable(lambda: (0, 0, 10, 10), region_sampler=churning, + config=_config()) + assert report.actionable is False and report.reason == "not stable" + + +def test_act_when_ready_calls_action_with_center(): + clicked = [] + result = act_when_ready(lambda point: clicked.append(point) or "ok", + lambda: (10, 10, 40, 20), config=_config()) + assert result == "ok" and clicked == [[30, 20]] + + +def test_act_when_ready_raises_on_timeout(): + from je_auto_control.utils.exception.exceptions import AutoControlActionException + with pytest.raises(AutoControlActionException): + act_when_ready(lambda point: "never", lambda: None, config=_config()) + + +# --- wiring --------------------------------------------------------------- + +def test_wiring(): + assert "AC_wait_actionable" in set(ac.executor.known_commands()) + from je_auto_control.utils.mcp_server.tools import build_default_tool_registry + names = {t.name for t in build_default_tool_registry()} + assert "ac_wait_actionable" in names + from je_auto_control.gui.script_builder.command_schema import _build_specs + specs = {s.command for s in _build_specs()} + assert "AC_wait_actionable" in specs + + +def test_facade_exports(): + for attr in ("wait_actionable", "act_when_ready", "ActionabilityReport", + "GateConfig"): + assert hasattr(ac, attr) and attr in ac.__all__ diff --git a/test/unit_test/headless/test_anchor_ordinal_batch.py b/test/unit_test/headless/test_anchor_ordinal_batch.py new file mode 100644 index 00000000..7803f4a2 --- /dev/null +++ b/test/unit_test/headless/test_anchor_ordinal_batch.py @@ -0,0 +1,74 @@ +"""Headless tests for anchor ordinal / locate-all. No Qt.""" +import je_auto_control as ac +from je_auto_control.utils.anchor_locator import ( + anchor_locate, anchor_locate_all, image_locator, +) +from je_auto_control.utils.anchor_locator import locator as locator_mod + + +def _patch(monkeypatch, anchor_point, bboxes): + monkeypatch.setattr(locator_mod, "_resolve_single", lambda _l: anchor_point) + monkeypatch.setattr( + locator_mod, "_resolve_candidates", + lambda _l: [locator_mod._Bbox(*tup) for tup in bboxes]) + + +_ANCHOR = image_locator("a.png") +_TARGET = image_locator("b.png") +# three targets below the anchor at (0, 0), at increasing distance +_BELOW = [(0, 100, 10, 110), (0, 200, 10, 210), (0, 300, 10, 310)] + + +def test_ordinal_selects_nth_nearest(monkeypatch): + _patch(monkeypatch, (0, 0), _BELOW) + first = anchor_locate(anchor=_ANCHOR, target=_TARGET, relation="below") + assert first.found and first.target_coords == (5, 105) # nearest + second = anchor_locate(anchor=_ANCHOR, target=_TARGET, relation="below", + ordinal=2) + assert second.found and second.target_coords == (5, 205) + third = anchor_locate(anchor=_ANCHOR, target=_TARGET, relation="below", + ordinal=3) + assert third.found and third.target_coords == (5, 305) + + +def test_ordinal_out_of_range_not_found(monkeypatch): + _patch(monkeypatch, (0, 0), _BELOW) + outcome = anchor_locate(anchor=_ANCHOR, target=_TARGET, relation="below", + ordinal=9) + assert outcome.found is False and "ordinal" in outcome.error + + +def test_locate_all_returns_sorted(monkeypatch): + _patch(monkeypatch, (0, 0), _BELOW) + results = anchor_locate_all(anchor=_ANCHOR, target=_TARGET, relation="below") + assert [r.target_coords for r in results] == [(5, 105), (5, 205), (5, 305)] + assert all(r.found for r in results) + + +def test_locate_all_empty_when_anchor_missing(monkeypatch): + _patch(monkeypatch, None, _BELOW) + assert anchor_locate_all(anchor=_ANCHOR, target=_TARGET) == [] + + +def test_relation_filters_before_ordinal(monkeypatch): + # one above, two below -> "below" ordinal=2 is the farther below one + _patch(monkeypatch, (0, 150), + [(0, 50, 10, 60), (0, 200, 10, 210), (0, 300, 10, 310)]) + out = anchor_locate(anchor=_ANCHOR, target=_TARGET, relation="below", + ordinal=2) + assert out.target_coords == (5, 305) + + +# --- wiring --------------------------------------------------------------- + +def test_wiring(): + known = ac.executor.known_commands() + assert "AC_anchor_locate_all" in set(known) + from je_auto_control.utils.mcp_server.tools import build_default_tool_registry + names = {t.name for t in build_default_tool_registry()} + assert "ac_anchor_locate_all" in names + + +def test_facade_exports(): + assert hasattr(ac, "anchor_locate_all") + assert "anchor_locate_all" in ac.__all__ diff --git a/test/unit_test/headless/test_checksum_batch.py b/test/unit_test/headless/test_checksum_batch.py new file mode 100644 index 00000000..f5697ea3 --- /dev/null +++ b/test/unit_test/headless/test_checksum_batch.py @@ -0,0 +1,90 @@ +"""Headless tests for check-digit algorithms. No Qt.""" +import je_auto_control as ac +from je_auto_control.utils.checksum import ( + damm_check_digit, damm_validate, luhn_check_digit, luhn_validate, + mod97_10_check_digits, mod97_10_validate, verhoeff_check_digit, + verhoeff_validate, +) + + +def test_luhn(): + assert luhn_validate("4111111111111111") is True + assert luhn_validate("4111111111111112") is False + assert luhn_check_digit("7992739871") == "3" # 79927398713 is valid + assert luhn_validate("79927398713") is True + assert luhn_validate("") is False # no digits + assert luhn_validate("49015420323751") is True # a valid IMEI + assert luhn_validate("3566 0020 2036 0505") is True # spaced input ignored + + +def test_verhoeff(): + assert verhoeff_check_digit("236") == "3" + assert verhoeff_validate("2363") is True + assert verhoeff_validate("2364") is False # wrong check digit + assert verhoeff_validate("2633") is False # transposition caught + + +def test_damm(): + assert damm_check_digit("572") == "4" + assert damm_validate("5724") is True + assert damm_validate("5720") is False + assert damm_validate("7524") is False # transposition caught + + +def test_mod97_10(): + # GB82WEST12345698765432 rearranged + letters->numbers ends ...% 97 == 1 + rearranged = "3214282912345698765432161182" + assert mod97_10_validate(rearranged) is True + assert mod97_10_validate("3214282912345698765432161183") is False + # appended check digits make the value satisfy MOD 97-10 + body = "123456789" + check = mod97_10_check_digits(body) + assert mod97_10_validate(body + check) is True + assert len(check) == 2 + + +def test_check_digit_round_trips(): + for scheme_validate, scheme_digit in [ + (luhn_validate, luhn_check_digit), + (verhoeff_validate, verhoeff_check_digit), + (damm_validate, damm_check_digit)]: + base = "12345678" + assert scheme_validate(base + scheme_digit(base)) is True + + +# --- wiring --------------------------------------------------------------- + +def test_executor_round_trip(): + rec = ac.execute_action([[ + "AC_checksum_validate", + {"scheme": "luhn", "number": "4111111111111111"}]]) + out = next(v for v in rec.values() if isinstance(v, dict)) + assert out["valid"] is True + rec2 = ac.execute_action([[ + "AC_checksum_digit", {"scheme": "damm", "partial": "572"}]]) + assert next(v for v in rec2.values() if isinstance(v, dict))["check_digit"] == "4" + + +def test_executor_unknown_scheme_errors(): + rec = ac.execute_action([[ + "AC_checksum_validate", {"scheme": "nope", "number": "1"}]]) + # the executor records the failure rather than returning a {valid} dict + assert not any(isinstance(v, dict) and v.get("valid") for v in rec.values()) + + +def test_wiring(): + known = ac.executor.known_commands() + assert {"AC_checksum_validate", "AC_checksum_digit"} <= set(known) + from je_auto_control.utils.mcp_server.tools import build_default_tool_registry + names = {t.name for t in build_default_tool_registry()} + assert {"ac_checksum_validate", "ac_checksum_digit"} <= names + from je_auto_control.gui.script_builder.command_schema import _build_specs + specs = {s.command for s in _build_specs()} + assert {"AC_checksum_validate", "AC_checksum_digit"} <= specs + + +def test_facade_exports(): + for attr in ("luhn_validate", "luhn_check_digit", "verhoeff_validate", + "verhoeff_check_digit", "damm_validate", "damm_check_digit", + "mod97_10_validate", "mod97_10_check_digits"): + assert hasattr(ac, attr) and attr in ac.__all__ diff --git a/test/unit_test/headless/test_color_region_batch.py b/test/unit_test/headless/test_color_region_batch.py new file mode 100644 index 00000000..f760d1c2 --- /dev/null +++ b/test/unit_test/headless/test_color_region_batch.py @@ -0,0 +1,67 @@ +"""Headless tests for colour-region location. No Qt.""" +import pytest + +import je_auto_control as ac + +np = pytest.importorskip("numpy") +pytest.importorskip("cv2") + +from je_auto_control.utils.color_region import ( # noqa: E402 + find_color_region, find_color_regions, +) + + +def _scene(): + img = np.zeros((100, 200, 3), dtype=np.uint8) + img[20:40, 30:80] = (0, 200, 0) # big green (50x20 = 1000) + img[60:70, 100:120] = (0, 205, 5) # small near-green (20x10 = 200) + img[80:90, 10:20] = (200, 0, 0) # red + return img + + +def test_finds_regions_sorted_by_area(): + greens = find_color_regions((0, 200, 0), haystack=_scene(), tolerance=20, + min_area=10) + boxes = [(r["x"], r["y"], r["width"], r["height"]) for r in greens] + assert boxes == [(30, 20, 50, 20), (100, 60, 20, 10)] # largest first + assert greens[0]["area"] == 1000 + + +def test_find_largest_center(): + assert find_color_region((0, 200, 0), haystack=_scene())["center"] == [54, 29] + + +def test_red_and_absent_colour(): + assert find_color_region((200, 0, 0), haystack=_scene(), + min_area=10)["center"] == [14, 84] + assert find_color_region((0, 0, 255), haystack=_scene()) is None + + +def test_min_area_filters_small_blobs(): + assert len(find_color_regions((0, 200, 0), haystack=_scene(), + tolerance=20, min_area=300)) == 1 + + +def test_tolerance_widens_match(): + # the near-green (0,205,5) only matches (0,200,0) within a wide tolerance + tight = find_color_regions((0, 200, 0), haystack=_scene(), tolerance=2, + min_area=10) + assert all(r["y"] < 50 for r in tight) # only the exact big block + + +# --- wiring --------------------------------------------------------------- + +def test_wiring(): + known = ac.executor.known_commands() + assert "AC_find_color_region" in set(known) + from je_auto_control.utils.mcp_server.tools import build_default_tool_registry + names = {t.name for t in build_default_tool_registry()} + assert "ac_find_color_region" in names + from je_auto_control.gui.script_builder.command_schema import _build_specs + specs = {s.command for s in _build_specs()} + assert "AC_find_color_region" in specs + + +def test_facade_exports(): + for attr in ("find_color_region", "find_color_regions"): + assert hasattr(ac, attr) and attr in ac.__all__ diff --git a/test/unit_test/headless/test_edge_lines_batch.py b/test/unit_test/headless/test_edge_lines_batch.py new file mode 100644 index 00000000..729a0cdb --- /dev/null +++ b/test/unit_test/headless/test_edge_lines_batch.py @@ -0,0 +1,73 @@ +"""Headless tests for Hough line / grid / separator detection. No Qt.""" +import pytest + +import je_auto_control as ac + +np = pytest.importorskip("numpy") +cv2 = pytest.importorskip("cv2") + +from je_auto_control.utils.edge_lines import ( # noqa: E402 + find_grid, find_lines, find_separators, +) + + +def _grid_image(): + img = np.full((240, 300), 255, dtype=np.uint8) + for x in (50, 150, 250): + cv2.line(img, (x, 40), (x, 200), 0, 2) + for y in (40, 120, 200): + cv2.line(img, (50, y), (250, y), 0, 2) + return img + + +def test_find_lines_classifies_orientation(): + lines = find_lines(_grid_image(), min_length=80) + assert lines + assert {line["orientation"] for line in lines} == {"horizontal", "vertical"} + assert all(line["length"] >= 80 for line in lines) + + +def test_find_lines_orientation_filter(): + horizontals = find_lines(_grid_image(), min_length=80, orientation="horizontal") + assert horizontals and all(h["orientation"] == "horizontal" for h in horizontals) + + +def test_find_grid_recovers_rows_cols_cells(): + grid = find_grid(_grid_image(), min_length=100) + assert grid["rows"] == [40, 120, 200] + assert grid["cols"] == [50, 150, 250] + assert len(grid["cells"]) == 4 # 2x2 cells from a 3x3 grid + first = grid["cells"][0] + assert first == {"x": 50, "y": 40, "width": 100, "height": 80} + + +def test_find_separators_both_axes(): + assert find_separators(_grid_image(), axis="horizontal", min_length=100) == [ + 40, 120, 200] + assert find_separators(_grid_image(), axis="vertical", min_length=100) == [ + 50, 150, 250] + + +def test_blank_image_has_no_lines(): + blank = np.full((100, 100), 255, dtype=np.uint8) + assert find_lines(blank) == [] + assert find_grid(blank)["cells"] == [] + assert find_separators(blank) == [] + + +# --- wiring --------------------------------------------------------------- + +def test_wiring(): + known = set(ac.executor.known_commands()) + assert {"AC_find_lines", "AC_find_grid", "AC_find_separators"} <= known + from je_auto_control.utils.mcp_server.tools import build_default_tool_registry + names = {t.name for t in build_default_tool_registry()} + assert {"ac_find_lines", "ac_find_grid", "ac_find_separators"} <= names + from je_auto_control.gui.script_builder.command_schema import _build_specs + specs = {s.command for s in _build_specs()} + assert {"AC_find_lines", "AC_find_grid", "AC_find_separators"} <= specs + + +def test_facade_exports(): + for attr in ("find_lines", "find_grid", "find_separators"): + assert hasattr(ac, attr) and attr in ac.__all__ diff --git a/test/unit_test/headless/test_element_parse_batch.py b/test/unit_test/headless/test_element_parse_batch.py new file mode 100644 index 00000000..6e315a05 --- /dev/null +++ b/test/unit_test/headless/test_element_parse_batch.py @@ -0,0 +1,88 @@ +"""Headless tests for element-box fusion / ordering. No Qt.""" +import pytest + +import je_auto_control as ac +from je_auto_control.utils.element_parse import ( + fuse_elements, iou, merge_boxes, reading_order, +) + + +def _b(x, y, w, h, **extra): + return dict(x=x, y=y, width=w, height=h, **extra) + + +def test_iou_full_half_none(): + assert iou(_b(0, 0, 10, 10), _b(0, 0, 10, 10)) == pytest.approx(1.0) + assert iou(_b(0, 0, 10, 10), _b(5, 0, 10, 10)) == pytest.approx(1 / 3) + assert iou(_b(0, 0, 10, 10), _b(100, 100, 10, 10)) == pytest.approx(0.0, abs=1e-9) + + +def test_merge_drops_near_duplicates(): + merged = merge_boxes([_b(0, 0, 100, 100), _b(1, 1, 100, 100), + _b(500, 500, 20, 20)]) + assert len(merged) == 2 + assert merged[0]["width"] == 100 # largest kept first + + +def test_merge_threshold_controls_strictness(): + boxes = [_b(0, 0, 100, 100), _b(40, 0, 100, 100)] # ~0.43 IoU + assert len(merge_boxes(boxes, iou_threshold=0.9)) == 2 # kept as distinct + assert len(merge_boxes(boxes, iou_threshold=0.3)) == 1 # merged + + +def test_fuse_prefers_higher_priority_source(): + fused = fuse_elements( + ocr_boxes=[_b(0, 0, 50, 20, text="OK")], + a11y_boxes=[_b(1, 0, 50, 20, text="OK", role="button")]) + assert len(fused) == 1 + assert fused[0]["source"] == "a11y" and fused[0]["role"] == "button" + + +def test_fuse_keeps_disjoint_boxes_from_all_sources(): + fused = fuse_elements( + ocr_boxes=[_b(0, 0, 20, 20)], + icon_boxes=[_b(100, 0, 20, 20)], + a11y_boxes=[_b(200, 0, 20, 20)]) + assert len(fused) == 3 + assert {f["source"] for f in fused} == {"ocr", "icon", "a11y"} + + +def test_fuse_custom_priority(): + fused = fuse_elements( + ocr_boxes=[_b(0, 0, 50, 20, text="from-ocr")], + icon_boxes=[_b(0, 0, 50, 20, text="from-icon")], + source_priority=("icon", "ocr", "a11y")) + assert len(fused) == 1 and fused[0]["source"] == "icon" + + +def test_reading_order_rows_then_columns(): + elements = [_b(100, 0, 20, 20, id="b"), _b(0, 0, 20, 20, id="a"), + _b(0, 100, 20, 20, id="c")] + ordered = reading_order(elements) + assert [(e["id"], e["index"]) for e in ordered] == [("a", 0), ("b", 1), + ("c", 2)] + + +def test_reading_order_row_tolerance_bands(): + # tops differ by 5px -> same row when tol >= 5 + elements = [_b(50, 5, 10, 10, id="r"), _b(0, 0, 10, 10, id="l")] + ordered = reading_order(elements, row_tol=8) + assert [e["id"] for e in ordered] == ["l", "r"] + + +# --- wiring --------------------------------------------------------------- + +def test_wiring(): + known = set(ac.executor.known_commands()) + assert {"AC_fuse_elements", "AC_reading_order"} <= known + from je_auto_control.utils.mcp_server.tools import build_default_tool_registry + names = {t.name for t in build_default_tool_registry()} + assert {"ac_fuse_elements", "ac_reading_order"} <= names + from je_auto_control.gui.script_builder.command_schema import _build_specs + specs = {s.command for s in _build_specs()} + assert {"AC_fuse_elements", "AC_reading_order"} <= specs + + +def test_facade_exports(): + for attr in ("iou", "merge_boxes", "fuse_elements", "reading_order"): + assert hasattr(ac, attr) and attr in ac.__all__ diff --git a/test/unit_test/headless/test_expect_poll_batch.py b/test/unit_test/headless/test_expect_poll_batch.py new file mode 100644 index 00000000..bc4148ad --- /dev/null +++ b/test/unit_test/headless/test_expect_poll_batch.py @@ -0,0 +1,86 @@ +"""Headless tests for retrying value assertions. No Qt; clock is injected.""" +import pytest + +import je_auto_control as ac +from je_auto_control.utils.expect_poll import ( + assert_poll, expect_poll, to_be_greater_than, to_be_stable, to_contain, + to_equal, to_match_regex, +) + + +class _Clock: + def __init__(self): + self.t = 0.0 + + def now(self): + return self.t + + def sleep(self, seconds): + self.t += seconds + + +def _cfg(): + clock = _Clock() + return {"timeout_s": 5.0, "interval_s": 0.25, "clock": clock.now, + "sleep": clock.sleep} + + +def test_matches_after_a_few_polls(): + seq = iter([1, 3, 5, 5]) + result = expect_poll(lambda: next(seq), to_equal(5), **_cfg()) + assert result.ok and result.attempts == 3 and result.value == 5 + + +def test_times_out_when_never_matching(): + result = expect_poll(lambda: 0, to_equal(9), **_cfg()) + assert result.ok is False and result.attempts == 21 # 20 sleeps + first + + +def test_contains_gt_regex_matchers(): + assert expect_poll(lambda: "done ok", to_contain("ok"), **_cfg()).ok + assert expect_poll(lambda: 42, to_be_greater_than(10), **_cfg()).ok + assert expect_poll(lambda: "total=42", to_match_regex(r"=\d+"), **_cfg()).ok + + +def test_to_be_stable_requires_repeats(): + vals = iter([7, 8, 8, 8, 8]) + # not stable until three equal in a row + result = expect_poll(lambda: next(vals), to_be_stable(3), **_cfg()) + assert result.ok and result.value == 8 + + +def test_assert_poll_raises_on_timeout(): + from je_auto_control.utils.exception.exceptions import AutoControlActionException + with pytest.raises(AutoControlActionException): + assert_poll(lambda: 0, to_equal(1), timeout_s=0) + + +def test_assert_poll_returns_result_on_success(): + result = assert_poll(lambda: "ready", to_equal("ready"), timeout_s=0) + assert result.ok and result.value == "ready" + + +# --- wiring --------------------------------------------------------------- + +def test_wiring(): + assert "AC_expect_poll" in set(ac.executor.known_commands()) + from je_auto_control.utils.mcp_server.tools import build_default_tool_registry + names = {t.name for t in build_default_tool_registry()} + assert "ac_expect_poll" in names + from je_auto_control.gui.script_builder.command_schema import _build_specs + specs = {s.command for s in _build_specs()} + assert "AC_expect_poll" in specs + + +def test_executor_polls_nested_action(): + # AC_fuse_elements with no args returns {count: 0, ...} — device-free + deterministic. + from je_auto_control.utils.executor.action_executor import _expect_poll + result = _expect_poll(["AC_fuse_elements"], key="count", op="equals", + expected=0, timeout_s=0.0) + assert result["ok"] is True and result["value"] == 0 + + +def test_facade_exports(): + for attr in ("expect_poll", "assert_poll", "to_equal", "to_contain", + "to_be_stable"): + assert hasattr(ac, attr) and attr in ac.__all__ diff --git a/test/unit_test/headless/test_feature_match_batch.py b/test/unit_test/headless/test_feature_match_batch.py new file mode 100644 index 00000000..e65aad84 --- /dev/null +++ b/test/unit_test/headless/test_feature_match_batch.py @@ -0,0 +1,78 @@ +"""Headless tests for ORB feature matching. No Qt.""" +import pytest + +import je_auto_control as ac + +np = pytest.importorskip("numpy") +cv2 = pytest.importorskip("cv2") + +from je_auto_control.utils.feature_match import ( # noqa: E402 + FeatureMatch, feature_match, +) + + +def _template(): + """A 60x60 feature-rich patch (corners/blobs ORB can latch onto).""" + tmpl = np.zeros((60, 60), dtype=np.uint8) + cv2.rectangle(tmpl, (5, 5), (25, 20), 255, -1) + cv2.rectangle(tmpl, (35, 10), (55, 40), 200, -1) + cv2.circle(tmpl, (20, 45), 10, 150, -1) + cv2.rectangle(tmpl, (40, 45), (52, 55), 90, 2) + return tmpl + + +def _scene(rotate=False): + """A 200x200 scene with the template pasted at (90, 60), optionally rotated.""" + scene = np.zeros((200, 200), dtype=np.uint8) + patch = _template() + if rotate: + matrix = cv2.getRotationMatrix2D((30, 30), 20, 1.0) + patch = cv2.warpAffine(patch, matrix, (60, 60)) + scene[60:120, 90:150] = patch + return scene + + +def test_locates_unrotated_template(): + hit = feature_match(_template(), haystack=_scene(), min_inliers=8) + assert hit is not None + # the template was pasted at columns 90..150, rows 60..120 -> centre ~ (120, 90) + assert 100 <= hit.center[0] <= 140 + assert 70 <= hit.center[1] <= 110 + assert hit.inliers >= 8 + assert len(hit.corners) == 4 + + +def test_survives_rotation(): + hit = feature_match(_template(), haystack=_scene(rotate=True), min_inliers=6) + assert hit is not None + assert hit.inliers >= 6 + + +def test_absent_template_returns_none(): + blank = np.zeros((200, 200), dtype=np.uint8) + assert feature_match(_template(), haystack=blank, min_inliers=8) is None + + +def test_score_is_inlier_fraction(): + hit = feature_match(_template(), haystack=_scene(), min_inliers=8) + assert 0.0 < hit.score <= 1.0 + assert hit.to_dict()["inliers"] == hit.inliers + + +# --- wiring --------------------------------------------------------------- + +def test_wiring(): + known = set(ac.executor.known_commands()) + assert "AC_feature_match" in known + from je_auto_control.utils.mcp_server.tools import build_default_tool_registry + names = {t.name for t in build_default_tool_registry()} + assert "ac_feature_match" in names + from je_auto_control.gui.script_builder.command_schema import _build_specs + specs = {s.command for s in _build_specs()} + assert "AC_feature_match" in specs + + +def test_facade_exports(): + assert hasattr(ac, "feature_match") and "feature_match" in ac.__all__ + assert isinstance(feature_match(_template(), haystack=_scene(), + min_inliers=8), FeatureMatch) diff --git a/test/unit_test/headless/test_field_entry_batch.py b/test/unit_test/headless/test_field_entry_batch.py new file mode 100644 index 00000000..881b4786 --- /dev/null +++ b/test/unit_test/headless/test_field_entry_batch.py @@ -0,0 +1,71 @@ +"""Headless tests for clear-then-type field entry. No Qt.""" +import je_auto_control as ac +from je_auto_control.utils.field_entry import plan_field_set, set_field_text + + +def test_default_plan_clears_then_types(): + plan = plan_field_set("hello") + assert plan == [ + {"op": "hotkey", "keys": ["ctrl", "a"]}, + {"op": "key", "key": "delete"}, + {"op": "type", "text": "hello"}, + ] + + +def test_paste_plan_uses_clipboard(): + plan = plan_field_set("café 🚀", paste=True) + ops = [event["op"] for event in plan] + assert ops == ["hotkey", "key", "set_clipboard", "hotkey"] + assert plan[2] == {"op": "set_clipboard", "text": "café 🚀"} + assert plan[-1]["keys"] == ["ctrl", "v"] + + +def test_clear_none_skips_clear(): + assert plan_field_set("x", clear="none") == [{"op": "type", "text": "x"}] + + +def test_modifier_override_for_macos(): + plan = plan_field_set("y", modifier="command") + assert plan[0]["keys"] == ["command", "a"] + + +def test_unknown_clear_raises(): + try: + plan_field_set("z", clear="bogus") + except ValueError as error: + assert "clear" in str(error) + else: # pragma: no cover + raise AssertionError("expected ValueError") + + +def test_set_field_text_dispatches_plan(): + events = [] + result = set_field_text("hi", sink=events.append) + assert result["ops"] == 3 + assert [e["op"] for e in events] == ["hotkey", "key", "type"] + + +# --- wiring --------------------------------------------------------------- + +def test_executor_adapter_planning(): + # the executor's default dispatch is device-bound; exercise the adapter + # planning + JSON-free path by calling the underlying with an injected sink. + events = [] + set_field_text("v", paste=True, sink=events.append) + assert events[-1] == {"op": "hotkey", "keys": ["ctrl", "v"]} + + +def test_wiring(): + known = ac.executor.known_commands() + assert "AC_set_field_text" in set(known) + from je_auto_control.utils.mcp_server.tools import build_default_tool_registry + names = {t.name for t in build_default_tool_registry()} + assert "ac_set_field_text" in names + from je_auto_control.gui.script_builder.command_schema import _build_specs + specs = {s.command for s in _build_specs()} + assert "AC_set_field_text" in specs + + +def test_facade_exports(): + for attr in ("plan_field_set", "set_field_text"): + assert hasattr(ac, attr) and attr in ac.__all__ diff --git a/test/unit_test/headless/test_grid_locator_batch.py b/test/unit_test/headless/test_grid_locator_batch.py new file mode 100644 index 00000000..e75ed40f --- /dev/null +++ b/test/unit_test/headless/test_grid_locator_batch.py @@ -0,0 +1,69 @@ +"""Headless tests for grid / table cell addressing. No Qt.""" +import json + +import je_auto_control as ac +from je_auto_control.utils.grid_locator import cluster_grid, locate_cell + +# a 2-row x 3-col grid, deliberately shuffled +_BOXES = [ + (210, 100, 20, 10), (10, 100, 20, 10), (110, 100, 20, 10), + (110, 200, 20, 10), (10, 200, 20, 10), (210, 200, 20, 10), +] + + +def test_cluster_groups_rows_and_orders_columns(): + grid = cluster_grid(_BOXES) + assert len(grid) == 2 and [len(r) for r in grid] == [3, 3] + assert [b[0] for b in grid[0]] == [10, 110, 210] # left-to-right + assert [b[1] for b in grid[0]] == [100, 100, 100] # top row + + +def test_locate_cell_returns_center(): + cell = locate_cell(_BOXES, 1, 2) + assert cell["found"] is True + assert cell["center"] == [220, 205] and cell["box"] == [210, 200, 20, 10] + assert cell["rows"] == 2 and cell["cols"] == 3 + + +def test_out_of_range(): + assert locate_cell(_BOXES, 5, 0)["found"] is False + assert locate_cell(_BOXES, 0, 9)["reason"] == "col out of range" + + +def test_row_tolerance_merges_near_y(): + # second row jitters by 4px in y -> still one row at tolerance 10 + jittered = [(10, 100, 20, 10), (110, 104, 20, 10), (210, 98, 20, 10)] + assert len(cluster_grid(jittered, row_tolerance=10)) == 1 + assert len(cluster_grid(jittered, row_tolerance=2)) >= 2 # splits + + +def test_empty_boxes(): + assert cluster_grid([]) == [] + assert locate_cell([], 0, 0)["found"] is False + + +# --- wiring --------------------------------------------------------------- + +def test_executor_round_trip(): + rec = ac.execute_action([[ + "AC_grid_cell", {"boxes": json.dumps([[10, 100, 20, 10], + [110, 100, 20, 10]]), + "row": 0, "col": 1}]]) + out = next(v for v in rec.values() if isinstance(v, dict)) + assert out["found"] is True and out["center"] == [120, 105] + + +def test_wiring(): + known = ac.executor.known_commands() + assert "AC_grid_cell" in set(known) + from je_auto_control.utils.mcp_server.tools import build_default_tool_registry + names = {t.name for t in build_default_tool_registry()} + assert "ac_grid_cell" in names + from je_auto_control.gui.script_builder.command_schema import _build_specs + specs = {s.command for s in _build_specs()} + assert "AC_grid_cell" in specs + + +def test_facade_exports(): + for attr in ("cluster_grid", "locate_cell"): + assert hasattr(ac, attr) and attr in ac.__all__ diff --git a/test/unit_test/headless/test_hsv_segment_batch.py b/test/unit_test/headless/test_hsv_segment_batch.py new file mode 100644 index 00000000..71c5480b --- /dev/null +++ b/test/unit_test/headless/test_hsv_segment_batch.py @@ -0,0 +1,70 @@ +"""Headless tests for HSV colour segmentation. No Qt.""" +import pytest + +import je_auto_control as ac + +np = pytest.importorskip("numpy") +pytest.importorskip("cv2") + +from je_auto_control.utils.hsv_segment import ( # noqa: E402 + color_mask, dominant_hue_regions, segment_hsv, +) + + +def _scene(): + """Bright red + dark red (same hue, different brightness) + a green patch.""" + img = np.zeros((100, 200, 3), dtype=np.uint8) + img[10:40, 10:60] = (255, 0, 0) # bright red, V=255 + img[60:90, 10:50] = (120, 0, 0) # dark red, V=120 + img[10:40, 120:170] = (0, 200, 0) # green + return img + + +def test_hue_band_catches_both_brightnesses_of_red(): + reds = dominant_hue_regions(_scene(), hue=0, hue_tol=10, sat_min=80, + val_min=80, min_area=100) + boxes = sorted((r["y"], r["x"], r["width"], r["height"]) for r in reds) + assert boxes == [(10, 10, 50, 30), (60, 10, 40, 30)] # bright AND dark red + + +def test_red_hue_excludes_green(): + reds = dominant_hue_regions(_scene(), hue=0, hue_tol=10, min_area=100) + assert all(r["x"] < 100 for r in reds) # green is at x=120 + + +def test_green_hue_found(): + greens = dominant_hue_regions(_scene(), hue=60, hue_tol=15, min_area=100) + assert len(greens) == 1 and greens[0]["x"] == 120 + + +def test_explicit_band_segment_and_mask(): + boxes = segment_hsv(_scene(), lower_hsv=[40, 80, 80], upper_hsv=[80, 255, 255], + min_area=100) + assert len(boxes) == 1 and boxes[0]["x"] == 120 # the green patch + mask = color_mask(_scene(), lower_hsv=[40, 80, 80], upper_hsv=[80, 255, 255]) + assert int(mask.sum()) > 0 + + +def test_val_floor_skips_dark(): + # require a high value floor -> the dark red (V=120) drops out, bright stays + reds = dominant_hue_regions(_scene(), hue=0, hue_tol=10, val_min=200, + min_area=100) + assert len(reds) == 1 and reds[0]["y"] == 10 # only the bright patch + + +# --- wiring --------------------------------------------------------------- + +def test_wiring(): + known = set(ac.executor.known_commands()) + assert {"AC_segment_hsv", "AC_dominant_hue_regions"} <= known + from je_auto_control.utils.mcp_server.tools import build_default_tool_registry + names = {t.name for t in build_default_tool_registry()} + assert {"ac_segment_hsv", "ac_dominant_hue_regions"} <= names + from je_auto_control.gui.script_builder.command_schema import _build_specs + specs = {s.command for s in _build_specs()} + assert {"AC_segment_hsv", "AC_dominant_hue_regions"} <= specs + + +def test_facade_exports(): + for attr in ("segment_hsv", "color_mask", "dominant_hue_regions"): + assert hasattr(ac, attr) and attr in ac.__all__ diff --git a/test/unit_test/headless/test_key_hold_batch.py b/test/unit_test/headless/test_key_hold_batch.py new file mode 100644 index 00000000..6cc0fffc --- /dev/null +++ b/test/unit_test/headless/test_key_hold_batch.py @@ -0,0 +1,77 @@ +"""Headless tests for key hold / auto-repeat. No Qt.""" +import pytest + +import je_auto_control as ac +from je_auto_control.utils.key_hold import hold_key, plan_key_hold + + +def test_hold_plan_press_wait_release(): + plan = plan_key_hold("key_d", 1.5) + assert plan == [ + {"op": "press", "key": "key_d"}, + {"op": "wait", "seconds": 1.5}, + {"op": "release", "key": "key_d"}, + ] + + +def test_repeat_plan_emits_n_key_events(): + plan = plan_key_hold("a", 1.0, rate_hz=20) + keys = [event for event in plan if event["op"] == "key"] + waits = [event for event in plan if event["op"] == "wait"] + assert len(keys) == 20 + assert len(waits) == 19 # one fewer gap than events + assert waits[0]["seconds"] == pytest.approx(0.05) # 1 / 20 Hz + + +def test_repeat_rounds_count_min_one(): + assert len([e for e in plan_key_hold("a", 0.01, rate_hz=10) + if e["op"] == "key"]) == 1 # round(0.1) -> 0 -> clamped to 1 + + +def test_hold_key_routes_waits_to_sleep(): + events, slept = [], [] + result = hold_key("a", 0.5, sink=events.append, sleep=slept.append) + assert [e["op"] for e in events] == ["press", "release"] + assert slept == [0.5] + assert result["ops"] == 3 + + +def test_validation(): + for bad in (("a", 0), ("a", -1)): + try: + plan_key_hold(*bad) + except ValueError: + pass + else: # pragma: no cover + raise AssertionError("expected ValueError") + try: + plan_key_hold("a", 1.0, rate_hz=0) + except ValueError: + pass + else: # pragma: no cover + raise AssertionError("expected ValueError for rate_hz=0") + + +# --- wiring --------------------------------------------------------------- + +def test_executor_adapter_planning(): + events, slept = [], [] + from je_auto_control.utils.key_hold import hold_key as _hk + _hk("space", 0.2, sink=events.append, sleep=slept.append) + assert events and events[0]["op"] == "press" and events[-1]["op"] == "release" + + +def test_wiring(): + known = ac.executor.known_commands() + assert "AC_hold_key" in set(known) + from je_auto_control.utils.mcp_server.tools import build_default_tool_registry + names = {t.name for t in build_default_tool_registry()} + assert "ac_hold_key" in names + from je_auto_control.gui.script_builder.command_schema import _build_specs + specs = {s.command for s in _build_specs()} + assert "AC_hold_key" in specs + + +def test_facade_exports(): + for attr in ("plan_key_hold", "hold_key"): + assert hasattr(ac, attr) and attr in ac.__all__ diff --git a/test/unit_test/headless/test_locator_chain_batch.py b/test/unit_test/headless/test_locator_chain_batch.py new file mode 100644 index 00000000..3260daab --- /dev/null +++ b/test/unit_test/headless/test_locator_chain_batch.py @@ -0,0 +1,77 @@ +"""Headless tests for composable/filtered candidate locators. No Qt.""" +import je_auto_control as ac +from je_auto_control.utils.locator_chain import Candidates, from_boxes + + +def _b(x, y, w, h, **extra): + return dict(x=x, y=y, width=w, height=h, **extra) + + +def _scene(): + return [_b(10, 10, 40, 20, text="Save"), _b(200, 10, 40, 20, text="Delete"), + _b(60, 10, 40, 20, text="Delete"), _b(10, 200, 40, 20, text="Cancel")] + + +def test_within_clips_to_region(): + inside = from_boxes(_scene()).within((0, 0, 300, 100)) + assert len(inside) == 3 # the y=200 Cancel is excluded + assert "Cancel" not in [b["text"] for b in inside] + + +def test_filter_has_text_case_insensitive(): + deletes = from_boxes(_scene()).filter(has_text="delete") + assert len(deletes) == 2 and all(b["text"] == "Delete" for b in deletes) + + +def test_chained_within_filter_nth(): + second = (from_boxes(_scene()).within((0, 0, 300, 100)) + .filter(has_text="Delete").sort_reading().nth(1)) + assert second.center() == [220, 20] # the right-hand Delete + + +def test_first_last_and_empty_nth(): + chain = from_boxes(_scene()).sort_reading() + assert chain.first().center() == [30, 20] # Save (leftmost top row) + assert chain.last().center() == [30, 210] # Cancel (bottom row) + assert from_boxes(_scene()).nth(99).center() is None + + +def test_filter_near_and_area_and_predicate(): + assert len(from_boxes(_scene()).filter(near=(30, 20, 30))) == 1 + assert len(from_boxes(_scene()).filter(min_area=10000)) == 0 + cancels = from_boxes(_scene()).filter(predicate=lambda b: b["text"][0] == "C") + assert len(cancels) == 1 + + +def test_immutable_chaining(): + base = from_boxes(_scene()) + assert isinstance(base, Candidates) + base.filter(has_text="Save") # does not mutate base + assert len(base) == 4 + + +# --- wiring --------------------------------------------------------------- + +def test_wiring(): + assert "AC_locate_chain" in set(ac.executor.known_commands()) + from je_auto_control.utils.mcp_server.tools import build_default_tool_registry + names = {t.name for t in build_default_tool_registry()} + assert "ac_locate_chain" in names + from je_auto_control.gui.script_builder.command_schema import _build_specs + specs = {s.command for s in _build_specs()} + assert "AC_locate_chain" in specs + + +def test_executor_applies_op_chain(): + from je_auto_control.utils.executor.action_executor import _locate_chain + result = _locate_chain(_scene(), ops=[ + {"op": "within", "region": [0, 0, 300, 100]}, + {"op": "filter", "has_text": "Delete"}, + {"op": "reading"}, + {"op": "nth", "index": 0}]) + assert result["count"] == 1 and result["center"] == [80, 20] + + +def test_facade_exports(): + for attr in ("Candidates", "from_boxes"): + assert hasattr(ac, attr) and attr in ac.__all__ diff --git a/test/unit_test/headless/test_masked_match_batch.py b/test/unit_test/headless/test_masked_match_batch.py new file mode 100644 index 00000000..50b9ff19 --- /dev/null +++ b/test/unit_test/headless/test_masked_match_batch.py @@ -0,0 +1,98 @@ +"""Headless tests for masked template matching. No Qt.""" +import pytest + +import je_auto_control as ac + +np = pytest.importorskip("numpy") +pytest.importorskip("cv2") + +from je_auto_control.utils.visual_match import ( # noqa: E402 + match_masked, match_masked_all, +) + + +def _textured(height, width, seed): + yy, xx = np.mgrid[0:height, 0:width] + return ((yy * 53 + xx * 97 + yy * xx * 17 + seed * 31) % 256).astype(np.uint8) + + +def _template_with_border(): + """20x20: a distinctive 10x10 core (seed 1) inside a border (seed 2).""" + tmpl = _textured(20, 20, 2) + tmpl[5:15, 5:15] = _textured(10, 10, 1) + mask = np.zeros((20, 20), dtype=np.uint8) + mask[5:15, 5:15] = 255 + return tmpl, mask + + +def _scene_with_different_border(): + """Haystack: the same core at (40, 30) but a DIFFERENT surrounding border.""" + hay = _textured(100, 100, 9) + patch = _textured(20, 20, 7) # border differs from template + patch[5:15, 5:15] = _textured(10, 10, 1) # core is identical + hay[30:50, 40:60] = patch + return hay + + +def test_mask_ignores_background(): + tmpl, mask = _template_with_border() + hit = match_masked(tmpl, mask=mask, haystack=_scene_with_different_border(), + min_score=0.9) + assert hit is not None + assert (hit.x, hit.y) == (40, 30) + assert hit.center == [50, 40] + + +def test_unmasked_is_dragged_down_by_border(): + """Without the mask the differing border lowers the score below threshold.""" + tmpl, _ = _template_with_border() + assert match_masked(tmpl, haystack=_scene_with_different_border(), + min_score=0.999) is None + + +def test_alpha_channel_is_the_implicit_mask(): + tmpl, mask = _template_with_border() + rgba = np.dstack([tmpl, tmpl, tmpl, mask]) # alpha == the core mask + hit = match_masked(rgba, haystack=_scene_with_different_border(), + min_score=0.9) + assert hit is not None and (hit.x, hit.y) == (40, 30) + + +def test_absent_template_returns_none(): + tmpl, mask = _template_with_border() + blank = np.zeros((100, 100), dtype=np.uint8) + assert match_masked(tmpl, mask=mask, haystack=blank, min_score=0.9) is None + + +def test_match_all_dedupes_to_one(): + tmpl, mask = _template_with_border() + hits = match_masked_all(tmpl, mask=mask, + haystack=_scene_with_different_border(), + min_score=0.99) # only the exact (1.0) core matches + assert len(hits) == 1 + assert (hits[0].x, hits[0].y) == (40, 30) + + +def test_mask_shape_mismatch_raises(): + tmpl, _ = _template_with_border() + bad = np.zeros((5, 5), dtype=np.uint8) + with pytest.raises(ValueError): + match_masked(tmpl, mask=bad, haystack=_scene_with_different_border()) + + +# --- wiring --------------------------------------------------------------- + +def test_wiring(): + known = set(ac.executor.known_commands()) + assert {"AC_match_masked", "AC_match_masked_all"} <= known + from je_auto_control.utils.mcp_server.tools import build_default_tool_registry + names = {t.name for t in build_default_tool_registry()} + assert {"ac_match_masked", "ac_match_masked_all"} <= names + from je_auto_control.gui.script_builder.command_schema import _build_specs + specs = {s.command for s in _build_specs()} + assert {"AC_match_masked", "AC_match_masked_all"} <= specs + + +def test_facade_exports(): + for attr in ("match_masked", "match_masked_all"): + assert hasattr(ac, attr) and attr in ac.__all__ diff --git a/test/unit_test/headless/test_modifier_state_batch.py b/test/unit_test/headless/test_modifier_state_batch.py new file mode 100644 index 00000000..2ef63c68 --- /dev/null +++ b/test/unit_test/headless/test_modifier_state_batch.py @@ -0,0 +1,65 @@ +"""Headless tests for holding modifiers across an action group. No Qt.""" +import je_auto_control as ac +from je_auto_control.utils.modifier_state import ( + hold_modifiers, plan_with_modifiers, +) + + +def test_plan_wraps_with_reversed_release(): + plan = plan_with_modifiers([{"op": "click"}], ["ctrl", "shift"]) + assert plan == [ + {"op": "press", "key": "ctrl"}, + {"op": "press", "key": "shift"}, + {"op": "click"}, + {"op": "release", "key": "shift"}, + {"op": "release", "key": "ctrl"}, + ] + + +def test_context_manager_press_then_release(): + events = [] + with hold_modifiers(["ctrl"], sink=events.append): + events.append({"op": "body"}) + assert [e["op"] for e in events] == ["press", "body", "release"] + assert events[0]["key"] == "ctrl" and events[-1]["key"] == "ctrl" + + +def test_release_even_on_exception(): + events = [] + try: + with hold_modifiers(["ctrl", "shift"], sink=events.append): + raise RuntimeError("boom") + except RuntimeError: + pass + releases = [e for e in events if e["op"] == "release"] + assert [r["key"] for r in releases] == ["shift", "ctrl"] # reversed + + +def test_yields_modifier_list(): + with hold_modifiers(["alt"], sink=lambda e: None) as held: + assert held == ["alt"] + + +# --- wiring --------------------------------------------------------------- + +def test_executor_adapter_modifier_parsing(): + # the nested-action run is device-bound; verify the modifier-string parsing + # the adapter does up front, using the pure plan as the oracle. + plan = plan_with_modifiers([], ["ctrl", "shift"]) + assert plan[0]["key"] == "ctrl" and plan[1]["key"] == "shift" + + +def test_wiring(): + known = ac.executor.known_commands() + assert "AC_with_modifiers" in set(known) + from je_auto_control.utils.mcp_server.tools import build_default_tool_registry + names = {t.name for t in build_default_tool_registry()} + assert "ac_with_modifiers" in names + from je_auto_control.gui.script_builder.command_schema import _build_specs + specs = {s.command for s in _build_specs()} + assert "AC_with_modifiers" in specs + + +def test_facade_exports(): + for attr in ("hold_modifiers", "plan_with_modifiers"): + assert hasattr(ac, attr) and attr in ac.__all__ diff --git a/test/unit_test/headless/test_monitor_layout_batch.py b/test/unit_test/headless/test_monitor_layout_batch.py new file mode 100644 index 00000000..5d059456 --- /dev/null +++ b/test/unit_test/headless/test_monitor_layout_batch.py @@ -0,0 +1,87 @@ +"""Headless tests for multi-monitor geometry. No Qt; provider is injected.""" +import je_auto_control as ac +from je_auto_control.utils.monitor_layout import ( + Monitor, enumerate_monitors, monitor_at_point, monitor_for_window, + primary_monitor, remap_point, to_local, to_virtual, virtual_bounds, +) + + +def _two_monitors(): + # primary 1920x1080 at origin; secondary 1280x1024 to the LEFT (negative x) + return [ + Monitor(0, 0, 0, 1920, 1080, scale=1.0, primary=True), + Monitor(1, -1280, 0, 1280, 1024, scale=1.25), + ] + + +def test_virtual_bounds_spans_negative_origin(): + assert virtual_bounds(_two_monitors()) == (-1280, 0, 3200, 1080) + + +def test_virtual_bounds_empty_raises(): + try: + virtual_bounds([]) + except ValueError: + return + raise AssertionError("expected ValueError") + + +def test_primary_monitor(): + assert primary_monitor(_two_monitors()).index == 0 + + +def test_monitor_at_point(): + mons = _two_monitors() + assert monitor_at_point(mons, 100, 100).index == 0 + assert monitor_at_point(mons, -200, 100).index == 1 # on the left monitor + assert monitor_at_point(mons, 5000, 5000) is None + + +def test_monitor_for_window_max_overlap(): + mons = _two_monitors() + # window mostly on the secondary (left) monitor + assert monitor_for_window((-1000, 50, 400, 300), mons).index == 1 + assert monitor_for_window((10, 10, 400, 300), mons).index == 0 + assert monitor_for_window((9000, 9000, 10, 10), mons) is None + + +def test_to_local_and_back(): + mons = _two_monitors() + assert to_local(mons, -200, 100) == (1, 1080, 100) # local within monitor 1 + assert to_virtual(mons[1], 1080, 100) == (-200, 100) + assert to_local(mons, 9000, 9000) is None + + +def test_remap_point_preserves_fraction(): + src, dst = _two_monitors() # 1920x1080 -> 1280x1024 + # centre of src maps to centre of dst + assert remap_point(src, dst, 960, 540) == (640, 512) + assert remap_point(src, dst, 0, 0) == (0, 0) + + +def test_enumerate_monitors_with_injected_provider(): + rows = [{"x": 0, "y": 0, "width": 1920, "height": 1080, "primary": True}, + {"x": 1920, "y": 0, "width": 1280, "height": 1024, "scale": 1.5}] + mons = enumerate_monitors(provider=lambda: rows) + assert [m.index for m in mons] == [0, 1] + assert mons[0].primary is True and abs(mons[1].scale - 1.5) < 1e-9 + assert mons[1].x == 1920 + + +# --- wiring --------------------------------------------------------------- + +def test_wiring(): + known = set(ac.executor.known_commands()) + assert {"AC_enumerate_monitors", "AC_monitor_at_point"} <= known + from je_auto_control.utils.mcp_server.tools import build_default_tool_registry + names = {t.name for t in build_default_tool_registry()} + assert {"ac_enumerate_monitors", "ac_monitor_at_point"} <= names + from je_auto_control.gui.script_builder.command_schema import _build_specs + specs = {s.command for s in _build_specs()} + assert {"AC_enumerate_monitors", "AC_monitor_at_point"} <= specs + + +def test_facade_exports(): + for attr in ("Monitor", "enumerate_monitors", "monitor_at_point", + "virtual_bounds", "remap_point", "to_local", "to_virtual"): + assert hasattr(ac, attr) and attr in ac.__all__ diff --git a/test/unit_test/headless/test_mouse_path_batch.py b/test/unit_test/headless/test_mouse_path_batch.py new file mode 100644 index 00000000..62411d8a --- /dev/null +++ b/test/unit_test/headless/test_mouse_path_batch.py @@ -0,0 +1,77 @@ +"""Headless tests for multi-waypoint mouse gestures. No Qt.""" +import json + +import je_auto_control as ac +from je_auto_control.utils.mouse_path import ( + drag_path, move_along_path, path_easings, plan_path, +) + + +def test_plan_path_through_waypoints_dedups_junctions(): + points = plan_path([(0, 0), (10, 0), (10, 10)], per_segment_steps=5) + assert points[0] == [0, 0] and points[-1] == [10, 10] + assert len(points) == 11 # 5 + 5 + shared junction once + assert points.count([10, 0]) == 1 # junction not duplicated + + +def test_plan_path_single_and_empty(): + assert plan_path([(3, 4)]) == [[3, 4]] + assert plan_path([]) == [] + + +def test_move_along_path_emits_only_moves(): + events = [] + result = move_along_path([(0, 0), (2, 2)], per_segment_steps=2, + sink=events.append) + assert all(e["op"] == "move" for e in events) + assert result["points"] == len(events) == 3 + + +def test_drag_path_press_first_release_last(): + events = [] + drag_path([(0, 0), (5, 5), (5, 0)], per_segment_steps=2, + sink=events.append) + assert events[0]["op"] == "press" and (events[0]["x"], events[0]["y"]) == (0, 0) + assert events[-1]["op"] == "release" and (events[-1]["x"], events[-1]["y"]) == (5, 0) + assert [e["op"] for e in events[1:-1]] == ["move"] * (len(events) - 2) + + +def test_drag_path_empty_waypoints_noop(): + events = [] + result = drag_path([], sink=events.append) + assert result["points"] == 0 and events == [] + + +def test_easing_changes_intermediate_points(): + linear = plan_path([(0, 0), (100, 0)], easing="linear", per_segment_steps=10) + eased = plan_path([(0, 0), (100, 0)], easing="ease_in_cubic", + per_segment_steps=10) + assert linear[0] == eased[0] and linear[-1] == eased[-1] + assert linear[5] != eased[5] # different curve mid-path + assert "linear" in path_easings() + + +# --- wiring --------------------------------------------------------------- + +def test_executor_waypoint_coercion(): + # the executor's backend dispatch is device-bound; exercise the adapter's + # JSON-string -> list coercion (the part that runs before any real input). + from je_auto_control.utils.executor.action_executor import _waypoints + assert _waypoints(json.dumps([[0, 0], [4, 4]])) == [[0, 0], [4, 4]] + assert _waypoints([[1, 2]]) == [[1, 2]] # already a list + + +def test_wiring(): + known = ac.executor.known_commands() + assert {"AC_move_along_path", "AC_drag_path"} <= set(known) + from je_auto_control.utils.mcp_server.tools import build_default_tool_registry + names = {t.name for t in build_default_tool_registry()} + assert {"ac_move_along_path", "ac_drag_path"} <= names + from je_auto_control.gui.script_builder.command_schema import _build_specs + specs = {s.command for s in _build_specs()} + assert {"AC_move_along_path", "AC_drag_path"} <= specs + + +def test_facade_exports(): + for attr in ("plan_path", "move_along_path", "drag_path", "path_easings"): + assert hasattr(ac, attr) and attr in ac.__all__ diff --git a/test/unit_test/headless/test_mouse_relative_batch.py b/test/unit_test/headless/test_mouse_relative_batch.py new file mode 100644 index 00000000..576d9055 --- /dev/null +++ b/test/unit_test/headless/test_mouse_relative_batch.py @@ -0,0 +1,54 @@ +"""Headless tests for relative mouse movement. No Qt.""" +import je_auto_control as ac +from je_auto_control.utils.exception.exceptions import AutoControlMouseException +from je_auto_control.utils.mouse_relative import ( + move_mouse_relative, relative_target, +) + + +def test_relative_target_arithmetic(): + assert relative_target((100, 100), -40, 12) == (60, 112) + assert relative_target((0, 0), 0, 0) == (0, 0) + + +def test_move_uses_current_position_plus_delta(): + moves = [] + result = move_mouse_relative( + 10, -5, get_position=lambda: (200, 200), + set_position=lambda x, y: moves.append((x, y))) + assert moves == [(210, 195)] + assert result == {"from": [200, 200], "to": [210, 195], "delta": [10, -5]} + + +def test_raises_when_position_unreadable(): + try: + move_mouse_relative(1, 1, get_position=lambda: None, + set_position=lambda x, y: None) + except AutoControlMouseException: + pass + else: # pragma: no cover + raise AssertionError("expected AutoControlMouseException") + + +# --- wiring --------------------------------------------------------------- + +def test_executor_adapter_planning(): + # the default backend get/set is device-bound; exercise the pure arithmetic + # the adapter relies on instead. + assert relative_target((5, 5), 3, 4) == (8, 9) + + +def test_wiring(): + known = ac.executor.known_commands() + assert "AC_move_mouse_relative" in set(known) + from je_auto_control.utils.mcp_server.tools import build_default_tool_registry + names = {t.name for t in build_default_tool_registry()} + assert "ac_move_mouse_relative" in names + from je_auto_control.gui.script_builder.command_schema import _build_specs + specs = {s.command for s in _build_specs()} + assert "AC_move_mouse_relative" in specs + + +def test_facade_exports(): + for attr in ("move_mouse_relative", "relative_target"): + assert hasattr(ac, attr) and attr in ac.__all__ diff --git a/test/unit_test/headless/test_preprocess_batch.py b/test/unit_test/headless/test_preprocess_batch.py new file mode 100644 index 00000000..9488be66 --- /dev/null +++ b/test/unit_test/headless/test_preprocess_batch.py @@ -0,0 +1,113 @@ +"""Headless tests for OCR/match image preprocessing. No Qt.""" +import pytest + +import je_auto_control as ac + +np = pytest.importorskip("numpy") +cv2 = pytest.importorskip("cv2") + +from je_auto_control.utils.preprocess import ( # noqa: E402 + binarize, denoise, deskew, detect_skew_angle, enhance_contrast, + preprocess_image, to_grayscale, upscale, +) + + +def _color(): + img = np.zeros((40, 60, 3), dtype=np.uint8) + img[:, :30] = (80, 80, 80) + img[:, 30:] = (200, 200, 200) + return img + + +def _skewed(angle): + """A dark horizontal bar on a light background, rotated by ``angle`` degrees.""" + canvas = np.full((120, 200), 255, dtype=np.uint8) + cv2.rectangle(canvas, (40, 55), (160, 65), 0, -1) + matrix = cv2.getRotationMatrix2D((100, 60), angle, 1.0) + return cv2.warpAffine(canvas, matrix, (200, 120), borderValue=255) + + +def test_grayscale_drops_channels(): + assert to_grayscale(_color()).shape == (40, 60) + + +def test_upscale_doubles_dimensions(): + assert upscale(_color(), scale=2.0).shape[:2] == (80, 120) + + +def test_upscale_rejects_unknown_interp(): + with pytest.raises(ValueError): + upscale(_color(), interp="magic") + + +def test_binarize_otsu_is_two_valued(): + assert sorted(set(binarize(_color()).flatten().tolist())) == [0, 255] + + +def test_binarize_adaptive_keeps_shape(): + assert binarize(_color(), method="adaptive_gaussian").shape == (40, 60) + + +def test_binarize_rejects_unknown_method(): + with pytest.raises(ValueError): + binarize(_color(), method="triangle") + + +def test_denoise_and_contrast_keep_grayscale_shape(): + assert denoise(_color()).shape == (40, 60) + assert enhance_contrast(_color()).shape == (40, 60) + + +def test_detect_skew_recovers_angle_magnitude(): + assert 5.0 <= abs(detect_skew_angle(_skewed(10))) <= 15.0 + + +def test_detect_skew_clamps_to_zero_beyond_max(): + assert detect_skew_angle(_skewed(10), max_angle=3.0) == pytest.approx(0.0, abs=1e-9) + + +def test_deskew_reduces_skew(): + rotated = _skewed(10) + before = abs(detect_skew_angle(rotated)) + after = abs(detect_skew_angle(deskew(rotated))) + assert after < before + + +def test_pipeline_chains_steps(): + out = preprocess_image(_color(), steps=("grayscale", "upscale", "binarize")) + assert out.ndim == 2 and out.shape == (80, 120) + assert sorted(set(out.flatten().tolist())) == [0, 255] + + +def test_pipeline_rejects_unknown_step(): + with pytest.raises(ValueError): + preprocess_image(_color(), steps=("sharpen",)) + + +# --- wiring --------------------------------------------------------------- + +def test_wiring(): + assert "AC_preprocess_image" in set(ac.executor.known_commands()) + from je_auto_control.utils.mcp_server.tools import build_default_tool_registry + names = {t.name for t in build_default_tool_registry()} + assert "ac_preprocess_image" in names + from je_auto_control.gui.script_builder.command_schema import _build_specs + specs = {s.command for s in _build_specs()} + assert "AC_preprocess_image" in specs + + +def test_facade_exports(): + for attr in ("preprocess_image", "to_grayscale", "binarize", "upscale", + "deskew", "enhance_contrast"): + assert hasattr(ac, attr) and attr in ac.__all__ + + +def test_executor_writes_output(tmp_path): + from je_auto_control.utils.executor.action_executor import _preprocess_image + src = tmp_path / "src.png" + cv2.imwrite(str(src), _color()) + out = tmp_path / "out.png" + result = _preprocess_image(str(out), source=str(src), + steps=["grayscale", "binarize"]) + assert result["path"] == str(out) and out.exists() + assert result["width"] == 60 and result["height"] == 40 diff --git a/test/unit_test/headless/test_remote_desktop_chat_and_multicursor.py b/test/unit_test/headless/test_remote_desktop_chat_and_multicursor.py index b5aace94..717de3cc 100644 --- a/test/unit_test/headless/test_remote_desktop_chat_and_multicursor.py +++ b/test/unit_test/headless/test_remote_desktop_chat_and_multicursor.py @@ -22,6 +22,19 @@ def jpeg_bytes(): return _make_jpeg() +def _await_clients(host, expected: int = 1, timeout: float = 5.0) -> None: + """Wait until the host has authenticated ``expected`` viewers. + + ``viewer.connect()`` returns once the *client* side is up; the host's accept + thread may not have authenticated the viewer yet, so a broadcast issued + immediately can reach zero clients (reliably so on a slow CI container). This + closes that race using the host's own authenticated-client count. + """ + deadline = time.monotonic() + timeout + while host.connected_clients < expected and time.monotonic() < deadline: + time.sleep(0.02) + + # --- Phase 5.2: chat ---------------------------------------------------- def test_host_broadcasts_chat_to_viewer(jpeg_bytes): @@ -44,6 +57,7 @@ def on_chat(sender: str, text: str) -> None: ) viewer.connect(timeout=5.0) try: + _await_clients(host) sent = host.broadcast_chat("hello viewer") assert sent == 1 deadline = time.monotonic() + 2.0 @@ -75,6 +89,7 @@ def host_on_chat(sender: str, text: str) -> None: viewer = RemoteDesktopViewer(host="127.0.0.1", port=host.port, token="t") viewer.connect(timeout=5.0) try: + _await_clients(host) viewer.send_chat("ping from viewer", sender="alice") deadline = time.monotonic() + 2.0 while not received and time.monotonic() < deadline: @@ -124,6 +139,7 @@ def test_viewer_cursor_payload_routes_to_separate_callback(jpeg_bytes): ) viewer.connect(timeout=5.0) try: + _await_clients(host) # Simulate MultiViewerHost relaying another operator's cursor. host.broadcast_viewer_cursor("alice", 200, 300) deadline = time.monotonic() + 2.0 diff --git a/test/unit_test/headless/test_rich_clipboard_batch.py b/test/unit_test/headless/test_rich_clipboard_batch.py new file mode 100644 index 00000000..bb055ad0 --- /dev/null +++ b/test/unit_test/headless/test_rich_clipboard_batch.py @@ -0,0 +1,76 @@ +"""Headless tests for rich (CF_HTML) clipboard. No Qt; I/O is Windows-only.""" +import re +import sys + +import pytest + +import je_auto_control as ac +from je_auto_control.utils.rich_clipboard import ( + build_cf_html, get_clipboard_html, parse_cf_html, set_clipboard_html, +) + + +def test_build_offsets_point_at_fragment(): + html = "Bold & café 文字" # multibyte UTF-8 + blob = build_cf_html(html) + assert isinstance(blob, bytes) + text = blob.decode("utf-8") + start = int(re.search(r"StartFragment:(\d+)", text).group(1)) + end = int(re.search(r"EndFragment:(\d+)", text).group(1)) + assert blob[start:end].decode("utf-8") == html # byte offsets are exact + + +def test_round_trip_via_markers(): + html = "hello world
" + assert parse_cf_html(build_cf_html(html)) == html + + +def test_parse_marker_only_payload(): + payload = "X" + assert parse_cf_html(payload) == "X" + + +def test_parse_offset_fallback_without_markers(): + fragment = "FRAG" + body = "" + fragment + "" + prefix = "StartFragment:{:010d}\r\nEndFragment:{:010d}\r\n".format( + len(""), len("") + len(fragment)) + # offsets are relative to the whole payload, so prepend the header length + header_len = len(prefix.encode("utf-8")) + payload = ("StartFragment:{:010d}\r\nEndFragment:{:010d}\r\n".format( + header_len + len(""), header_len + len("") + len(fragment)) + + body).encode("utf-8") + assert parse_cf_html(payload) == fragment + + +def test_build_rejects_non_string(): + with pytest.raises(TypeError): + build_cf_html(b"bytes") + + +@pytest.mark.skipif(sys.platform.startswith("win"), + reason="HTML clipboard I/O is supported on Windows") +def test_io_raises_off_windows(): + with pytest.raises(RuntimeError): + set_clipboard_html("x") + with pytest.raises(RuntimeError): + get_clipboard_html() + + +# --- wiring --------------------------------------------------------------- + +def test_wiring(): + known = set(ac.executor.known_commands()) + assert {"AC_set_clipboard_html", "AC_get_clipboard_html"} <= known + from je_auto_control.utils.mcp_server.tools import build_default_tool_registry + names = {t.name for t in build_default_tool_registry()} + assert {"ac_set_clipboard_html", "ac_get_clipboard_html"} <= names + from je_auto_control.gui.script_builder.command_schema import _build_specs + specs = {s.command for s in _build_specs()} + assert {"AC_set_clipboard_html", "AC_get_clipboard_html"} <= specs + + +def test_facade_exports(): + for attr in ("build_cf_html", "parse_cf_html", "get_clipboard_html", + "set_clipboard_html"): + assert hasattr(ac, attr) and attr in ac.__all__ diff --git a/test/unit_test/headless/test_shape_locator_batch.py b/test/unit_test/headless/test_shape_locator_batch.py new file mode 100644 index 00000000..2ff2f66e --- /dev/null +++ b/test/unit_test/headless/test_shape_locator_batch.py @@ -0,0 +1,75 @@ +"""Headless tests for edge/contour shape location. No Qt.""" +import pytest + +import je_auto_control as ac + +np = pytest.importorskip("numpy") +cv2 = pytest.importorskip("cv2") + +from je_auto_control.utils.shape_locator import ( # noqa: E402 + find_rectangles, find_shapes, +) + + +def _scene(): + """300x200 scene: a wide button, a square outline, and a circle.""" + img = np.zeros((200, 300, 3), dtype=np.uint8) + cv2.rectangle(img, (20, 20), (120, 70), (255, 255, 255), -1) # 100x50 button + cv2.rectangle(img, (160, 30), (200, 70), (200, 200, 200), 2) # 40x40 outline + cv2.circle(img, (80, 150), 30, (180, 180, 180), -1) # not a rectangle + return img + + +def _near(value, target, tol=4): + return abs(value - target) <= tol + + +def test_find_shapes_returns_all_three(): + shapes = find_shapes(_scene(), min_area=300) + assert len(shapes) == 3 # button, outline, circle bbox + biggest = shapes[0] # largest first + assert _near(biggest["width"], 100) and _near(biggest["height"], 50) + + +def test_find_rectangles_excludes_the_circle(): + rects = find_rectangles(_scene(), min_area=300) + assert len(rects) == 2 # circle dropped + assert all(0.7 < r["aspect"] for r in rects) + + +def test_aspect_range_keeps_only_wide_button(): + wide = find_rectangles(_scene(), min_area=300, aspect_range=(1.5, 8.0)) + assert len(wide) == 1 + box = wide[0] + assert _near(box["x"], 20) and _near(box["y"], 20) + assert _near(box["center"][0], 70) and _near(box["center"][1], 45) + + +def test_max_area_filters_large_shapes(): + small = find_shapes(_scene(), min_area=300, max_area=3000) + assert all(s["area"] <= 3000 for s in small) + assert small # the ~40x40 outline survives + + +def test_min_area_filters_specks(): + img = np.zeros((100, 100, 3), dtype=np.uint8) + cv2.rectangle(img, (10, 10), (15, 15), (255, 255, 255), -1) # tiny speck + assert find_shapes(img, min_area=500) == [] + + +# --- wiring --------------------------------------------------------------- + +def test_wiring(): + known = set(ac.executor.known_commands()) + assert {"AC_find_shapes", "AC_find_rectangles"} <= known + from je_auto_control.utils.mcp_server.tools import build_default_tool_registry + names = {t.name for t in build_default_tool_registry()} + assert {"ac_find_shapes", "ac_find_rectangles"} <= names + from je_auto_control.gui.script_builder.command_schema import _build_specs + specs = {s.command for s in _build_specs()} + assert {"AC_find_shapes", "AC_find_rectangles"} <= specs + + +def test_facade_exports(): + for attr in ("find_shapes", "find_rectangles"): + assert hasattr(ac, attr) and attr in ac.__all__ diff --git a/test/unit_test/headless/test_ssim_compare_batch.py b/test/unit_test/headless/test_ssim_compare_batch.py new file mode 100644 index 00000000..6b81cca2 --- /dev/null +++ b/test/unit_test/headless/test_ssim_compare_batch.py @@ -0,0 +1,79 @@ +"""Headless tests for SSIM structural comparison. No Qt.""" +import pytest + +import je_auto_control as ac + +np = pytest.importorskip("numpy") +pytest.importorskip("cv2") + +from je_auto_control.utils.ssim import ( # noqa: E402 + ssim_changed_regions, ssim_compare, +) + + +def _base(): + """A textured 80x80 reference image (not flat — SSIM needs structure).""" + yy, xx = np.mgrid[0:80, 0:80] + return ((yy * 7 + xx * 11 + yy * xx) % 256).astype(np.uint8) + + +def _with_block(): + """The reference with a 20x20 block overwritten at (40, 30).""" + changed = _base() + changed[30:50, 40:60] = 0 + return changed + + +def test_identical_scores_one(): + base = _base() + assert ssim_compare(base, base.copy()) == pytest.approx(1.0) + + +def test_change_lowers_score(): + assert ssim_compare(_base(), _with_block()) < 0.99 + + +def test_ignore_region_restores_score(): + # masking out the changed block lifts the score back towards identical + without = ssim_compare(_base(), _with_block()) + with_ignore = ssim_compare(_base(), _with_block(), ignore=[[40, 30, 20, 20]]) + assert with_ignore > without + assert with_ignore > 0.98 + + +def test_changed_regions_locates_the_block(): + regions = ssim_changed_regions(_base(), _with_block(), min_area=20) + assert len(regions) == 1 + box = regions[0] + # the changed blob overlaps the (40,30)-(60,50) block + assert 30 <= box["x"] <= 50 and 20 <= box["y"] <= 40 + assert box["area"] >= 20 + + +def test_changed_regions_empty_when_identical(): + base = _base() + assert ssim_changed_regions(base, base.copy(), min_area=20) == [] + + +def test_size_mismatch_raises(): + small = np.zeros((40, 40), dtype=np.uint8) + with pytest.raises(ValueError): + ssim_compare(_base(), small) + + +# --- wiring --------------------------------------------------------------- + +def test_wiring(): + known = set(ac.executor.known_commands()) + assert {"AC_ssim_compare", "AC_ssim_changed_regions"} <= known + from je_auto_control.utils.mcp_server.tools import build_default_tool_registry + names = {t.name for t in build_default_tool_registry()} + assert {"ac_ssim_compare", "ac_ssim_changed_regions"} <= names + from je_auto_control.gui.script_builder.command_schema import _build_specs + specs = {s.command for s in _build_specs()} + assert {"AC_ssim_compare", "AC_ssim_changed_regions"} <= specs + + +def test_facade_exports(): + for attr in ("ssim_compare", "ssim_changed_regions"): + assert hasattr(ac, attr) and attr in ac.__all__ diff --git a/test/unit_test/headless/test_text_regions_batch.py b/test/unit_test/headless/test_text_regions_batch.py new file mode 100644 index 00000000..df7abff9 --- /dev/null +++ b/test/unit_test/headless/test_text_regions_batch.py @@ -0,0 +1,70 @@ +"""Headless tests for MSER text-region detection. No Qt.""" +import pytest + +import je_auto_control as ac + +np = pytest.importorskip("numpy") +cv2 = pytest.importorskip("cv2") + +from je_auto_control.utils.text_regions import ( # noqa: E402 + find_text_lines, find_text_regions, +) + + +def _two_rows(): + img = np.full((120, 300), 235, dtype=np.uint8) + for i, ch in enumerate("ABCD"): + cv2.putText(img, ch, (20 + i * 30, 40), cv2.FONT_HERSHEY_SIMPLEX, 1.0, 0, 2) + for i, ch in enumerate("XY"): + cv2.putText(img, ch, (20 + i * 30, 90), cv2.FONT_HERSHEY_SIMPLEX, 1.0, 0, 2) + return img + + +def test_finds_glyph_regions(): + regions = find_text_regions(_two_rows(), min_area=60) + assert len(regions) >= 4 # at least the four top-row glyphs + assert all("center" in r and r["area"] >= 60 for r in regions) + + +def test_groups_into_two_lines(): + lines = find_text_lines(_two_rows(), y_tolerance=8) + assert len(lines) == 2 + top, bottom = sorted(lines, key=lambda line: line["y"]) + assert top["width"] > bottom["width"] # ABCD wider than XY + assert bottom["y"] > top["y"] + + +def test_blank_image_has_no_text(): + blank = np.full((100, 100), 235, dtype=np.uint8) + assert find_text_regions(blank) == [] + assert find_text_lines(blank) == [] + + +def test_min_area_filters_small(): + big = find_text_regions(_two_rows(), min_area=100000) + assert big == [] + + +def test_merge_reduces_region_count(): + scene = _two_rows() + unmerged = find_text_regions(scene, min_area=40, merge=False) + merged = find_text_regions(scene, min_area=40, merge=True) + assert len(merged) <= len(unmerged) + + +# --- wiring --------------------------------------------------------------- + +def test_wiring(): + known = set(ac.executor.known_commands()) + assert {"AC_find_text_regions", "AC_find_text_lines"} <= known + from je_auto_control.utils.mcp_server.tools import build_default_tool_registry + names = {t.name for t in build_default_tool_registry()} + assert {"ac_find_text_regions", "ac_find_text_lines"} <= names + from je_auto_control.gui.script_builder.command_schema import _build_specs + specs = {s.command for s in _build_specs()} + assert {"AC_find_text_regions", "AC_find_text_lines"} <= specs + + +def test_facade_exports(): + for attr in ("find_text_regions", "find_text_lines"): + assert hasattr(ac, attr) and attr in ac.__all__ diff --git a/test/unit_test/headless/test_text_unicode_batch.py b/test/unit_test/headless/test_text_unicode_batch.py new file mode 100644 index 00000000..f2c80141 --- /dev/null +++ b/test/unit_test/headless/test_text_unicode_batch.py @@ -0,0 +1,58 @@ +"""Headless tests for Unicode text entry via clipboard. No Qt.""" +import je_auto_control as ac +from je_auto_control.utils.text_unicode import ( + plan_paste, type_unicode, unicode_code_units, +) + + +def test_code_units_ascii_and_bmp(): + assert unicode_code_units("Hi") == [72, 105] + assert unicode_code_units("値") == [0x5024] + assert unicode_code_units("") == [] + + +def test_code_units_astral_surrogate_pair(): + assert unicode_code_units("🚀") == [0xD83D, 0xDE80] # rocket > U+FFFF + assert len(unicode_code_units("a🚀b")) == 4 # 1 + 2 + 1 + + +def test_plan_paste_is_clipboard_then_hotkey(): + assert plan_paste("café 🚀") == [ + {"op": "set_clipboard", "text": "café 🚀"}, + {"op": "hotkey", "keys": ["ctrl", "v"]}, + ] + assert plan_paste("x", modifier="command")[1]["keys"] == ["command", "v"] + + +def test_type_unicode_dispatches_plan(): + events = [] + result = type_unicode("café 🚀", sink=events.append) + assert [e["op"] for e in events] == ["set_clipboard", "hotkey"] + assert events[0]["text"] == "café 🚀" + assert result["ops"] == 2 and result["code_units"] == 7 + + +# --- wiring --------------------------------------------------------------- + +def test_executor_adapter_dispatches_via_sink(): + # the executor default dispatch is device-bound; verify the planning the + # adapter delegates to instead. + events = [] + type_unicode("値", sink=events.append, modifier="ctrl") + assert events[-1] == {"op": "hotkey", "keys": ["ctrl", "v"]} + + +def test_wiring(): + known = ac.executor.known_commands() + assert "AC_type_unicode" in set(known) + from je_auto_control.utils.mcp_server.tools import build_default_tool_registry + names = {t.name for t in build_default_tool_registry()} + assert "ac_type_unicode" in names + from je_auto_control.gui.script_builder.command_schema import _build_specs + specs = {s.command for s in _build_specs()} + assert "AC_type_unicode" in specs + + +def test_facade_exports(): + for attr in ("type_unicode", "plan_paste", "unicode_code_units"): + assert hasattr(ac, attr) and attr in ac.__all__ diff --git a/test/unit_test/headless/test_visual_match_batch.py b/test/unit_test/headless/test_visual_match_batch.py new file mode 100644 index 00000000..d47988ac --- /dev/null +++ b/test/unit_test/headless/test_visual_match_batch.py @@ -0,0 +1,87 @@ +"""Headless tests for confidence-returning template matching. No Qt.""" +import pytest + +import je_auto_control as ac + +np = pytest.importorskip("numpy") +pytest.importorskip("cv2") + +from je_auto_control.utils.visual_match import ( # noqa: E402 + Match, best_matches, match_template, match_template_all, +) + + +def _patch(): + """A 20x20 textured patch (non-constant, so CCOEFF_NORMED is well-defined).""" + return np.tile(np.arange(0, 200, 10, dtype=np.uint8), (20, 1)) + + +def _haystack(): + hay = np.zeros((100, 200), dtype=np.uint8) + patch = _patch() + hay[30:50, 50:70] = patch + hay[30:50, 120:140] = patch + return hay + + +def test_match_returns_score_and_box(): + match = match_template(_patch(), haystack=_haystack(), min_score=0.9) + assert isinstance(match, Match) + assert (match.x, match.y, match.width, match.height) == (50, 30, 20, 20) + assert match.score == pytest.approx(1.0) + assert match.center == [60, 40] + + +def test_no_match_below_threshold(): + other = np.tile(np.arange(200, 0, -10, dtype=np.uint8), (20, 1))[:8, :8] + assert match_template(other, haystack=_haystack(), min_score=0.99) is None + + +def test_match_all_with_nms(): + matches = match_template_all(_patch(), haystack=_haystack(), min_score=0.9) + assert len(matches) == 2 # two patches, neighbours merged + xs = sorted(m.x for m in matches) + assert xs == [50, 120] + assert all(m.score == pytest.approx(1.0) for m in matches) + + +def test_multiscale_finds_scaled_template(): + import cv2 + yy, xx = np.mgrid[0:20, 0:20] # non-self-similar texture + tmpl = ((yy * 53 + xx * 97 + yy * xx * 17) % 256).astype(np.uint8) + big = cv2.resize(tmpl, (40, 40)) # == _resize(tmpl, 2.0) + hay = np.zeros((120, 200), dtype=np.uint8) + hay[10:50, 60:100] = big # embed only the 2x version + match = match_template(tmpl, haystack=hay, scales=(1.0, 2.0), min_score=0.9) + assert match is not None and match.scale == pytest.approx(2.0) + + +def test_unknown_method_raises(): + with pytest.raises(ValueError): + match_template(_patch(), haystack=_haystack(), method="bogus") + + +def test_to_dict_has_center(): + match = match_template(_patch(), haystack=_haystack(), min_score=0.9) + data = match.to_dict() + assert data["center"] == [60, 40] and data["score"] == pytest.approx(1.0) + + +# --- wiring --------------------------------------------------------------- + +def test_wiring(): + known = ac.executor.known_commands() + assert {"AC_match_template", "AC_match_template_all"} <= set(known) + from je_auto_control.utils.mcp_server.tools import build_default_tool_registry + names = {t.name for t in build_default_tool_registry()} + assert {"ac_match_template", "ac_match_template_all"} <= names + from je_auto_control.gui.script_builder.command_schema import _build_specs + specs = {s.command for s in _build_specs()} + assert {"AC_match_template", "AC_match_template_all"} <= specs + + +def test_facade_exports(): + for attr in ("TemplateMatch", "match_template", "match_template_all", + "best_matches"): + assert hasattr(ac, attr) and attr in ac.__all__ + assert callable(best_matches) diff --git a/test/unit_test/headless/test_wait_color_batch.py b/test/unit_test/headless/test_wait_color_batch.py new file mode 100644 index 00000000..195f2113 --- /dev/null +++ b/test/unit_test/headless/test_wait_color_batch.py @@ -0,0 +1,81 @@ +"""Headless tests for region-colour waits. No Qt.""" +import pytest + +import je_auto_control as ac +from je_auto_control.utils.smart_waits import ( + Frame, WaitOutcome, wait_until_color, +) + + +def _frame(color, count, *, other=(0, 0, 0), other_count=0): + """Build a 1-row Frame: ``count`` px of ``color`` then ``other_count`` of other.""" + pixels = bytes(list(color) * count + list(other) * other_count) + return Frame(width=count + other_count, height=1, pixels=pixels) + + +def test_succeeds_when_colour_reaches_fraction(): + green = _frame((0, 200, 0), 8, other=(0, 0, 0), other_count=2) # 80% green + outcome = wait_until_color( + target_rgb=(0, 200, 0), tolerance=10, min_fraction=0.5, present=True, + timeout_s=1.0, poll_interval_s=0.001, sampler=lambda region: green) + assert isinstance(outcome, WaitOutcome) and outcome.succeeded is True + + +def test_times_out_when_colour_absent(): + blank = _frame((0, 0, 0), 10) + outcome = wait_until_color( + target_rgb=(255, 0, 0), min_fraction=0.5, present=True, + timeout_s=0.03, poll_interval_s=0.001, sampler=lambda region: blank) + assert outcome.succeeded is False and "timeout" in outcome.reason + + +def test_vanish_succeeds_when_below_fraction(): + blank = _frame((0, 0, 0), 10) + outcome = wait_until_color( + target_rgb=(255, 0, 0), min_fraction=0.5, present=False, + timeout_s=1.0, poll_interval_s=0.001, sampler=lambda region: blank) + assert outcome.succeeded is True # colour absent -> "gone" met + + +def test_tolerance_band(): + near = _frame((5, 198, 3), 10) # all near (0,200,0) + outcome = wait_until_color( + target_rgb=(0, 200, 0), tolerance=10, min_fraction=1.0, present=True, + timeout_s=1.0, poll_interval_s=0.001, sampler=lambda region: near) + assert outcome.succeeded is True + + +def test_validation(): + for bad in ({"timeout_s": 0}, {"poll_interval_s": 0}): + try: + wait_until_color(target_rgb=(0, 0, 0), **bad) + except ValueError: + pass + else: # pragma: no cover + raise AssertionError("expected ValueError") + + +# --- wiring --------------------------------------------------------------- + +def test_executor_color_fraction_helper(): + # the adapter's screen dispatch is device-bound; exercise the pure pixel- + # counting helper the wait relies on instead. + from je_auto_control.utils.smart_waits.waits import _color_fraction + half = _frame((0, 200, 0), 5, other=(0, 0, 0), other_count=5) + assert _color_fraction(half, (0, 200, 0), 10) == pytest.approx(0.5) + assert _color_fraction(half, (255, 255, 255), 10) == pytest.approx(0.0) + + +def test_wiring(): + known = ac.executor.known_commands() + assert "AC_wait_color" in set(known) + from je_auto_control.utils.mcp_server.tools import build_default_tool_registry + names = {t.name for t in build_default_tool_registry()} + assert "ac_wait_color" in names + from je_auto_control.gui.script_builder.command_schema import _build_specs + specs = {s.command for s in _build_specs()} + assert "AC_wait_color" in specs + + +def test_facade_exports(): + assert hasattr(ac, "wait_until_color") and "wait_until_color" in ac.__all__ diff --git a/test/unit_test/headless/test_wait_gone_batch.py b/test/unit_test/headless/test_wait_gone_batch.py new file mode 100644 index 00000000..dabb2419 --- /dev/null +++ b/test/unit_test/headless/test_wait_gone_batch.py @@ -0,0 +1,68 @@ +"""Headless tests for blocking wait-until-vanish. No Qt.""" +import je_auto_control as ac +from je_auto_control.utils.smart_waits import WaitOutcome, wait_until_gone + + +def test_returns_when_predicate_becomes_false(): + # present() is True for the first 2 polls, then False. + calls = {"n": 0} + + def present(): + calls["n"] += 1 + return calls["n"] <= 2 + + outcome = wait_until_gone(present, timeout_s=5.0, poll_interval_s=0.001) + assert isinstance(outcome, WaitOutcome) + assert outcome.succeeded is True + assert outcome.reason == "target gone" + assert outcome.samples_taken == 3 + + +def test_already_gone_returns_immediately(): + outcome = wait_until_gone(lambda: False, timeout_s=5.0, + poll_interval_s=0.001) + assert outcome.succeeded is True and outcome.samples_taken == 1 + + +def test_timeout_when_always_present(): + outcome = wait_until_gone(lambda: True, timeout_s=0.05, + poll_interval_s=0.001) + assert outcome.succeeded is False + assert "timeout" in outcome.reason + + +def test_validation(): + for bad in ({"timeout_s": 0}, {"poll_interval_s": 0}, {"gone_for_s": -1}): + try: + wait_until_gone(lambda: False, **bad) + except ValueError: + pass + else: # pragma: no cover + raise AssertionError(f"expected ValueError for {bad}") + + +def test_gone_for_requires_sustained_absence(): + # flips False once then True again -> must NOT count as gone with gone_for_s + seq = iter([True, False, True, True, True, True, True, True]) + outcome = wait_until_gone(lambda: next(seq, True), timeout_s=0.05, + poll_interval_s=0.001, gone_for_s=10.0) + assert outcome.succeeded is False # never gone long enough + + +# --- wiring --------------------------------------------------------------- + +def test_wiring(): + known = ac.executor.known_commands() + assert {"AC_wait_image_gone", "AC_wait_text_gone"} <= set(known) + from je_auto_control.utils.mcp_server.tools import build_default_tool_registry + names = {t.name for t in build_default_tool_registry()} + assert {"ac_wait_image_gone", "ac_wait_text_gone"} <= names + from je_auto_control.gui.script_builder.command_schema import _build_specs + specs = {s.command for s in _build_specs()} + assert {"AC_wait_image_gone", "AC_wait_text_gone"} <= specs + + +def test_facade_exports(): + for attr in ("wait_until_gone", "wait_until_image_gone", + "wait_until_text_gone"): + assert hasattr(ac, attr) and attr in ac.__all__ diff --git a/test/unit_test/headless/test_wait_window_title_batch.py b/test/unit_test/headless/test_wait_window_title_batch.py new file mode 100644 index 00000000..65a7c6da --- /dev/null +++ b/test/unit_test/headless/test_wait_window_title_batch.py @@ -0,0 +1,67 @@ +"""Headless tests for wait-until-window-title. No Qt.""" +import je_auto_control as ac +from je_auto_control.utils.smart_waits import ( + WaitOutcome, wait_until_window_title, +) + + +def test_regex_title_appears(): + # title list flips to include a match on the 2nd poll + state = {"polls": 0} + + def lister(): + state["polls"] += 1 + return ["Editor"] if state["polls"] < 2 else ["Shop — Checkout"] + + outcome = wait_until_window_title(r".*— Checkout$", timeout_s=5.0, + poll_interval_s=0.001, title_lister=lister) + assert isinstance(outcome, WaitOutcome) and outcome.succeeded is True + + +def test_substring_mode(): + outcome = wait_until_window_title( + "Checkout", regex=False, timeout_s=5.0, poll_interval_s=0.001, + title_lister=lambda: ["My Shop Checkout Page"]) + assert outcome.succeeded is True + + +def test_wait_for_vanish(): + outcome = wait_until_window_title( + "Loading", present=False, timeout_s=5.0, poll_interval_s=0.001, + title_lister=lambda: ["Done"]) + assert outcome.succeeded is True # no title matches -> vanished + + +def test_timeout_when_never_matches(): + outcome = wait_until_window_title( + "Nope", timeout_s=0.03, poll_interval_s=0.001, + title_lister=lambda: ["Editor", "Browser"]) + assert outcome.succeeded is False and "timeout" in outcome.reason + + +def test_validation(): + for bad in ({"timeout_s": 0}, {"poll_interval_s": 0}): + try: + wait_until_window_title("x", title_lister=lambda: [], **bad) + except ValueError: + pass + else: # pragma: no cover + raise AssertionError("expected ValueError") + + +# --- wiring --------------------------------------------------------------- + +def test_wiring(): + known = ac.executor.known_commands() + assert "AC_wait_window_title" in set(known) + from je_auto_control.utils.mcp_server.tools import build_default_tool_registry + names = {t.name for t in build_default_tool_registry()} + assert "ac_wait_window_title" in names + from je_auto_control.gui.script_builder.command_schema import _build_specs + specs = {s.command for s in _build_specs()} + assert "AC_wait_window_title" in specs + + +def test_facade_exports(): + assert hasattr(ac, "wait_until_window_title") + assert "wait_until_window_title" in ac.__all__ diff --git a/test/unit_test/headless/test_window_arrange_batch.py b/test/unit_test/headless/test_window_arrange_batch.py new file mode 100644 index 00000000..6fb8ef4e --- /dev/null +++ b/test/unit_test/headless/test_window_arrange_batch.py @@ -0,0 +1,89 @@ +"""Headless tests for multi-window arrangers. No Qt; mover is injected.""" +import je_auto_control as ac +from je_auto_control.utils.window_capture import arrange_cascade, arrange_grid + + +def _recorder(): + """Return (mover, calls) where mover records every move and reports success.""" + calls = [] + + def mover(title, x, y, width, height): + calls.append((title, x, y, width, height)) + return True + + return mover, calls + + +def _screen(): + return lambda: (1920, 1080) + + +def test_arrange_grid_tiles_four_windows_2x2(): + mover, calls = _recorder() + moved = arrange_grid(["a", "b", "c", "d"], mover=mover, screen_size=_screen()) + assert moved == 4 + assert calls == [ + ("a", 0, 0, 960, 540), ("b", 960, 0, 960, 540), + ("c", 0, 540, 960, 540), ("d", 960, 540, 960, 540)] + + +def test_arrange_grid_auto_shape_three_windows(): + mover, calls = _recorder() + # 3 windows -> near-square 2x2 grid; the first three cells are used. + moved = arrange_grid(["a", "b", "c"], mover=mover, screen_size=_screen()) + assert moved == 3 + assert [c[0] for c in calls] == ["a", "b", "c"] + + +def test_arrange_grid_explicit_rows_cols_and_gap(): + mover, calls = _recorder() + arrange_grid(["a", "b", "c"], rows=1, cols=3, gap=10, mover=mover, + screen_size=_screen()) + assert calls[0] == ("a", 10, 10, 620, 1060) # 640-wide cell, gap 10 + assert len(calls) == 3 + + +def test_arrange_grid_empty_is_noop(): + mover, calls = _recorder() + assert arrange_grid([], mover=mover, screen_size=_screen()) == 0 + assert calls == [] + + +def test_arrange_cascade_staggers_windows(): + mover, calls = _recorder() + moved = arrange_cascade(["a", "b", "c"], offset=40, mover=mover, + screen_size=_screen()) + assert moved == 3 + assert calls[0][1:3] == (0, 0) + assert calls[1][1:3] == (40, 40) + assert calls[2][1:3] == (80, 80) + assert all(c[3] == 1152 and c[4] == 648 for c in calls) # 60% of 1920x1080 + + +def test_arrange_counts_only_successful_moves(): + calls = [] + + def flaky(title, x, y, width, height): + calls.append(title) + return title != "b" # "b" fails to move + + moved = arrange_grid(["a", "b", "c", "d"], mover=flaky, screen_size=_screen()) + assert moved == 3 and len(calls) == 4 + + +# --- wiring --------------------------------------------------------------- + +def test_wiring(): + known = set(ac.executor.known_commands()) + assert {"AC_arrange_grid", "AC_arrange_cascade"} <= known + from je_auto_control.utils.mcp_server.tools import build_default_tool_registry + names = {t.name for t in build_default_tool_registry()} + assert {"ac_arrange_grid", "ac_arrange_cascade"} <= names + from je_auto_control.gui.script_builder.command_schema import _build_specs + specs = {s.command for s in _build_specs()} + assert {"AC_arrange_grid", "AC_arrange_cascade"} <= specs + + +def test_facade_exports(): + for attr in ("arrange_grid", "arrange_cascade"): + assert hasattr(ac, attr) and attr in ac.__all__ diff --git a/test/unit_test/headless/test_window_layout_batch.py b/test/unit_test/headless/test_window_layout_batch.py new file mode 100644 index 00000000..2207d675 --- /dev/null +++ b/test/unit_test/headless/test_window_layout_batch.py @@ -0,0 +1,104 @@ +"""Headless tests for the window tiling/layout geometry planner. No Qt.""" +import je_auto_control as ac +from je_auto_control.utils.window_layout import ( + WindowRect, available_slots, cascade_rects, grid_rects, tile_rect, +) + +_SCREEN = (0, 0, 1920, 1080) + + +def test_tile_halves_and_quadrants(): + assert tile_rect(_SCREEN, "left").as_tuple() == (0, 0, 960, 1080) + assert tile_rect(_SCREEN, "right").as_tuple() == (960, 0, 960, 1080) + assert tile_rect(_SCREEN, "top").as_tuple() == (0, 0, 1920, 540) + assert tile_rect(_SCREEN, "bottom_right").as_tuple() == (960, 540, 960, 540) + assert tile_rect(_SCREEN, "full").as_tuple() == (0, 0, 1920, 1080) + + +def test_thirds_split_the_width(): + left = tile_rect(_SCREEN, "left_third") + center = tile_rect(_SCREEN, "center_third") + right = tile_rect(_SCREEN, "right_third") + assert left.x == 0 and center.x == 640 and right.x == 1280 + assert left.width == center.width == right.width == 640 + + +def test_screen_offset_is_honoured(): + rect = tile_rect((100, 50, 800, 600), "left") + assert rect.as_tuple() == (100, 50, 400, 600) + + +def test_gap_insets_all_sides(): + rect = tile_rect(_SCREEN, "left", gap=10) + assert rect.as_tuple() == (10, 10, 940, 1060) + + +def test_unknown_slot_raises(): + try: + tile_rect(_SCREEN, "diagonal") + except ValueError: + pass + else: + raise AssertionError("expected ValueError for unknown slot") + + +def test_grid_rects_row_major_and_tiled(): + rects = grid_rects(_SCREEN, 2, 2) + assert [r.as_tuple() for r in rects] == [ + (0, 0, 960, 540), (960, 0, 960, 540), + (0, 540, 960, 540), (960, 540, 960, 540)] + + +def test_grid_rejects_zero_dimensions(): + for rows, cols in ((0, 2), (2, 0)): + try: + grid_rects(_SCREEN, rows, cols) + except ValueError: + continue + raise AssertionError("expected ValueError for non-positive grid") + + +def test_cascade_staggers_and_clamps(): + rects = cascade_rects(_SCREEN, 3, offset=40, size=(800, 600)) + assert rects[0].as_tuple() == (0, 0, 800, 600) + assert rects[1].as_tuple() == (40, 40, 800, 600) + assert all(r.x + r.width <= 1920 and r.y + r.height <= 1080 for r in rects) + + +def test_cascade_default_size_is_60_percent(): + rect = cascade_rects(_SCREEN, 1)[0] + assert rect.width == 1152 and rect.height == 648 # 0.6 * 1920 / 1080 + + +def test_available_slots_lists_known_names(): + slots = available_slots() + assert {"left", "right", "center", "left_third"} <= set(slots) + + +# --- wiring --------------------------------------------------------------- + +def test_wiring(): + known = set(ac.executor.known_commands()) + assert {"AC_tile_rect", "AC_grid_rects", "AC_cascade_rects"} <= known + from je_auto_control.utils.mcp_server.tools import build_default_tool_registry + names = {t.name for t in build_default_tool_registry()} + assert {"ac_tile_rect", "ac_grid_rects", "ac_cascade_rects"} <= names + from je_auto_control.gui.script_builder.command_schema import _build_specs + specs = {s.command for s in _build_specs()} + assert {"AC_tile_rect", "AC_grid_rects", "AC_cascade_rects"} <= specs + + +def test_executor_adapters_return_plans(): + from je_auto_control.utils.executor.action_executor import ( + _cascade_rects, _grid_rects, _tile_rect) + assert _tile_rect("left", screen=[0, 0, 100, 80])["rect"] == { + "x": 0, "y": 0, "width": 50, "height": 80} + assert _grid_rects(2, 2, screen=[0, 0, 100, 80])["count"] == 4 + assert _cascade_rects(3, screen=[0, 0, 100, 80])["count"] == 3 + + +def test_facade_exports(): + for attr in ("tile_rect", "grid_rects", "cascade_rects", "available_slots", + "WindowRect"): + assert hasattr(ac, attr) and attr in ac.__all__ + assert isinstance(tile_rect(_SCREEN, "left"), WindowRect)