diff --git a/README/WHATS_NEW_zh-CN.md b/README/WHATS_NEW_zh-CN.md index ed021acd..d154ac7b 100644 --- a/README/WHATS_NEW_zh-CN.md +++ b/README/WHATS_NEW_zh-CN.md @@ -1,5 +1,11 @@ # 本次更新 — AutoControl +## 本次更新 (2026-06-23) — 等待区域颜色 + +阻塞直到某颜色填满(或离开)屏幕区域。完整参考:[`docs/source/Zh/doc/new_features/v121_features_doc.rst`](../docs/source/Zh/doc/new_features/v121_features_doc.rst)。 + +- **`wait_until_color`**(`AC_wait_color`):`wait_for_pixel` 精确比对单点、`wait_until_pixel_changes` 检测单点任何变化——两者都无法等「状态灯变绿」/「进度条填满」/「红色横幅消失」。本功能计数区域中接近 `target_rgb`(在 `tolerance` 内)的像素,当比例越过 `min_fraction`(或 `present=False` 时低于)即成功。可注入 sampler、无头可测。纯标准库。 + ## 本次更新 (2026-06-23) — 相对鼠标移动 从当前位置将指针位移一个增量。完整参考:[`docs/source/Zh/doc/new_features/v120_features_doc.rst`](../docs/source/Zh/doc/new_features/v120_features_doc.rst)。 diff --git a/README/WHATS_NEW_zh-TW.md b/README/WHATS_NEW_zh-TW.md index c2c5b1e5..c0513727 100644 --- a/README/WHATS_NEW_zh-TW.md +++ b/README/WHATS_NEW_zh-TW.md @@ -1,5 +1,11 @@ # 本次更新 — AutoControl +## 本次更新 (2026-06-23) — 等待區域顏色 + +阻塞直到某顏色填滿(或離開)螢幕區域。完整參考:[`docs/source/Zh/doc/new_features/v121_features_doc.rst`](../docs/source/Zh/doc/new_features/v121_features_doc.rst)。 + +- **`wait_until_color`**(`AC_wait_color`):`wait_for_pixel` 精確比對單點、`wait_until_pixel_changes` 偵測單點任何變化——兩者都無法等「狀態燈變綠」/「進度條填滿」/「紅色橫幅消失」。本功能計數區域中接近 `target_rgb`(在 `tolerance` 內)的像素,當比例越過 `min_fraction`(或 `present=False` 時低於)即成功。可注入 sampler、無頭可測。純標準函式庫。 + ## 本次更新 (2026-06-23) — 相對滑鼠移動 從目前位置將指標位移一個增量。完整參考:[`docs/source/Zh/doc/new_features/v120_features_doc.rst`](../docs/source/Zh/doc/new_features/v120_features_doc.rst)。 diff --git a/WHATS_NEW.md b/WHATS_NEW.md index b19d3407..a4373ff0 100644 --- a/WHATS_NEW.md +++ b/WHATS_NEW.md @@ -1,5 +1,11 @@ # What's New — AutoControl +## What's new (2026-06-23) — Wait for Region Colour + +Block until a colour fills (or leaves) a screen region. Full reference: [`docs/source/Eng/doc/new_features/v121_features_doc.rst`](docs/source/Eng/doc/new_features/v121_features_doc.rst). + +- **`wait_until_color`** (`AC_wait_color`): `wait_for_pixel` matches one point exactly and `wait_until_pixel_changes` detects any change at one point — neither waits for "the status light turns green" / "the progress bar fills" / "the red banner is gone". This counts pixels within `tolerance` of `target_rgb` over a region and succeeds when that fraction crosses `min_fraction` (or drops below it, `present=False`). Injectable sampler, headless-testable. Pure-stdlib. + ## What's new (2026-06-23) — Relative Mouse Movement Nudge the pointer by a delta from where it is. Full reference: [`docs/source/Eng/doc/new_features/v120_features_doc.rst`](docs/source/Eng/doc/new_features/v120_features_doc.rst). diff --git a/docs/source/Eng/doc/new_features/v121_features_doc.rst b/docs/source/Eng/doc/new_features/v121_features_doc.rst new file mode 100644 index 00000000..1960eccd --- /dev/null +++ b/docs/source/Eng/doc/new_features/v121_features_doc.rst @@ -0,0 +1,39 @@ +Wait for Region Colour +====================== + +``wait_for_pixel`` matches a single point exactly and ``wait_until_pixel_changes`` +detects *any* change at one point — neither answers "wait until this status light +turns green", "until the progress bar is mostly filled", or "until the red error +banner is gone". This adds a region-colour wait to the ``smart_waits`` family. + +The pixel counting is a pure helper and :func:`wait_until_color` takes an +injectable ``sampler``, so the loop is headless-testable without a real screen. +Imports no ``PySide6``. + +Headless API +------------ + +.. code-block:: python + + from je_auto_control import wait_until_color + + # wait until ≥ 60% of the region is (near) green + wait_until_color(region=[10, 10, 210, 40], target_rgb=[0, 200, 0], + tolerance=15, min_fraction=0.6, timeout_s=20) + + # wait until a red banner disappears + wait_until_color(region=[0, 0, 800, 60], target_rgb=[200, 0, 0], + present=False, timeout_s=10) + +Pixels within ``tolerance`` (per channel) of ``target_rgb`` are counted. With +``present=True`` the wait succeeds once that fraction reaches ``min_fraction``; +with ``present=False`` once it drops below it. The result is a ``WaitOutcome`` +(``succeeded`` / ``reason`` / ``elapsed_s`` / ``samples_taken``). + +Executor commands +----------------- + +``AC_wait_color`` takes ``target_rgb`` (and optional ``region``) as JSON arrays +plus ``tolerance`` / ``min_fraction`` / ``present`` / ``timeout_s`` / +``poll_interval_s``, and returns the ``WaitOutcome`` dict. It is exposed as the +MCP tool ``ac_wait_color`` and as a Script Builder command under **Flow**. diff --git a/docs/source/Eng/eng_index.rst b/docs/source/Eng/eng_index.rst index 28c6c940..12e06cca 100644 --- a/docs/source/Eng/eng_index.rst +++ b/docs/source/Eng/eng_index.rst @@ -143,6 +143,7 @@ Comprehensive guides for all AutoControl features. doc/new_features/v118_features_doc doc/new_features/v119_features_doc doc/new_features/v120_features_doc + doc/new_features/v121_features_doc doc/ocr_backends/ocr_backends_doc doc/observability/observability_doc doc/operations_layer/operations_layer_doc diff --git a/docs/source/Zh/doc/new_features/v121_features_doc.rst b/docs/source/Zh/doc/new_features/v121_features_doc.rst new file mode 100644 index 00000000..438335ba --- /dev/null +++ b/docs/source/Zh/doc/new_features/v121_features_doc.rst @@ -0,0 +1,35 @@ +等待區域顏色 +============ + +``wait_for_pixel`` 精確比對單一點,``wait_until_pixel_changes`` 偵測單點的*任何*變化——兩者都無法回答 +「等到狀態燈變綠」、「等到進度條大致填滿」或「等到紅色錯誤橫幅消失」。本功能為 ``smart_waits`` 家族加入 +區域顏色等待。 + +像素計數為純函式輔助,:func:`wait_until_color` 接受可注入的 ``sampler``,因此迴圈可在無真實螢幕下測試。 +不匯入 ``PySide6``。 + +無頭 API +-------- + +.. code-block:: python + + from je_auto_control import wait_until_color + + # 等到區域中 ≥ 60% 為(接近)綠色 + wait_until_color(region=[10, 10, 210, 40], target_rgb=[0, 200, 0], + tolerance=15, min_fraction=0.6, timeout_s=20) + + # 等到紅色橫幅消失 + wait_until_color(region=[0, 0, 800, 60], target_rgb=[200, 0, 0], + present=False, timeout_s=10) + +在 ``tolerance``(各通道)內接近 ``target_rgb`` 的像素會被計數。``present=True`` 時,當該比例達到 +``min_fraction`` 即成功;``present=False`` 時,當其低於該值即成功。結果為 ``WaitOutcome`` +(``succeeded`` / ``reason`` / ``elapsed_s`` / ``samples_taken``)。 + +執行器命令 +---------- + +``AC_wait_color`` 接受 ``target_rgb``(與選用的 ``region``)為 JSON 陣列,以及 ``tolerance`` / ``min_fraction`` / +``present`` / ``timeout_s`` / ``poll_interval_s``,並回傳 ``WaitOutcome`` dict。它以 MCP 工具 ``ac_wait_color`` +以及 Script Builder 中 **Flow** 分類下的命令提供。 diff --git a/docs/source/Zh/zh_index.rst b/docs/source/Zh/zh_index.rst index 5b507654..f91c3262 100644 --- a/docs/source/Zh/zh_index.rst +++ b/docs/source/Zh/zh_index.rst @@ -143,6 +143,7 @@ AutoControl 所有功能的完整使用指南。 doc/new_features/v118_features_doc doc/new_features/v119_features_doc doc/new_features/v120_features_doc + doc/new_features/v121_features_doc doc/ocr_backends/ocr_backends_doc doc/observability/observability_doc doc/operations_layer/operations_layer_doc diff --git a/je_auto_control/__init__.py b/je_auto_control/__init__.py index ee64490a..13e9cd35 100644 --- a/je_auto_control/__init__.py +++ b/je_auto_control/__init__.py @@ -610,10 +610,11 @@ ) # Smart waits (frame-diff replacements for time.sleep) from je_auto_control.utils.smart_waits import ( - WaitOutcome, wait_until_clipboard_changes, wait_until_file, - wait_until_gone, wait_until_image_gone, wait_until_pixel_changes, - wait_until_port, wait_until_process, wait_until_region_idle, - wait_until_screen_stable, wait_until_text_gone, wait_until_window_closed, + WaitOutcome, wait_until_clipboard_changes, wait_until_color, + wait_until_file, wait_until_gone, wait_until_image_gone, + wait_until_pixel_changes, wait_until_port, wait_until_process, + wait_until_region_idle, wait_until_screen_stable, wait_until_text_gone, + wait_until_window_closed, ) # Visual regression (golden-image comparison) from je_auto_control.utils.visual_regression import ( @@ -1242,6 +1243,7 @@ def start_autocontrol_gui(*args, **kwargs): "wait_until_clipboard_changes", "wait_until_window_closed", "wait_until_file", "wait_until_port", "wait_until_process", "wait_until_gone", "wait_until_image_gone", "wait_until_text_gone", + "wait_until_color", # Visual regression + state machine "take_golden", "compare_to_golden", "image_difference", "DiffResult", "MaskRegion", diff --git a/je_auto_control/gui/script_builder/command_schema.py b/je_auto_control/gui/script_builder/command_schema.py index 93c18018..764f7771 100644 --- a/je_auto_control/gui/script_builder/command_schema.py +++ b/je_auto_control/gui/script_builder/command_schema.py @@ -478,6 +478,22 @@ def _add_flow_specs(specs: List[CommandSpec]) -> None: ), description="Block until on-screen text (OCR) disappears.", )) + specs.append(CommandSpec( + "AC_wait_color", "Flow", "Wait for Region Colour", + fields=( + FieldSpec("target_rgb", FieldType.STRING, placeholder="[0, 200, 0]"), + FieldSpec("region", FieldType.STRING, optional=True, + placeholder="[left, top, right, bottom]"), + FieldSpec("tolerance", FieldType.INT, optional=True, default=10), + FieldSpec("min_fraction", FieldType.FLOAT, optional=True, + default=0.5, min_value=0.0, max_value=1.0), + FieldSpec("present", FieldType.BOOL, optional=True, default=True), + FieldSpec("timeout_s", FieldType.FLOAT, optional=True, default=10.0), + FieldSpec("poll_interval_s", FieldType.FLOAT, optional=True, + default=0.2, min_value=0.01), + ), + description="Block until a colour fills (or leaves) a screen region.", + )) specs.append(CommandSpec( "AC_loop", "Flow", "Loop (N times)", fields=( diff --git a/je_auto_control/utils/executor/action_executor.py b/je_auto_control/utils/executor/action_executor.py index 3711be1c..d805fa9a 100644 --- a/je_auto_control/utils/executor/action_executor.py +++ b/je_auto_control/utils/executor/action_executor.py @@ -384,6 +384,24 @@ def _wait_text_gone(text: str, timeout_s: float = 10.0, ).to_dict() +def _wait_color(target_rgb: Any, region: Any = None, + tolerance: int = 10, min_fraction: float = 0.5, + present: bool = True, timeout_s: float = 10.0, + poll_interval_s: float = 0.2) -> Dict[str, Any]: + """Executor adapter: wait until a colour fills/leaves a region.""" + import json + from je_auto_control.utils.smart_waits import wait_until_color + if isinstance(target_rgb, str): + target_rgb = json.loads(target_rgb) + if isinstance(region, str): + region = json.loads(region) if region.strip() else None + return wait_until_color( + region=region, target_rgb=target_rgb, tolerance=int(tolerance), + min_fraction=float(min_fraction), present=bool(present), + timeout_s=float(timeout_s), poll_interval_s=float(poll_interval_s), + ).to_dict() + + def _wait_window_closed(title: str, case_sensitive: bool = False, timeout_s: float = 10.0, poll_interval_s: float = 0.2) -> Dict[str, Any]: @@ -5012,6 +5030,7 @@ def __init__(self): "AC_wait_clipboard_change": _wait_clipboard_change, "AC_wait_image_gone": _wait_image_gone, "AC_wait_text_gone": _wait_text_gone, + "AC_wait_color": _wait_color, "AC_wait_window_closed": _wait_window_closed, # Cost telemetry (LLM token + USD tracking) diff --git a/je_auto_control/utils/mcp_server/tools/_factories.py b/je_auto_control/utils/mcp_server/tools/_factories.py index 9207f965..e419914b 100644 --- a/je_auto_control/utils/mcp_server/tools/_factories.py +++ b/je_auto_control/utils/mcp_server/tools/_factories.py @@ -1305,6 +1305,24 @@ def smart_wait_tools() -> List[MCPTool]: handler=h.wait_text_gone, annotations=READ_ONLY, ), + MCPTool( + name="ac_wait_color", + description=("Block until 'target_rgb' [r,g,b] covers >= " + "'min_fraction' of 'region' [l,t,r,b] within " + "'tolerance' (present=True), or drops below it " + "(present=False). A status-light / progress-bar wait."), + input_schema=schema({ + "target_rgb": {"type": "array", "items": {"type": "integer"}}, + "region": {"type": "array", "items": {"type": "integer"}}, + "tolerance": {"type": "integer"}, + "min_fraction": {"type": "number"}, + "present": {"type": "boolean"}, + "timeout_s": {"type": "number"}, + "poll_interval_s": {"type": "number"}}, + required=["target_rgb"]), + handler=h.wait_color, + annotations=READ_ONLY, + ), MCPTool( name="ac_wait_screen_stable", description=("Block until the screen stops moving (consecutive " diff --git a/je_auto_control/utils/mcp_server/tools/_handlers.py b/je_auto_control/utils/mcp_server/tools/_handlers.py index 228a3a7d..83224972 100644 --- a/je_auto_control/utils/mcp_server/tools/_handlers.py +++ b/je_auto_control/utils/mcp_server/tools/_handlers.py @@ -2443,6 +2443,13 @@ def wait_text_gone(text: str, timeout_s: float = 10.0, return _wait_text_gone(text, timeout_s, poll_interval_s, gone_for_s) +def wait_color(target_rgb, region=None, tolerance=10, min_fraction=0.5, + present=True, timeout_s=10.0, poll_interval_s=0.2): + from je_auto_control.utils.executor.action_executor import _wait_color + return _wait_color(target_rgb, region, tolerance, min_fraction, + present, timeout_s, poll_interval_s) + + def wait_for_file(path: str, timeout_s: float = 30.0, poll_interval_s: float = 0.25, stable_for_s: float = 1.0, diff --git a/je_auto_control/utils/smart_waits/__init__.py b/je_auto_control/utils/smart_waits/__init__.py index 24e51427..2b179962 100644 --- a/je_auto_control/utils/smart_waits/__init__.py +++ b/je_auto_control/utils/smart_waits/__init__.py @@ -10,7 +10,7 @@ from je_auto_control.utils.smart_waits.waits import ( ClipboardReader, FileStatReader, Frame, PortConnector, ProcessLister, ScreenSampler, WaitOutcome, WindowFinder, wait_until_clipboard_changes, - wait_until_file, wait_until_gone, wait_until_image_gone, + wait_until_color, wait_until_file, wait_until_gone, wait_until_image_gone, wait_until_pixel_changes, wait_until_port, wait_until_process, wait_until_region_idle, wait_until_screen_stable, wait_until_text_gone, wait_until_window_closed, @@ -20,8 +20,9 @@ __all__ = [ "ClipboardReader", "FileStatReader", "Frame", "PortConnector", "ProcessLister", "ScreenSampler", "WaitOutcome", "WindowFinder", - "wait_until_clipboard_changes", "wait_until_file", "wait_until_gone", - "wait_until_image_gone", "wait_until_pixel_changes", "wait_until_port", - "wait_until_process", "wait_until_region_idle", "wait_until_screen_stable", - "wait_until_text_gone", "wait_until_window_closed", + "wait_until_clipboard_changes", "wait_until_color", "wait_until_file", + "wait_until_gone", "wait_until_image_gone", "wait_until_pixel_changes", + "wait_until_port", "wait_until_process", "wait_until_region_idle", + "wait_until_screen_stable", "wait_until_text_gone", + "wait_until_window_closed", ] diff --git a/je_auto_control/utils/smart_waits/waits.py b/je_auto_control/utils/smart_waits/waits.py index 87212545..3a7cd7fd 100644 --- a/je_auto_control/utils/smart_waits/waits.py +++ b/je_auto_control/utils/smart_waits/waits.py @@ -446,6 +446,55 @@ def wait_until_text_gone(text: str, *, timeout_s: float = 10.0, poll_interval_s=poll_interval_s, gone_for_s=gone_for_s) +def _color_fraction(frame: "Frame", target: Sequence[int], + tolerance: int) -> float: + """Fraction of the frame's pixels within ``tolerance`` of ``target`` RGB.""" + pixels = frame.pixels + total = len(pixels) // 3 + if total == 0: + return 0.0 + red, green, blue = int(target[0]), int(target[1]), int(target[2]) + tol = int(tolerance) + matched = 0 + for offset in range(0, total * 3, 3): + if (abs(pixels[offset] - red) <= tol + and abs(pixels[offset + 1] - green) <= tol + and abs(pixels[offset + 2] - blue) <= tol): + matched += 1 + return matched / total + + +def wait_until_color(*, region: Optional[Sequence[int]] = None, + target_rgb: Sequence[int], tolerance: int = 10, + min_fraction: float = 0.5, present: bool = True, + timeout_s: float = 10.0, poll_interval_s: float = 0.2, + sampler: Optional[ScreenSampler] = None) -> WaitOutcome: + """Wait until ``target_rgb`` covers (or stops covering) a fraction of ``region``. + + Counts pixels within ``tolerance`` (per channel) of ``target_rgb``. With + ``present=True`` the wait succeeds once that fraction reaches + ``min_fraction`` (a status light turns green, a progress bar fills); with + ``present=False`` it succeeds once the fraction drops below it (the colour + disappears). ``sampler`` is injectable for headless tests. + """ + if timeout_s <= 0: + raise ValueError(_TIMEOUT_POSITIVE) + if poll_interval_s <= 0: + raise ValueError(_POLL_POSITIVE) + grab = sampler or _default_sampler + started = time.monotonic() + deadline = started + float(timeout_s) + samples = 0 + while time.monotonic() < deadline: + samples += 1 + fraction = _color_fraction(grab(region), target_rgb, tolerance) + reached = fraction >= float(min_fraction) + if reached == bool(present): + return _finish(True, "colour condition met", started, samples) + time.sleep(float(poll_interval_s)) + return _finish(False, "timeout while waiting for colour", started, samples) + + def _default_process_lister(name: str) -> List[str]: """List running process names matching ``name`` (requires psutil).""" from je_auto_control.utils.assertion.assertions import _running_process_names diff --git a/test/unit_test/headless/test_wait_color_batch.py b/test/unit_test/headless/test_wait_color_batch.py new file mode 100644 index 00000000..195f2113 --- /dev/null +++ b/test/unit_test/headless/test_wait_color_batch.py @@ -0,0 +1,81 @@ +"""Headless tests for region-colour waits. No Qt.""" +import pytest + +import je_auto_control as ac +from je_auto_control.utils.smart_waits import ( + Frame, WaitOutcome, wait_until_color, +) + + +def _frame(color, count, *, other=(0, 0, 0), other_count=0): + """Build a 1-row Frame: ``count`` px of ``color`` then ``other_count`` of other.""" + pixels = bytes(list(color) * count + list(other) * other_count) + return Frame(width=count + other_count, height=1, pixels=pixels) + + +def test_succeeds_when_colour_reaches_fraction(): + green = _frame((0, 200, 0), 8, other=(0, 0, 0), other_count=2) # 80% green + outcome = wait_until_color( + target_rgb=(0, 200, 0), tolerance=10, min_fraction=0.5, present=True, + timeout_s=1.0, poll_interval_s=0.001, sampler=lambda region: green) + assert isinstance(outcome, WaitOutcome) and outcome.succeeded is True + + +def test_times_out_when_colour_absent(): + blank = _frame((0, 0, 0), 10) + outcome = wait_until_color( + target_rgb=(255, 0, 0), min_fraction=0.5, present=True, + timeout_s=0.03, poll_interval_s=0.001, sampler=lambda region: blank) + assert outcome.succeeded is False and "timeout" in outcome.reason + + +def test_vanish_succeeds_when_below_fraction(): + blank = _frame((0, 0, 0), 10) + outcome = wait_until_color( + target_rgb=(255, 0, 0), min_fraction=0.5, present=False, + timeout_s=1.0, poll_interval_s=0.001, sampler=lambda region: blank) + assert outcome.succeeded is True # colour absent -> "gone" met + + +def test_tolerance_band(): + near = _frame((5, 198, 3), 10) # all near (0,200,0) + outcome = wait_until_color( + target_rgb=(0, 200, 0), tolerance=10, min_fraction=1.0, present=True, + timeout_s=1.0, poll_interval_s=0.001, sampler=lambda region: near) + assert outcome.succeeded is True + + +def test_validation(): + for bad in ({"timeout_s": 0}, {"poll_interval_s": 0}): + try: + wait_until_color(target_rgb=(0, 0, 0), **bad) + except ValueError: + pass + else: # pragma: no cover + raise AssertionError("expected ValueError") + + +# --- wiring --------------------------------------------------------------- + +def test_executor_color_fraction_helper(): + # the adapter's screen dispatch is device-bound; exercise the pure pixel- + # counting helper the wait relies on instead. + from je_auto_control.utils.smart_waits.waits import _color_fraction + half = _frame((0, 200, 0), 5, other=(0, 0, 0), other_count=5) + assert _color_fraction(half, (0, 200, 0), 10) == pytest.approx(0.5) + assert _color_fraction(half, (255, 255, 255), 10) == pytest.approx(0.0) + + +def test_wiring(): + known = ac.executor.known_commands() + assert "AC_wait_color" in set(known) + from je_auto_control.utils.mcp_server.tools import build_default_tool_registry + names = {t.name for t in build_default_tool_registry()} + assert "ac_wait_color" in names + from je_auto_control.gui.script_builder.command_schema import _build_specs + specs = {s.command for s in _build_specs()} + assert "AC_wait_color" in specs + + +def test_facade_exports(): + assert hasattr(ac, "wait_until_color") and "wait_until_color" in ac.__all__