From ded786846cc6b96fe8815f01401c7ff89c39c102 Mon Sep 17 00:00:00 2001 From: JeffreyChen Date: Tue, 23 Jun 2026 03:18:50 +0800 Subject: [PATCH] Add locate-on-screen-regions-by-colour --- README/WHATS_NEW_zh-CN.md | 6 ++ README/WHATS_NEW_zh-TW.md | 6 ++ WHATS_NEW.md | 6 ++ .../doc/new_features/v128_features_doc.rst | 41 ++++++++++ docs/source/Eng/eng_index.rst | 1 + .../Zh/doc/new_features/v128_features_doc.rst | 34 ++++++++ docs/source/Zh/zh_index.rst | 1 + je_auto_control/__init__.py | 6 ++ .../gui/script_builder/command_schema.py | 11 +++ .../utils/color_region/__init__.py | 6 ++ .../utils/color_region/color_region.py | 82 +++++++++++++++++++ .../utils/executor/action_executor.py | 17 ++++ .../utils/mcp_server/tools/_factories.py | 22 ++++- .../utils/mcp_server/tools/_handlers.py | 6 ++ .../headless/test_color_region_batch.py | 67 +++++++++++++++ 15 files changed, 311 insertions(+), 1 deletion(-) create mode 100644 docs/source/Eng/doc/new_features/v128_features_doc.rst create mode 100644 docs/source/Zh/doc/new_features/v128_features_doc.rst create mode 100644 je_auto_control/utils/color_region/__init__.py create mode 100644 je_auto_control/utils/color_region/color_region.py create mode 100644 test/unit_test/headless/test_color_region_batch.py diff --git a/README/WHATS_NEW_zh-CN.md b/README/WHATS_NEW_zh-CN.md index eef34573..7d4f81d2 100644 --- a/README/WHATS_NEW_zh-CN.md +++ b/README/WHATS_NEW_zh-CN.md @@ -1,5 +1,11 @@ # 本次更新 — AutoControl +## 本次更新 (2026-06-23) — 依颜色定位屏幕区域 + +依颜色找出绿色状态药丸 / 红色横幅。完整参考:[`docs/source/Zh/doc/new_features/v128_features_doc.rst`](../docs/source/Zh/doc/new_features/v128_features_doc.rst)。 + +- **`find_color_region` / `find_color_regions`**(`AC_find_color_region`):`color_stats` 只描述区域颜色、`assert_pixel` 检查单点——两者都不*定位*彩色区域。本功能将接近目标 RGB(在 `tolerance` 内)的像素遮罩起来,返回相连区块的框(`{x,y,width,height,area,center}`,由大到小)——用于模板脆弱的状态灯、进度填充、错误横幅。可注入 haystack → 无头可测;OpenCV/NumPy 透过 `je_open_cv`。 + ## 本次更新 (2026-06-23) — 具信心分数的模板匹配 返回分数、搜索多尺度、找出所有出现处的模板匹配。完整参考:[`docs/source/Zh/doc/new_features/v127_features_doc.rst`](../docs/source/Zh/doc/new_features/v127_features_doc.rst)。 diff --git a/README/WHATS_NEW_zh-TW.md b/README/WHATS_NEW_zh-TW.md index d1397e79..45fe97f8 100644 --- a/README/WHATS_NEW_zh-TW.md +++ b/README/WHATS_NEW_zh-TW.md @@ -1,5 +1,11 @@ # 本次更新 — AutoControl +## 本次更新 (2026-06-23) — 依顏色定位螢幕區域 + +依顏色找出綠色狀態藥丸 / 紅色橫幅。完整參考:[`docs/source/Zh/doc/new_features/v128_features_doc.rst`](../docs/source/Zh/doc/new_features/v128_features_doc.rst)。 + +- **`find_color_region` / `find_color_regions`**(`AC_find_color_region`):`color_stats` 只描述區域顏色、`assert_pixel` 檢查單點——兩者都不*定位*彩色區域。本功能將接近目標 RGB(在 `tolerance` 內)的像素遮罩起來,回傳相連區塊的框(`{x,y,width,height,area,center}`,由大到小)——用於模板脆弱的狀態燈、進度填充、錯誤橫幅。可注入 haystack → 無頭可測;OpenCV/NumPy 透過 `je_open_cv`。 + ## 本次更新 (2026-06-23) — 具信心分數的模板比對 回傳分數、搜尋多尺度、找出所有出現處的模板比對。完整參考:[`docs/source/Zh/doc/new_features/v127_features_doc.rst`](../docs/source/Zh/doc/new_features/v127_features_doc.rst)。 diff --git a/WHATS_NEW.md b/WHATS_NEW.md index 6ef10d69..0a3b8233 100644 --- a/WHATS_NEW.md +++ b/WHATS_NEW.md @@ -1,5 +1,11 @@ # What's New — AutoControl +## What's new (2026-06-23) — Locate On-Screen Regions by Colour + +Find the green status pill / red banner by colour. Full reference: [`docs/source/Eng/doc/new_features/v128_features_doc.rst`](docs/source/Eng/doc/new_features/v128_features_doc.rst). + +- **`find_color_region` / `find_color_regions`** (`AC_find_color_region`): `color_stats` only describes a region's colour and `assert_pixel` checks one point — neither *locates* a coloured region. This masks pixels within `tolerance` of a target RGB and returns the connected blobs' boxes (`{x,y,width,height,area,center}`, largest first) — for status lights, progress fills, error banners where a template is brittle. Injectable haystack → headless-testable; OpenCV/NumPy via `je_open_cv`. + ## What's new (2026-06-23) — Confidence-Returning Template Matching Template matching that returns the score, searches multiple scales, and finds all occurrences. Full reference: [`docs/source/Eng/doc/new_features/v127_features_doc.rst`](docs/source/Eng/doc/new_features/v127_features_doc.rst). diff --git a/docs/source/Eng/doc/new_features/v128_features_doc.rst b/docs/source/Eng/doc/new_features/v128_features_doc.rst new file mode 100644 index 00000000..efe84a21 --- /dev/null +++ b/docs/source/Eng/doc/new_features/v128_features_doc.rst @@ -0,0 +1,41 @@ +Locate On-Screen Regions by Colour +================================== + +``color_stats`` only *describes* a region's dominant / average colour and +``assert_pixel`` checks a single point with a tolerance — neither *locates* a +coloured region. Template matching is brittle when only the colour is the signal +(a status light, a progress-bar fill, a red error banner). This masks pixels +within a tolerance of a target RGB and returns the bounding boxes of the +connected blobs. + +The masking + connected-components run on an injectable ``haystack`` image +(ndarray / path / PIL), so it is unit-testable on synthetic arrays without a real +screen. OpenCV + NumPy come in via the project's ``je_open_cv`` dependency. +Imports no ``PySide6``. + +Headless API +------------ + +.. code-block:: python + + from je_auto_control import find_color_region, find_color_regions + + pill = find_color_region([0, 200, 0], tolerance=25) # the green status pill + if pill: + click(*pill["center"]) + + for banner in find_color_regions([200, 0, 0], min_area=500): + print(banner["x"], banner["y"], banner["area"]) # every red blob + +``find_color_regions`` returns ``{x, y, width, height, area, center}`` for each +blob within ``tolerance`` (per channel) of ``rgb`` and at least ``min_area`` +pixels, largest first; ``find_color_region`` returns just the largest (or +``None``). ``haystack`` defaults to a screen grab of the optional ``region``. + +Executor commands +----------------- + +``AC_find_color_region`` takes ``rgb`` (a JSON ``[r, g, b]`` array) plus +``tolerance`` / ``min_area`` / ``region`` and returns ``{count, regions, best}``. +It is exposed as the MCP tool ``ac_find_color_region`` and as a Script Builder +command under **Image**. diff --git a/docs/source/Eng/eng_index.rst b/docs/source/Eng/eng_index.rst index 97c2fe05..119c23e7 100644 --- a/docs/source/Eng/eng_index.rst +++ b/docs/source/Eng/eng_index.rst @@ -150,6 +150,7 @@ Comprehensive guides for all AutoControl features. doc/new_features/v125_features_doc doc/new_features/v126_features_doc doc/new_features/v127_features_doc + doc/new_features/v128_features_doc doc/ocr_backends/ocr_backends_doc doc/observability/observability_doc doc/operations_layer/operations_layer_doc diff --git a/docs/source/Zh/doc/new_features/v128_features_doc.rst b/docs/source/Zh/doc/new_features/v128_features_doc.rst new file mode 100644 index 00000000..df55846d --- /dev/null +++ b/docs/source/Zh/doc/new_features/v128_features_doc.rst @@ -0,0 +1,34 @@ +依顏色定位螢幕區域 +================== + +``color_stats`` 只*描述*區域的主要 / 平均顏色,``assert_pixel`` 檢查單一點及容差——兩者都不*定位*彩色區域。 +當唯一訊號是顏色時(狀態燈、進度條填充、紅色錯誤橫幅),模板比對很脆弱。本功能將接近目標 RGB(在容差內)的 +像素遮罩起來,並回傳相連區塊的邊界框。 + +遮罩與相連元件分析在可注入的 ``haystack`` 影像(ndarray / 路徑 / PIL)上執行,因此可在無真實螢幕下對合成陣列 +做單元測試。OpenCV + NumPy 透過專案的 ``je_open_cv`` 相依引入。不匯入 ``PySide6``。 + +無頭 API +-------- + +.. code-block:: python + + from je_auto_control import find_color_region, find_color_regions + + pill = find_color_region([0, 200, 0], tolerance=25) # 綠色狀態藥丸 + if pill: + click(*pill["center"]) + + for banner in find_color_regions([200, 0, 0], min_area=500): + print(banner["x"], banner["y"], banner["area"]) # 每個紅色區塊 + +``find_color_regions`` 為每個在 ``tolerance``(各通道)內接近 ``rgb`` 且至少 ``min_area`` 像素的區塊回傳 +``{x, y, width, height, area, center}``,由大到小;``find_color_region`` 僅回傳最大的(或 ``None``)。 +``haystack`` 預設為對選用 ``region`` 的螢幕擷取。 + +執行器命令 +---------- + +``AC_find_color_region`` 接受 ``rgb``(JSON ``[r, g, b]`` 陣列)以及 ``tolerance`` / ``min_area`` / ``region``, +並回傳 ``{count, regions, best}``。它以 MCP 工具 ``ac_find_color_region`` 以及 Script Builder 中 **Image** +分類下的命令提供。 diff --git a/docs/source/Zh/zh_index.rst b/docs/source/Zh/zh_index.rst index 0fc69a0e..3843cf92 100644 --- a/docs/source/Zh/zh_index.rst +++ b/docs/source/Zh/zh_index.rst @@ -150,6 +150,7 @@ AutoControl 所有功能的完整使用指南。 doc/new_features/v125_features_doc doc/new_features/v126_features_doc doc/new_features/v127_features_doc + doc/new_features/v128_features_doc doc/ocr_backends/ocr_backends_doc doc/observability/observability_doc doc/operations_layer/operations_layer_doc diff --git a/je_auto_control/__init__.py b/je_auto_control/__init__.py index 83441a97..7d6a639d 100644 --- a/je_auto_control/__init__.py +++ b/je_auto_control/__init__.py @@ -278,6 +278,10 @@ best_matches, match_template, match_template_all, ) from je_auto_control.utils.visual_match import Match as TemplateMatch +# Locate on-screen regions by colour (mask + connected components) +from je_auto_control.utils.color_region import ( + find_color_region, find_color_regions, +) # CI workflow annotations (GitHub Actions) from je_auto_control.utils.ci_annotations import ( emit_annotations, format_annotation, @@ -1071,6 +1075,8 @@ def start_autocontrol_gui(*args, **kwargs): "match_template", "match_template_all", "best_matches", + "find_color_region", + "find_color_regions", "emit_annotations", "format_annotation", "ClipboardHistory", "default_clipboard_history", "analyze_heal_log", "heal_stats", "scan_secrets", diff --git a/je_auto_control/gui/script_builder/command_schema.py b/je_auto_control/gui/script_builder/command_schema.py index 2536dd85..02814974 100644 --- a/je_auto_control/gui/script_builder/command_schema.py +++ b/je_auto_control/gui/script_builder/command_schema.py @@ -278,6 +278,17 @@ def _add_image_specs(specs: List[CommandSpec]) -> None: ), description="Find every occurrence of a template (scored, NMS-deduped).", )) + specs.append(CommandSpec( + "AC_find_color_region", "Image", "Find Colour Region", + fields=( + FieldSpec("rgb", FieldType.STRING, placeholder="[0, 200, 0]"), + FieldSpec("tolerance", FieldType.INT, optional=True, default=20), + FieldSpec("min_area", FieldType.INT, optional=True, default=50), + FieldSpec("region", FieldType.STRING, optional=True, + placeholder="[left, top, right, bottom]"), + ), + description="Locate regions by colour (status light / banner / fill).", + )) def _add_ocr_specs(specs: List[CommandSpec]) -> None: diff --git a/je_auto_control/utils/color_region/__init__.py b/je_auto_control/utils/color_region/__init__.py new file mode 100644 index 00000000..c59b80a5 --- /dev/null +++ b/je_auto_control/utils/color_region/__init__.py @@ -0,0 +1,6 @@ +"""Locate on-screen regions by colour (mask + connected components).""" +from je_auto_control.utils.color_region.color_region import ( + find_color_region, find_color_regions, +) + +__all__ = ["find_color_region", "find_color_regions"] diff --git a/je_auto_control/utils/color_region/color_region.py b/je_auto_control/utils/color_region/color_region.py new file mode 100644 index 00000000..bf96f325 --- /dev/null +++ b/je_auto_control/utils/color_region/color_region.py @@ -0,0 +1,82 @@ +"""Locate on-screen regions by colour — find the green pill, the red banner. + +``color_stats`` only *describes* a region's dominant / average colour and +``assert_pixel`` checks a single point with a tolerance; neither *locates* a +coloured region. Template matching is brittle when only the colour is the signal +(a status light, a progress fill, an error banner). This masks pixels within a +tolerance of a target RGB and returns the bounding boxes of the connected blobs. + +The masking + connected-components run on an injectable ``haystack`` image +(ndarray / path / PIL), so it is unit-testable on synthetic arrays without a real +screen. OpenCV + NumPy come in via the project's ``je_open_cv`` dependency and are +imported lazily. Imports no ``PySide6``. +""" +from typing import Any, Dict, List, Optional, Sequence + +ImageSource = Any + + +def _to_rgb(source: ImageSource): + """Load a path / ndarray / PIL image as an RGB ndarray.""" + import cv2 + import numpy as np + if hasattr(source, "shape"): + return np.asarray(source) + if isinstance(source, (str, bytes)) or hasattr(source, "__fspath__"): + bgr = cv2.imread(str(source), cv2.IMREAD_COLOR) + if bgr is None: + raise ValueError(f"could not read image: {source!r}") + return cv2.cvtColor(bgr, cv2.COLOR_BGR2RGB) + return np.asarray(source) + + +def _grab_rgb(region: Optional[Sequence[int]]): + import numpy as np + from je_auto_control.utils.cv2_utils.screenshot import pil_screenshot + image = pil_screenshot(screen_region=list(region) if region else None) + return np.asarray(image.convert("RGB")) + + +def find_color_regions(rgb: Sequence[int], *, + haystack: Optional[ImageSource] = None, + region: Optional[Sequence[int]] = None, + tolerance: int = 20, + min_area: int = 50) -> List[Dict[str, Any]]: + """Return bounding boxes of blobs within ``tolerance`` of ``rgb``, largest first. + + Each result is ``{x, y, width, height, area, center}``. ``tolerance`` is the + per-channel band around ``rgb``; ``min_area`` drops specks. ``haystack`` is an + RGB ndarray / path / PIL image (default: grab the screen / ``region``). + """ + import cv2 + import numpy as np + image = _to_rgb(haystack) if haystack is not None else _grab_rgb(region) + red, green, blue = (int(channel) for channel in rgb[:3]) + tol = int(tolerance) + lower = np.array([max(0, red - tol), max(0, green - tol), + max(0, blue - tol)], dtype=np.uint8) + upper = np.array([min(255, red + tol), min(255, green + tol), + min(255, blue + tol)], dtype=np.uint8) + mask = cv2.inRange(image, lower, upper) + count, _labels, stats, centroids = cv2.connectedComponentsWithStats( + mask, connectivity=8) + regions: List[Dict[str, Any]] = [] + for index in range(1, count): # 0 is the background + x, y, width, height, area = (int(v) for v in stats[index]) + if area >= int(min_area): + cx, cy = centroids[index] + regions.append({"x": x, "y": y, "width": width, "height": height, + "area": area, "center": [int(cx), int(cy)]}) + regions.sort(key=lambda item: item["area"], reverse=True) + return regions + + +def find_color_region(rgb: Sequence[int], *, + haystack: Optional[ImageSource] = None, + region: Optional[Sequence[int]] = None, + tolerance: int = 20, + min_area: int = 50) -> Optional[Dict[str, Any]]: + """Return the largest blob within ``tolerance`` of ``rgb`` (or ``None``).""" + regions = find_color_regions(rgb, haystack=haystack, region=region, + tolerance=tolerance, min_area=min_area) + return regions[0] if regions else None diff --git a/je_auto_control/utils/executor/action_executor.py b/je_auto_control/utils/executor/action_executor.py index d4575367..8b60a90b 100644 --- a/je_auto_control/utils/executor/action_executor.py +++ b/je_auto_control/utils/executor/action_executor.py @@ -3229,6 +3229,22 @@ def _match_template_all(template: str, min_score: Any = 0.8, return {"count": len(matches), "matches": [m.to_dict() for m in matches]} +def _find_color_region(rgb: Any, tolerance: Any = 20, min_area: Any = 50, + region: Any = None) -> Dict[str, Any]: + """Adapter: locate coloured regions on the screen, largest first.""" + import json + from je_auto_control.utils.color_region import find_color_regions + if isinstance(rgb, str): + rgb = json.loads(rgb) + if isinstance(region, str): + region = json.loads(region) if region.strip() else None + regions = find_color_regions(list(rgb), region=region, + tolerance=int(tolerance), + min_area=int(min_area)) + return {"count": len(regions), "regions": regions, + "best": regions[0] if regions else None} + + def _with_modifiers(modifiers: Any, actions: Any) -> Dict[str, Any]: """Adapter: run nested actions while modifier keys are held down.""" import json @@ -4948,6 +4964,7 @@ def __init__(self): "AC_grid_cell": _grid_cell, "AC_match_template": _match_template, "AC_match_template_all": _match_template_all, + "AC_find_color_region": _find_color_region, "AC_detect_drift": _detect_drift, "AC_categorical_drift": _categorical_drift, "AC_diff_rows": _diff_rows, diff --git a/je_auto_control/utils/mcp_server/tools/_factories.py b/je_auto_control/utils/mcp_server/tools/_factories.py index de40c5be..208b8b8f 100644 --- a/je_auto_control/utils/mcp_server/tools/_factories.py +++ b/je_auto_control/utils/mcp_server/tools/_factories.py @@ -2628,6 +2628,26 @@ def tween_drag_tools() -> List[MCPTool]: ] +def color_region_tools() -> List[MCPTool]: + return [ + MCPTool( + name="ac_find_color_region", + description=("Locate on-screen regions matching 'rgb' [r,g,b] within " + "'tolerance' (per channel), blobs >= 'min_area'. " + "Returns {count, regions, best} (largest first). " + "For status lights / progress bars / coloured banners."), + input_schema=schema({ + "rgb": {"type": "array", "items": {"type": "integer"}}, + "tolerance": {"type": "integer"}, + "min_area": {"type": "integer"}, + "region": {"type": "array", "items": {"type": "integer"}}}, + required=["rgb"]), + handler=h.find_color_region, + annotations=READ_ONLY, + ), + ] + + def visual_match_tools() -> List[MCPTool]: return [ MCPTool( @@ -6055,7 +6075,7 @@ def media_assert_tools() -> List[MCPTool]: process_doc_tools, tween_drag_tools, mouse_path_tools, field_entry_tools, key_hold_tools, mouse_relative_tools, text_unicode_tools, modifier_state_tools, grid_locator_tools, visual_match_tools, - plugin_sdk_tools, governance_tools, + color_region_tools, plugin_sdk_tools, governance_tools, credential_lease_tools, egress_tools, approval_testing_tools, trajectory_eval_tools, compliance_tools, agent_trace_tools, video_report_tools, fuzzy_tools, artifact_store_tools, image_dedup_tools, diff --git a/je_auto_control/utils/mcp_server/tools/_handlers.py b/je_auto_control/utils/mcp_server/tools/_handlers.py index 5014fd8c..4e961c82 100644 --- a/je_auto_control/utils/mcp_server/tools/_handlers.py +++ b/je_auto_control/utils/mcp_server/tools/_handlers.py @@ -2081,6 +2081,12 @@ def match_template_all(template, min_score=0.8, max_results=20, nms_iou=0.3, return _match_template_all(template, min_score, max_results, nms_iou, region) +def find_color_region(rgb, tolerance=20, min_area=50, region=None): + from je_auto_control.utils.executor.action_executor import ( + _find_color_region) + return _find_color_region(rgb, tolerance, min_area, region) + + def detect_drift(reference, current, threshold=0.25, bins=10): from je_auto_control.utils.executor.action_executor import _detect_drift return _detect_drift(reference, current, threshold, bins) diff --git a/test/unit_test/headless/test_color_region_batch.py b/test/unit_test/headless/test_color_region_batch.py new file mode 100644 index 00000000..f760d1c2 --- /dev/null +++ b/test/unit_test/headless/test_color_region_batch.py @@ -0,0 +1,67 @@ +"""Headless tests for colour-region location. No Qt.""" +import pytest + +import je_auto_control as ac + +np = pytest.importorskip("numpy") +pytest.importorskip("cv2") + +from je_auto_control.utils.color_region import ( # noqa: E402 + find_color_region, find_color_regions, +) + + +def _scene(): + img = np.zeros((100, 200, 3), dtype=np.uint8) + img[20:40, 30:80] = (0, 200, 0) # big green (50x20 = 1000) + img[60:70, 100:120] = (0, 205, 5) # small near-green (20x10 = 200) + img[80:90, 10:20] = (200, 0, 0) # red + return img + + +def test_finds_regions_sorted_by_area(): + greens = find_color_regions((0, 200, 0), haystack=_scene(), tolerance=20, + min_area=10) + boxes = [(r["x"], r["y"], r["width"], r["height"]) for r in greens] + assert boxes == [(30, 20, 50, 20), (100, 60, 20, 10)] # largest first + assert greens[0]["area"] == 1000 + + +def test_find_largest_center(): + assert find_color_region((0, 200, 0), haystack=_scene())["center"] == [54, 29] + + +def test_red_and_absent_colour(): + assert find_color_region((200, 0, 0), haystack=_scene(), + min_area=10)["center"] == [14, 84] + assert find_color_region((0, 0, 255), haystack=_scene()) is None + + +def test_min_area_filters_small_blobs(): + assert len(find_color_regions((0, 200, 0), haystack=_scene(), + tolerance=20, min_area=300)) == 1 + + +def test_tolerance_widens_match(): + # the near-green (0,205,5) only matches (0,200,0) within a wide tolerance + tight = find_color_regions((0, 200, 0), haystack=_scene(), tolerance=2, + min_area=10) + assert all(r["y"] < 50 for r in tight) # only the exact big block + + +# --- wiring --------------------------------------------------------------- + +def test_wiring(): + known = ac.executor.known_commands() + assert "AC_find_color_region" in set(known) + from je_auto_control.utils.mcp_server.tools import build_default_tool_registry + names = {t.name for t in build_default_tool_registry()} + assert "ac_find_color_region" in names + from je_auto_control.gui.script_builder.command_schema import _build_specs + specs = {s.command for s in _build_specs()} + assert "AC_find_color_region" in specs + + +def test_facade_exports(): + for attr in ("find_color_region", "find_color_regions"): + assert hasattr(ac, attr) and attr in ac.__all__