diff --git a/README/WHATS_NEW_zh-CN.md b/README/WHATS_NEW_zh-CN.md index e565fb5d..75ffabfd 100644 --- a/README/WHATS_NEW_zh-CN.md +++ b/README/WHATS_NEW_zh-CN.md @@ -1,5 +1,11 @@ # 本次更新 — AutoControl +## 本次更新 (2026-06-23) — HSV 色彩空间分割 + +不论光照都能找出「任一色阶的红色」。完整参考:[`docs/source/Zh/doc/new_features/v139_features_doc.rst`](../docs/source/Zh/doc/new_features/v139_features_doc.rst)。 + +- **`dominant_hue_regions` / `segment_hsv` / `color_mask`**(`AC_dominant_hue_regions`、`AC_segment_hsv`):`find_color_region` 在 RGB 以各通道 ± 框遮罩——无法匹配「同一颜色但不同亮度」(状态灯、强调色、主题色调)。HSV 把色相与亮度分离,因此「色相带 + 饱和度 / 明度下限」可在不同光照下捕捉所有色阶。`dominant_hue_regions(hue=…)` 自动处理红色 0/180 环绕;`segment_hsv` 接受明确带;两者皆返回 `{x,y,width,height,area,center}` 区块并沿用共用连通元件辅助函数。可注入 haystack → 无头可测。 + ## 本次更新 (2026-06-23) — 融合并排序屏幕元素框 把原始的 OCR + 图标 + a11y 框转成一份干净、已编号的元素列表。完整参考:[`docs/source/Zh/doc/new_features/v138_features_doc.rst`](../docs/source/Zh/doc/new_features/v138_features_doc.rst)。 diff --git a/README/WHATS_NEW_zh-TW.md b/README/WHATS_NEW_zh-TW.md index b0cbc148..97a0d8eb 100644 --- a/README/WHATS_NEW_zh-TW.md +++ b/README/WHATS_NEW_zh-TW.md @@ -1,5 +1,11 @@ # 本次更新 — AutoControl +## 本次更新 (2026-06-23) — HSV 色彩空間分割 + +不論光照都能找出「任一色階的紅色」。完整參考:[`docs/source/Zh/doc/new_features/v139_features_doc.rst`](../docs/source/Zh/doc/new_features/v139_features_doc.rst)。 + +- **`dominant_hue_regions` / `segment_hsv` / `color_mask`**(`AC_dominant_hue_regions`、`AC_segment_hsv`):`find_color_region` 在 RGB 以各通道 ± 框遮罩——無法比對「同一顏色但不同亮度」(狀態燈、強調色、主題色調)。HSV 把色相與亮度分離,因此「色相帶 + 飽和度 / 明度下限」可在不同光照下捕捉所有色階。`dominant_hue_regions(hue=…)` 自動處理紅色 0/180 環繞;`segment_hsv` 接受明確帶;兩者皆回傳 `{x,y,width,height,area,center}` 區塊並沿用共用連通元件輔助函式。可注入 haystack → 無頭可測。 + ## 本次更新 (2026-06-23) — 融合並排序螢幕元素框 把原始的 OCR + 圖示 + a11y 框轉成一份乾淨、已編號的元素清單。完整參考:[`docs/source/Zh/doc/new_features/v138_features_doc.rst`](../docs/source/Zh/doc/new_features/v138_features_doc.rst)。 diff --git a/WHATS_NEW.md b/WHATS_NEW.md index 1e6e3775..bdd52c75 100644 --- a/WHATS_NEW.md +++ b/WHATS_NEW.md @@ -1,5 +1,11 @@ # What's New — AutoControl +## What's new (2026-06-23) — HSV Colour-Space Segmentation + +Find "any shade of red" regardless of lighting. Full reference: [`docs/source/Eng/doc/new_features/v139_features_doc.rst`](docs/source/Eng/doc/new_features/v139_features_doc.rst). + +- **`dominant_hue_regions` / `segment_hsv` / `color_mask`** (`AC_dominant_hue_regions`, `AC_segment_hsv`): `find_color_region` masks in RGB with a per-channel ± box — it can't match "the same colour at a different brightness" (status lights, highlights, theme tints). HSV separates hue from brightness, so a hue band + saturation/value floor catches every shade across lighting. `dominant_hue_regions(hue=…)` handles red's 0/180 wrap automatically; `segment_hsv` takes an explicit band; both return `{x,y,width,height,area,center}` blobs reusing the shared connected-components helper. Injectable haystack → headless-testable. + ## What's new (2026-06-23) — Fuse & Order On-Screen Element Boxes Turn raw OCR + icon + a11y boxes into one clean, numbered element list. Full reference: [`docs/source/Eng/doc/new_features/v138_features_doc.rst`](docs/source/Eng/doc/new_features/v138_features_doc.rst). diff --git a/docs/source/Eng/doc/new_features/v139_features_doc.rst b/docs/source/Eng/doc/new_features/v139_features_doc.rst new file mode 100644 index 00000000..877ed990 --- /dev/null +++ b/docs/source/Eng/doc/new_features/v139_features_doc.rst @@ -0,0 +1,43 @@ +HSV Colour-Space Segmentation +============================= + +``find_color_region`` masks in RGB with a per-channel ± tolerance box, which fails +the canonical case: a status light, highlight or theme tint that is "the same colour" +but at a different *brightness*. HSV separates hue from saturation/value, so a hue +band with a saturation/value floor catches every shade of a colour across lighting. +This adds HSV masking and blob boxes, reusing the shared connected-component helper, +with correct hue-wrap handling for red (which straddles the 0/180 boundary). + +Runs on an injectable ``haystack`` (ndarray / path / PIL, RGB), so it is headless- +testable on synthetic arrays. OpenCV + NumPy come in via ``je_open_cv``. Imports no +``PySide6``. + +Headless API +------------ + +.. code-block:: python + + from je_auto_control import dominant_hue_regions, segment_hsv, color_mask + + # Every red region — bright or dark — regardless of lighting (red wrap handled). + for r in dominant_hue_regions(hue=0, hue_tol=10, sat_min=80, val_min=80): + click(*r["center"]) + + # Or an explicit HSV band (H 0-179, S/V 0-255). + greens = segment_hsv(lower_hsv=[40, 80, 80], upper_hsv=[80, 255, 255]) + mask = color_mask(lower_hsv=[40, 80, 80], upper_hsv=[80, 255, 255]) + +``dominant_hue_regions`` constrains only the hue (± ``hue_tol``) plus a ``sat_min`` / +``val_min`` floor to skip greys, returning ``{x, y, width, height, area, center}`` per +blob largest first — so it finds a colour at any brightness, unlike the RGB box. +``segment_hsv`` takes an explicit ``lower_hsv`` / ``upper_hsv`` band; ``color_mask`` +returns the raw uint8 mask. + +Executor commands +----------------- + +``AC_segment_hsv`` (``lower_hsv`` / ``upper_hsv`` / ``min_area`` / ``region``) and +``AC_dominant_hue_regions`` (``hue`` / ``hue_tol`` / ``sat_min`` / ``val_min`` / +``min_area`` / ``region``), both returning ``{count, regions, best}``. They are +exposed as the MCP tools ``ac_segment_hsv`` / ``ac_dominant_hue_regions`` and as +Script Builder commands under **Image**. diff --git a/docs/source/Eng/eng_index.rst b/docs/source/Eng/eng_index.rst index e73b2af6..71f07108 100644 --- a/docs/source/Eng/eng_index.rst +++ b/docs/source/Eng/eng_index.rst @@ -161,6 +161,7 @@ Comprehensive guides for all AutoControl features. doc/new_features/v136_features_doc doc/new_features/v137_features_doc doc/new_features/v138_features_doc + doc/new_features/v139_features_doc doc/ocr_backends/ocr_backends_doc doc/observability/observability_doc doc/operations_layer/operations_layer_doc diff --git a/docs/source/Zh/doc/new_features/v139_features_doc.rst b/docs/source/Zh/doc/new_features/v139_features_doc.rst new file mode 100644 index 00000000..cc0d9003 --- /dev/null +++ b/docs/source/Zh/doc/new_features/v139_features_doc.rst @@ -0,0 +1,36 @@ +HSV 色彩空間分割 +================ + +``find_color_region`` 在 RGB 以各通道 ± 容差框遮罩,這在經典情境會失效:狀態燈、強調色或主題色調是「同一個顏色」 +但*亮度*不同。HSV 把色相與飽和度 / 明度分離,因此「色相帶 + 飽和度 / 明度下限」可在不同光照下捕捉某顏色的所有色階。 +本功能加入 HSV 遮罩與區塊框,沿用共用的連通元件輔助函式,並正確處理紅色的色相環繞(跨越 0/180 邊界)。 + +在可注入的 ``haystack``(ndarray / 路徑 / PIL,RGB)上執行,因此可對合成陣列做無頭測試。OpenCV + NumPy 透過 +``je_open_cv`` 引入。不匯入 ``PySide6``。 + +無頭 API +-------- + +.. code-block:: python + + from je_auto_control import dominant_hue_regions, segment_hsv, color_mask + + # 每個紅色區域——不論明暗、不論光照(已處理紅色環繞)。 + for r in dominant_hue_regions(hue=0, hue_tol=10, sat_min=80, val_min=80): + click(*r["center"]) + + # 或明確的 HSV 帶(H 0-179、S/V 0-255)。 + greens = segment_hsv(lower_hsv=[40, 80, 80], upper_hsv=[80, 255, 255]) + mask = color_mask(lower_hsv=[40, 80, 80], upper_hsv=[80, 255, 255]) + +``dominant_hue_regions`` 只限制色相(± ``hue_tol``)再加 ``sat_min`` / ``val_min`` 下限以略過灰階,為每個區塊回傳 +``{x, y, width, height, area, center}``,由大到小——因此能在任何亮度下找到某顏色,不像 RGB 框。``segment_hsv`` 接受 +明確的 ``lower_hsv`` / ``upper_hsv`` 帶;``color_mask`` 回傳原始 uint8 遮罩。 + +執行器命令 +---------- + +``AC_segment_hsv``(``lower_hsv`` / ``upper_hsv`` / ``min_area`` / ``region``)與 +``AC_dominant_hue_regions``(``hue`` / ``hue_tol`` / ``sat_min`` / ``val_min`` / ``min_area`` / ``region``), +皆回傳 ``{count, regions, best}``。它們以 MCP 工具 ``ac_segment_hsv`` / ``ac_dominant_hue_regions`` 以及 Script +Builder 中 **Image** 分類下的命令提供。 diff --git a/docs/source/Zh/zh_index.rst b/docs/source/Zh/zh_index.rst index ff1ca6ec..011e0a2e 100644 --- a/docs/source/Zh/zh_index.rst +++ b/docs/source/Zh/zh_index.rst @@ -161,6 +161,7 @@ AutoControl 所有功能的完整使用指南。 doc/new_features/v136_features_doc doc/new_features/v137_features_doc doc/new_features/v138_features_doc + doc/new_features/v139_features_doc doc/ocr_backends/ocr_backends_doc doc/observability/observability_doc doc/operations_layer/operations_layer_doc diff --git a/je_auto_control/__init__.py b/je_auto_control/__init__.py index 12e4f679..e375fba4 100644 --- a/je_auto_control/__init__.py +++ b/je_auto_control/__init__.py @@ -316,6 +316,10 @@ from je_auto_control.utils.element_parse import ( fuse_elements, iou, merge_boxes, reading_order, ) +# HSV colour-space segmentation (lighting-robust colour masking + blob boxes) +from je_auto_control.utils.hsv_segment import ( + color_mask, dominant_hue_regions, segment_hsv, +) # CI workflow annotations (GitHub Actions) from je_auto_control.utils.ci_annotations import ( emit_annotations, format_annotation, @@ -1149,6 +1153,9 @@ def start_autocontrol_gui(*args, **kwargs): "merge_boxes", "fuse_elements", "reading_order", + "segment_hsv", + "color_mask", + "dominant_hue_regions", "emit_annotations", "format_annotation", "ClipboardHistory", "default_clipboard_history", "analyze_heal_log", "heal_stats", "scan_secrets", diff --git a/je_auto_control/gui/script_builder/command_schema.py b/je_auto_control/gui/script_builder/command_schema.py index 3bbdb75d..3fb97a93 100644 --- a/je_auto_control/gui/script_builder/command_schema.py +++ b/je_auto_control/gui/script_builder/command_schema.py @@ -44,6 +44,7 @@ class CommandSpec: _MOUSE_BUTTONS = ("mouse_left", "mouse_right", "mouse_middle") +_REGION_PLACEHOLDER = "[left, top, right, bottom]" def _build_specs() -> List[CommandSpec]: @@ -262,7 +263,7 @@ def _add_image_specs(specs: List[CommandSpec]) -> None: FieldSpec("scales", FieldType.STRING, optional=True, placeholder="[0.9, 1.0, 1.1]"), FieldSpec("region", FieldType.STRING, optional=True, - placeholder="[left, top, right, bottom]"), + placeholder=_REGION_PLACEHOLDER), ), description="Locate a template and return its confidence score + scale.", )) @@ -286,7 +287,7 @@ def _add_image_specs(specs: List[CommandSpec]) -> None: FieldSpec("min_score", FieldType.FLOAT, optional=True, default=0.9, min_value=0.0, max_value=1.0), FieldSpec("region", FieldType.STRING, optional=True, - placeholder="[left, top, right, bottom]"), + placeholder=_REGION_PLACEHOLDER), ), description="Match counting only opaque/masked pixels (alpha or mask).", )) @@ -310,7 +311,7 @@ def _add_image_specs(specs: List[CommandSpec]) -> None: FieldSpec("tolerance", FieldType.INT, optional=True, default=20), FieldSpec("min_area", FieldType.INT, optional=True, default=50), FieldSpec("region", FieldType.STRING, optional=True, - placeholder="[left, top, right, bottom]"), + placeholder=_REGION_PLACEHOLDER), ), description="Locate regions by colour (status light / banner / fill).", )) @@ -322,7 +323,7 @@ def _add_image_specs(specs: List[CommandSpec]) -> None: FieldSpec("ignore", FieldType.STRING, optional=True, placeholder="[[x, y, w, h], ...]"), FieldSpec("region", FieldType.STRING, optional=True, - placeholder="[left, top, right, bottom]"), + placeholder=_REGION_PLACEHOLDER), ), description="Structural-similarity score (0..1) vs reference / screen.", )) @@ -337,7 +338,7 @@ def _add_image_specs(specs: List[CommandSpec]) -> None: min_value=0.0, max_value=1.0), FieldSpec("min_area", FieldType.INT, optional=True, default=50), FieldSpec("region", FieldType.STRING, optional=True, - placeholder="[left, top, right, bottom]"), + placeholder=_REGION_PLACEHOLDER), ), description="Boxes of the regions that structurally changed.", )) @@ -346,7 +347,7 @@ def _add_image_specs(specs: List[CommandSpec]) -> None: fields=( FieldSpec("template", FieldType.FILE_PATH), FieldSpec("region", FieldType.STRING, optional=True, - placeholder="[left, top, right, bottom]"), + placeholder=_REGION_PLACEHOLDER), FieldSpec("max_features", FieldType.INT, optional=True, default=500), FieldSpec("ratio", FieldType.FLOAT, optional=True, default=0.75, min_value=0.0, max_value=1.0), @@ -358,7 +359,7 @@ def _add_image_specs(specs: List[CommandSpec]) -> None: "AC_find_shapes", "Image", "Find Shapes", fields=( FieldSpec("region", FieldType.STRING, optional=True, - placeholder="[left, top, right, bottom]"), + placeholder=_REGION_PLACEHOLDER), FieldSpec("min_area", FieldType.INT, optional=True, default=400), FieldSpec("max_area", FieldType.INT, optional=True), ), @@ -368,7 +369,7 @@ def _add_image_specs(specs: List[CommandSpec]) -> None: "AC_find_rectangles", "Image", "Find Rectangles", fields=( FieldSpec("region", FieldType.STRING, optional=True, - placeholder="[left, top, right, bottom]"), + placeholder=_REGION_PLACEHOLDER), FieldSpec("min_area", FieldType.INT, optional=True, default=400), FieldSpec("max_area", FieldType.INT, optional=True), FieldSpec("aspect_range", FieldType.STRING, optional=True, @@ -387,10 +388,34 @@ def _add_image_specs(specs: List[CommandSpec]) -> None: placeholder="grayscale,upscale,binarize"), FieldSpec("scale", FieldType.FLOAT, optional=True, default=2.0), FieldSpec("region", FieldType.STRING, optional=True, - placeholder="[left, top, right, bottom]"), + placeholder=_REGION_PLACEHOLDER), ), description="Clean up an image for OCR / matching (grayscale/binarize/…).", )) + specs.append(CommandSpec( + "AC_segment_hsv", "Image", "Segment by HSV", + fields=( + FieldSpec("lower_hsv", FieldType.STRING, placeholder="[40, 80, 80]"), + FieldSpec("upper_hsv", FieldType.STRING, placeholder="[80, 255, 255]"), + FieldSpec("min_area", FieldType.INT, optional=True, default=50), + FieldSpec("region", FieldType.STRING, optional=True, + placeholder=_REGION_PLACEHOLDER), + ), + description="Locate regions inside an HSV band (lighting-robust).", + )) + specs.append(CommandSpec( + "AC_dominant_hue_regions", "Image", "Find Hue Regions", + fields=( + FieldSpec("hue", FieldType.INT, placeholder="0=red, 60=green, 120=blue"), + FieldSpec("hue_tol", FieldType.INT, optional=True, default=10), + FieldSpec("sat_min", FieldType.INT, optional=True, default=80), + FieldSpec("val_min", FieldType.INT, optional=True, default=80), + FieldSpec("min_area", FieldType.INT, optional=True, default=50), + FieldSpec("region", FieldType.STRING, optional=True, + placeholder=_REGION_PLACEHOLDER), + ), + description="Locate any shade of a hue, any brightness (handles red wrap).", + )) specs.append(CommandSpec( "AC_fuse_elements", "Image", "Fuse Element Boxes", fields=( @@ -684,7 +709,7 @@ def _add_flow_specs(specs: List[CommandSpec]) -> None: FieldSpec("min_score", FieldType.FLOAT, optional=True, default=0.8, min_value=0.0, max_value=1.0), FieldSpec("region", FieldType.STRING, optional=True, - placeholder="[left, top, right, bottom]"), + placeholder=_REGION_PLACEHOLDER), ), description="Wait until a target is visible + stable before acting.", )) @@ -742,7 +767,7 @@ def _add_flow_specs(specs: List[CommandSpec]) -> None: fields=( FieldSpec("target_rgb", FieldType.STRING, placeholder="[0, 200, 0]"), FieldSpec("region", FieldType.STRING, optional=True, - placeholder="[left, top, right, bottom]"), + placeholder=_REGION_PLACEHOLDER), FieldSpec("tolerance", FieldType.INT, optional=True, default=10), FieldSpec("min_fraction", FieldType.FLOAT, optional=True, default=0.5, min_value=0.0, max_value=1.0), diff --git a/je_auto_control/utils/executor/action_executor.py b/je_auto_control/utils/executor/action_executor.py index fe76826c..21d1c95e 100644 --- a/je_auto_control/utils/executor/action_executor.py +++ b/je_auto_control/utils/executor/action_executor.py @@ -3505,6 +3505,38 @@ def _reading_order(elements: Any, row_tol: Any = 12) -> Dict[str, Any]: return {"count": len(ordered), "elements": ordered} +def _segment_hsv(lower_hsv: Any, upper_hsv: Any, min_area: Any = 50, + region: Any = None) -> Dict[str, Any]: + """Adapter: locate blobs inside an explicit HSV band on the screen.""" + import json + from je_auto_control.utils.hsv_segment import segment_hsv + if isinstance(lower_hsv, str): + lower_hsv = json.loads(lower_hsv) + if isinstance(upper_hsv, str): + upper_hsv = json.loads(upper_hsv) + if isinstance(region, str): + region = json.loads(region) if region.strip() else None + boxes = segment_hsv(region=region, lower_hsv=list(lower_hsv), + upper_hsv=list(upper_hsv), min_area=int(min_area)) + return {"count": len(boxes), "regions": boxes, + "best": boxes[0] if boxes else None} + + +def _dominant_hue_regions(hue: Any, hue_tol: Any = 10, sat_min: Any = 80, + val_min: Any = 80, min_area: Any = 50, + region: Any = None) -> Dict[str, Any]: + """Adapter: locate any-brightness regions near a hue on the screen.""" + import json + from je_auto_control.utils.hsv_segment import dominant_hue_regions + if isinstance(region, str): + region = json.loads(region) if region.strip() else None + boxes = dominant_hue_regions(region=region, hue=int(hue), hue_tol=int(hue_tol), + sat_min=int(sat_min), val_min=int(val_min), + min_area=int(min_area)) + return {"count": len(boxes), "regions": boxes, + "best": boxes[0] if boxes else None} + + def _with_modifiers(modifiers: Any, actions: Any) -> Dict[str, Any]: """Adapter: run nested actions while modifier keys are held down.""" import json @@ -5237,6 +5269,8 @@ def __init__(self): "AC_wait_actionable": _wait_actionable, "AC_fuse_elements": _fuse_elements, "AC_reading_order": _reading_order, + "AC_segment_hsv": _segment_hsv, + "AC_dominant_hue_regions": _dominant_hue_regions, "AC_tile_rect": _tile_rect, "AC_grid_rects": _grid_rects, "AC_cascade_rects": _cascade_rects, diff --git a/je_auto_control/utils/hsv_segment/__init__.py b/je_auto_control/utils/hsv_segment/__init__.py new file mode 100644 index 00000000..0ee58aa7 --- /dev/null +++ b/je_auto_control/utils/hsv_segment/__init__.py @@ -0,0 +1,6 @@ +"""HSV colour-space segmentation (lighting-robust colour masking + blob boxes).""" +from je_auto_control.utils.hsv_segment.hsv_segment import ( + color_mask, dominant_hue_regions, segment_hsv, +) + +__all__ = ["color_mask", "dominant_hue_regions", "segment_hsv"] diff --git a/je_auto_control/utils/hsv_segment/hsv_segment.py b/je_auto_control/utils/hsv_segment/hsv_segment.py new file mode 100644 index 00000000..8dc7029f --- /dev/null +++ b/je_auto_control/utils/hsv_segment/hsv_segment.py @@ -0,0 +1,85 @@ +"""HSV colour-space segmentation — find "any shade of red" regardless of lighting. + +``color_region`` masks in RGB with a per-channel ± tolerance box, which fails the +canonical case: a status light, highlight or theme tint that is "the same colour" but +at a different brightness. HSV separates hue from saturation/value, so a hue band with +a saturation/value floor catches every shade of a colour across lighting. This adds +HSV masking + blob boxes, reusing the shared connected-component helper, with correct +hue-wrap handling for red (which straddles the 0/180 boundary). + +Runs on an injectable ``haystack`` (ndarray / path / PIL, RGB), so it is headless- +testable on synthetic arrays. OpenCV + NumPy come in via ``je_open_cv``. Imports no +``PySide6``. +""" +from typing import Any, Dict, List, Optional, Sequence + +# Reuse the RGB loader / screen grab from color_region (single source of truth). +from je_auto_control.utils.color_region.color_region import _grab_rgb, _to_rgb + +ImageSource = Any + + +def _hsv(haystack: Optional[ImageSource], region: Optional[Sequence[int]]): + import cv2 + rgb = _to_rgb(haystack) if haystack is not None else _grab_rgb(region) + return cv2.cvtColor(rgb, cv2.COLOR_RGB2HSV) + + +def color_mask(haystack: Optional[ImageSource] = None, *, + region: Optional[Sequence[int]] = None, + lower_hsv: Sequence[int], upper_hsv: Sequence[int]): + """Return a uint8 mask of pixels inside the ``lower_hsv``..``upper_hsv`` band. + + HSV ranges are OpenCV's: H in 0..179, S and V in 0..255. + """ + import cv2 + import numpy as np + hsv = _hsv(haystack, region) + lower = np.array([int(value) for value in lower_hsv], dtype=np.uint8) + upper = np.array([int(value) for value in upper_hsv], dtype=np.uint8) + return cv2.inRange(hsv, lower, upper) + + +def segment_hsv(haystack: Optional[ImageSource] = None, *, + region: Optional[Sequence[int]] = None, + lower_hsv: Sequence[int], upper_hsv: Sequence[int], + min_area: int = 50) -> List[Dict[str, Any]]: + """Return blob boxes for pixels inside an explicit HSV band, largest first.""" + from je_auto_control.utils.cv2_utils.blobs import connected_boxes + mask = color_mask(haystack, region=region, lower_hsv=lower_hsv, + upper_hsv=upper_hsv) + return connected_boxes(mask, int(min_area)) + + +def _hue_mask(hsv, low_h: int, high_h: int, sat_min: int, val_min: int): + """Build an inRange mask for a hue band, OR-ing the two parts when it wraps 0/180.""" + import cv2 + import numpy as np + floor = [int(sat_min), int(val_min)] + top = [255, 255] + + def band(start: int, end: int): + return cv2.inRange(hsv, np.array([start, *floor], dtype=np.uint8), + np.array([end, *top], dtype=np.uint8)) + + if low_h < 0: + return cv2.bitwise_or(band(180 + low_h, 179), band(0, high_h)) + if high_h > 179: + return cv2.bitwise_or(band(low_h, 179), band(0, high_h - 180)) + return band(low_h, high_h) + + +def dominant_hue_regions(haystack: Optional[ImageSource] = None, *, + region: Optional[Sequence[int]] = None, hue: int, + hue_tol: int = 10, sat_min: int = 80, val_min: int = 80, + min_area: int = 50) -> List[Dict[str, Any]]: + """Return blob boxes for any pixel near ``hue`` (± ``hue_tol``), any brightness. + + Only the hue is constrained (plus a ``sat_min`` / ``val_min`` floor to skip greys), + so this catches every shade of a colour across lighting — unlike ``color_region``'s + RGB box. Red's 0/180 hue wrap is handled automatically. + """ + from je_auto_control.utils.cv2_utils.blobs import connected_boxes + mask = _hue_mask(_hsv(haystack, region), int(hue) - int(hue_tol), + int(hue) + int(hue_tol), int(sat_min), int(val_min)) + return connected_boxes(mask, int(min_area)) diff --git a/je_auto_control/utils/mcp_server/tools/_factories.py b/je_auto_control/utils/mcp_server/tools/_factories.py index 8b74943d..476bb2f9 100644 --- a/je_auto_control/utils/mcp_server/tools/_factories.py +++ b/je_auto_control/utils/mcp_server/tools/_factories.py @@ -2899,6 +2899,43 @@ def element_parse_tools() -> List[MCPTool]: ] +def hsv_segment_tools() -> List[MCPTool]: + return [ + MCPTool( + name="ac_segment_hsv", + description=("Locate on-screen blobs inside an explicit HSV band " + "(H 0-179, S/V 0-255): 'lower_hsv' [h,s,v] .. 'upper_hsv' " + "[h,s,v], blobs >= 'min_area'. Returns {count, regions, " + "best}. HSV is lighting-robust where RGB tolerance is not."), + input_schema=schema({ + "lower_hsv": {"type": "array", "items": {"type": "integer"}}, + "upper_hsv": {"type": "array", "items": {"type": "integer"}}, + "min_area": {"type": "integer"}, + "region": {"type": "array", "items": {"type": "integer"}}}, + required=["lower_hsv", "upper_hsv"]), + handler=h.segment_hsv, + annotations=READ_ONLY, + ), + MCPTool( + name="ac_dominant_hue_regions", + description=("Locate regions of ANY shade of a colour near 'hue' " + "(0-179, ± 'hue_tol'), any brightness, with a 'sat_min' / " + "'val_min' floor. Red's 0/180 wrap is handled. Returns " + "{count, regions, best}."), + input_schema=schema({ + "hue": {"type": "integer"}, + "hue_tol": {"type": "integer"}, + "sat_min": {"type": "integer"}, + "val_min": {"type": "integer"}, + "min_area": {"type": "integer"}, + "region": {"type": "array", "items": {"type": "integer"}}}, + required=["hue"]), + handler=h.dominant_hue_regions, + annotations=READ_ONLY, + ), + ] + + def ssim_tools() -> List[MCPTool]: return [ MCPTool( @@ -6403,7 +6440,7 @@ def media_assert_tools() -> List[MCPTool]: color_region_tools, ssim_tools, feature_match_tools, shape_locator_tools, window_layout_tools, window_arrange_tools, preprocess_tools, monitor_layout_tools, actionability_tools, element_parse_tools, - plugin_sdk_tools, governance_tools, + hsv_segment_tools, plugin_sdk_tools, governance_tools, credential_lease_tools, egress_tools, approval_testing_tools, trajectory_eval_tools, compliance_tools, agent_trace_tools, video_report_tools, fuzzy_tools, artifact_store_tools, image_dedup_tools, diff --git a/je_auto_control/utils/mcp_server/tools/_handlers.py b/je_auto_control/utils/mcp_server/tools/_handlers.py index 6093a542..cafe8b49 100644 --- a/je_auto_control/utils/mcp_server/tools/_handlers.py +++ b/je_auto_control/utils/mcp_server/tools/_handlers.py @@ -2188,6 +2188,18 @@ def reading_order(elements, row_tol=12): return _reading_order(elements, row_tol) +def segment_hsv(lower_hsv, upper_hsv, min_area=50, region=None): + from je_auto_control.utils.executor.action_executor import _segment_hsv + return _segment_hsv(lower_hsv, upper_hsv, min_area, region) + + +def dominant_hue_regions(hue, hue_tol=10, sat_min=80, val_min=80, min_area=50, + region=None): + from je_auto_control.utils.executor.action_executor import ( + _dominant_hue_regions) + return _dominant_hue_regions(hue, hue_tol, sat_min, val_min, min_area, region) + + def detect_drift(reference, current, threshold=0.25, bins=10): from je_auto_control.utils.executor.action_executor import _detect_drift return _detect_drift(reference, current, threshold, bins) diff --git a/test/unit_test/headless/test_hsv_segment_batch.py b/test/unit_test/headless/test_hsv_segment_batch.py new file mode 100644 index 00000000..71c5480b --- /dev/null +++ b/test/unit_test/headless/test_hsv_segment_batch.py @@ -0,0 +1,70 @@ +"""Headless tests for HSV colour segmentation. No Qt.""" +import pytest + +import je_auto_control as ac + +np = pytest.importorskip("numpy") +pytest.importorskip("cv2") + +from je_auto_control.utils.hsv_segment import ( # noqa: E402 + color_mask, dominant_hue_regions, segment_hsv, +) + + +def _scene(): + """Bright red + dark red (same hue, different brightness) + a green patch.""" + img = np.zeros((100, 200, 3), dtype=np.uint8) + img[10:40, 10:60] = (255, 0, 0) # bright red, V=255 + img[60:90, 10:50] = (120, 0, 0) # dark red, V=120 + img[10:40, 120:170] = (0, 200, 0) # green + return img + + +def test_hue_band_catches_both_brightnesses_of_red(): + reds = dominant_hue_regions(_scene(), hue=0, hue_tol=10, sat_min=80, + val_min=80, min_area=100) + boxes = sorted((r["y"], r["x"], r["width"], r["height"]) for r in reds) + assert boxes == [(10, 10, 50, 30), (60, 10, 40, 30)] # bright AND dark red + + +def test_red_hue_excludes_green(): + reds = dominant_hue_regions(_scene(), hue=0, hue_tol=10, min_area=100) + assert all(r["x"] < 100 for r in reds) # green is at x=120 + + +def test_green_hue_found(): + greens = dominant_hue_regions(_scene(), hue=60, hue_tol=15, min_area=100) + assert len(greens) == 1 and greens[0]["x"] == 120 + + +def test_explicit_band_segment_and_mask(): + boxes = segment_hsv(_scene(), lower_hsv=[40, 80, 80], upper_hsv=[80, 255, 255], + min_area=100) + assert len(boxes) == 1 and boxes[0]["x"] == 120 # the green patch + mask = color_mask(_scene(), lower_hsv=[40, 80, 80], upper_hsv=[80, 255, 255]) + assert int(mask.sum()) > 0 + + +def test_val_floor_skips_dark(): + # require a high value floor -> the dark red (V=120) drops out, bright stays + reds = dominant_hue_regions(_scene(), hue=0, hue_tol=10, val_min=200, + min_area=100) + assert len(reds) == 1 and reds[0]["y"] == 10 # only the bright patch + + +# --- wiring --------------------------------------------------------------- + +def test_wiring(): + known = set(ac.executor.known_commands()) + assert {"AC_segment_hsv", "AC_dominant_hue_regions"} <= known + from je_auto_control.utils.mcp_server.tools import build_default_tool_registry + names = {t.name for t in build_default_tool_registry()} + assert {"ac_segment_hsv", "ac_dominant_hue_regions"} <= names + from je_auto_control.gui.script_builder.command_schema import _build_specs + specs = {s.command for s in _build_specs()} + assert {"AC_segment_hsv", "AC_dominant_hue_regions"} <= specs + + +def test_facade_exports(): + for attr in ("segment_hsv", "color_mask", "dominant_hue_regions"): + assert hasattr(ac, attr) and attr in ac.__all__