From 4b7f996e212bd158b992e8875212b89dded33da6 Mon Sep 17 00:00:00 2001 From: JeffreyChen Date: Wed, 24 Jun 2026 06:27:39 +0800 Subject: [PATCH] Add heading_segment: heading vs body classification + document outline Nothing mapped line height to heading levels or built a section outline; ocr/structure and element_parse are positional and text_blocks doesn't rank. Apply the standard heuristic: a line taller than heading_ratio x the median line height is a heading, and distinct heading heights become levels (tallest = 1). classify_lines tags each line; outline returns the headings in order. --- README/WHATS_NEW_zh-CN.md | 6 ++ README/WHATS_NEW_zh-TW.md | 6 ++ WHATS_NEW.md | 6 ++ .../doc/new_features/v176_features_doc.rst | 37 +++++++++++ docs/source/Eng/eng_index.rst | 1 + .../Zh/doc/new_features/v176_features_doc.rst | 35 +++++++++++ docs/source/Zh/zh_index.rst | 1 + je_auto_control/__init__.py | 6 ++ .../gui/script_builder/command_schema.py | 20 ++++++ .../utils/executor/action_executor.py | 22 +++++++ .../utils/heading_segment/__init__.py | 6 ++ .../utils/heading_segment/heading_segment.py | 63 +++++++++++++++++++ .../utils/mcp_server/tools/_factories.py | 25 ++++++++ .../utils/mcp_server/tools/_handlers.py | 10 +++ .../headless/test_heading_segment_batch.py | 58 +++++++++++++++++ 15 files changed, 302 insertions(+) create mode 100644 docs/source/Eng/doc/new_features/v176_features_doc.rst create mode 100644 docs/source/Zh/doc/new_features/v176_features_doc.rst create mode 100644 je_auto_control/utils/heading_segment/__init__.py create mode 100644 je_auto_control/utils/heading_segment/heading_segment.py create mode 100644 test/unit_test/headless/test_heading_segment_batch.py diff --git a/README/WHATS_NEW_zh-CN.md b/README/WHATS_NEW_zh-CN.md index a22ef05c..56d7d827 100644 --- a/README/WHATS_NEW_zh-CN.md +++ b/README/WHATS_NEW_zh-CN.md @@ -1,5 +1,11 @@ # 本次更新 — AutoControl +## 本次更新 (2026-06-24) — 标题与正文分类 + 文档大纲 + +以高度区分标题与正文,并建立文档大纲。完整参考:[`docs/source/Zh/doc/new_features/v176_features_doc.rst`](../docs/source/Zh/doc/new_features/v176_features_doc.rst)。 + +- **`classify_lines` / `outline`**(`AC_classify_lines`、`AC_outline`):框架中没有功能把行高对应到标题层级或建立章节大纲——`ocr/structure` / `element_parse` 纯属位置性,`text_blocks` 不排序。本功能套用标准启发法:行高超过 `heading_ratio` × 中位行高者为标题,不同标题高度成为层级(最高 = 1)。`classify_lines` 为每行标记 `{box, text, role, level}`;`outline` 依序返回标题作为目录。纯标准库,作用于行字典;不导入 `PySide6`。 + ## 本次更新 (2026-06-24) — 变化量序列的稳定检测 判断 UI 何时安定下来——以纯粹、可测试的函数作用于变化序列。完整参考:[`docs/source/Zh/doc/new_features/v175_features_doc.rst`](../docs/source/Zh/doc/new_features/v175_features_doc.rst)。 diff --git a/README/WHATS_NEW_zh-TW.md b/README/WHATS_NEW_zh-TW.md index 624a9d32..8f8e14c2 100644 --- a/README/WHATS_NEW_zh-TW.md +++ b/README/WHATS_NEW_zh-TW.md @@ -1,5 +1,11 @@ # 本次更新 — AutoControl +## 本次更新 (2026-06-24) — 標題與內文分類 + 文件大綱 + +以高度區分標題與內文,並建立文件大綱。完整參考:[`docs/source/Zh/doc/new_features/v176_features_doc.rst`](../docs/source/Zh/doc/new_features/v176_features_doc.rst)。 + +- **`classify_lines` / `outline`**(`AC_classify_lines`、`AC_outline`):框架中沒有功能把行高對應到標題層級或建立章節大綱——`ocr/structure` / `element_parse` 純屬位置性,`text_blocks` 不排序。本功能套用標準啟發法:行高超過 `heading_ratio` × 中位行高者為標題,不同標題高度成為層級(最高 = 1)。`classify_lines` 為每行標記 `{box, text, role, level}`;`outline` 依序回傳標題作為目錄。純標準函式庫,作用於行字典;不匯入 `PySide6`。 + ## 本次更新 (2026-06-24) — 變化量序列的穩定偵測 判斷 UI 何時安定下來——以純粹、可測試的函式作用於變化序列。完整參考:[`docs/source/Zh/doc/new_features/v175_features_doc.rst`](../docs/source/Zh/doc/new_features/v175_features_doc.rst)。 diff --git a/WHATS_NEW.md b/WHATS_NEW.md index 7e1cb2d9..e264f073 100644 --- a/WHATS_NEW.md +++ b/WHATS_NEW.md @@ -1,5 +1,11 @@ # What's New — AutoControl +## What's new (2026-06-24) — Heading vs Body Classification + Document Outline + +Tell headings from body text by height and build a document outline. Full reference: [`docs/source/Eng/doc/new_features/v176_features_doc.rst`](docs/source/Eng/doc/new_features/v176_features_doc.rst). + +- **`classify_lines` / `outline`** (`AC_classify_lines`, `AC_outline`): nothing mapped line height to heading levels or built a section outline — `ocr/structure` / `element_parse` are positional and `text_blocks` doesn't rank. This applies the standard heuristic: a line taller than `heading_ratio` × the median line height is a heading, and distinct heading heights become levels (tallest = 1). `classify_lines` tags each line `{box, text, role, level}`; `outline` returns the headings in order as a table of contents. Pure-stdlib over line dicts; no `PySide6`. + ## What's new (2026-06-24) — Settle Detection Over a Churn Series Decide when the UI has gone quiet — as a pure, testable function over a change series. Full reference: [`docs/source/Eng/doc/new_features/v175_features_doc.rst`](docs/source/Eng/doc/new_features/v175_features_doc.rst). diff --git a/docs/source/Eng/doc/new_features/v176_features_doc.rst b/docs/source/Eng/doc/new_features/v176_features_doc.rst new file mode 100644 index 00000000..bf58d1ce --- /dev/null +++ b/docs/source/Eng/doc/new_features/v176_features_doc.rst @@ -0,0 +1,37 @@ +Heading vs Body Classification + Document Outline +================================================= + +Nothing in the framework maps line height to heading levels or builds a section outline — +``ocr/structure`` and ``element_parse`` are purely positional, and ``text_blocks`` groups +paragraphs / lists but does not rank them. ``heading_segment`` adds the standard heuristic: +a line whose height exceeds ``heading_ratio`` times the median line height is a heading, and +distinct heading heights become heading *levels* (the tallest is level 1). From that it emits +a flat document outline. + +Pure-stdlib over plain line dicts (text + bbox); fully unit-testable with no image and no OCR +engine. Reuses ``table_grid_fill``'s box-bounds reader. Imports no ``PySide6``. + +Headless API +------------ + +.. code-block:: python + + from je_auto_control import classify_lines, outline + + for item in classify_lines(ocr_lines, heading_ratio=1.2): + print(item["role"], item["level"], item["text"]) + + for heading in outline(ocr_lines): + print(" " * (heading["level"] - 1) + heading["text"]) + +``classify_lines`` tags each line ``{box, text, role, level}`` — ``role`` is ``"heading"`` or +``"body"``, ``level`` is the heading level (1 = tallest, 0 for body). ``outline`` returns just +the headings in top-to-bottom order as ``{level, text, top}`` — a document table of contents. + +Executor commands +----------------- + +``AC_classify_lines`` (``lines`` / ``heading_ratio`` → ``{count, lines}``) and ``AC_outline`` +(``lines`` / ``heading_ratio`` → ``{count, headings}``). They are exposed as the MCP tools +``ac_classify_lines`` / ``ac_outline`` (read-only) and as the Script Builder commands +**Classify Headings vs Body** / **Document Outline** under **OCR**. diff --git a/docs/source/Eng/eng_index.rst b/docs/source/Eng/eng_index.rst index e9906164..22d10531 100644 --- a/docs/source/Eng/eng_index.rst +++ b/docs/source/Eng/eng_index.rst @@ -198,6 +198,7 @@ Comprehensive guides for all AutoControl features. doc/new_features/v173_features_doc doc/new_features/v174_features_doc doc/new_features/v175_features_doc + doc/new_features/v176_features_doc doc/ocr_backends/ocr_backends_doc doc/observability/observability_doc doc/operations_layer/operations_layer_doc diff --git a/docs/source/Zh/doc/new_features/v176_features_doc.rst b/docs/source/Zh/doc/new_features/v176_features_doc.rst new file mode 100644 index 00000000..c37aecd5 --- /dev/null +++ b/docs/source/Zh/doc/new_features/v176_features_doc.rst @@ -0,0 +1,35 @@ +標題與內文分類 + 文件大綱 +========================== + +框架中沒有任何功能把行高對應到標題層級或建立章節大綱——``ocr/structure`` 與 ``element_parse`` +純屬位置性,``text_blocks`` 把段落 / 清單分組但不對其排序。``heading_segment`` 補上標準啟發法: +行高超過 ``heading_ratio`` 乘以中位行高者為標題,且不同的標題高度成為標題*層級*(最高為第 1 級)。 +由此輸出扁平的文件大綱。 + +純標準函式庫,作用於純行字典(text + bbox);可在無影像、無 OCR 引擎下完整單元測試。重用 +``table_grid_fill`` 的框邊界讀取器。不匯入 ``PySide6``。 + +無頭 API +-------- + +.. code-block:: python + + from je_auto_control import classify_lines, outline + + for item in classify_lines(ocr_lines, heading_ratio=1.2): + print(item["role"], item["level"], item["text"]) + + for heading in outline(ocr_lines): + print(" " * (heading["level"] - 1) + heading["text"]) + +``classify_lines`` 為每行標記 ``{box, text, role, level}``——``role`` 為 ``"heading"`` 或 +``"body"``,``level`` 為標題層級(1 = 最高,內文為 0)。``outline`` 只回傳依上到下順序的標題, +為 ``{level, text, top}``——即文件目錄。 + +執行器指令 +---------- + +``AC_classify_lines``(``lines`` / ``heading_ratio`` → ``{count, lines}``)與 ``AC_outline`` +(``lines`` / ``heading_ratio`` → ``{count, headings}``)。兩者以 MCP 工具 ``ac_classify_lines`` / +``ac_outline``(唯讀)及 Script Builder 指令 **Classify Headings vs Body** / **Document Outline** +(位於 **OCR** 分類下)形式提供。 diff --git a/docs/source/Zh/zh_index.rst b/docs/source/Zh/zh_index.rst index 7acbd579..d3826633 100644 --- a/docs/source/Zh/zh_index.rst +++ b/docs/source/Zh/zh_index.rst @@ -198,6 +198,7 @@ AutoControl 所有功能的完整使用指南。 doc/new_features/v173_features_doc doc/new_features/v174_features_doc doc/new_features/v175_features_doc + doc/new_features/v176_features_doc doc/ocr_backends/ocr_backends_doc doc/observability/observability_doc doc/operations_layer/operations_layer_doc diff --git a/je_auto_control/__init__.py b/je_auto_control/__init__.py index 497ea8ac..3ebf7817 100644 --- a/je_auto_control/__init__.py +++ b/je_auto_control/__init__.py @@ -323,6 +323,10 @@ from je_auto_control.utils.text_blocks import ( detect_lists, group_paragraphs, ) +# Classify OCR lines as headings vs body and build a document outline +from je_auto_control.utils.heading_segment import ( + classify_lines, outline, +) # Associate form labels with values (multi-direction) + checkbox state from je_auto_control.utils.form_fields import ( associate_fields, checkbox_state, match_labels_to_widgets, @@ -1289,6 +1293,8 @@ def start_autocontrol_gui(*args, **kwargs): "to_blocks", "group_paragraphs", "detect_lists", + "classify_lines", + "outline", "associate_fields", "match_labels_to_widgets", "checkbox_state", diff --git a/je_auto_control/gui/script_builder/command_schema.py b/je_auto_control/gui/script_builder/command_schema.py index 58d4e45f..deae98bb 100644 --- a/je_auto_control/gui/script_builder/command_schema.py +++ b/je_auto_control/gui/script_builder/command_schema.py @@ -827,6 +827,26 @@ def _add_ocr_specs(specs: List[CommandSpec]) -> None: ), description="Detect bulleted / numbered list items among OCR lines.", )) + specs.append(CommandSpec( + "AC_classify_lines", "OCR", "Classify Headings vs Body", + fields=( + FieldSpec("lines", FieldType.STRING, + placeholder='[{"x":0,"y":0,"width":200,"height":40,' + '"text":"Title"}]'), + FieldSpec("heading_ratio", FieldType.FLOAT, optional=True, default=1.2), + ), + description="Tag OCR lines as heading/body by height; assign heading levels.", + )) + specs.append(CommandSpec( + "AC_outline", "OCR", "Document Outline", + fields=( + FieldSpec("lines", FieldType.STRING, + placeholder='[{"x":0,"y":0,"width":200,"height":40,' + '"text":"Title"}]'), + FieldSpec("heading_ratio", FieldType.FLOAT, optional=True, default=1.2), + ), + description="Headings in order with levels (document outline) from OCR lines.", + )) specs.append(CommandSpec( "AC_scroll_to_find", "OCR", "Scroll Until Visible", fields=( diff --git a/je_auto_control/utils/executor/action_executor.py b/je_auto_control/utils/executor/action_executor.py index df1b0ccb..e93b06a4 100644 --- a/je_auto_control/utils/executor/action_executor.py +++ b/je_auto_control/utils/executor/action_executor.py @@ -3537,6 +3537,26 @@ def _detect_lists(lines: Any) -> Dict[str, Any]: return {"count": len(items), "items": items} +def _classify_lines(lines: Any, heading_ratio: Any = 1.2) -> Dict[str, Any]: + """Adapter: classify OCR lines as headings vs body with levels.""" + import json + from je_auto_control.utils.heading_segment import classify_lines + if isinstance(lines, str): + lines = json.loads(lines) + classified = classify_lines(lines, heading_ratio=float(heading_ratio)) + return {"count": len(classified), "lines": classified} + + +def _outline(lines: Any, heading_ratio: Any = 1.2) -> Dict[str, Any]: + """Adapter: the document outline (headings in order) from OCR lines.""" + import json + from je_auto_control.utils.heading_segment import outline + if isinstance(lines, str): + lines = json.loads(lines) + headings = outline(lines, heading_ratio=float(heading_ratio)) + return {"count": len(headings), "headings": headings} + + def _find_color_region(rgb: Any, tolerance: Any = 20, min_area: Any = 50, region: Any = None) -> Dict[str, Any]: """Adapter: locate coloured regions on the screen, largest first.""" @@ -6089,6 +6109,8 @@ def __init__(self): "AC_xy_cut": _xy_cut, "AC_group_paragraphs": _group_paragraphs, "AC_detect_lists": _detect_lists, + "AC_classify_lines": _classify_lines, + "AC_outline": _outline, "AC_ssim_compare": _ssim_compare, "AC_ssim_changed_regions": _ssim_changed_regions, "AC_feature_match": _feature_match, diff --git a/je_auto_control/utils/heading_segment/__init__.py b/je_auto_control/utils/heading_segment/__init__.py new file mode 100644 index 00000000..638ff8e5 --- /dev/null +++ b/je_auto_control/utils/heading_segment/__init__.py @@ -0,0 +1,6 @@ +"""Classify OCR lines as headings vs body and build a document outline.""" +from je_auto_control.utils.heading_segment.heading_segment import ( + classify_lines, outline, +) + +__all__ = ["classify_lines", "outline"] diff --git a/je_auto_control/utils/heading_segment/heading_segment.py b/je_auto_control/utils/heading_segment/heading_segment.py new file mode 100644 index 00000000..52068c8d --- /dev/null +++ b/je_auto_control/utils/heading_segment/heading_segment.py @@ -0,0 +1,63 @@ +"""Classify OCR lines as headings vs body and build a document outline. + +Nothing in the framework maps line height to heading levels or builds a section outline — +``ocr/structure`` and ``element_parse`` are purely positional, and ``text_blocks`` groups +paragraphs / lists but does not rank them. ``heading_segment`` adds the standard heuristic: +a line whose height exceeds ``heading_ratio`` times the median line height is a heading, and +distinct heading heights become heading *levels* (the tallest is level 1). From that it emits +a flat document outline. + +Pure-stdlib over plain line dicts (text + bbox); fully unit-testable with no image and no OCR +engine. Reuses ``table_grid_fill``'s box-bounds reader. Imports no ``PySide6``. +""" +from typing import Any, Dict, List, Sequence + +from je_auto_control.utils.table_grid_fill.table_grid_fill import _box_bounds + +Line = Dict[str, Any] + + +def _height(line: Line) -> int: + _, top, _, bottom = _box_bounds(line) + return bottom - top + + +def _box(line: Line) -> Dict[str, int]: + left, top, right, bottom = _box_bounds(line) + return {"left": left, "top": top, "right": right, "bottom": bottom} + + +def classify_lines(lines: Sequence[Line], *, + heading_ratio: float = 1.2) -> List[Dict[str, Any]]: + """Tag each line as a heading or body line with a heading ``level``. + + A line taller than ``heading_ratio`` x the median line height is a heading; distinct + heading heights map to levels (tallest = 1). Body lines get ``level`` 0. Returns + ``{box, text, role, level}`` per line, in input order. + """ + if not lines: + return [] + heights = sorted(_height(line) for line in lines) + threshold = heights[len(heights) // 2] * float(heading_ratio) + heading_heights = sorted({_height(line) for line in lines + if _height(line) > threshold}, reverse=True) + level_of = {height: index + 1 for index, height in enumerate(heading_heights)} + classified: List[Dict[str, Any]] = [] + for line in lines: + height = _height(line) + is_heading = height > threshold + classified.append({"box": _box(line), + "text": str(line.get("text", "")), + "role": "heading" if is_heading else "body", + "level": level_of.get(height, 0) if is_heading else 0}) + return classified + + +def outline(lines: Sequence[Line], *, + heading_ratio: float = 1.2) -> List[Dict[str, Any]]: + """Return the document outline: the headings in top-to-bottom order with levels.""" + headings = [item for item in classify_lines(lines, heading_ratio=heading_ratio) + if item["role"] == "heading"] + headings.sort(key=lambda item: item["box"]["top"]) + return [{"level": item["level"], "text": item["text"], + "top": item["box"]["top"]} for item in headings] diff --git a/je_auto_control/utils/mcp_server/tools/_factories.py b/je_auto_control/utils/mcp_server/tools/_factories.py index 784fc086..09067846 100644 --- a/je_auto_control/utils/mcp_server/tools/_factories.py +++ b/je_auto_control/utils/mcp_server/tools/_factories.py @@ -3990,6 +3990,31 @@ def screen_grid_tools() -> List[MCPTool]: handler=h.detect_lists, annotations=READ_ONLY, ), + MCPTool( + name="ac_classify_lines", + description=("Classify OCR 'lines' as headings vs body by height: a line " + "taller than 'heading_ratio' x the median line height is a " + "heading, and distinct heading heights become levels (tallest " + "= 1). Returns {count, lines:[{box,text,role,level}]}."), + input_schema=schema({ + "lines": {"type": "array", "items": {"type": "object"}}, + "heading_ratio": {"type": "number"}}, + required=["lines"]), + handler=h.classify_lines, + annotations=READ_ONLY, + ), + MCPTool( + name="ac_outline", + description=("Return the document outline from OCR 'lines' — the headings " + "in top-to-bottom order with levels. Returns {count, " + "headings:[{level,text,top}]}."), + input_schema=schema({ + "lines": {"type": "array", "items": {"type": "object"}}, + "heading_ratio": {"type": "number"}}, + required=["lines"]), + handler=h.outline, + annotations=READ_ONLY, + ), ] diff --git a/je_auto_control/utils/mcp_server/tools/_handlers.py b/je_auto_control/utils/mcp_server/tools/_handlers.py index 8876b59d..d101ba4a 100644 --- a/je_auto_control/utils/mcp_server/tools/_handlers.py +++ b/je_auto_control/utils/mcp_server/tools/_handlers.py @@ -2205,6 +2205,16 @@ def detect_lists(lines): return _detect_lists(lines) +def classify_lines(lines, heading_ratio=1.2): + from je_auto_control.utils.executor.action_executor import _classify_lines + return _classify_lines(lines, heading_ratio) + + +def outline(lines, heading_ratio=1.2): + from je_auto_control.utils.executor.action_executor import _outline + return _outline(lines, heading_ratio) + + def find_color_region(rgb, tolerance=20, min_area=50, region=None): from je_auto_control.utils.executor.action_executor import ( _find_color_region) diff --git a/test/unit_test/headless/test_heading_segment_batch.py b/test/unit_test/headless/test_heading_segment_batch.py new file mode 100644 index 00000000..67add083 --- /dev/null +++ b/test/unit_test/headless/test_heading_segment_batch.py @@ -0,0 +1,58 @@ +"""Headless tests for heading vs body classification + outline (pure stdlib).""" +import je_auto_control as ac +from je_auto_control.utils.heading_segment import classify_lines, outline + + +def _line(y, text, h=20, x=0, w=200): + return {"x": x, "y": y, "width": w, "height": h, "text": text} + + +def _doc(): + # one big title (h=40), some body (h=20), a smaller heading (h=30) + return [_line(0, "Title", h=40), _line(50, "body one"), + _line(75, "body two"), _line(110, "Subsection", h=30), + _line(145, "more body")] + + +def test_classify_marks_headings_and_levels(): + by_text = {c["text"]: c for c in classify_lines(_doc(), heading_ratio=1.2)} + assert by_text["Title"]["role"] == "heading" + assert by_text["Subsection"]["role"] == "heading" + assert by_text["body one"]["role"] == "body" + # tallest heading is level 1, the next distinct height is level 2 + assert by_text["Title"]["level"] == 1 + assert by_text["Subsection"]["level"] == 2 + + +def test_body_only_has_no_headings(): + lines = [_line(0, "a"), _line(25, "b"), _line(50, "c")] + assert all(c["role"] == "body" for c in classify_lines(lines)) + + +def test_outline_lists_headings_in_order(): + result = outline(_doc(), heading_ratio=1.2) + assert [h["text"] for h in result] == ["Title", "Subsection"] + assert [h["level"] for h in result] == [1, 2] + + +def test_empty(): + assert classify_lines([]) == [] + assert outline([]) == [] + + +# --- wiring --------------------------------------------------------------- + +def test_wiring(): + known = set(ac.executor.known_commands()) + assert {"AC_classify_lines", "AC_outline"} <= known + from je_auto_control.utils.mcp_server.tools import build_default_tool_registry + names = {t.name for t in build_default_tool_registry()} + assert {"ac_classify_lines", "ac_outline"} <= names + from je_auto_control.gui.script_builder.command_schema import _build_specs + specs = {s.command for s in _build_specs()} + assert {"AC_classify_lines", "AC_outline"} <= specs + + +def test_facade_exports(): + for name in ("classify_lines", "outline"): + assert hasattr(ac, name) and name in ac.__all__