fix: render Click \b verbatim blocks in CLI reference docs (#866) (#867)

* fix: render Click \b verbatim blocks in CLI reference docs (#866)

The docs generator was silently dropping \b-delimited sections
(Modes, Best practices, Detection notes, Rewrites) from m fix async
and m fix genslots. These blocks were buried inside the Raises section
by the docstring parser and never rendered, causing a fidelity gap
with --help output.

Adds _extract_verbatim_blocks() to split on \x08 directly from the
raw help text, independent of the section parser. Three new tests
verify the extraction logic and that the affected content now appears
in the generated reference page.

* fix: render bullet \b blocks as markdown lists not code fences

Best practices, Detection notes and Rewrites sections in \b blocks
are bullet lists and should render as markdown, not monospace code.
Modes retains a code fence as it uses columnar alignment.

Continuation lines wrapped across multiple source lines are joined
back into their parent bullet.

* fix: render two-column \b blocks as markdown tables

Click-style two-column aligned blocks (e.g. Modes in m fix async) were
rendered as code fences. The generator now detects the name + 2-space +
description pattern and emits a markdown table, preserving the original
terminal-aligned docstring format while rendering cleanly in HTML docs.

* fix: render two-column blocks as bold-name bullets not tables

Table cells render in a smaller font than body text in Mintlify,
creating inconsistency with bullet-list blocks. Render two-column
\b blocks as `- **\`name\`** — description` bullets instead:
consistent font, no invented column headers.

Author: planetf1

tooling/docs-autogen/generate_cli_reference.py

@@ -149,6 +149,79 @@ def _get_click_app():
     return typer.main.get_command(cli)
 
 
+def _parse_two_column_block(content: str) -> list[tuple[str, str]] | None:
+    """Parse a Click-style two-column aligned block into (name, description) pairs.
+
+    Click authors sometimes format ``\\b`` blocks as a fixed-width two-column
+    table (name left-aligned, description right-aligned with consistent padding)
+    for legible ``--help`` output.  This function detects that format and returns
+    ``(name, description)`` pairs so the generator can emit a proper markdown
+    table instead of a raw code fence.
+
+    Returns ``None`` if the content does not match the two-column pattern.
+
+    Example input::
+
+        add-await-result  Adds await_result=True to each call.
+                          Continuation of the description.
+        add-stream-loop   Inserts a while loop after each call.
+    """
+    pairs: list[tuple[str, str]] = []
+    current_name: str | None = None
+    current_desc_parts: list[str] = []
+    desc_col: int | None = None
+
+    for raw in content.splitlines():
+        if not raw.strip():
+            continue
+        # Detect a new entry: leading whitespace, a name (no spaces), then 2+
+        # spaces, then description text — all starting before the desc column.
+        m = re.match(r"^(\s+)(\S+)(\s{2,})(\S.*)", raw)
+        if m and (desc_col is None or m.start(4) == desc_col):
+            if current_name is not None:
+                pairs.append((current_name, " ".join(current_desc_parts)))
+            desc_col = m.start(4)
+            current_name = m.group(2)
+            current_desc_parts = [m.group(4)]
+            continue
+        # Continuation line: non-space content starts at or near desc_col
+        if desc_col is not None and current_name is not None:
+            leading = len(raw) - len(raw.lstrip())
+            if leading >= desc_col - 1:
+                current_desc_parts.append(raw.strip())
+                continue
+        # Does not fit either pattern — not a two-column block
+        return None
+
+    if current_name is not None:
+        pairs.append((current_name, " ".join(current_desc_parts)))
+
+    return pairs if pairs else None
+
+
+def _extract_verbatim_blocks(help_text: str) -> list[str]:
+    """Extract Click ``\\b`` verbatim blocks from help text.
+
+    Click uses the backspace character (``\\x08``, written as ``\\b`` in Python
+    source string literals) as a marker to prevent paragraph rewrapping in
+    ``--help`` output.  The generator must extract these blocks independently
+    because the section parser buries them inside whatever named section
+    (e.g. ``Raises``) happens to precede them, and that section is never
+    rendered.
+
+    Returns a list of stripped block strings (first line is typically the
+    block title, e.g. ``"Modes:"``, followed by indented content lines).
+    """
+    # In memory the docstring contains actual \x08 chars; split on them.
+    parts = re.split(r"\x08\s*\n", help_text)
+    blocks: list[str] = []
+    for part in parts[1:]:  # parts[0] is content before the first \b
+        block = part.rstrip()
+        if block.strip():
+            blocks.append(block)
+    return blocks
+
+
 def _format_default(value: Any) -> str:
     """Format a parameter default for display."""
     if value is None:
@@ -301,6 +374,57 @@ def _render_command(
             lines.append(f"| {flags} | {ptype} | {default} | {help_text} |")
         lines.append("")
 
+    # \b verbatim blocks — Click-style preformatted sections (e.g. "Modes:",
+    # "Best practices:") that appear in --help but are buried inside the
+    # Raises/Args sections of the parsed docstring and would otherwise be lost.
+    if cmd.help:
+        vb_blocks = _extract_verbatim_blocks(cmd.help)
+        for block in vb_blocks:
+            first_line, _, rest = block.partition("\n")
+            title = first_line.strip()
+            if title:
+                lines.append(f"**{title}**")
+                lines.append("")
+            if rest.strip():
+                # Bullet lists render as markdown; columnar/aligned content
+                # (e.g. mode tables) keeps a code fence for monospace alignment.
+                first_content = next(
+                    (line.strip() for line in rest.splitlines() if line.strip()), ""
+                )
+                if first_content.startswith("- "):
+                    # Bullet list — join wrapped continuation lines into bullets
+                    bullets: list[str] = []
+                    current: str | None = None
+                    for raw in rest.splitlines():
+                        s = raw.strip()
+                        if not s:
+                            if current is not None:
+                                bullets.append(current)
+                                current = None
+                        elif s.startswith("- "):
+                            if current is not None:
+                                bullets.append(current)
+                            current = s
+                        else:
+                            current = (current + " " + s) if current is not None else s
+                    if current is not None:
+                        bullets.append(current)
+                    for b in bullets:
+                        lines.append(b)
+                else:
+                    # Try two-column aligned format → markdown table
+                    pairs = _parse_two_column_block(rest)
+                    if pairs:
+                        # Render as bold-name bullets — consistent font with
+                        # other bullet blocks, no invented column headers.
+                        for name, desc in pairs:
+                            lines.append(f"- **`{name}`** — {desc}")
+                    else:
+                        lines.append("```")
+                        lines.append(rest.rstrip())
+                        lines.append("```")
+                lines.append("")
+
     # Output
     output = _rst_to_md(sections.get("Output", ""))
     if output:

tooling/docs-autogen/test_cli_reference.py

@@ -191,3 +191,59 @@ def test_no_mdx_or_framework_specific_syntax(generated_md):
     assert "<Callout" not in generated_md
     assert "<Tab" not in generated_md
     assert "import " not in generated_md.split("---")[-1]  # After frontmatter
+
+
+def test_verbatim_blocks_rendered(generated_md):
+    """Click \\b verbatim blocks must appear in the generated docs.
+
+    ``m fix async`` and ``m fix genslots`` contain \\b-delimited sections
+    (Modes, Best practices, Detection notes, Rewrites) that are visible in
+    ``--help`` output.  These were previously silently dropped because they
+    appear after ``Raises:`` in the docstring, which the generator never
+    renders.  They should now appear correctly formatted.
+    """
+    assert "**Modes:**" in generated_md, "fix async Modes block missing"
+    assert "add-await-result" in generated_md, "fix async mode value missing"
+    assert "**Best practices:**" in generated_md, "Best practices block missing"
+    assert "**Detection notes:**" in generated_md, "Detection notes block missing"
+    assert "**Rewrites:**" in generated_md, "fix genslots Rewrites block missing"
+    assert "GenerativeStub" in generated_md, "fix genslots rewrite target missing"
+
+
+def test_bullet_blocks_not_in_code_fence(generated_md):
+    """Bullet-list \\b blocks must render as markdown lists, not code fences.
+
+    Best practices / Detection notes / Rewrites start with ``- `` items and
+    should be plain markdown bullets so they render properly in the browser,
+    not as monospace preformatted blocks.
+    """
+    # Find the Best practices section and verify the bullet follows as plain text
+    idx = generated_md.index("**Best practices:**")
+    # Next non-empty line after the heading should be a markdown bullet, not ```
+    after = generated_md[idx:].split("\n")
+    content_lines = [ln for ln in after[1:] if ln.strip()]
+    assert content_lines[0].startswith("- "), (
+        f"Expected markdown bullet after Best practices, got: {content_lines[0]!r}"
+    )
+    assert content_lines[0] != "```", "Best practices content is inside a code fence"
+
+
+def test_extract_verbatim_blocks_basic():
+    """_extract_verbatim_blocks should split on \\x08 and return each block."""
+    from generate_cli_reference import _extract_verbatim_blocks
+
+    text = "Summary.\n\nRaises:\n    SomeError: ...\n\x08\nModes:\n  foo  bar\n\x08\nBest practices:\n  - do X\n"
+    blocks = _extract_verbatim_blocks(text)
+    assert len(blocks) == 2
+    assert blocks[0].startswith("Modes:")
+    assert "foo  bar" in blocks[0]
+    assert blocks[1].startswith("Best practices:")
+    assert "do X" in blocks[1]
+
+
+def test_extract_verbatim_blocks_empty():
+    """No \\b chars means no verbatim blocks."""
+    from generate_cli_reference import _extract_verbatim_blocks
+
+    assert _extract_verbatim_blocks("plain text with no backspace") == []
+    assert _extract_verbatim_blocks("") == []