|
| 1 | +"""Fail the docs build when a page's cross-references did not resolve. |
| 2 | +
|
| 3 | +Zensical (0.0.48) stays green even under `--strict` on two failure modes |
| 4 | +MkDocs strict mode aborted on: |
| 5 | +
|
| 6 | +- An unresolvable `[text][identifier]` cross-reference renders as literal |
| 7 | + bracket text (mkdocs-autorefs used to warn). The generated API index and |
| 8 | + the docstring cross-references rely on such references resolving. |
| 9 | +- A failed `objects.inv` inventory download is logged as an ERROR record and |
| 10 | + otherwise ignored, silently degrading every link through that inventory |
| 11 | + (thousands of standard-library links alone) to plain text. |
| 12 | +
|
| 13 | +Both are caught from the built site itself, so no log-wording change can |
| 14 | +disarm the check: an unresolved reference leaves a tell-tale bracket |
| 15 | +sequence in prose text (code blocks legitimately contain `][`, e.g. dict |
| 16 | +indexing, so only text outside `<pre>`/`<code>` counts), and every inventory |
| 17 | +declared in `mkdocs.yml` must contribute at least one resolved reference — |
| 18 | +an `autorefs-external` anchor, which hand-authored prose links to the same |
| 19 | +host never carry — to the site (an inventory that contributes none is dead |
| 20 | +config and fails too). |
| 21 | +
|
| 22 | +Offline contributors can skip the inventory check by setting |
| 23 | +`DOCS_ALLOW_INVENTORY_FAILURE=1`; CI (`CI=true`) never skips it. |
| 24 | +
|
| 25 | +Usage: |
| 26 | + python scripts/docs/check_crossrefs.py --site-dir site |
| 27 | +""" |
| 28 | + |
| 29 | +from __future__ import annotations |
| 30 | + |
| 31 | +import argparse |
| 32 | +import os |
| 33 | +import re |
| 34 | +import sys |
| 35 | +from html.parser import HTMLParser |
| 36 | +from pathlib import Path |
| 37 | +from urllib.parse import urlsplit |
| 38 | + |
| 39 | +import yaml |
| 40 | + |
| 41 | +ROOT = Path(__file__).parent.parent.parent |
| 42 | + |
| 43 | +# Unresolved cross-reference tell-tales in extracted prose (`\x00` marks a |
| 44 | +# skipped code element, see _ProseTextExtractor): the two-part |
| 45 | +# `[text][identifier]` reconstruction — the identifier part is always plain |
| 46 | +# text, so a code mark inside the second brackets means indexing prose like |
| 47 | +# `data[`x`][`y`]`, not a reference — and the shortcut `[`identifier`]` form, |
| 48 | +# which extracts as `[\x00]` unless a preceding word character or bracket |
| 49 | +# makes it a subscript like `list[`str`]`. |
| 50 | +_UNRESOLVED = re.compile(r"\]\[[^\]\s\x00]*\]|(?<![\w\]\x00])\[\x00\]") |
| 51 | + |
| 52 | +# An inventory-resolved reference renders as an anchor with the |
| 53 | +# `autorefs-external` class; a hand-authored prose link to the same host has |
| 54 | +# no autorefs class and must not satisfy the inventory check. |
| 55 | +_EXTERNAL_REF = re.compile(r"<a\s[^>]*autorefs-external[^>]*>") |
| 56 | + |
| 57 | + |
| 58 | +class _ProseTextExtractor(HTMLParser): |
| 59 | + """Collect text outside <pre>/<code>/<script>/<style> elements. |
| 60 | +
|
| 61 | + A skipped element leaves a `\\x00` mark and every other tag boundary |
| 62 | + breaks the text with a newline, so bracket sequences cannot be |
| 63 | + synthesized by joining text from unrelated elements (`x]</td><td>[y`) |
| 64 | + while sequences genuinely adjacent in one text node stay intact. A |
| 65 | + block-level tag resets the skip state, so an unclosed inline `<code>` |
| 66 | + in authored raw HTML cannot hide the rest of the page. |
| 67 | + """ |
| 68 | + |
| 69 | + _SKIP = frozenset({"pre", "code", "script", "style"}) |
| 70 | + _BLOCK = frozenset( |
| 71 | + {"article", "blockquote", "div", "h1", "h2", "h3", "h4", "h5", "h6", "li", "p", "section", "table", "td", "th"} |
| 72 | + ) |
| 73 | + |
| 74 | + def __init__(self) -> None: |
| 75 | + super().__init__(convert_charrefs=True) |
| 76 | + self._skip_depth = 0 |
| 77 | + self.chunks: list[str] = [] |
| 78 | + |
| 79 | + def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None: |
| 80 | + if tag in self._SKIP: |
| 81 | + if not self._skip_depth: |
| 82 | + self.chunks.append("\x00") |
| 83 | + self._skip_depth += 1 |
| 84 | + elif tag in self._BLOCK: |
| 85 | + self._skip_depth = 0 |
| 86 | + self.chunks.append("\n") |
| 87 | + elif not self._skip_depth: |
| 88 | + self.chunks.append("\n") |
| 89 | + |
| 90 | + def handle_endtag(self, tag: str) -> None: |
| 91 | + if tag in self._SKIP: |
| 92 | + if self._skip_depth: |
| 93 | + self._skip_depth -= 1 |
| 94 | + elif not self._skip_depth: |
| 95 | + self.chunks.append("\n") |
| 96 | + |
| 97 | + def handle_data(self, data: str) -> None: |
| 98 | + if not self._skip_depth: |
| 99 | + self.chunks.append(data) |
| 100 | + |
| 101 | + |
| 102 | +def unresolved_refs(html: str) -> list[str]: |
| 103 | + """Return the unresolved cross-reference fragments rendered in `html`.""" |
| 104 | + parser = _ProseTextExtractor() |
| 105 | + parser.feed(html) |
| 106 | + return [fragment.replace("\x00", "<code>") for fragment in _UNRESOLVED.findall("".join(parser.chunks))] |
| 107 | + |
| 108 | + |
| 109 | +def _inventory_origins() -> set[str]: |
| 110 | + """The scheme+host origins of the inventories declared in mkdocs.yml.""" |
| 111 | + config = yaml.safe_load((ROOT / "mkdocs.yml").read_text(encoding="utf-8")) |
| 112 | + for plugin in config["plugins"]: |
| 113 | + if isinstance(plugin, dict) and "mkdocstrings" in plugin: |
| 114 | + inventories = plugin["mkdocstrings"]["handlers"]["python"].get("inventories", []) |
| 115 | + return {_origin(entry["url"] if isinstance(entry, dict) else entry) for entry in inventories} |
| 116 | + return set() |
| 117 | + |
| 118 | + |
| 119 | +def _origin(url: str) -> str: |
| 120 | + parts = urlsplit(url) |
| 121 | + return f"{parts.scheme}://{parts.netloc}" |
| 122 | + |
| 123 | + |
| 124 | +def main() -> None: |
| 125 | + parser = argparse.ArgumentParser(description=__doc__) |
| 126 | + parser.add_argument("--site-dir", default="site", help="The built site directory to scan.") |
| 127 | + args = parser.parse_args() |
| 128 | + |
| 129 | + unlinked = _inventory_origins() |
| 130 | + failures: list[str] = [] |
| 131 | + for page in sorted(Path(args.site_dir).rglob("*.html")): |
| 132 | + html = page.read_text(encoding="utf-8") |
| 133 | + if unlinked and "autorefs-external" in html: |
| 134 | + for tag in _EXTERNAL_REF.finditer(html): |
| 135 | + unlinked -= {origin for origin in unlinked if f'href="{origin}' in tag.group(0)} |
| 136 | + # Both tell-tales have a literal signature in the raw HTML ("][" for |
| 137 | + # the two-part form, "[<code" for the shortcut form); skip the parse |
| 138 | + # for the majority of pages that contain neither. |
| 139 | + if "][" in html or "[<code" in html: |
| 140 | + failures.extend(f"{page}: {fragment}" for fragment in unresolved_refs(html)) |
| 141 | + if failures: |
| 142 | + print("error: unresolved cross-references rendered as literal text:", file=sys.stderr) |
| 143 | + print("\n".join(failures[:20]), file=sys.stderr) |
| 144 | + raise SystemExit(1) |
| 145 | + offline_ok = os.environ.get("DOCS_ALLOW_INVENTORY_FAILURE") == "1" and os.environ.get("CI") != "true" |
| 146 | + if unlinked and not offline_ok: |
| 147 | + print( |
| 148 | + "error: no page links into these declared inventories (download failed, or dead" |
| 149 | + " inventory config?): " + ", ".join(sorted(unlinked)), |
| 150 | + file=sys.stderr, |
| 151 | + ) |
| 152 | + print("set DOCS_ALLOW_INVENTORY_FAILURE=1 to build offline", file=sys.stderr) |
| 153 | + raise SystemExit(1) |
| 154 | + |
| 155 | + |
| 156 | +if __name__ == "__main__": |
| 157 | + main() |
0 commit comments