From 14521645f4834d468ec900e873e690717512a2ac Mon Sep 17 00:00:00 2001 From: Esteban Zimanyi Date: Fri, 17 Jul 2026 00:08:20 +0200 Subject: [PATCH] Scan pgtypes for @ingroup and skip preprocessor lines in the fndef MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The exported base-type functions (text_out, date_in, timestamp_cmp_internal, …) live in the vendored pgtypes/ tree at the repo root, not under meos/src, and carry @ingroup meos_base_* there. Scan pgtypes/ alongside meos/src so the catalog picks up the base surface too: _name_to_group now accepts several source roots and run.py passes both. Make _FNDEF robust to the layouts those base sources use. A doxygen block and the function it labels can be separated by a #if MEOS guard, a multi-line comment, and blank lines (cstring_to_text is all three), and some sources ship CRLF. The fndef scanner now skips any run of blank / preprocessor / comment lines before the return-type line, and tolerates carriage returns. The meos/src group map is unchanged (verified: identical assignments across 3419 groups). --- parser/doxygroup.py | 52 ++++++++++---- run.py | 8 ++- tests/test_doxygroup.py | 154 ++++++++++++++++++++++++++++++++++++++++ 3 files changed, 198 insertions(+), 16 deletions(-) create mode 100644 tests/test_doxygroup.py diff --git a/parser/doxygroup.py b/parser/doxygroup.py index b809d13..831db08 100644 --- a/parser/doxygroup.py +++ b/parser/doxygroup.py @@ -7,6 +7,12 @@ catalog lets every binding organize its generated surface the SAME way the manual does, so a function is found in the same place across all tools. +The base-type functions the MEOS umbrella headers export (`text_out`, `date_in`, +`timestamp_cmp_internal`, …) live in the vendored `pgtypes/` tree at the repo +root, NOT under `meos/src`, and they ALSO carry `@ingroup meos_base_*` tags. +`_name_to_group` therefore scans every root it is given, so a binding can pass +both `meos/src` and `pgtypes/` and pick up the base surface too. + Adds per function (when found): `group`. The `meos_internal_*` groups are MEOS-internal, not user-facing — they are tagged like any other so a binding can filter them out, but they are NOT a separate concept here. @@ -15,26 +21,44 @@ from pathlib import Path _INGROUP = re.compile(r"@ingroup\s+(meos_\w+)") -# Same shape as sqlfn._FNDEF: after the doxygen close, an optional return-type -# line (no parens/braces/;/=), then `name(`. -_FNDEF = re.compile(r"\*/\s*\n(?:[^\n(){};=]+\n)?(\w+)\s*\(") +# After the doxygen close `*/`, the function definition can be separated from the +# comment by preprocessor directives (`#if MEOS` … `#endif`) and/or ordinary C +# comments — the vendored pgtypes files guard the MEOS-build twin of a symbol +# that way. Skip any run of such lines, then an optional return-type line (no +# parens/braces/;/=), then `name(`. DOTALL lets a multi-line `/* … */` comment +# be skipped as one unit. +# A skippable line is blank or holds only a preprocessor directive / comment +# (the directive/comment part is optional so a blank line matches too). `[^\S\n]` +# is horizontal whitespace only — it matches `\r` so CRLF sources work, without +# letting a unit swallow the following return-type or name line. +_SKIP = r"(?:[^\S\n]*(?:\#[^\n]*|//[^\n]*|/\*.*?\*/)?[^\S\n]*\n)*" +_FNDEF = re.compile( + r"\*/[^\S\n]*\n" + _SKIP + r"(?:[^\n(){};=]+\n)?(\w+)\s*\(", + re.DOTALL, +) + +def _name_to_group(*srcs): + """MEOS-C function name -> doxygen @ingroup group (first occurrence wins). -def _name_to_group(meos_src): - """MEOS-C function name -> doxygen @ingroup group (first occurrence wins).""" + Scans every source root given (e.g. `meos/src` and `pgtypes/`). + """ out = {} - for cf in Path(meos_src).rglob("*.c"): - text = cf.read_text(errors="ignore") - for m in _INGROUP.finditer(text): - grp = m.group(1) - fm = _FNDEF.search(text, m.end()) - if fm: - out.setdefault(fm.group(1), grp) + for src in srcs: + if not src or not Path(src).exists(): + continue + for cf in Path(src).rglob("*.c"): + text = cf.read_text(errors="ignore") + for m in _INGROUP.finditer(text): + grp = m.group(1) + fm = _FNDEF.search(text, m.end()) + if fm: + out.setdefault(fm.group(1), grp) return out -def attach_groups(idl, meos_src): - n2g = _name_to_group(meos_src) +def attach_groups(idl, *srcs): + n2g = _name_to_group(*srcs) n = 0 for f in idl["functions"]: g = n2g.get(f["name"]) diff --git a/run.py b/run.py index b4f794c..09a012e 100644 --- a/run.py +++ b/run.py @@ -207,9 +207,13 @@ def main(): print(f" {fn}({pn}) — {reason}", file=sys.stderr) # 5. Attach the doxygen module group (@ingroup) from the vendored source, so - # bindings organize their generated surface like the reference manual. + # bindings organize their generated surface like the reference manual. The + # exported base-type functions (text_out, date_in, timestamp_cmp_internal, + # …) live in the vendored `pgtypes/` tree at the repo root and carry + # `@ingroup meos_base_*` there, so scan it alongside meos/src. + PGTYPES_SRC = SRC_ROOT / "pgtypes" if MEOS_SRC.exists(): - idl, ngrp = attach_groups(idl, MEOS_SRC) + idl, ngrp = attach_groups(idl, MEOS_SRC, PGTYPES_SRC) print(f"[5/5] Attached {ngrp} doxygen @ingroup groups", file=sys.stderr) # Surface any forward-declared external ABI struct pointer in the API, so a diff --git a/tests/test_doxygroup.py b/tests/test_doxygroup.py new file mode 100644 index 0000000..bc5a318 --- /dev/null +++ b/tests/test_doxygroup.py @@ -0,0 +1,154 @@ +"""Unit tests for parser/doxygroup.py. + +Runs without libclang or pytest: python3 tests/test_doxygroup.py + +Focuses on `_FNDEF` robustness — the doxygen `@ingroup` block and the function +definition it labels can be separated by preprocessor guards (`#if MEOS` … +`#endif`), ordinary comments, and blank lines, especially in the vendored +`pgtypes/` base-type sources — and on the multi-root scan that lets a binding +pick up both `meos/src` and `pgtypes/`. +""" + +import sys +import tempfile +import unittest +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from parser.doxygroup import _name_to_group, attach_groups + + +def _write(root, rel, text): + p = Path(root) / rel + p.parent.mkdir(parents=True, exist_ok=True) + p.write_bytes(text.encode()) + return p + + +# The plain case: return type on its own line, name right after. +PLAIN = """\ +/** + * @ingroup meos_setspan_accessor + * @brief doc + */ +struct Set * +set_out(const struct Set *s) +{ + return NULL; +} +""" + +# A `#if MEOS` guard plus a multi-line comment between the doxygen close and the +# MEOS-build twin — the exact shape of `cstring_to_text` in pgtypes/varlena.c. +GUARDED = """\ +/** + * @ingroup meos_base_text + * @brief doc + */ +#if MEOS +/* In the extension build libpostgres exports this same symbol; use the + * backend's copy there to keep allocation consistent with it. */ +text * +cstring_to_text(const char *str) +{ + return NULL; +} +#endif +text * +pg_cstring_to_text(const char *str) +{ + return NULL; +} +""" + +# A blank line between the doxygen close and the return type (common in json/). +BLANK_LINE = """\ +/** + * @ingroup meos_json_inout + * @brief doc + */ + +Temporal * +tjsonb_from_mfjson(const char *mfjson) +{ + return NULL; +} +""" + +# CRLF line endings must be tolerated (jsonbset.c ships CRLF). +CRLF = ( + "/**\r\n" + " * @ingroup meos_json_set_accessor\r\n" + " * @brief doc\r\n" + " */\r\n" + "\r\n" + "bool\r\n" + "jsonbset_value_n(const Set *s, int n, Jsonb **result)\r\n" + "{\r\n" + " return true;\r\n" + "}\r\n" +) + + +class TestFndefRobustness(unittest.TestCase): + def setUp(self): + self._tmp = tempfile.TemporaryDirectory() + self.root = self._tmp.name + + def tearDown(self): + self._tmp.cleanup() + + def _map(self): + return _name_to_group(self.root) + + def test_plain(self): + _write(self.root, "meos/src/set.c", PLAIN) + self.assertEqual(self._map().get("set_out"), "meos_setspan_accessor") + + def test_guarded_twin_and_comment(self): + _write(self.root, "pgtypes/varlena.c", GUARDED) + m = self._map() + # The @ingroup labels the guarded MEOS twin, not the pg_ fallback. + self.assertEqual(m.get("cstring_to_text"), "meos_base_text") + + def test_blank_line(self): + _write(self.root, "meos/src/json/tjsonb.c", BLANK_LINE) + self.assertEqual(self._map().get("tjsonb_from_mfjson"), "meos_json_inout") + + def test_crlf(self): + _write(self.root, "meos/src/json/jsonbset.c", CRLF) + self.assertEqual( + self._map().get("jsonbset_value_n"), "meos_json_set_accessor") + + +class TestMultiRootScan(unittest.TestCase): + def test_scans_every_root(self): + with tempfile.TemporaryDirectory() as a, tempfile.TemporaryDirectory() as b: + _write(a, "set.c", PLAIN) + _write(b, "varlena.c", GUARDED) + m = _name_to_group(a, b) + self.assertEqual(m.get("set_out"), "meos_setspan_accessor") + self.assertEqual(m.get("cstring_to_text"), "meos_base_text") + + def test_missing_root_is_skipped(self): + with tempfile.TemporaryDirectory() as a: + _write(a, "set.c", PLAIN) + # A non-existent second root must not raise. + m = _name_to_group(a, "/no/such/path") + self.assertEqual(m.get("set_out"), "meos_setspan_accessor") + + +class TestAttachGroups(unittest.TestCase): + def test_attach_sets_group_field(self): + with tempfile.TemporaryDirectory() as a: + _write(a, "set.c", PLAIN) + idl = {"functions": [{"name": "set_out"}, {"name": "unknown_fn"}]} + idl, n = attach_groups(idl, a) + self.assertEqual(n, 1) + self.assertEqual(idl["functions"][0]["group"], "meos_setspan_accessor") + self.assertNotIn("group", idl["functions"][1]) + + +if __name__ == "__main__": + unittest.main()