diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index b96dca35b..24ab185ce 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -24,6 +24,12 @@ jobs: with: python-version: '3.13' + # vhdl-dump-ast generates the machine-readable VHDL API reference (tools/build_api_reference.py) + - name: '⚙️ Setup vhdl-dump-ast' + run: | + cargo install vhdl-dump-ast --version 0.1.0 --locked + echo "$HOME/.cargo/bin" >> "$GITHUB_PATH" + - name: Install dependencies run: | pip install -U pip --progress-bar off diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 000000000..2f9ed999c --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,46 @@ +# AGENTS.md + +Guidance for AI coding agents working on the VUnit repository. + +To *use* VUnit rather than develop it, read the [documentation](https://vunit.github.io), all of it as plain text in +[llms-full.txt](https://vunit.github.io/llms-full.txt), and the machine-readable +[VHDL API](https://vunit.github.io/api/vunit_lib.json) and [Python API](https://vunit.github.io/api/python.json). + +## Repository layout + +- `vunit/`: the Python package, with the run script API in `vunit/ui/`, the simulator interfaces in `vunit/sim_if/` and the test runner in `vunit/test/`. +- `vunit/vhdl/`: the VHDL libraries compiled into `vunit_lib` (`run`, `check`, `logging`, `com`, `data_types`, `verification_components`, ...). `vunit/vhdl/osvvm` is a third-party git submodule. +- `vunit/verilog/`: the SystemVerilog support. +- `tests/unit/`, `tests/acceptance/`, `tests/lint/`: see "Testing" in `docs/contributing.rst`. +- `examples/`, `docs/` (Sphinx) and `tools/` (documentation and release scripts). + +## Commands + +- **Unit tests:** `pytest tests/unit/`, or `tox -e py313-unit`. +- **Lint** (pycodestyle, pylint, mypy, license headers): `tox -e py313-lint`. +- **Formatting:** black, with the line length from `pyproject.toml`. Check only the files you changed, e.g. `python -m black --check `. `tox -e py313-fmt` formats every file in the repository. +- **Acceptance tests with a simulator:** `tox -e py313-acceptance-`, where `` is `activehdl`, `ghdl`, `modelsim`, `nvc` or `rivierapro`. +- **Verification components:** `tox -e py313-vcomponents-`. +- **Documentation:** `tox -e py313-docs`. + - The build needs the git tags for the release notes. + - It needs `vhdl-dump-ast` for the API reference (`cargo install vhdl-dump-ast --version 0.1.0 --locked`). Set `VUNIT_DOCS_SKIP_API=1` to build without it. +- **All tox environments:** `tox -l`. + +## Rules + +Details are in `docs/contributing.rst`. + +- **News fragments:** a change visible to users needs a news fragment `docs/news.d/..rst`. `` is one of `breaking`, `bugfix`, `doc`, `deprecation`, `feature` or `misc`. +- **License header:** Python, VHDL and Verilog source files start with the Mozilla Public License 2.0 header with the copyright assigned to Lars Asplund. Copy it from a file next to the new one. `tests/lint/test_license.py` checks it. +- **Tests:** new functionality comes with tests. + +## Generated files + +Change the generator and run it again instead of editing its output by hand: + +- `vunit/vhdl/check/tools/generate_check_equal.py`, `generate_check_equal_2008p.py` and `generate_check_match.py` generate parts of `vunit/vhdl/check/src/check_api*.vhd` and `check*.vhd`, and their testbenches. +- `vunit/vhdl/data_types/tools/generate_dict.py` generates the typed `set_*`/`get_*` subprograms and their aliases in `dict_pkg`. + +## Pitfalls + +- Running a `run.py` inside the repository creates a `vunit_out/` directory next to it. Do not commit it. diff --git a/context7.json b/context7.json index 809cfe6b2..6e130ad93 100644 --- a/context7.json +++ b/context7.json @@ -1,4 +1,22 @@ { + "$schema": "https://context7.com/schema/context7.json", "url": "https://context7.com/vunit/vunit", - "public_key": "pk_lFI1bFQ8pHgYDeBRtvnEp" + "public_key": "pk_lFI1bFQ8pHgYDeBRtvnEp", + "projectTitle": "VUnit", + "description": "Unit testing framework for VHDL and SystemVerilog: a Python test runner (run.py) and VHDL libraries for checks, logging, message passing, data types and verification components.", + "folders": [ + "docs", + "examples" + ], + "excludeFolders": [ + "docs/blog", + "docs/release_notes", + "docs/news.d" + ], + "excludeFiles": [], + "rules": [ + "Complete VHDL API signatures: https://vunit.github.io/api/vunit_lib.json", + "Complete Python API signatures: https://vunit.github.io/api/python.json", + "All documentation as plain text, including the VHDL package headers: https://vunit.github.io/llms-full.txt" + ] } diff --git a/docs/news.d/1222.doc.rst b/docs/news.d/1222.doc.rst new file mode 100644 index 000000000..29dc1d836 --- /dev/null +++ b/docs/news.d/1222.doc.rst @@ -0,0 +1,3 @@ +The website now publishes, for AI agents, all documentation and the declarations of all VHDL packages as plain text in +``llms-full.txt``, and a machine-readable reference of the VHDL and Python APIs (``api/vunit_lib.json`` and +``api/python.json``). ``AGENTS.md`` and ``context7.json`` were added to the repository. diff --git a/pyproject.toml b/pyproject.toml index d135dd713..f96dba4aa 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -103,7 +103,9 @@ isolated_build = True [testenv] recreate=True -passenv=ALDEC_LICENSE_FILE +passenv= + ALDEC_LICENSE_FILE + VUNIT_DOCS_SKIP_API deps= fmt: black @@ -136,7 +138,9 @@ commands= fmt: {envpython} -m black ./ --exclude .eggs|.git|.hg|.mypy_cache|.nox|.tox|.venv|_build|buck-out|build|dist' {posargs} unit: {envpython} -m pytest -v -ra tests/unit {posargs} lint: {envpython} -m pytest -v -ra tests/lint {posargs} + docs: {envpython} -m pytest -ra tests/unit/test_api_reference.py docs: {envpython} tools/build_docs.py {envtmpdir}/docsbuild {posargs} + docs: {envpython} tools/build_api_reference.py {envtmpdir}/docsbuild acceptance: {envpython} -m pytest -v -ra tests/acceptance {posargs} vcomponents: {envpython} vunit/vhdl/verification_components/run.py --clean coverage: {envpython} -m coverage run --branch --source vunit/ -m pytest tests/ diff --git a/tests/unit/test_api_reference.py b/tests/unit/test_api_reference.py new file mode 100644 index 000000000..25d863261 --- /dev/null +++ b/tests/unit/test_api_reference.py @@ -0,0 +1,355 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. +# +# Copyright (c) 2014-2026, Lars Asplund lars.anders.asplund@gmail.com + +""" +Test of the machine-readable VHDL and Python API reference generator. + +The VHDL-facing tests run small inline snippets through the real +vhdl-dump-ast binary (skipped when it is not on PATH, e.g. plain unit CI). +Error paths that do not need real parsing (missing binary, version +mismatch, non-zero exit) are covered separately with mocks. +""" + +import json +import shutil +import subprocess +import sys +import tempfile +import types +import unittest +from pathlib import Path +from unittest import mock + +from tests.common import with_tempdir +from tools import build_api_reference as bar + +HAS_DUMP_AST = shutil.which("vhdl-dump-ast") is not None + + +def _convert(vhdl, name="my_pkg", path="f.vhd"): + """Run vhdl-dump-ast on vhdl (a full source file's text) and convert the named package.""" + with tempfile.TemporaryDirectory() as tmpdir: + file_path = Path(tmpdir) / "f.vhd" + file_path.write_text(vhdl, encoding="utf-8") + tree = json.loads(bar.run_vhdl_dump_ast(file_path)) + return bar.convert_package_file(tree, Path(path), name, path) + + +def _pkg(body, name="my_pkg", preamble=""): + """A full package source: optional text directly before "package", then body, then "end package;".""" + return f"{preamble}package {name} is\n{body}\nend package;\n" + + +@unittest.skipUnless(HAS_DUMP_AST, "vhdl-dump-ast not installed") +class TestSubprograms(unittest.TestCase): + def test_procedures_and_functions(self): + pkg = _convert(_pkg(""" + -- Does a thing. + procedure p(value : in integer := 0); + impure function f(target : rec_t) return integer; + function "+" (l, r : rec_t) return rec_t; +""")) + p, f, plus = pkg["subprograms"] + self.assertEqual(p["kind"], "procedure") + self.assertIsNone(p["pure"]) + self.assertFalse(p["operator"]) + self.assertEqual(p["doc"], "Does a thing.") + self.assertEqual( + p["parameters"][0], {"name": "value", "class": "constant", "mode": "in", "type": "integer", "default": "0"} + ) + self.assertEqual(f["kind"], "function") + self.assertFalse(f["pure"]) # impure + self.assertEqual(f["return_type"], "integer") + self.assertTrue(plus["operator"]) + self.assertEqual(plus["name"], "+") + self.assertTrue(plus["pure"]) # no "impure" keyword + self.assertEqual([p["name"] for p in plus["parameters"]], ["l", "r"]) + + def test_parameter_classes_modes_and_file(self): + pkg = _convert(_pkg(""" + procedure p( + signal net : inout t; + variable pass_ : out boolean; + stat : out checker_stat_t; + file f : text; + signal event1, event2 : inout any_event_t; + data : std_logic_vector(7 downto 0)); +""")) + params = {p["name"]: p for p in pkg["subprograms"][0]["parameters"]} + self.assertEqual((params["net"]["class"], params["net"]["mode"]), ("signal", "inout")) + self.assertEqual((params["pass_"]["class"], params["pass_"]["mode"]), ("variable", "out")) + # Default class for an "out" mode (no explicit class keyword) is "variable". + self.assertEqual(params["stat"]["class"], "variable") + self.assertEqual((params["f"]["class"], params["f"]["mode"]), ("file", "inout")) + # A multi-identifier parameter expands to one entry per name, sharing the type. + self.assertEqual([params["event1"]["type"], params["event2"]["type"]], ["any_event_t", "any_event_t"]) + self.assertEqual(params["data"]["type"], "std_logic_vector(7 downto 0)") + + def test_line_number_skips_blank_lines_and_counts_from_1(self): + pkg = _convert(_pkg("\n\n procedure p;")) + self.assertEqual(pkg["subprograms"][0]["line"], 4) + + +@unittest.skipUnless(HAS_DUMP_AST, "vhdl-dump-ast not installed") +class TestAliases(unittest.TestCase): + def test_subprogram_alias_with_signature(self): + # A signature's "return " (a function alias) is not extracted by the + # current converter (it looks inside TypeMarkList; vhdl-dump-ast puts + # ReturnType as a sibling of it) -- return_type stays None, matching + # the real reference output for VUnit's own function aliases. + pkg = _convert(_pkg(" alias alias_get is func_get[rec_t return integer];")) + entry = pkg["aliases"][0] + self.assertEqual(entry["target"], "func_get") + self.assertEqual(entry["signature"], ["rec_t"]) + self.assertIsNone(entry["return_type"]) + + def test_procedure_alias_has_no_return_type(self): + pkg = _convert(_pkg(" alias alias_set is proc_set[rec_t, integer];")) + entry = pkg["aliases"][0] + self.assertEqual(entry["signature"], ["rec_t", "integer"]) + self.assertIsNone(entry["return_type"]) + + def test_type_alias_has_no_signature(self): + pkg = _convert(_pkg(" alias null_vec_ptr is null_ptr;")) + entry = pkg["aliases"][0] + self.assertEqual(entry["target"], "null_ptr") + self.assertEqual(entry["signature"], []) + + +@unittest.skipUnless(HAS_DUMP_AST, "vhdl-dump-ast not installed") +class TestTypesSubtypesConstantsObjects(unittest.TestCase): + def test_record_type_expands_multi_identifier_elements(self): + pkg = _convert(_pkg(" type rec_t is record\n a, b : integer;\n end record;")) + entry = pkg["types"][0] + self.assertEqual(entry["kind"], "record") + self.assertEqual(entry["elements"], [{"name": "a", "type": "integer"}, {"name": "b", "type": "integer"}]) + + def test_enumeration_type(self): + pkg = _convert(_pkg(" type color_t is (red, green, blue);")) + entry = pkg["types"][0] + self.assertEqual(entry["kind"], "enumeration") + self.assertEqual(entry["literals"], ["red", "green", "blue"]) + + def test_protected_type_with_documented_method(self): + pkg = _convert( + _pkg( + " type protected_t is protected\n -- Sets the value.\n procedure set(value : integer);\n end protected;" + ) + ) + entry = pkg["types"][0] + self.assertEqual(entry["kind"], "protected") + self.assertEqual(entry["subprograms"][0]["name"], "set") + self.assertEqual(entry["subprograms"][0]["doc"], "Sets the value.") + + def test_protected_type_unhandled_member_kind_raises(self): + with self.assertRaises(bar.ApiReferenceError) as ctx: + _convert(_pkg(" type protected_t is protected\n use work.foo_pkg.all;\n end protected;")) + self.assertIn("UseClauseDeclaration", str(ctx.exception)) + + def test_other_type_keeps_source_text_definition(self): + pkg = _convert(_pkg(" type vec_t is array (natural range <>) of rec_t;")) + entry = pkg["types"][0] + self.assertEqual(entry["kind"], "other") + self.assertEqual(entry["definition"], "array (natural range <>) of rec_t") + + def test_subtype_definition(self): + pkg = _convert(_pkg(" subtype small_t is integer range 0 to 15;")) + self.assertEqual(pkg["subtypes"][0]["definition"], "integer range 0 to 15") + + def test_constant_multi_identifier_and_deferred(self): + pkg = _convert( + _pkg(" -- A constant.\n constant c_max, c_min : integer := 15;\n constant c_deferred : boolean;") + ) + c_max, c_min, c_deferred = pkg["constants"] + self.assertEqual((c_max["value"], c_min["value"]), ("15", "15")) + self.assertEqual(c_max["doc"], "A constant.") + self.assertIsNone(c_deferred["value"]) + + def test_objects(self): + pkg = _convert(_pkg(" signal sig : std_logic;\n shared variable v : integer;\n file fh : text;")) + classes = {o["name"]: o["class"] for o in pkg["objects"]} + self.assertEqual(classes, {"sig": "signal", "v": "shared variable", "fh": "file"}) + + +@unittest.skipUnless(HAS_DUMP_AST, "vhdl-dump-ast not installed") +class TestDocCommentsAndPackage(unittest.TestCase): + def test_blank_line_ends_the_doc_block(self): + pkg = _convert(_pkg(" -- Unrelated comment\n\n constant c : integer := 1;")) + self.assertIsNone(pkg["constants"][0]["doc"]) + + def test_license_header_excluded_from_package_doc(self): + preamble = "-- This Source Code Form is subject to the terms of the Mozilla Public\n-- License, v. 2.0.\n" + pkg = _convert(_pkg("", preamble=preamble)) + self.assertIsNone(pkg["doc"]) + + def test_non_license_comment_kept_for_package_doc(self): + pkg = _convert(_pkg("", preamble="-- A package for testing\n")) + self.assertEqual(pkg["doc"], "A package for testing") + + def test_private_package_flag(self): + name = "data_types_private_pkg" + self.assertTrue(_convert(_pkg("", name=name), name=name)["private"]) + self.assertFalse(_convert(_pkg(""))["private"]) + + def test_p_prefixed_name_not_flagged_private(self): + pkg = _convert(_pkg(" constant p_internal : integer := 1;")) + self.assertFalse(pkg["private"]) + self.assertEqual(pkg["constants"][0]["name"], "p_internal") + + def test_unhandled_declaration_kind_raises_with_file_and_line(self): + with self.assertRaises(bar.ApiReferenceError) as ctx: + _convert(_pkg(" component comp_x is\n end component;"), path="/a/my_pkg.vhd") + message = str(ctx.exception) + self.assertIn("ComponentDeclaration", message) + self.assertIn("/a/my_pkg.vhd", message) + + def test_missing_package_name_returns_none(self): + with tempfile.TemporaryDirectory() as tmpdir: + file_path = Path(tmpdir) / "f.vhd" + file_path.write_text(_pkg(""), encoding="utf-8") + tree = json.loads(bar.run_vhdl_dump_ast(file_path)) + self.assertIsNone(bar.convert_package_file(tree, file_path, "other_pkg", "f.vhd")) + + +class TestPackageScope(unittest.TestCase): + """find_package_files scans file text for "package X is" -- no vhdl-dump-ast needed.""" + + @with_tempdir + def test_finds_declarations_excludes_body_instantiation_and_osvvm(self, tempdir): + write = lambda relpath, content: (tempdir / relpath).parent.mkdir(parents=True, exist_ok=True) or ( + tempdir / relpath + ).write_text(content, encoding="utf-8") + write("vunit/vhdl/data_types/src/queue_pkg.vhd", "package queue_pkg is\nend package;\n") + write( + "vunit/vhdl/foo/src/foo_pkg.vhd", + "package foo_generic_pkg is\nend package;\n" + "package foo_pkg is new work.foo_generic_pkg generic map (t => integer);\n" + "package body foo_generic_pkg is\nend package body;\n", + ) + write("vunit/vhdl/osvvm/src/SomePkg.vhd", "package SomePkg is\nend package;\n") + found = bar.find_package_files(tempdir) + self.assertEqual(sorted(name for _, name in found), ["foo_generic_pkg", "queue_pkg"]) + + +class TestCompleteness(unittest.TestCase): + def test_passes_when_sets_match(self): + bar.check_completeness(["a", "b"], ["b", "a"]) + + def test_raises_listing_missing_and_extra(self): + with self.assertRaises(bar.ApiReferenceError) as ctx: + bar.check_completeness(["a", "c"], ["a", "b"]) + message = str(ctx.exception) + self.assertIn("'b'", message) + self.assertIn("'c'", message) + + +class TestErrorHandling(unittest.TestCase): + """Missing binary, version pin and non-zero exit -- mocked, no vhdl-dump-ast needed.""" + + def test_vhdl_dump_ast_not_on_path(self): + with mock.patch.object(shutil, "which", return_value=None): + with self.assertRaises(bar.ApiReferenceError) as ctx: + bar.check_vhdl_dump_ast_available() + self.assertIn("VUNIT_DOCS_SKIP_API", str(ctx.exception)) + + def test_version_mismatch_raises(self): + fake = subprocess.CompletedProcess([], returncode=0, stdout="vhdl-dump-ast 0.2.0\n", stderr="") + with mock.patch.object(subprocess, "run", return_value=fake): + with self.assertRaises(bar.ApiReferenceError) as ctx: + bar.check_vhdl_dump_ast_version() + self.assertIn("0.2.0", str(ctx.exception)) + + def test_correct_version_returns_version_string(self): + fake = subprocess.CompletedProcess([], returncode=0, stdout="vhdl-dump-ast 0.1.0\n", stderr="") + with mock.patch.object(subprocess, "run", return_value=fake): + self.assertEqual(bar.check_vhdl_dump_ast_version(), "vhdl-dump-ast 0.1.0") + + def test_nonzero_exit_names_file_and_first_stderr_line(self): + fake = subprocess.CompletedProcess([], returncode=2, stdout="", stderr="12..18 unexpected token\nmore\n") + with mock.patch.object(subprocess, "run", return_value=fake): + with self.assertRaises(bar.ApiReferenceError) as ctx: + bar.run_vhdl_dump_ast(Path("somefile.vhd")) + message = str(ctx.exception) + self.assertIn("somefile.vhd", message) + self.assertIn("unexpected token", message) + + +# --------------------------------------------------------------------------- +# Python API +# --------------------------------------------------------------------------- + + +class SampleClass: + """A sample class for the Python API test.""" + + def method_one(self, value): + """Doubles value.""" + return value * 2 + + @property + def read_only(self): + """A read-only property.""" + return 1 + + def _private_method(self): + pass + + +def sample_function(a, b=1): + """Adds a and b.""" + return a + b + + +class TestPythonApi(unittest.TestCase): + @with_tempdir + def test_finds_autoclass_and_automodule_targets(self, tempdir): + docs = tempdir / "docs" + docs.mkdir() + (docs / "vunit.rst").write_text( + ".. autoclass:: vunit.ui.VUnit()\n\n.. autoclass:: vunit.ui.library.Library()\n :exclude-members: package\n", + encoding="utf-8", + ) + (docs / "ui.rst").write_text(".. automodule:: vunit.vunit_cli\n", encoding="utf-8") + (docs / "opts.rst").write_text( + ".. automodule:: tests.unit.test_api_reference\n :members:\n", encoding="utf-8" + ) + as_dict = { + (directive, dotted): has_members for directive, dotted, has_members in bar.find_python_targets(tempdir) + } + self.assertEqual(as_dict[("autoclass", "vunit.ui.VUnit")], False) + self.assertEqual(as_dict[("automodule", "vunit.vunit_cli")], False) + self.assertEqual(as_dict[("automodule", "tests.unit.test_api_reference")], True) + + def test_class_object_collects_public_members_defined_on_the_class(self): + obj = bar.build_class_object(f"{__name__}.SampleClass", SampleClass) + members = {m["name"]: m for m in obj["members"]} + self.assertNotIn("_private_method", members) + self.assertEqual(members["read_only"]["kind"], "property") + self.assertEqual(members["method_one"]["kind"], "method") + self.assertIn("(self, value)", members["method_one"]["signature"]) + self.assertEqual(members["method_one"]["doc"], "Doubles value.") + + def test_module_object_without_members_option_has_no_members(self): + obj = bar.build_module_object(__name__, sys.modules[__name__], has_members=False) + self.assertEqual(obj["members"], []) + + def test_module_object_with_members_option_collects_classes_and_functions(self): + obj = bar.build_module_object(__name__, sys.modules[__name__], has_members=True) + names = {m["name"] for m in obj["members"]} + self.assertTrue({"SampleClass", "sample_function"} <= names) + + def test_no_public_objects_found_raises(self): + empty_module = types.ModuleType("vunit_test_empty_module") + with self.assertRaises(bar.ApiReferenceError) as ctx: + bar.build_module_object("vunit_test_empty_module", empty_module, has_members=True) + self.assertIn("vunit_test_empty_module", str(ctx.exception)) + + def test_import_dotted_class(self): + self.assertIs(bar.import_dotted(f"{__name__}.SampleClass"), SampleClass) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/unit/test_build_docs.py b/tests/unit/test_build_docs.py new file mode 100644 index 000000000..af41ccd37 --- /dev/null +++ b/tests/unit/test_build_docs.py @@ -0,0 +1,76 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. +# +# Copyright (c) 2014-2026, Lars Asplund lars.anders.asplund@gmail.com + +""" +Test the generation of llms-full.txt from the text build of the documentation +""" + +import unittest +from pathlib import Path +from tests.common import with_tempdir +from tools.build_docs import vhdl_package_declarations, write_llms_full_txt + + +class TestLlmsFullTxt(unittest.TestCase): + """ + Test the generation of llms-full.txt + """ + + @with_tempdir + def test_concatenates_documentation_pages_front_page_first(self, tempdir): + text_path = Path(tempdir) / "text" + pages = { + "index": "VUnit\n*****\n\nIntro.\n", + "cli": "Command Line Interface\n**********************\n\nOptions.\n", + "data_types/queue": "*queue* package\n***************\n\n package queue_pkg is\n", + "blog/2023_03_31_vunit_events": "VUnit Events\n************\n", + "genindex": "Index\n*****\n", + } + for docname, text in pages.items(): + path = text_path / f"{docname}.txt" + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(text, encoding="utf-8") + + write_llms_full_txt(text_path, Path(tempdir) / "llms-full.txt") + + self.assertEqual( + (Path(tempdir) / "llms-full.txt").read_text(encoding="utf-8"), + "# VUnit\n\nSource: https://vunit.github.io/index.html\n\nVUnit\n*****\n\nIntro.\n\n" + "# Command Line Interface\n\nSource: https://vunit.github.io/cli.html\n\n" + "Command Line Interface\n**********************\n\nOptions.\n\n" + "# *queue* package\n\nSource: https://vunit.github.io/data_types/queue.html\n\n" + "*queue* package\n***************\n\n package queue_pkg is\n", + ) + + @with_tempdir + def test_package_declarations_without_license_bodies_or_private_packages(self, tempdir): + src = Path(tempdir) / "vunit" / "vhdl" / "data_types" / "src" + src.mkdir(parents=True) + (src / "queue_pkg.vhd").write_text( + "-- This Source Code Form is subject to the terms of the Mozilla Public\n" + "-- License, v. 2.0.\n\n" + "use work.integer_vector_ptr_pkg.all;\n\n" + "-- A queue\npackage queue_pkg is\n" + " type queue_t is record\n p : natural;\n end record;\n" + " procedure push(queue : queue_t; value : integer);\nend package;\n\n" + "package body queue_pkg is\nend package body;\n\n" + "package queue_private_pkg is\nend package;\n\n" + "package queue_types_pkg is\n subtype index_t is natural;\nend queue_types_pkg;\n", + encoding="utf-8", + ) + (src / "queue_pkg_inst.vhd").write_text("package int_queue_pkg is new work.generic_pkg;\n", encoding="utf-8") + + self.assertEqual( + vhdl_package_declarations(tempdir), + "# VHDL package queue_pkg\n\n" + "Source: https://github.com/VUnit/vunit/blob/master/vunit/vhdl/data_types/src/queue_pkg.vhd\n\n" + "use work.integer_vector_ptr_pkg.all;\n\n-- A queue\npackage queue_pkg is\n" + " type queue_t is record\n p : natural;\n end record;\n" + " procedure push(queue : queue_t; value : integer);\nend package;\n\n" + "# VHDL package queue_types_pkg\n\n" + "Source: https://github.com/VUnit/vunit/blob/master/vunit/vhdl/data_types/src/queue_pkg.vhd\n\n" + "package queue_types_pkg is\n subtype index_t is natural;\nend queue_types_pkg;\n", + ) diff --git a/tools/build_api_reference.py b/tools/build_api_reference.py new file mode 100644 index 000000000..f482f3a6d --- /dev/null +++ b/tools/build_api_reference.py @@ -0,0 +1,475 @@ +#!/usr/bin/env python3 + +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. +# +# Copyright (c) 2014-2026, Lars Asplund lars.anders.asplund@gmail.com + +""" +Command line utility to build the machine-readable VHDL and Python API +reference: /api/vunit_lib.json and /api/python.json. + +Usage: python tools/build_api_reference.py + +Set VUNIT_DOCS_SKIP_API=1 to skip the build (prints a warning, writes nothing, exits 0), +e.g. to build the rest of the documentation without vhdl-dump-ast installed. +""" + +import inspect +import json +import os +import pkgutil +import re +import shutil +import subprocess +import sys +from pathlib import Path + +ROOT = Path(__file__).parent.parent + +VHDL_DUMP_AST_VERSION = "vhdl-dump-ast 0.1.0" +VHDL_STANDARD = "2008" +LIBRARY_NAME = "vunit_lib" + + +class ApiReferenceError(Exception): + """Raised for every error of the API reference generator.""" + + +# "package is", not "package body", and not a package instantiation ("is new"). +_PACKAGE_RE = re.compile(r"^[ \t]*package\s+(?!body\b)(\w+)\s+is\b\s*(new\b)?", re.MULTILINE) + + +def find_package_files(root): + """(path, package name) of every package declaration in vunit/vhdl/**/src, except OSVVM, sorted by name.""" + vhdl_root = root / "vunit" / "vhdl" + found = [] + for path in sorted(vhdl_root.glob("**/src/**/*.vhd")): + if "osvvm" not in path.relative_to(vhdl_root).parts: + text = path.read_text(encoding="utf-8") + found += [(path, match.group(1)) for match in _PACKAGE_RE.finditer(text) if match.group(2) is None] + return sorted(found, key=lambda item: item[1]) + + +def check_vhdl_dump_ast_available(): + """Raise ApiReferenceError if vhdl-dump-ast is not on PATH.""" + if shutil.which("vhdl-dump-ast") is None: + raise ApiReferenceError( + "vhdl-dump-ast is required to build the API reference; install it with " + "`cargo install vhdl-dump-ast --version 0.1.0 --locked` or set VUNIT_DOCS_SKIP_API=1" + ) + + +def check_vhdl_dump_ast_version(): + """Return the vhdl-dump-ast version, raising ApiReferenceError unless it is the syntax tree format pin.""" + proc = subprocess.run(["vhdl-dump-ast", "--version"], capture_output=True, text=True, check=False) + version_line = (proc.stdout or "").strip() + if version_line != VHDL_DUMP_AST_VERSION: + raise ApiReferenceError( + f"vhdl-dump-ast reports '{version_line}', expected '{VHDL_DUMP_AST_VERSION}'; " + "the converter must be checked against the new syntax tree format" + ) + return version_line + + +def run_vhdl_dump_ast(file_path): + """The JSON syntax tree of file_path, as a string.""" + proc = subprocess.run( + ["vhdl-dump-ast", str(file_path), "--trivia", "--no-pretty"], capture_output=True, text=True, check=False + ) + if proc.returncode != 0: + first_line = (proc.stderr or "").partition("\n")[0] or "(no output)" + raise ApiReferenceError(f"vhdl-dump-ast failed for {file_path} (exit {proc.returncode}): {first_line}") + return proc.stdout + + +def _normalize(tree): + """ + Unwrap the {"Node": ...}/{"Token": ...} wrappers of a vhdl-dump-ast tree into + nodes {"kind", "children"} and tokens {"kind", "text", "trivia", "line"}. + A keyword token's kind is the keyword name, e.g. "Impure". + """ + line = 1 + + def walk(element): + nonlocal line + if "Token" not in element: + node = element.get("Node", element) + return {"kind": node["kind"], "children": [walk(child) for child in node.get("children", [])]} + token = element["Token"] + line += sum(_newlines(trivia) for trivia in token["leading_trivia"]) + kind = token["kind"] + result = { + "kind": kind.get("Keyword") if isinstance(kind, dict) else kind, + "text": token["text"], + "trivia": token["leading_trivia"], + "line": line, + } + line += token["text"].count("\n") + return result + + return walk(tree) + + +def _comment(trivia): + """The text of a comment trivia entry, or "" for whitespace.""" + value = next((value for value in trivia.values() if isinstance(value, dict) and "value" in value), None) + return bytes(value["value"]).decode(value.get("encoding", "utf-8"), errors="replace") if value else "" + + +def _newlines(trivia): + return trivia["LineFeeds"] if "LineFeeds" in trivia else _comment(trivia).count("\n") + + +def _tokens(node): + """The tokens under node (or node itself), in document order.""" + if "children" not in node: + yield node + else: + for child in node["children"]: + yield from _tokens(child) + + +def _find_all(node, *kinds): + """All nodes of the given kinds under node (or node itself), in document order.""" + if "children" not in node: + return [] + return ([node] if node["kind"] in kinds else []) + [ + found for child in node["children"] for found in _find_all(child, *kinds) + ] + + +def _child(node, kind): + """The first direct child of the given kind, or None (also for a None node).""" + return next((child for child in node["children"] if child["kind"] == kind), None) if node else None + + +def _text(node, skip=0): + """The source text of node (without its first skip children), whitespace and comments as single spaces.""" + if node is None: + return None + pieces = [] + for child in node["children"][skip:] if skip else [node]: + for token in _tokens(child): + pieces += [" ", token["text"]] if pieces and token["trivia"] else [token["text"]] + return "".join(pieces) + + +def _names(node): + """The identifier tokens of a declaration (from its identifier list, if it has one).""" + return [child for child in (_child(node, "IdentifierList") or node)["children"] if child["kind"] == "Identifier"] + + +def _doc(node): + """The block of "--" comments directly above node, not separated from it by a blank line, or None.""" + block = [] + for trivia in next(_tokens(node))["trivia"]: + if "LineComment" in trivia: + block.append(_comment(trivia)[2:].removeprefix(" ")) + elif _newlines(trivia) >= 2: + block = [] + return "\n".join(block) if block else None + + +def _unhandled(node, file_path): + return ApiReferenceError( + f"Unhandled declaration kind '{node['kind']}' at {file_path}:{next(_tokens(node))['line']}" + ) + + +_MODES = ("In", "Out", "Inout", "Buffer", "Linkage") +_CLASSES = ("Constant", "Variable", "Signal", "File") +_OBJECT_CLASSES = {"SignalDeclaration": "signal", "VariableDeclaration": "shared variable", "FileDeclaration": "file"} + + +def _subprogram(decl): + """The subprogram dict of a SubprogramDeclaration.""" + spec = decl["children"][0] + function = spec["kind"] == "FunctionSpecification" + name = next(child for child in spec["children"] if child["kind"] in ("Identifier", "StringLiteral")) + parameters = [] + for parameter in _find_all(spec, "InterfaceObjectDeclaration", "InterfaceFileDeclaration"): + keywords = {child["kind"] for child in parameter["children"]} + file = parameter["kind"] == "InterfaceFileDeclaration" + # VHDL defaults: mode "in" ("inout" for a file); class "constant" for mode "in", else "variable" + mode = next((mode.lower() for mode in _MODES if mode in keywords), "inout" if file else "in") + default_class = "file" if file else "constant" if mode == "in" else "variable" + parameter_class = next((c.lower() for c in _CLASSES if c in keywords and not file), default_class) + parameters += [ + { + "name": identifier["text"], + "class": parameter_class, + "mode": mode, + "type": _text(_child(parameter, "SubtypeIndication")), + "default": _text(_child(parameter, "InitialValue"), skip=1), + } + for identifier in _names(parameter) + ] + return { + "kind": "function" if function else "procedure", + "name": name["text"].strip('"'), + "operator": name["kind"] == "StringLiteral", + "pure": "Impure" not in {child["kind"] for child in spec["children"]} if function else None, + "line": name["line"], + "doc": _doc(decl), + "parameters": parameters, + "return_type": _text(_child(spec, "Name")) if function else None, + } + + +def _declarations(decl, file_path): + """(package dict key, list of entries) for one declaration of a package.""" + kind, names, doc = decl["kind"], _names(decl), _doc(decl) + subtype = _text(_child(decl, "SubtypeIndication")) + if kind == "SubprogramDeclaration": + return "subprograms", [_subprogram(decl)] + if kind == "ConstantDeclaration": + value = _text(_child(decl, "InitialValue"), skip=1) + return "constants", [ + {"name": name["text"], "line": name["line"], "doc": doc, "type": subtype, "value": value} for name in names + ] + if kind in _OBJECT_CLASSES: + return "objects", [ + {"name": name["text"], "class": _OBJECT_CLASSES[kind], "line": name["line"], "doc": doc, "type": subtype} + for name in names + ] + entry = {"name": names[0]["text"], "line": names[0]["line"]} if names else {} + if kind == "SubtypeDeclaration": + return "subtypes", [{**entry, "doc": doc, "definition": subtype}] + if kind == "AliasDeclaration": + marks = _child(_child(decl, "Signature"), "TypeMarkList") + signature = [_text(mark) for mark in marks["children"] if mark["kind"] == "Name"] if marks else [] + return "aliases", [ + {**entry, "target": _text(_child(decl, "Name")), "signature": signature, "return_type": None} + ] + if kind == "FullTypeDeclaration": + return "types", [_type(decl, {**entry, "doc": doc}, file_path)] + raise _unhandled(decl, file_path) + + +def _type(decl, entry, file_path): + """The type dict of a FullTypeDeclaration, extending entry (its name, line and doc).""" + definition = next((child for child in decl["children"] if "children" in child), None) + definition_kind = definition and definition["kind"] + if definition_kind == "RecordTypeDefinition": + elements = [ + {"name": name["text"], "type": _text(_child(element, "SubtypeIndication"))} + for element in _find_all(definition, "ElementDeclaration") + for name in _names(element) + ] + return {**entry, "kind": "record", "elements": elements} + if definition_kind == "EnumerationTypeDefinition": + literals = (_child(definition, "EnumerationList") or {"children": []})["children"] + literals = [literal["text"] for literal in literals if literal["kind"] in ("Identifier", "CharacterLiteral")] + return {**entry, "kind": "enumeration", "literals": literals} + if definition_kind == "ProtectedTypeDeclaration": + members = (_child(definition, "ProtectedTypeDeclarativePart") or {"children": []})["children"] + for member in members: + if member["kind"] != "SubprogramDeclaration": + raise _unhandled(member, file_path) + return {**entry, "kind": "protected", "subprograms": [_subprogram(member) for member in members]} + # Array, access and file types + return {**entry, "kind": "other", "definition": _text(definition)} + + +def convert_package_file(tree, file_path, package_name, repo_relative_path): + """The package dict of package_name in a vhdl-dump-ast syntax tree, or None if it is not declared there.""" + for package in _find_all(_normalize(tree), "PackageDeclaration"): + preamble = _child(package, "PackagePreamble") + name = _names(preamble)[0] + if name["text"] != package_name: + continue + doc = _doc(preamble) + result = { + "library": LIBRARY_NAME, + "name": package_name, + "source": repo_relative_path, + "line": name["line"], + "private": package_name.endswith("_private_pkg"), + "doc": None if "This Source Code Form" in (doc or "") else doc, # not the license header + **{key: [] for key in ("types", "subtypes", "constants", "objects", "subprograms", "aliases")}, + } + for decl in (_child(package, "PackageDeclarativePart") or {"children": []})["children"]: + if "children" in decl: + key, entries = _declarations(decl, file_path) + result[key] += entries + return result + return None + + +def check_completeness(json_package_names, expected_package_names): + """Raise ApiReferenceError listing missing/extra names if the two sets differ.""" + found, expected = set(json_package_names), set(expected_package_names) + if found != expected: + raise ApiReferenceError( + "Package set in the JSON differs from the package declarations found in the source files: " + f"missing {sorted(expected - found)}, extra {sorted(found - expected)}" + ) + + +def build_vhdl_api(root, vunit_version): + """The VHDL symbol table of every in-scope package, extracted with vhdl-dump-ast.""" + package_files = find_package_files(root) + check_vhdl_dump_ast_available() + generator = check_vhdl_dump_ast_version() + packages = [] + for file_path, name in package_files: + tree = json.loads(run_vhdl_dump_ast(file_path)) + package = convert_package_file(tree, file_path, name, file_path.relative_to(root).as_posix()) + if package is not None: + packages.append(package) + check_completeness((pkg["name"] for pkg in packages), (name for _, name in package_files)) + packages.sort(key=lambda pkg: pkg["name"]) + return { + "schema_version": 1, + "vunit_version": vunit_version, + "vhdl_standard": VHDL_STANDARD, + "generator": generator, + "packages": packages, + } + + +_DIRECTIVE_RE = re.compile(r"^\.\.\s+(auto\w+)::\s*(\S+)", re.MULTILINE) + + +def _has_members_option(text, after_pos): + """True if a ":members:" option line follows the directive ending at after_pos.""" + for line in text[after_pos:].split("\n")[1:]: + if line.strip() == "": + break + if not line[:1].isspace(): + break + if line.strip().startswith(":members:"): + return True + return False + + +def find_python_targets(root): + """ + Return a list of (directive, dotted_path, has_members) for every + autoclass/autofunction/automodule directive under docs/**/*.rst. + """ + targets = [] + for rst_path in sorted((root / "docs").glob("**/*.rst")): + text = rst_path.read_text(encoding="utf-8") + for match in _DIRECTIVE_RE.finditer(text): + directive, target = match.groups() + target = target.rstrip("()") + targets.append((directive, target, _has_members_option(text, match.end()))) + return targets + + +def import_dotted(dotted): + """Import the object with the dotted name of an autodoc target, e.g. "a.b.C".""" + try: + return pkgutil.resolve_name(dotted) + except (ImportError, AttributeError) as exc: + raise ApiReferenceError(f"Could not import Python autodoc target '{dotted}': {exc}") from exc + + +def _member_dict(name, member_kind, member, has_signature): + """A Python API member dict; signature is None unless has_signature.""" + sig = str(inspect.signature(member)) if has_signature else None + return {"name": name, "kind": member_kind, "signature": sig, "doc": inspect.getdoc(member)} + + +def _class_members(cls): + """Public methods and properties defined directly on cls (not inherited).""" + prefix = cls.__qualname__ + "." + members = [] + for name in sorted(dir(cls)): + if name.startswith("_"): + continue + member = getattr(cls, name) + if isinstance(member, property): + fget = member.fget + if fget is not None and getattr(fget, "__qualname__", "").startswith(prefix): + members.append(_member_dict(name, "property", member, False)) + elif inspect.isfunction(member) and member.__qualname__.startswith(prefix): + members.append(_member_dict(name, "method", member, True)) + return members + + +def build_class_object(dotted, cls): + """The Python API object for an autoclass target.""" + return {"name": dotted, "kind": "class", "doc": inspect.getdoc(cls), "members": _class_members(cls)} + + +def build_module_object(dotted, module, has_members): + """The Python API object for an automodule target.""" + members = [] + if has_members: + for name in sorted(vars(module)): + member = vars(module)[name] + if name.startswith("_") or getattr(member, "__module__", None) != module.__name__: + continue + if inspect.isclass(member): + members.append(_member_dict(name, "class", member, False)) + elif inspect.isfunction(member): + members.append(_member_dict(name, "function", member, True)) + if not members: + raise ApiReferenceError(f"No public objects found for Python autodoc target '{dotted}'") + return {"name": dotted, "kind": "module", "doc": inspect.getdoc(module), "members": members} + + +def build_python_api(root, vunit_version): + """Collect the Python API objects for every autodoc target under docs/.""" + objects = [] + for directive, dotted, has_members in find_python_targets(root): + obj = import_dotted(dotted) + if directive == "autoclass": + objects.append(build_class_object(dotted, obj)) + elif directive == "automodule": + objects.append(build_module_object(dotted, obj, has_members)) + else: + raise ApiReferenceError(f"Unsupported autodoc directive '{directive}' for '{dotted}' in docs/") + + objects.sort(key=lambda item: item["name"]) + return {"schema_version": 1, "vunit_version": vunit_version, "objects": objects} + + +def _write_json(path, data): + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(data, indent=1, sort_keys=False, ensure_ascii=False) + "\n", encoding="utf-8") + + +def build(docs_dir, root=ROOT): + """Build both API JSON files into /api/.""" + from vunit.about import version # pylint: disable=import-outside-toplevel + + vunit_version = version() + vhdl_api = build_vhdl_api(root, vunit_version) + python_api = build_python_api(root, vunit_version) + + api_dir = Path(docs_dir) / "api" + _write_json(api_dir / "vunit_lib.json", vhdl_api) + _write_json(api_dir / "python.json", python_api) + + +def main(argv=None): + """CLI entry point: build_api_reference.py .""" + argv = sys.argv[1:] if argv is None else argv + + if os.environ.get("VUNIT_DOCS_SKIP_API") == "1": + print( + "WARNING: VUNIT_DOCS_SKIP_API=1, skipping the API reference build (no api/*.json written)", + file=sys.stderr, + ) + return 0 + + if len(argv) != 1: + print("usage: build_api_reference.py ", file=sys.stderr) + return 1 + + try: + build(argv[0]) + except ApiReferenceError as exc: + print(f"error: {exc}", file=sys.stderr) + return 1 + + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tools/build_docs.py b/tools/build_docs.py index 3c64f4af2..b0328fe34 100644 --- a/tools/build_docs.py +++ b/tools/build_docs.py @@ -10,30 +10,93 @@ Command line utility to build documentation/website """ +import re from subprocess import check_call from pathlib import Path from sys import argv, executable - +from tempfile import TemporaryDirectory ROOT = Path(__file__).parent.parent +# Pages that do not document how to use VUnit, left out of llms-full.txt +LLMS_FULL_EXCLUDED = ("about", "blog/", "contributing", "genindex", "license", "release_notes", "testimonials/") + + +def sphinx(builder, output_path, args): + """ + Build the documentation with a Sphinx builder + """ + check_call([executable, "-m", "sphinx"] + args + ["-TEWanb", builder, ROOT / "docs", output_path]) + + +def write_llms_full_txt(text_path, output_file): + """ + Concatenate the pages of a text build into one file for AI agents, the front page first + """ + pages = [] + for path in Path(text_path).rglob("*.txt"): + docname = path.relative_to(text_path).with_suffix("").as_posix() + if any(docname == name or (name.endswith("/") and docname.startswith(name)) for name in LLMS_FULL_EXCLUDED): + continue + text = path.read_text(encoding="utf-8") + title = text.splitlines()[0] + pages.append((docname, f"# {title}\n\nSource: https://vunit.github.io/{docname}.html\n\n{text.rstrip()}\n")) + + pages.sort(key=lambda page: (page[0] != "index", page[0])) + Path(output_file).write_text("\n".join(page for _, page in pages), encoding="utf-8") + + +# A package declaration, not a body or an instantiation +_PACKAGE_RE = re.compile(r"^[ \t]*package\s+(?!body\b)(\w+)\s+is\b(?!\s*new\b)", re.MULTILINE | re.IGNORECASE) +_LICENSE_RE = re.compile(r"\A(?:--[^\n]*\n)*?--[^\n]*Mozilla Public[^\n]*\n(?:--[^\n]*\n)*") + + +def vhdl_package_declarations(root=ROOT): + """ + The declarations of the public vunit_lib packages, with their context and doc comments, + as llms-full.txt pages. Most are not included by any documentation page. + """ + vhdl_root = Path(root) / "vunit" / "vhdl" + pages = [] + for path in sorted(vhdl_root.glob("**/src/**/*.vhd")): + if "osvvm" in path.relative_to(vhdl_root).parts: + continue + text = path.read_text(encoding="utf-8") + # The first package with the context and comments above it, the others from their declaration + license_header = _LICENSE_RE.match(text) + start = license_header.end() if license_header else 0 + for match in _PACKAGE_RE.finditer(text): + start = match.start() if start is None else start + name = match.group(1) + end_re = re.compile(rf"^[ \t]*end(\s+package)?(\s+{name})?\s*;[^\n]*", re.MULTILINE | re.IGNORECASE) + end = end_re.search(text, match.end()).end() + if not name.endswith("_private_pkg"): + source = path.relative_to(root).as_posix() + pages.append( + ( + name, + f"# VHDL package {name}\n\nSource: https://github.com/VUnit/vunit/blob/master/{source}\n\n" + f"{text[start:end].strip()}\n", + ) + ) + start = None + return "\n".join(page for _, page in sorted(pages)) + def main(): """ - Build documentation/website - """ - check_call( - [ - executable, - "-m", - "sphinx" - ] + ([] if len(argv) < 2 else argv[2:]) + [ - "-TEWanb", - "html", - Path(__file__).parent.parent / "docs", - argv[1], - ] - ) + Build documentation/website, and all pages and VHDL package declarations as plain text in + llms-full.txt for AI agents + """ + output_path = Path(argv[1]) + args = argv[2:] + sphinx("html", output_path, args) + with TemporaryDirectory() as temp_path: + text_path = Path(temp_path) / "text" + sphinx("text", text_path, args + ["-d", str(Path(temp_path) / "doctrees")]) + write_llms_full_txt(text_path, output_path / "llms-full.txt") + with (output_path / "llms-full.txt").open("a", encoding="utf-8") as fptr: + fptr.write("\n" + vhdl_package_declarations()) if __name__ == "__main__":