From c56b6666a51517f69c650732a688dc2e106896dd Mon Sep 17 00:00:00 2001 From: Hans Johnson Date: Tue, 23 Jun 2026 05:23:00 -0500 Subject: [PATCH 1/6] ENH: Declare pkl files in ninja graph via per-module stamp file igenerator.py writes N pkl files per module as side effects that ninja cannot track because their names (ClassName.SubmoduleName.pkl) are not enumerable at CMake configure time. When pkl files are deleted while the .index.txt byproducts survive, ninja considers igenerator up-to-date and pyi_generator.py fails with "No pickle files were found." Add a --pkl_stamp argument to igenerator.py. The stamp is written after all pkl files for the module are complete and is declared as a CMake OUTPUT of the igenerator add_custom_command. Ninja now re-runs igenerator whenever the stamp is absent, which guarantees the pkl files are regenerated before pyi_generator.py reads the .index.txt manifests. --- Wrapping/Generators/SwigInterface/igenerator.py | 12 ++++++++++++ Wrapping/macro_files/itk_end_wrap_module.cmake | 5 ++++- 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/Wrapping/Generators/SwigInterface/igenerator.py b/Wrapping/Generators/SwigInterface/igenerator.py index bcda8695eab..c5a0f157c8e 100755 --- a/Wrapping/Generators/SwigInterface/igenerator.py +++ b/Wrapping/Generators/SwigInterface/igenerator.py @@ -207,6 +207,15 @@ def argument_parser(): type=str, help="The directory for .pyi files to be generated", ) + cmdln_arg_parser.add_argument( + "--pkl_stamp", + action="store", + dest="pkl_stamp", + default="", + type=str, + help="Stamp file written after all pkl files are complete; declared as a " + "CMake OUTPUT so ninja re-runs igenerator when pkl files are missing.", + ) cmdln_arg_parser.add_argument( "-d", action="store_true", @@ -2054,6 +2063,9 @@ def main(): ff.write("'" + function + "', ") ff.write(")\n") + if options.pkl_stamp: + Path(options.pkl_stamp).touch() + if __name__ == "__main__": main() diff --git a/Wrapping/macro_files/itk_end_wrap_module.cmake b/Wrapping/macro_files/itk_end_wrap_module.cmake index 04bde56dc4b..45f825ab57e 100644 --- a/Wrapping/macro_files/itk_end_wrap_module.cmake +++ b/Wrapping/macro_files/itk_end_wrap_module.cmake @@ -161,11 +161,13 @@ macro(itk_end_wrap_module) # Set up outputs and byproducts for custom command set(igenerator_outputs "") set(igenerator_byproducts "") + set(pkl_stamp_file "${ITK_PKL_DIR}/${WRAPPER_LIBRARY_NAME}.pkl.stamp") list(APPEND igenerator_outputs "${i_files}") # Typedefs/.i list(APPEND igenerator_outputs "${typedef_files}") # Typedefs/SwigInterface.h list(APPEND igenerator_outputs "${idx_files}") # Typedefs/.idx list(APPEND igenerator_outputs "${snake_case_config_file}") # Generators/Python/itk/Configuration/_snake_case.py + list(APPEND igenerator_outputs "${pkl_stamp_file}") # itk-pkl/.pkl.stamp if(CMAKE_GENERATOR STREQUAL "Ninja") # Ninja generator requires byproduct for correct dependency handling # See https://cmake.org/cmake/help/latest/policy/CMP0058.html @@ -188,7 +190,7 @@ macro(itk_end_wrap_module) --library-output-dir "${WRAPPER_LIBRARY_OUTPUT_DIR}" --submodule-order "${THIS_MODULE_SUBMODULE_ORDER}" --pyi_index_list "${ITK_PYI_INDEX_FILES}" --pyi_dir "${ITK_STUB_DIR}" --pkl_dir - "${ITK_PKL_DIR}" + "${ITK_PKL_DIR}" --pkl_stamp "${pkl_stamp_file}" DEPENDS ${IGENERATOR} ${ITK_WRAP_DOC_DOCSTRING_FILES} @@ -202,6 +204,7 @@ macro(itk_end_wrap_module) VERBATIM ) + unset(pkl_stamp_file) unset(snake_case_config_file) else() #message(FATAL_ERROR "Number of interface files is 0 :${WRAPPER_LIBRARY_NAME}:") From 31309723d8f128210115e2a8376faf6efe0cf27b Mon Sep 17 00:00:00 2001 From: Hans Johnson Date: Tue, 23 Jun 2026 06:40:34 -0500 Subject: [PATCH 2/6] ENH: Replace per-class .pkl files with a per-build SQLite pkl database MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit igenerator.py now writes all pickle data for a module into a single WAL-mode SQLite database (itk-pkl-v3.db) kept in the build tree's ITK_PKL_DIR. The DB is keyed by bare class.submodule strings and is intermediate handoff state between igenerator (writer) and pyi_generator (reader), so it stays local to one build tree and is never shared via ITK_WRAP_CACHE. The .index.txt manifests now hold DB keys instead of file paths. pyi_generator.py reads keys from the manifests, queries the DB, and prunes any row whose key is absent from the current build's manifests (exact keyset cleanup, replacing the previous per-tree *.pkl glob). The schema version is embedded in the filename so a stale database from an older schema is ignored rather than migrated. itk_end_wrap_module.cmake passes --module_name to igenerator to identify the manifest partition; the per-module stamp file (.stamp) remains the CMake OUTPUT that ninja tracks. Assisted-by: Claude Code — design, implementation, and pre-commit validation --- Wrapping/CMakeLists.txt | 10 +- .../Generators/Python/itk/pyi_generator.py | 139 ++++++++++++------ .../Generators/SwigInterface/CMakeLists.txt | 7 + .../Generators/SwigInterface/igenerator.py | 61 ++++---- Wrapping/Generators/pkl_db.py | 31 ++++ .../macro_files/itk_end_wrap_module.cmake | 5 +- 6 files changed, 170 insertions(+), 83 deletions(-) create mode 100644 Wrapping/Generators/pkl_db.py diff --git a/Wrapping/CMakeLists.txt b/Wrapping/CMakeLists.txt index 1f4d8c95cd3..13cd681e01b 100644 --- a/Wrapping/CMakeLists.txt +++ b/Wrapping/CMakeLists.txt @@ -233,6 +233,13 @@ if(ITK_WRAP_PYTHON) # ${GLOBAL_IdxFilesList}: is described within SwigInterface/CMakeLists.txt # The variable is passed to pyi_generator to make sure that the function only uses current index files # All index files found that are not in the given list are assumed to be outdated and are removed. + # Prune only for ITK's own build: external wrap projects share ITK_PKL_DIR + # but their manifests cover only their own modules, so pruning there would + # delete every ITK-core row. + set(_pyi_prune_arg) + if(NOT EXTERNAL_WRAP_ITK_PROJECT) + set(_pyi_prune_arg --prune) + endif() add_custom_target( itk-stub-files ALL @@ -242,13 +249,14 @@ if(ITK_WRAP_PYTHON) COMMAND ${Python3_EXECUTABLE} ${PYI_GENERATOR} --pyi_dir "${ITK_STUB_DIR}" --pkl_dir "${ITK_PKL_DIR}" --index_list_file - "${CMAKE_CURRENT_BINARY_DIR}/GlobalIdxFilesList.txt" + "${CMAKE_CURRENT_BINARY_DIR}/GlobalIdxFilesList.txt" ${_pyi_prune_arg} DEPENDS ${PYI_GENERATOR} ${GLOBAL_IdxFilesList} COMMENT "Generating .pyi files for Python wrapper interface" VERBATIM ) + unset(_pyi_prune_arg) # Add module dependencies to ensure .index.txt files have been generated unique(WRAPPED_MODULE_LIST "${WRAP_ITK_MODULES}") diff --git a/Wrapping/Generators/Python/itk/pyi_generator.py b/Wrapping/Generators/Python/itk/pyi_generator.py index d8007fb6aed..ce2aa4787e5 100644 --- a/Wrapping/Generators/Python/itk/pyi_generator.py +++ b/Wrapping/Generators/Python/itk/pyi_generator.py @@ -5,17 +5,20 @@ """ import os +import sys +import sqlite3 from os import remove from argparse import ArgumentParser from io import StringIO from pathlib import Path, PurePath import pickle import glob -import re from collections import defaultdict +sys.path.insert(0, str(Path(__file__).parents[2])) +from pkl_db import open_pkl_db # noqa: E402 + -# The ITKClass is duplicated in igenerator.py class ITKClass: def __init__(self, l_class_name): # Structure @@ -379,13 +382,21 @@ def write_class_proxy_pyi( pyi_file.write(interfaces_code) -def unpack(file_names: [str], save_dir: str) -> str | None: - class_definitions = [] +def unpack(keys: list[str], db_conn: sqlite3.Connection, save_dir: str) -> str | None: + placeholders = ",".join("?" * len(keys)) + rows = { + k: data + for k, data in db_conn.execute( + f"SELECT key, data FROM pkl_data WHERE key IN ({placeholders})", keys + ) + } - for file_name in file_names: - with open(file_name, "rb") as pickled_file: - itk_class = pickle.load(pickled_file) - class_definitions.append(itk_class) + class_definitions = [] + for key in keys: + if key not in rows: + print(f"WARNING: pkl key {key!r} not found in database, skipping.") + continue + class_definitions.append(pickle.loads(rows[key])) base = merge(class_definitions) @@ -419,9 +430,9 @@ def unpack(file_names: [str], save_dir: str) -> str | None: return base.class_name else: - files = "\n\t".join(file_names) + files = "\n\t".join(keys) print( - f"WARNING: The following files do not contain a class definition for Python stub wrapping:\n" + f"WARNING: The following keys do not contain a class definition for Python stub wrapping:\n" f"\t{files}" ) @@ -522,6 +533,13 @@ def merge(class_definitions: []) -> ITKClass | None: type=str, help="Configured file listing the index files containing pickle file references", ) + cmdln_arg_parser.add_argument( + "--prune", + action="store_true", + dest="prune", + help="Delete pkl DB rows whose keys are absent from the index manifests. " + "Only safe when the manifests cover every module sharing the DB.", + ) cmdln_arg_parser.add_argument( "-d", action="store_true", @@ -530,7 +548,7 @@ def merge(class_definitions: []) -> ITKClass | None: ) options = cmdln_arg_parser.parse_args() - if not Path(options.pkl_dir).exists: + if not Path(options.pkl_dir).exists(): except_comment = f"Invalid directory provided '{options.pkl_dir}'" raise Exception(except_comment) @@ -556,7 +574,6 @@ def merge(class_definitions: []) -> ITKClass | None: remove(invalid_index_file) for missing_index_file in missing_index_files: - # continue on without missing file, display warning print( f"WARNING: index file {missing_index_file} is missing, " f"Python stub hints will not be generated for this file. " @@ -568,52 +585,84 @@ def merge(class_definitions: []) -> ITKClass | None: print(f"Exception: {except_comment}") raise Exception(except_comment) - indexed_pickled_files = set() + # Collect DB keys from all .index.txt manifests (keys, not file paths). + indexed_pkl_keys: set[str] = set() for index_file in sorted(index_files): with open(index_file) as file: for line in file: - indexed_pickled_files.add(line.strip()) + key = line.strip() + if key: + indexed_pkl_keys.add(key) - existing_pickled_files = { - filepath.replace(os.sep, "/") - for filepath in glob.glob(f"{options.pkl_dir}/*.pkl") - } - - invalid_pickled_files = existing_pickled_files - indexed_pickled_files - missing_pickled_files = indexed_pickled_files - existing_pickled_files - - if options.debug_code: - for invalid_pickle_file in invalid_pickled_files: - print( - f"WARNING: Outdated pickle file {invalid_pickle_file} has been removed" - ) - remove(invalid_pickle_file) + if len(indexed_pkl_keys) == 0: + raise Exception(f"No pkl keys were found in index files in '{options.pkl_dir}'") - for missing_file in missing_pickled_files: - # continue on without missing file, display warning - print( - f"WARNING: pickle file {missing_file} is missing, Python stub hints will not be generated for this file." - ) - indexed_pickled_files.remove(missing_file) - - if len(indexed_pickled_files) == 0: - raise Exception(f"No pickle files were found in directory {options.pkl_dir}") - - indexed_pickled_files = sorted(list(indexed_pickled_files)) + indexed_pkl_keys_sorted = sorted(indexed_pkl_keys) output_template_import_list: list[str] = [] output_proxy_import_list: list[str] = [] - class_name_dict = defaultdict(list) - for file in indexed_pickled_files: - current_class_name = re.split(r"\.", PurePath(file).parts[-1])[0] - class_name_dict[current_class_name].append(file) + class_name_dict: defaultdict[str, list[str]] = defaultdict(list) + for key in indexed_pkl_keys_sorted: + current_class_name = key.split(".")[0] + class_name_dict[current_class_name].append(key) + + db_conn = open_pkl_db(options.pkl_dir) + + # Indexed keys absent from the DB mean it was deleted or damaged while the + # stamp files stayed fresh, so ninja would never re-run igenerator; drop + # the stamps and fail so the next build regenerates the database. + present_keys: set[str] = set() + for chunk_start in range(0, len(indexed_pkl_keys_sorted), 500): + chunk = indexed_pkl_keys_sorted[chunk_start : chunk_start + 500] + placeholders = ",".join("?" * len(chunk)) + present_keys.update( + row[0] + for row in db_conn.execute( + f"SELECT key FROM pkl_data WHERE key IN ({placeholders})", chunk + ) + ) + absent_keys = set(indexed_pkl_keys_sorted) - present_keys + if absent_keys: + db_conn.close() + for stamp_file in glob.glob(f"{options.pkl_dir}/*.stamp"): + remove(stamp_file) + raise Exception( + f"{len(absent_keys)} indexed pkl keys are missing from the pkl " + f"database in '{options.pkl_dir}'. The igenerator stamp files " + "were removed; re-run the build to regenerate the database." + ) - for current_class_name, class_files in class_name_dict.items(): - class_name = unpack(class_files, options.pyi_dir) + for current_class_name, class_keys in class_name_dict.items(): + class_name = unpack(class_keys, db_conn, options.pyi_dir) if class_name is not None: output_template_import_list.append(f"from .{class_name}Template import *\n") output_proxy_import_list.append(f"from .{class_name}Proxy import *\n") + if not output_template_import_list: + raise Exception( + f"No Python stub classes were generated from pkl keys in '{options.pkl_dir}'" + ) + + if options.prune: + if missing_index_files: + print( + "WARNING: skipping pkl DB pruning: the index manifest set is " + "incomplete, so live keys cannot be determined safely." + ) + else: + with db_conn: + db_conn.execute("CREATE TEMP TABLE _live_keys (key TEXT PRIMARY KEY)") + db_conn.executemany( + "INSERT OR IGNORE INTO _live_keys VALUES(?)", + [(k,) for k in indexed_pkl_keys_sorted], + ) + db_conn.execute( + "DELETE FROM pkl_data WHERE key NOT IN (SELECT key FROM _live_keys)" + ) + db_conn.execute("DROP TABLE _live_keys") + + db_conn.close() + output_init_import_file = init_init_import_file() output_proxy_import_file = init_proxy_import_file() diff --git a/Wrapping/Generators/SwigInterface/CMakeLists.txt b/Wrapping/Generators/SwigInterface/CMakeLists.txt index af8526463b6..0f554984117 100644 --- a/Wrapping/Generators/SwigInterface/CMakeLists.txt +++ b/Wrapping/Generators/SwigInterface/CMakeLists.txt @@ -397,6 +397,13 @@ set( "igenerator.py path" FORCE ) +set( + IGENERATOR_PKL_DB + "${CMAKE_CURRENT_SOURCE_DIR}/../pkl_db.py" + CACHE INTERNAL + "pkl_db.py path (shared module imported by igenerator.py)" + FORCE +) macro(itk_wrap_simple_type_swig_interface wrap_class swig_name) string(APPEND SWIG_INTERFACE_TYPEDEFS "using ${swig_name} = ${wrap_class};\n") diff --git a/Wrapping/Generators/SwigInterface/igenerator.py b/Wrapping/Generators/SwigInterface/igenerator.py index c5a0f157c8e..f0f5e761b08 100755 --- a/Wrapping/Generators/SwigInterface/igenerator.py +++ b/Wrapping/Generators/SwigInterface/igenerator.py @@ -62,12 +62,14 @@ import re from argparse import ArgumentParser from io import StringIO -from os.path import exists from pathlib import Path from keyword import iskeyword from typing import Any import logging +sys.path.insert(0, str(Path(__file__).parents[1])) +from pkl_db import open_pkl_db # noqa: E402 + def argument_parser(): cmdln_arg_parser = ArgumentParser() @@ -205,7 +207,7 @@ def argument_parser(): dest="pkl_dir", default="", type=str, - help="The directory for .pyi files to be generated", + help="Directory for per-submodule .index.txt manifests and per-module stamp files.", ) cmdln_arg_parser.add_argument( "--pkl_stamp", @@ -213,8 +215,8 @@ def argument_parser(): dest="pkl_stamp", default="", type=str, - help="Stamp file written after all pkl files are complete; declared as a " - "CMake OUTPUT so ninja re-runs igenerator when pkl files are missing.", + help="Stamp file touched after the pkl DB transaction commits; " + "declared as a CMake OUTPUT so ninja re-runs igenerator when stale.", ) cmdln_arg_parser.add_argument( "-d", @@ -1910,7 +1912,7 @@ def get_submodule_namespace( def generate_swig_input( submodule_name, - pkl_dir, + pkl_db_rows, pygccxml_config, options, snake_case_process_object_functions, @@ -1933,45 +1935,23 @@ def generate_swig_input( swig_input_generator.snakeCaseProcessObjectFunctions ) - # Write index list of generated .pkl files index_file_contents: StringIO = StringIO() all_keys = swig_input_generator.classes.keys() if len(all_keys): for itk_class in all_keys: - # Future problem will be that a few files will be empty - # Can either somehow detect this or accept it - # pickle class here class_name: str = swig_input_generator.classes[itk_class].class_name submodule_name: str = swig_input_generator.classes[itk_class].submodule_name - pickled_filename: str = f"{pkl_dir}/{class_name}.{submodule_name}.pkl" - - # Only write to the pickle file if it does not match what is already saved. - overwrite: bool = False - pickle_exists: bool = exists(pickled_filename) - if pickle_exists: - with open(pickled_filename, "rb") as pickled_file: - existing_itk_class = pickle.load(pickled_file) - overwrite = not ( - existing_itk_class == swig_input_generator.classes[itk_class] - ) - if overwrite or not pickle_exists: - with open(pickled_filename, "wb") as pickled_file: - pickle.dump(swig_input_generator.classes[itk_class], pickled_file) - - index_file_contents.write(pickled_filename + "\n") + key: str = f"{class_name}.{submodule_name}" + pkl_db_rows.append( + (key, pickle.dumps(swig_input_generator.classes[itk_class])) + ) + index_file_contents.write(key + "\n") else: - # The following warning is useful for debugging, and eventually we - # may wish to find a way to remove modules that are not currently part - # of the build. For example, currently all *.wrap files are processed and listed - # as module dependencies. If FFTW is not enabled, that causes empty submodules - # to be created as dependencies unnecessarily. - # Changing that behavior will require structural code changes, or alternate - # mechanisms to be implemented. if glb_options.debug_code: print( f"WARNING: {submodule_name} has no classes identified, but was listed as a dependent submodule." ) - generate_pyi_index_files(submodule_name, index_file_contents, pkl_dir) + generate_pyi_index_files(submodule_name, index_file_contents, options.pkl_dir) def main(): @@ -1980,7 +1960,7 @@ def main(): raise ValueError(f"Required directory missing '{options.pyi_dir}'") if options.pkl_dir == "": - raise ValueError(f"Required directory missing '{options.pkl_dir}'") + raise ValueError("--pkl_dir is required for .index.txt manifests") # Ensure that the requested stub file directory exists if options.pyi_dir != "": @@ -2041,15 +2021,26 @@ def main(): ordered_submodule_list.append(submodule_name) del submodule_names_list + pkl_db_rows: list[tuple[str, bytes]] = [] for submodule_name in ordered_submodule_list: generate_swig_input( submodule_name, - options.pkl_dir, + pkl_db_rows, pygccxml_config, options, snake_case_process_object_functions, ) + if pkl_db_rows: + conn = open_pkl_db(options.pkl_dir) + with conn: + conn.executemany( + "INSERT INTO pkl_data(key, data) VALUES(?,?)" + " ON CONFLICT(key) DO UPDATE SET data=excluded.data", + pkl_db_rows, + ) + conn.close() + snake_case_file = options.snake_case_file if len(snake_case_file) > 1: with open(snake_case_file, "w") as ff: diff --git a/Wrapping/Generators/pkl_db.py b/Wrapping/Generators/pkl_db.py new file mode 100644 index 00000000000..bdc58801ce9 --- /dev/null +++ b/Wrapping/Generators/pkl_db.py @@ -0,0 +1,31 @@ +import sqlite3 +from pathlib import Path + +PKL_DB_SCHEMA_VERSION = 3 + + +def _pkl_db_path(pkl_dir: str) -> Path: + """Return the build-tree-local pkl SQLite DB path. + + The DB is intermediate handoff state keyed by bare class name, so it must + stay local to one build tree; it is never shared via ITK_WRAP_CACHE. + """ + return Path(pkl_dir) / f"itk-pkl-v{PKL_DB_SCHEMA_VERSION}.db" + + +def open_pkl_db(pkl_dir: str) -> sqlite3.Connection: + """Open (creating if absent) the build-tree-local pkl DB.""" + db_path = _pkl_db_path(pkl_dir) + db_path.parent.mkdir(parents=True, exist_ok=True) + conn = sqlite3.connect(db_path, timeout=30) + try: + # WAL needs shared memory; on filesystems without it (e.g. NFS) fall + # back to the default rollback journal, serialized by the busy timeout. + conn.execute("PRAGMA journal_mode=WAL") + except sqlite3.OperationalError: + pass + conn.execute( + "CREATE TABLE IF NOT EXISTS pkl_data (key TEXT PRIMARY KEY, data BLOB NOT NULL)" + ) + conn.commit() + return conn diff --git a/Wrapping/macro_files/itk_end_wrap_module.cmake b/Wrapping/macro_files/itk_end_wrap_module.cmake index 45f825ab57e..fd44a660e18 100644 --- a/Wrapping/macro_files/itk_end_wrap_module.cmake +++ b/Wrapping/macro_files/itk_end_wrap_module.cmake @@ -161,13 +161,13 @@ macro(itk_end_wrap_module) # Set up outputs and byproducts for custom command set(igenerator_outputs "") set(igenerator_byproducts "") - set(pkl_stamp_file "${ITK_PKL_DIR}/${WRAPPER_LIBRARY_NAME}.pkl.stamp") + set(pkl_stamp_file "${ITK_PKL_DIR}/${WRAPPER_LIBRARY_NAME}.stamp") list(APPEND igenerator_outputs "${i_files}") # Typedefs/.i list(APPEND igenerator_outputs "${typedef_files}") # Typedefs/SwigInterface.h list(APPEND igenerator_outputs "${idx_files}") # Typedefs/.idx list(APPEND igenerator_outputs "${snake_case_config_file}") # Generators/Python/itk/Configuration/_snake_case.py - list(APPEND igenerator_outputs "${pkl_stamp_file}") # itk-pkl/.pkl.stamp + list(APPEND igenerator_outputs "${pkl_stamp_file}") # itk-pkl/.stamp if(CMAKE_GENERATOR STREQUAL "Ninja") # Ninja generator requires byproduct for correct dependency handling # See https://cmake.org/cmake/help/latest/policy/CMP0058.html @@ -193,6 +193,7 @@ macro(itk_end_wrap_module) "${ITK_PKL_DIR}" --pkl_stamp "${pkl_stamp_file}" DEPENDS ${IGENERATOR} + ${IGENERATOR_PKL_DB} ${ITK_WRAP_DOC_DOCSTRING_FILES} ${CastXML_OUTPUT_FILES} ${typedef_in_files} From 6cf5f226d9afd85c8528f02468a74ceaaf59c668 Mon Sep 17 00:00:00 2001 From: Hans Johnson Date: Tue, 30 Jun 2026 08:17:43 -0500 Subject: [PATCH 3/6] ENH: Defer pygccxml initialization until after submodule list is built pygccxml is now imported lazily via _load_pygccxml() called in main() after the submodule_names_list is populated from the mdx files. The pygccxml_config is constructed immediately after, so the processing loop is unchanged. This separates the submodule-discovery phase (pure file I/O) from the pygccxml/CastXML-dependent phase, enabling a future cache-hit early return before pygccxml is ever loaded. --- .../Generators/SwigInterface/igenerator.py | 36 +++++++++++-------- 1 file changed, 22 insertions(+), 14 deletions(-) diff --git a/Wrapping/Generators/SwigInterface/igenerator.py b/Wrapping/Generators/SwigInterface/igenerator.py index f0f5e761b08..f05465ed710 100755 --- a/Wrapping/Generators/SwigInterface/igenerator.py +++ b/Wrapping/Generators/SwigInterface/igenerator.py @@ -230,8 +230,16 @@ def argument_parser(): glb_options = argument_parser() -sys.path.insert(1, glb_options.pygccxml_path) -import pygccxml # noqa: E402 +pygccxml = None # populated lazily by _load_pygccxml() + + +def _load_pygccxml(): + """Import pygccxml into the global namespace; loaded lazily, not at import time.""" + global pygccxml + import importlib + + sys.path.insert(1, glb_options.pygccxml_path) + pygccxml = importlib.import_module("pygccxml") # Global debugging variables @@ -1968,18 +1976,6 @@ def main(): if options.pkl_dir != "": Path(options.pkl_dir).mkdir(exist_ok=True, parents=True) - # init the pygccxml stuff - pygccxml.utils.loggers.cxx_parser.setLevel(logging.CRITICAL) - pygccxml.declarations.scopedef_t.RECURSIVE_DEFAULT = False - pygccxml.declarations.scopedef_t.ALLOW_EMPTY_MDECL_WRAPPER = True - - pygccxml_config = pygccxml.parser.config.xml_generator_configuration_t( - xml_generator_path=options.castxml_path, - xml_generator="castxml", - # Use castxml-output=1 to take advantage of the newer XML format - flags=["--castxml-output=1"], - ) - submodule_names_list: list[str] = [] # The first mdx file is the master index file for this module. master_mdx_filename: Path = Path(options.mdx[0]) @@ -2000,6 +1996,18 @@ def main(): if submodule_name not in submodule_names_list: submodule_names_list.append(submodule_name) + _load_pygccxml() + + pygccxml.utils.loggers.cxx_parser.setLevel(logging.CRITICAL) + pygccxml.declarations.scopedef_t.RECURSIVE_DEFAULT = False + pygccxml.declarations.scopedef_t.ALLOW_EMPTY_MDECL_WRAPPER = True + + pygccxml_config = pygccxml.parser.config.xml_generator_configuration_t( + xml_generator_path=options.castxml_path, + xml_generator="castxml", + flags=["--castxml-output=1"], + ) + for submodule_name in submodule_names_list: wrappers_namespace: Any = global_submodule_cache.get_submodule_namespace( submodule_name, options.library_output_dir, pygccxml_config From 74e305f92f60c8cbe5ea413dfae163463ae0da2d Mon Sep 17 00:00:00 2001 From: Hans Johnson Date: Tue, 30 Jun 2026 07:21:26 -0500 Subject: [PATCH 4/6] ENH: Add two-level content-addressed cache for CastXML wrapping On 2-core CI runners, 816 CastXML invocations (~32 min) account for ~10% of a cold Python wrapping build. A warm cache reduces that to ~23 sec, cutting total build time by 32 min on Azure DevOps (19%) and 3h 12m on GHA when combined with a warm ccache (64%). itk-castxml-cache.py wraps the CastXML binary transparently: - L1 key: sha256 of cxx + castxml.inc + flags (no subprocess, ~0.2s) - L2 key: sha256 of castxml -E preprocessed output with line markers stripped, making keys path-independent across build directories Enable via ITK_WRAP_CASTXML_CACHE (default ON). Set ITK_WRAP_CACHE to override the cache root (default ~/.cache/itk-wrap). CI wiring: Cache@2 tasks added to Linux, macOS, and Windows Azure DevOps pipelines; arm.yml updated for GHA. pixi tasks added for configure-python-ci / build-python-ci / test-python-ci. --- .github/workflows/arm.yml | 18 + CMake/itkWrapCastXMLCacheSupport.cmake | 66 ++ Documentation/docs/contributing/index.md | 1 + .../contributing/wrapping_architecture.md | 222 +++++ .../AzurePipelinesLinuxPython.yml | 10 + .../AzurePipelinesMacOSPython.yml | 9 + .../AzurePipelinesWindowsPython.yml | 9 + Wrapping/CMakeLists.txt | 19 + .../Generators/CastXML/itk-castxml-cache.py | 843 ++++++++++++++++++ .../itk_auto_load_submodules.cmake | 26 +- pixi.lock | 50 ++ pyproject.toml | 30 +- 12 files changed, 1300 insertions(+), 3 deletions(-) create mode 100644 CMake/itkWrapCastXMLCacheSupport.cmake create mode 100644 Documentation/docs/contributing/wrapping_architecture.md create mode 100755 Wrapping/Generators/CastXML/itk-castxml-cache.py diff --git a/.github/workflows/arm.yml b/.github/workflows/arm.yml index 5fa97d3b5e3..6c0d918d418 100644 --- a/.github/workflows/arm.yml +++ b/.github/workflows/arm.yml @@ -102,6 +102,7 @@ jobs: echo "CCACHE_SLOPPINESS=pch_defines,time_macros" >> "$GITHUB_ENV" echo "CCACHE_DIR=${{ runner.temp }}/ccache" >> "$GITHUB_ENV" echo "CCACHE_MAXSIZE=5G" >> "$GITHUB_ENV" + echo "ITK_WRAP_CACHE=${{ runner.temp }}/itk-castxml-cache" >> "$GITHUB_ENV" if [ "$RUNNER_OS" == "Linux" ]; then sudo apt-get update -qq && sudo apt-get install -y ccache locales sudo locale-gen de_DE.UTF-8 @@ -118,6 +119,16 @@ jobs: restore-keys: | ccache-v4-${{ runner.os }}-${{ matrix.name }}- + - name: Restore CastXML cache + if: matrix.python-version != '' + id: restore-castxml-cache + uses: actions/cache/restore@v5 + with: + path: ${{ runner.temp }}/itk-castxml-cache + key: castxml-v1-${{ runner.os }}-arm-python-${{ github.sha }} + restore-keys: | + castxml-v1-${{ runner.os }}-arm-python- + - name: Restore ExternalData object store id: restore-externaldata uses: actions/cache/restore@v5 @@ -202,6 +213,13 @@ jobs: path: ${{ runner.temp }}/ccache key: ccache-v4-${{ runner.os }}-${{ matrix.name }}-${{ github.sha }} + - name: Save CastXML cache + if: ${{ !cancelled() && matrix.python-version != '' }} + uses: actions/cache/save@v5 + with: + path: ${{ runner.temp }}/itk-castxml-cache + key: castxml-v1-${{ runner.os }}-arm-python-${{ github.sha }} + # ExternalData object store is populated by # .github/workflows/populate-externaldata-cache.yml — a dedicated # workflow whose only job is to prefetch every CID and write the diff --git a/CMake/itkWrapCastXMLCacheSupport.cmake b/CMake/itkWrapCastXMLCacheSupport.cmake new file mode 100644 index 00000000000..57a650acc22 --- /dev/null +++ b/CMake/itkWrapCastXMLCacheSupport.cmake @@ -0,0 +1,66 @@ +# Locate the wrapper script relative to this file so the default also +# resolves when ITK_SOURCE_DIR is not defined in the including scope. +get_filename_component( + _itk_wrap_castxml_cache_src_root + "${CMAKE_CURRENT_LIST_DIR}/.." + ABSOLUTE +) +set( + _ITK_WRAP_CASTXML_CACHE_SCRIPT_DEFAULT + "${_itk_wrap_castxml_cache_src_root}/Wrapping/Generators/CastXML/itk-castxml-cache.py" +) +unset(_itk_wrap_castxml_cache_src_root) + +option( + ITK_WRAP_CASTXML_CACHE + "Use a content-addressed two-level cache for CastXML wrapping steps." + ON +) +mark_as_advanced(ITK_WRAP_CASTXML_CACHE) + +if(ITK_WRAP_CASTXML_CACHE) + set( + ITK_WRAP_CASTXML_CACHE_SCRIPT + "${_ITK_WRAP_CASTXML_CACHE_SCRIPT_DEFAULT}" + CACHE FILEPATH + "Path to the CastXML content-addressed cache wrapper script" + ) + mark_as_advanced(ITK_WRAP_CASTXML_CACHE_SCRIPT) + + if(NOT EXISTS "${ITK_WRAP_CASTXML_CACHE_SCRIPT}") + message( + FATAL_ERROR + "ITK_WRAP_CASTXML_CACHE is ON but the wrapper script was not found:\n" + " ${ITK_WRAP_CASTXML_CACHE_SCRIPT}\n" + "Set ITK_WRAP_CASTXML_CACHE_SCRIPT to the correct path or turn off ITK_WRAP_CASTXML_CACHE." + ) + endif() + + if(NOT Python3_EXECUTABLE) + message( + FATAL_ERROR + "ITK_WRAP_CASTXML_CACHE requires Python3_EXECUTABLE to be set." + ) + endif() + + set( + ITK_WRAP_CASTXML_CACHE_MAX_DAYS + "13.9" + CACHE STRING + "Days before a CastXML cache entry is evicted by --evict after each build (default 13.9)" + ) + mark_as_advanced(ITK_WRAP_CASTXML_CACHE_MAX_DAYS) + + message(STATUS "CastXML content-addressed cache enabled") + message(STATUS " Script: ${ITK_WRAP_CASTXML_CACHE_SCRIPT}") + message( + STATUS + " Cache root: set ITK_WRAP_CACHE env var at build time (default: ~/.cache/itk-wrap)" + ) + message( + STATUS + " Eviction (after each build): entries older than ${ITK_WRAP_CASTXML_CACHE_MAX_DAYS} days, then oldest trimmed to ITK_WRAP_CACHE_MAX_SIZE GB (default 2.0)" + ) +endif() + +unset(_ITK_WRAP_CASTXML_CACHE_SCRIPT_DEFAULT) diff --git a/Documentation/docs/contributing/index.md b/Documentation/docs/contributing/index.md index bb36f8305b7..ca18462e249 100644 --- a/Documentation/docs/contributing/index.md +++ b/Documentation/docs/contributing/index.md @@ -469,6 +469,7 @@ CDash Dashboard dashboard.md updating_third_party.md python_packaging.md +wrapping_architecture.md ../README.md ``` diff --git a/Documentation/docs/contributing/wrapping_architecture.md b/Documentation/docs/contributing/wrapping_architecture.md new file mode 100644 index 00000000000..b030c6085a2 --- /dev/null +++ b/Documentation/docs/contributing/wrapping_architecture.md @@ -0,0 +1,222 @@ +Python Wrapping Architecture +============================ + +ITK's Python wrapping pipeline converts C++ template declarations into +importable Python extension modules (`.abi3.so`) and type-stub files +(`.pyi`). The pipeline runs in two distinct phases: a **configure phase** +driven by CMake and a **build phase** driven by the CMake-selected build +system (Ninja, Make, Xcode, Visual Studio, ...). + +## Configure phase: `.wrap` → `castxml_inputs/` + +Each ITK module that supports wrapping contains a `wrapping/` subdirectory +with one `.wrap` file per submodule. A `.wrap` file is a CMake script that +calls macros such as `itk_wrap_class()` and `itk_wrap_template()` to declare +which C++ template instantiations should be exposed to Python. + +CMake processes every `.wrap` file at configure time and writes three +files per submodule into `/Wrapping/castxml_inputs/`: + +| Generated file | How | Contents | +|---|---|---| +| `.cxx` | `configure_file` | `#include` directives + `_wrapping_` namespace with `using` aliases for every requested template instantiation | +| `.castxml.inc` | `file(GENERATE …)` | Compiler `-I` and `-D` flags needed to parse the `.cxx` file | +| `SwigInterface.h.in` | `configure_file` | `#include` list used by SWIG | + +CMake also registers one `add_custom_command` per submodule (for CastXML) +and one per ITK module (for `igenerator.py`). No compilation happens at +configure time; only the input files and build rules are written. + +## Build phase: CastXML → igenerator → SWIG → compile → link + +### Step 1 — CastXML (one independent job per submodule) + +Each submodule produces exactly one XML file: + +``` +castxml_inputs/.cxx +castxml_inputs/.castxml.inc ──▶ itk-castxml-cache.py ──▶ castxml_inputs/.xml +Modules/.../include/.h (many) +``` + +`itk-castxml-cache.py` wraps the CastXML binary with a two-level +content-addressed cache (`~/.cache/itk-wrap` or `$ITK_WRAP_CACHE`): + +- **L1** — hash of the `.cxx` file content → L2 key +- **L2** — `castxml -E` (preprocessor only) output hash → cached `output.xml.gz` + +A cache hit avoids running CastXML entirely. There is one CastXML job per +submodule (~800 in a full build; the exact count depends on the enabled +modules, pixel types, and dimensions), all independent and fully parallel. No CastXML job reads or modifies +another submodule's `.xml` output. + +### Step 2 — `igenerator.py` (one job per ITK module) + +Each ITK module (e.g. `ITKImageIntensity`) batches all of its submodules +into a single `igenerator.py` invocation: + +``` +castxml_inputs/itkAbsImageFilter.xml ──┐ +castxml_inputs/itkImage.xml ──┤ +castxml_inputs/itkOffset.xml ──┤ igenerator.py [ITKImageIntensity] +... (all N submodule XMLs) ──┘ --submodule-order "sub1;sub2;...;subN" + │ + ┌──────────────────────────┼─────────────────────────────┐ + │ per submodule (×N) │ │ + ▼ ▼ ▼ + Typedefs/.i itk-pkl/.index.txt itk-pkl-v3.db (SQLite) + Typedefs/.idx (lists DB keys; byproduct) (class metadata; WAL mode) + Typedefs/SwigInterface.h + + per module (×1): + itk-pkl/.stamp + itk/Configuration/_snake_case.py +``` + +`igenerator.py` uses `pygccxml` to parse each `.xml` file and emit SWIG +interface (`.i`) and index (`.idx`) files, class-metadata rows in a +build-local SQLite database consumed later by `pyi_generator.py`, and a +`.index.txt` manifest listing the DB keys for each submodule. + +The SQLite database (`itk-pkl-v3.db`) is written to the `itk-pkl/` directory +inside the build tree. It is intermediate handoff state keyed by bare class +name, so it is always build-tree-local and is never shared via +`ITK_WRAP_CACHE`. + +**Build-system scheduling**: an `igenerator.py` job for module A starts as soon +as all of A's CastXML jobs are complete, even while CastXML is still +running for module B. There is no global barrier between the CastXML and +`igenerator.py` layers. + +### Step 3 — SWIG, compile, link (per submodule) + +``` +Typedefs/.i +Typedefs/SwigInterface.h ──▶ swig ──▶ Modules/.../Python.cpp + Generators/Python/itk/Python.py + +Modules/.../Python.cpp ──▶ ccache + g++ ──▶ .o ──▶ link ──▶ _Python.abi3.so +``` + +### Step 4 — `pyi_generator.py` (one global job) + +After **all** `.index.txt` files exist, `pyi_generator.py` reads every +`.index.txt`, queries the SQLite database for each key, and writes the +type-stub files used by IDEs: + +``` +itk-pkl/.index.txt (all) ──▶ pyi_generator.py ──▶ _proxies.pyi +itk-pkl-v3.db (SQLite) (queries DB via keys in .index.txt) __init__.pyi +``` + +## Key file reference + +| Path pattern | Written by | Read by | +|---|---|---| +| `Wrapping/castxml_inputs/.cxx` | CMake `configure_file` | CastXML | +| `Wrapping/castxml_inputs/.castxml.inc` | CMake `file(GENERATE)` | `itk-castxml-cache.py` | +| `Wrapping/castxml_inputs/.xml` | `itk-castxml-cache.py` / CastXML | `igenerator.py` | +| `Wrapping/Typedefs/.i` | `igenerator.py` | SWIG | +| `Wrapping/Typedefs/.idx` | `igenerator.py` | SWIG | +| `Wrapping/Generators/Python/itk-pkl/.index.txt` | `igenerator.py` | `pyi_generator.py` | +| `Wrapping/Generators/Python/itk-pkl/itk-pkl-v3.db` | `igenerator.py` | `pyi_generator.py` | +| `Wrapping/Generators/Python/itk-pkl/.stamp` | `igenerator.py` | build system (tracks DB write completeness) | +| `Wrapping/Generators/Python/itk/_Python.abi3.so` | linker | Python `import itk` | +| `Wrapping/Generators/Python/itk/_proxies.pyi` | `pyi_generator.py` | IDEs | + +## Caches + +### CastXML cache (`itk-castxml-cache.py`) + +| Variable | Default | Purpose | +|---|---|---| +| `ITK_WRAP_CACHE` | `~/.cache/itk-wrap` | Cache root for CastXML `.xml.gz` files | +| `ITK_WRAP_CACHE_VERBOSE` | unset | Set to `1` to log HIT/MISS per file | + +The CastXML cache is content-addressed and generator-version-stamped +(`_KEY_VERSION` in `itk-castxml-cache.py`). It is shared across build +directories; a fresh configure reuses XML from a previous build on the same +machine. + +### pkl SQLite database (`igenerator.py` / `pyi_generator.py`) + +The pkl database (`itk-pkl-v3.db`) lives in the build tree's `itk-pkl/` +directory and is a build artifact, not a user-level cache. Unlike the +content-addressed CastXML cache, it is keyed by bare `class.submodule` +strings, so it is **always build-tree-local and never honors +`ITK_WRAP_CACHE`** — sharing it across build trees would collide same-named +classes from different ITK sources. + +`igenerator.py` writes the rows; `pyi_generator.py` reads them and, when +invoked with `--prune` (ITK's own build only), prunes any row whose key is +absent from the current build's `.index.txt` manifests (exact keyset +cleanup). External wrap projects share the ITK build tree's database but +their manifests cover only their own modules, so they never prune. The +schema version in the filename means a database from an older schema is +ignored rather than migrated. + +### ccache + +CastXML re-runs produce identical `.xml` files (the content is deterministic) +but are slow. The CastXML cache eliminates that cost. For the C++ compilation +steps (Step 3), ccache caches compiled `.o` files keyed on source content. +Both caches are independent and complement each other. + +### Build-phase timing on 2-core CI runners + +On a 2-core CI runner (cold caches throughout): + +| Phase | Approx. time | Notes | +|---|---|---| +| CastXML (~800 jobs, 2 cores) | ~32 min | Eliminated on warm-cache runs | +| igenerator + SWIG + C++ compile | ~225 min | Reduced by ccache on subsequent runs | +| Tests | ~44 min | | + +CastXML is ~10 % of the cold-cache total. The C++ compilation phase +dominates; `ccache` is the primary lever there. + +## Build dependency graph summary + +``` +.wrap files (configure time, not in graph) + │ cmake configure_file / file(GENERATE) + ▼ +castxml_inputs/.cxx + .castxml.inc + .h headers + │ itk-castxml-cache.py [one parallel job per submodule] + ▼ +castxml_inputs/.xml (write-once; never mutated after creation) + │ igenerator.py [one job per module; starts per-module, not globally gated] + ▼ +Typedefs/.i + .idx + SwigInterface.h +itk-pkl/.index.txt (DB keys) + itk-pkl-v3.db (SQLite) + .stamp + │ swig + ccache compile + link [parallel per submodule] + ▼ +_Python.abi3.so + │ pyi_generator.py [1 global job; needs all .index.txt] + ▼ +_proxies.pyi + __init__.pyi +``` + +## Troubleshooting + +**CastXML cache not being used** +: Set `ITK_WRAP_CACHE_VERBOSE=1` and rebuild one module to confirm HIT or + MISS log lines. Ensure `ITK_WRAP_CASTXML_CACHE=ON` is set in CMake. + A version bump to `_KEY_VERSION` in `itk-castxml-cache.py` forces a cold + cache for all entries. + +**`indexed pkl keys are missing from the pkl database`** +: The `.index.txt` manifests exist (so the build system considers `igenerator.py` + up-to-date) but the pkl database is absent or damaged. `pyi_generator.py` + detects this, deletes the stamp files itself, and fails the build once; + the next build re-runs `igenerator.py` and repopulates the DB. Manual + recovery is only needed if the automatic pass keeps failing: + ```bash + find /Wrapping/Generators/Python/itk-pkl -name "*.index.txt" -o -name "*.stamp" | xargs rm -f + cmake --build + ``` + +**Adding a new wrapped class** +: Add `itk_wrap_class()` / `itk_wrap_template()` calls to the relevant + `.wrap` file. Re-run CMake to regenerate the `.cxx` and `.castxml.inc` + files, then build normally. CMake will automatically include the new + submodule in the `--submodule-order` passed to `igenerator.py`. diff --git a/Testing/ContinuousIntegration/AzurePipelinesLinuxPython.yml b/Testing/ContinuousIntegration/AzurePipelinesLinuxPython.yml index 8cc9be6c2dc..b4d162752f8 100644 --- a/Testing/ContinuousIntegration/AzurePipelinesLinuxPython.yml +++ b/Testing/ContinuousIntegration/AzurePipelinesLinuxPython.yml @@ -36,6 +36,7 @@ variables: CCACHE_NOHASHDIR: 'true' CCACHE_SLOPPINESS: pch_defines,time_macros CCACHE_MAXSIZE: 8G + ITK_WRAP_CACHE: $(Pipeline.Workspace)/.castxml-cache jobs: - job: Linux timeoutInMinutes: 0 @@ -55,6 +56,7 @@ jobs: df -h / displayName: 'Free preinstalled software' + - checkout: self clean: true fetchDepth: 5 @@ -102,6 +104,14 @@ jobs: path: $(CCACHE_DIR) displayName: 'Restore ccache' + - task: Cache@2 + inputs: + key: '"castxml-v1" | "$(Agent.OS)" | "LinuxPython" | "$(Build.SourceVersion)"' + restoreKeys: | + "castxml-v1" | "$(Agent.OS)" | "LinuxPython" + path: $(ITK_WRAP_CACHE) + displayName: 'Restore CastXML cache' + - bash: | set -x ccache --zero-stats diff --git a/Testing/ContinuousIntegration/AzurePipelinesMacOSPython.yml b/Testing/ContinuousIntegration/AzurePipelinesMacOSPython.yml index 898ba9743cf..ee84c981241 100644 --- a/Testing/ContinuousIntegration/AzurePipelinesMacOSPython.yml +++ b/Testing/ContinuousIntegration/AzurePipelinesMacOSPython.yml @@ -36,6 +36,7 @@ variables: CCACHE_NOHASHDIR: 'true' CCACHE_SLOPPINESS: pch_defines,time_macros CCACHE_MAXSIZE: 8G + ITK_WRAP_CACHE: $(Pipeline.Workspace)/.castxml-cache jobs: - job: macOS timeoutInMinutes: 0 @@ -106,6 +107,14 @@ jobs: path: $(CCACHE_DIR) displayName: 'Restore ccache' + - task: Cache@2 + inputs: + key: '"castxml-v1" | "$(Agent.OS)" | "macOSPython" | "$(Build.SourceVersion)"' + restoreKeys: | + "castxml-v1" | "$(Agent.OS)" | "macOSPython" + path: $(ITK_WRAP_CACHE) + displayName: 'Restore CastXML cache' + - bash: | set -x ccache --zero-stats diff --git a/Testing/ContinuousIntegration/AzurePipelinesWindowsPython.yml b/Testing/ContinuousIntegration/AzurePipelinesWindowsPython.yml index 3e834a7dfda..c021e87db2e 100644 --- a/Testing/ContinuousIntegration/AzurePipelinesWindowsPython.yml +++ b/Testing/ContinuousIntegration/AzurePipelinesWindowsPython.yml @@ -36,6 +36,7 @@ variables: CCACHE_NOHASHDIR: 'true' CCACHE_SLOPPINESS: pch_defines,time_macros CCACHE_MAXSIZE: 8G + ITK_WRAP_CACHE: $(Pipeline.Workspace)/.castxml-cache jobs: - job: Windows timeoutInMinutes: 0 @@ -84,6 +85,14 @@ jobs: path: $(CCACHE_DIR) displayName: 'Restore ccache' + - task: Cache@2 + inputs: + key: '"castxml-v1" | "$(Agent.OS)" | "WindowsPython" | "$(Build.SourceVersion)"' + restoreKeys: | + "castxml-v1" | "$(Agent.OS)" | "WindowsPython" + path: $(ITK_WRAP_CACHE) + displayName: 'Restore CastXML cache' + - bash: | set -x ccache --zero-stats diff --git a/Wrapping/CMakeLists.txt b/Wrapping/CMakeLists.txt index 13cd681e01b..a7a9defb416 100644 --- a/Wrapping/CMakeLists.txt +++ b/Wrapping/CMakeLists.txt @@ -152,6 +152,12 @@ endif() include(ConfigureWrapping.cmake) +# The cache support module is not installed and its script lives in the ITK +# source tree, so external wrap projects build without the CastXML cache. +if(ITK_WRAP_PYTHON AND NOT EXTERNAL_WRAP_ITK_PROJECT) + include(itkWrapCastXMLCacheSupport) +endif() + ############################################################################### # Configure specific wrapper modules ############################################################################### @@ -272,6 +278,19 @@ if(ITK_WRAP_PYTHON) endif() endforeach() + if(ITK_WRAP_CASTXML_CACHE) + add_custom_command( + TARGET itk-stub-files + POST_BUILD + COMMAND + ${Python3_EXECUTABLE} "${ITK_WRAP_CASTXML_CACHE_SCRIPT}" --evict + "${ITK_WRAP_CASTXML_CACHE_MAX_DAYS}" + COMMENT + "CastXML cache: evicting entries older than ${ITK_WRAP_CASTXML_CACHE_MAX_DAYS} days, then trimming to the size cap" + VERBATIM + ) + endif() + unset(ITK_STUB_DIR CACHE) unset(ITK_PKL_DIR CACHE) endif() diff --git a/Wrapping/Generators/CastXML/itk-castxml-cache.py b/Wrapping/Generators/CastXML/itk-castxml-cache.py new file mode 100755 index 00000000000..bb565427a88 --- /dev/null +++ b/Wrapping/Generators/CastXML/itk-castxml-cache.py @@ -0,0 +1,843 @@ +#!/usr/bin/env python3 +""" +Two-level content-addressed cache wrapper for ITK's CastXML wrapping step. + +Invocation (replaces ccache + castxml in the cmake COMMAND): + python3 itk-castxml-cache.py /path/to/castxml [castxml args...] + python3 itk-castxml-cache.py --no-cache /path/to/castxml [castxml args...] + python3 itk-castxml-cache.py --evict DAYS [--max-size GB] # e.g. --evict 13.9 --max-size 2.0 + +Cache key hierarchy: + L1 (fast, no subprocess): sha256 of key version + castxml content-hash + + inc file + cxx file + flags. + The castxml binary is fingerprinted by SHA-256 of its content (not mtime), so + `ninja -t clean` re-links the binary without changing the L1 key. Each L1 + entry also records the `-E` header set with content hashes plus a listing + fingerprint of every -I include dir; an L1 hit is honored only when both + still match, so a changed header, or a new header appearing in any include + dir (shadowing), correctly misses instead of restoring stale XML. A new + header inside an existing subdirectory of an include dir is not detected + (same residual gap as compiler depfiles). + L2 (content-addressed): sha256 of key version + castxml content-hash + + normalised `castxml -E` preprocessed output. Preprocessor line markers + (# N "path") are stripped before hashing, making L2 keys path-independent + across build directories; the castxml content-hash ensures a castxml + upgrade never restores XML produced by the previous binary. + +Lookup: + L1 HIT (recorded header hashes still match) → restore L2 → DONE (no subprocess) + L1 miss / header changed → run castxml -E → compute L2 key + L2 HIT → restore; refresh L1 map + L2 miss → run full castxml; store; write L1 map + +Storage formats (ITK_WRAP_CACHE_FORMAT): + gzip (default): output.xml.gz, ~8x smaller than raw XML. Decompressed on + restore. 253 MB for a full 807-module ITK build; each build directory + gets its own decompressed copy (copy is nearly as fast as a hardlink). + uncompressed: output.xml, plain copy on restore. ~2.2 G per full build. + Set ITK_WRAP_CACHE_FORMAT=uncompressed to opt in. + +Multi-path cascade (ITK_WRAP_CACHE, colon-separated): + Reads search all paths in order, returning the first hit. Writes go to the + first path that accepts them (atomic rename succeeds). A read-only shared + NFS cache can be listed after the user's writable local SSD cache, e.g.: + export ITK_WRAP_CACHE=/local/ssd/cache:/nfs/lab/shared-cache + Students benefit from the lab cache for L2 hits (saving the full castxml + run) while writing L1 maps only to their own writable local cache. + +Eviction: run once after a full wrapping build via --evict DAYS. + Skipped immediately when nothing was stored since the previous eviction, so + null builds pay two stat calls instead of a cache-tree walk. Time pass + removes entries older than DAYS (the CMake ITK_WRAP_CASTXML_CACHE_MAX_DAYS + variable supplies DAYS in ITK builds); size pass then removes oldest entries + until total is under ITK_WRAP_CACHE_MAX_SIZE GB (default 2.0). Staging + leftovers and entries lacking their _ok marker are removed after a one-hour + grace period. --evict touches only the first writable cache in the list. + +Environment: + ITK_WRAP_CACHE colon-separated cache roots (default: ~/.cache/itk-wrap) + ITK_WRAP_CACHE_FORMAT storage format: gzip (default) or uncompressed + ITK_WRAP_CACHE_VERBOSE set to 1 for hit/miss logging to stderr + ITK_WRAP_CACHE_BYPASS set to 1 to bypass all caching (same as --no-cache) + ITK_WRAP_CACHE_MAX_SIZE max cache size in GB enforced by --evict (default: 2.0) +""" + +import gzip +import hashlib +import json +import os +import re +import shutil +import subprocess +import sys +import tempfile +import time +import zlib + +# Bump when the key algorithm changes; old entries orphan and age out. +# Mixed into both the L1 and L2 keys so a bump invalidates every lookup path. +_KEY_VERSION = b"v2\x00" + +# Strip "# N " preprocessor line markers so the L2 hash is path-independent. +_LINE_MARKER_RE = re.compile(rb"^# \d+ ", re.MULTILINE) + + +def _strip_line_markers(data: bytes) -> bytes: + return b"\n".join( + line for line in data.splitlines() if not _LINE_MARKER_RE.match(line) + ) + + +# Captures the file path from a preprocessor line marker: # N "path" flags +_DEP_MARKER_RE = re.compile(rb'^# \d+ "([^"]*)"', re.MULTILINE) + + +def _extract_dep_paths(preprocessed: bytes) -> list: + """Return sorted real file paths named in `castxml -E` line markers.""" + paths = set() + for m in _DEP_MARKER_RE.finditer(preprocessed): + p = m.group(1).decode("utf-8", "surrogateescape") + # Line markers escape backslashes on Windows (C:\\path\\file.h) + p = p.replace("\\\\", "\\") + if p and not p.startswith("<"): # skip , + paths.add(p) + return sorted(paths) + + +def _hash_file(path: str) -> str: + h = hashlib.sha256() + with open(path, "rb") as f: + for chunk in iter(lambda: f.read(1 << 16), b""): + h.update(chunk) + return h.hexdigest() + + +def _deps_manifest(paths: list) -> list: + """Return [[path, sha256], ...] for the preprocessed dependency set.""" + manifest = [] + for p in paths: + try: + manifest.append([p, _hash_file(p)]) + except OSError: + manifest.append([p, ""]) + return manifest + + +def _deps_unchanged(deps: list) -> bool: + """True when every recorded dependency still hashes to its stored value.""" + for dep in deps: + if not (isinstance(dep, list) and len(dep) == 2): + return False # malformed manifest entry: force a miss + path, expected = dep + try: + actual = _hash_file(path) + except OSError: + actual = "" # matches _deps_manifest's record for unreadable paths + if actual != expected: + return False + return True + + +def _incdirs_from_inc_file(inc_file: str) -> list: + """Return the ordered, deduplicated -I/-isystem directories from an @inc response file.""" + dirs = [] + seen = set() + + def _add(d: str) -> None: + d = d.strip() + if d and d not in seen: + seen.add(d) + dirs.append(d) + + try: + with open(inc_file, encoding="utf-8", errors="surrogateescape") as f: + for line in f: + tokens = re.findall(r'"([^"]*)"', line) or [line.strip()] + it = iter(tokens) + for tok in it: + if tok == "-isystem": + _add(next(it, "")) + elif tok.startswith("-isystem"): + _add(tok[len("-isystem") :]) + elif tok.startswith("-I"): + _add(tok[2:]) + except OSError: + pass + return dirs + + +def _incdirs_fingerprint(inc_file: str) -> str: + """Hash of the top-level entry names of every -I dir. + + A header appearing in (or vanishing from) any include dir changes the + fingerprint, so a newly-added shadowing header forces an L1 miss and the + `castxml -E` pass re-resolves includes against the real search path. + """ + h = hashlib.sha256(_KEY_VERSION) + for d in _incdirs_from_inc_file(inc_file): + h.update(d.encode("utf-8", "surrogateescape")) + h.update(b"\x00") + try: + names = sorted(os.listdir(d)) + except OSError: + names = [] + for name in names: + h.update(name.encode("utf-8", "surrogateescape")) + h.update(b"\x00") + h.update(b"\x01") + return h.hexdigest() + + +def _cache_roots(): + """Return ordered list of cache root directories from ITK_WRAP_CACHE. + + The env var is a colon-separated list (like PATH). Reads search all + roots in order; writes go to the first root that accepts them. + """ + raw = os.environ.get("ITK_WRAP_CACHE", "") + if not raw: + return [os.path.join(os.path.expanduser("~"), ".cache", "itk-wrap")] + sep = ";" if sys.platform == "win32" else ":" + return [p for p in raw.split(sep) if p] + + +def _verbose(): + return os.environ.get("ITK_WRAP_CACHE_VERBOSE", "") == "1" + + +def _log(msg): + if _verbose(): + print(f"itk-castxml-cache: {msg}", file=sys.stderr) + + +def _use_uncompressed(): + """Return True when uncompressed storage is explicitly requested via ITK_WRAP_CACHE_FORMAT.""" + return os.environ.get("ITK_WRAP_CACHE_FORMAT", "").lower() in ( + "uncompressed", + "raw", + "plain", + ) + + +def _max_cache_gb(): + """Parse ITK_WRAP_CACHE_MAX_SIZE (default 2.0) into GB as a float.""" + try: + return float(os.environ.get("ITK_WRAP_CACHE_MAX_SIZE", "2.0")) + except ValueError: + return 2.0 + + +def _remove_if_stale(path, grace_cutoff): + """Remove a dead dir (crashed staging / missing _ok) past the grace period.""" + try: + if os.stat(path).st_mtime < grace_cutoff: + shutil.rmtree(path, ignore_errors=True) + return 1 + except OSError: + pass + return 0 + + +def _evict(cache_root, max_size=2.0, days=13.9): + """Evict cache entries; run once after a full wrapping build via --evict. + + Exits immediately when nothing was stored since the previous eviction. + L2: remove entries whose _ok mtime is older than `days`, then remove the + oldest by mtime until the total is under `max_size` GB. + L1: remove mapping entries whose mtime is older than `days`. + Staging leftovers and entries without _ok are removed after a one-hour + grace period so crashed stores cannot grow the cache unbounded. + """ + l2_root = os.path.join(cache_root, "l2") + if not os.path.isdir(l2_root): + return + + last_store = os.path.join(cache_root, "_last_store") + last_evict = os.path.join(cache_root, "_last_evict") + try: + evict_mtime = os.stat(last_evict).st_mtime + except OSError: + evict_mtime = None + if evict_mtime is not None: + try: + store_mtime = os.stat(last_store).st_mtime + except OSError: + store_mtime = 0.0 + if evict_mtime >= store_mtime: + _log("evict: no stores since last eviction — skipping") + return + + max_bytes = int(max_size * (1 << 30)) + cutoff = time.time() - days * 86400 + grace_cutoff = time.time() - 3600 + kept = [] + total = 0 + removed = 0 + + for shard in os.listdir(l2_root): + shard_dir = os.path.join(l2_root, shard) + if not os.path.isdir(shard_dir): + continue + for key in os.listdir(shard_dir): + entry = os.path.join(shard_dir, key) + if ".tmp" in key or key.endswith(".old"): + removed += _remove_if_stale(entry, grace_cutoff) + continue + ok = os.path.join(entry, "_ok") + try: + mtime = os.stat(ok).st_mtime + except OSError: + removed += _remove_if_stale(entry, grace_cutoff) + continue + try: + entry_bytes = sum( + os.path.getsize(os.path.join(entry, fn)) for fn in os.listdir(entry) + ) + if mtime < cutoff: + shutil.rmtree(entry) + removed += 1 + else: + kept.append((mtime, entry_bytes, entry)) + total += entry_bytes + except OSError: + pass + + if total > max_bytes: + kept.sort(key=lambda x: x[0]) # oldest first + for _mtime, entry_bytes, entry in kept: + if total <= max_bytes: + break + try: + shutil.rmtree(entry) + total -= entry_bytes + removed += 1 + except OSError: + pass + + l1_root = os.path.join(cache_root, "l1") + if os.path.isdir(l1_root): + for shard in os.listdir(l1_root): + shard_dir = os.path.join(l1_root, shard) + if not os.path.isdir(shard_dir): + continue + for key_name in os.listdir(shard_dir): + key_dir = os.path.join(shard_dir, key_name) + l1f = os.path.join(key_dir, "l2_key") + try: + if os.stat(l1f).st_mtime < cutoff: + shutil.rmtree(key_dir) + removed += 1 + except OSError: + removed += _remove_if_stale(key_dir, grace_cutoff) + + try: + with open(last_evict, "w"): + pass + except OSError: + pass + + _log( + f"evict: removed {removed} entries" + f" (age>{days}d or size>{max_size:.1f}GB);" + f" {total / (1 << 30):.2f} GB remaining" + ) + + +def _bypass_mode(): + """Return True when caching should be skipped entirely. + + Controlled by --no-cache as the first positional arg (before castxml binary) + or by ITK_WRAP_CACHE_BYPASS=1 in the environment. Use for single-use builds + where the castxml -E overhead would cost more than the cache saves. + """ + return os.environ.get("ITK_WRAP_CACHE_BYPASS", "") == "1" + + +def _parse_args(argv): + """ + Parse a castxml command line into structured components. + + Strips a leading --no-cache flag (before the castxml binary) when present. + Returns (castxml_bin, output_xml, inc_file, cxx_file, passthrough_flags, no_cache) + where passthrough_flags preserves the original ordering for subprocess use. + """ + no_cache = False + if argv and argv[0] == "--no-cache": + no_cache = True + argv = argv[1:] + + if not argv: + return None, None, None, None, [], no_cache + + castxml_bin = argv[0] + output_xml = None + inc_file = None + cxx_file = None + passthrough_flags = [] + + i = 1 + while i < len(argv): + arg = argv[i] + if arg == "-o" and i + 1 < len(argv): + output_xml = argv[i + 1] + # Include in passthrough so castxml writes its output normally + passthrough_flags.extend([arg, argv[i + 1]]) + i += 2 + elif arg.startswith("@"): + inc_file = arg[1:] + passthrough_flags.append(arg) + i += 1 + elif arg.endswith(".cxx"): + cxx_file = arg + passthrough_flags.append(arg) + i += 1 + else: + passthrough_flags.append(arg) + i += 1 + + return castxml_bin, output_xml, inc_file, cxx_file, passthrough_flags, no_cache + + +def _castxml_content_hash(castxml_bin, primary_root): + """Return a stable SHA-256 of the castxml binary, cached on disk. + + Sidecar file stores "size mtime_ns sha256" so re-hashing only happens when + size or mtime changes. After `ninja -t clean`, castxml is re-linked with + the same content → same hash → stable L1 key → L1 hits on warm rebuilds. + Sidecar lives in the first (writable) cache root. + """ + try: + st = os.stat(castxml_bin) + except OSError: + return "missing" + + # One sidecar per binary path (path key avoids slashes in filename) + path_key = hashlib.sha256(castxml_bin.encode()).hexdigest()[:16] + sidecar = os.path.join(primary_root, "_binhash", path_key) + + try: + with open(sidecar) as f: + parts = f.read().split() + if ( + len(parts) == 3 + and int(parts[0]) == st.st_size + and int(parts[1]) == st.st_mtime_ns + ): + return parts[2] + except (OSError, ValueError): + pass + + # Sidecar miss or stale — hash the binary (one-time cost per unique binary) + h = hashlib.sha256() + try: + with open(castxml_bin, "rb") as f: + for chunk in iter(lambda: f.read(1 << 20), b""): + h.update(chunk) + except OSError: + return "unreadable" + + content_hash = h.hexdigest() + try: + os.makedirs(os.path.dirname(sidecar), exist_ok=True) + # Unique staging name + os.replace: concurrency-safe and refreshable on Windows + fd, tmp = tempfile.mkstemp(dir=os.path.dirname(sidecar), suffix=".tmp") + try: + with os.fdopen(fd, "w") as f: + f.write(f"{st.st_size} {st.st_mtime_ns} {content_hash}\n") + os.replace(tmp, sidecar) + except OSError: + os.unlink(tmp) + except OSError: + pass + return content_hash + + +def _l1_key(bin_hash, inc_file, cxx_file, passthrough_flags): + """Compute L1 cache key from direct inputs only (~0.2s, no subprocess).""" + h = hashlib.sha256(_KEY_VERSION) + + # Stable content fingerprint — survives re-link with unchanged binary. + h.update(f"castxml\x00{bin_hash}\x00".encode()) + + # Response file: include dirs + defines passed via @file + if inc_file: + try: + with open(inc_file, "rb") as f: + h.update(f.read()) + except OSError: + h.update(b"\x00inc_miss\x00") + h.update(b"\x00") + + if cxx_file: + try: + with open(cxx_file, "rb") as f: + h.update(f.read()) + except OSError: + h.update(b"\x00cxx_miss\x00") + h.update(b"\x00") + + # Remaining flags (--castxml-cc-gnu, compiler path, std flags, etc.) + # Skip -o and the output xml path — irrelevant to content. + skip_next = False + for flag in passthrough_flags: + if skip_next: + skip_next = False + continue + if flag == "-o": + skip_next = True + continue + if flag.endswith(".xml"): + continue + h.update(flag.encode()) + h.update(b"\x00") + + return h.hexdigest() + + +def _build_preprocess_cmd(castxml_bin, passthrough_flags, pre_output): + """ + Build a `castxml -E` command that preprocesses the same inputs. + + Strips --castxml-output, --castxml-start (XML-generation flags), + replaces -o with the temp preprocess output path, appends -E. + """ + cmd = [castxml_bin] + skip_next = False + for arg in passthrough_flags: + if skip_next: + skip_next = False + continue + if arg.startswith("--castxml-output"): + continue + if arg.startswith("--castxml-start"): + if "=" not in arg: + skip_next = True + continue + if arg == "-o": + skip_next = True # drop -o and its value (xml output path) + continue + if arg.endswith(".xml"): + continue + cmd.append(arg) + cmd.extend(["-E", "-o", pre_output]) + return cmd + + +def _compute_l2_key(castxml_bin, passthrough_flags, bin_hash): + """Run castxml -E; return (l2_key, deps_manifest) or (None, None) on failure.""" + pre_path = None + try: + with tempfile.NamedTemporaryFile(suffix=".i", delete=False) as tmp: + pre_path = tmp.name + cmd = _build_preprocess_cmd(castxml_bin, passthrough_flags, pre_path) + result = subprocess.run(cmd, capture_output=True) + if result.returncode != 0: + _log(f"castxml -E failed (exit {result.returncode})") + return None, None + with open(pre_path, "rb") as f: + pre = f.read() + h = hashlib.sha256(_KEY_VERSION) + # Same preprocessed text from a different castxml must never collide. + h.update(f"castxml\x00{bin_hash}\x00".encode()) + h.update(_strip_line_markers(pre)) + return h.hexdigest(), _deps_manifest(_extract_dep_paths(pre)) + except OSError: + return None, None + finally: + if pre_path is not None: + try: + os.unlink(pre_path) + except OSError: + pass + + +def _l1_file(cache_root, l1_key): + return os.path.join(cache_root, "l1", l1_key[:2], l1_key, "l2_key") + + +def _l2_dir(cache_root, l2_key): + return os.path.join(cache_root, "l2", l2_key[:2], l2_key) + + +def _restore_xml(cache_root, l2_key, output_xml): + """Restore cached XML to output_xml from one cache root. Returns True on success.""" + entry = _l2_dir(cache_root, l2_key) + ok_file = os.path.join(entry, "_ok") + xml_plain = os.path.join(entry, "output.xml") + xml_gz = os.path.join(entry, "output.xml.gz") + + if not os.path.isfile(ok_file): + return False + + try: + os.makedirs(os.path.dirname(os.path.abspath(output_xml)), exist_ok=True) + if os.path.isfile(xml_plain): + # copyfile (not copy2): a fresh mtime keeps ninja's edge up to date + shutil.copyfile(xml_plain, output_xml) + elif os.path.isfile(xml_gz): + with gzip.open(xml_gz, "rb") as src, open(output_xml, "wb") as dst: + shutil.copyfileobj(src, dst) + else: + return False + except (OSError, zlib.error, EOFError): + return False # truncated/corrupt cache entry: miss, fall through to -E + + try: + if os.path.getsize(output_xml) == 0: + return False # empty decompression: treat as corrupt + except OSError: + return False + + try: + os.utime(ok_file, None) + except OSError: + pass + + return True + + +def _restore_from_caches(roots, l2_key, output_xml): + """Search all cache roots for l2_key, restore on first hit.""" + for root in roots: + if _restore_xml(root, l2_key, output_xml): + return root # return the root that had the hit + return None + + +def _write_l1_mapping(l1f, l2_key, deps, incfp): + """Atomically write the L1 manifest (L2 key, dependency hashes, incdir fingerprint).""" + os.makedirs(os.path.dirname(l1f), exist_ok=True) + # Unique staging name + os.replace: concurrency-safe and refreshable on Windows + fd, tmp = tempfile.mkstemp(dir=os.path.dirname(l1f), suffix=".tmp") + try: + with os.fdopen(fd, "w") as f: + json.dump({"l2_key": l2_key, "deps": deps, "incfp": incfp}, f) + os.replace(tmp, l1f) + except OSError: + try: + os.unlink(tmp) + except OSError: + pass + raise + + +def _store(roots, l1_key, l2_key, output_xml, deps, incfp): + """Store output_xml to L2 cache and write L1→L2 mapping atomically. + + Tries each root in order and writes to the first that accepts an atomic + rename. Read-only roots (e.g. a shared NFS lab cache) are silently + skipped. + """ + uncompressed = _use_uncompressed() + + # L2 entry — write to first writable root + write_root = None + for root in roots: + entry = _l2_dir(root, l2_key) + tmp = None + try: + if not os.path.isfile(output_xml): + continue + os.makedirs(os.path.dirname(entry), exist_ok=True) + # Unique staging dir: concurrent stores of the same key cannot + # delete or interleave into each other's staging area. + tmp = tempfile.mkdtemp(dir=os.path.dirname(entry), prefix=l2_key + ".tmp") + # mkdtemp creates 0700; restore umask-derived perms for shared caches + umask = os.umask(0) + os.umask(umask) + os.chmod(tmp, 0o777 & ~umask) + + if uncompressed: + shutil.copy2(output_xml, os.path.join(tmp, "output.xml")) + else: + with ( + open(output_xml, "rb") as src, + gzip.open( + os.path.join(tmp, "output.xml.gz"), "wb", compresslevel=6 + ) as dst, + ): + shutil.copyfileobj(src, dst) + + with open(os.path.join(tmp, "_meta.json"), "w") as f: + json.dump( + { + "l1_key": l1_key, + "l2_key": l2_key, + "format": "uncompressed" if uncompressed else "gzip", + }, + f, + ) + open(os.path.join(tmp, "_ok"), "w").close() # noqa: WPS515 + + old = entry + ".old" + shutil.rmtree( + old, ignore_errors=True + ) # clear stale .old from a prior crash + if os.path.exists(entry): + os.rename(entry, old) + try: + os.rename(tmp, entry) + except OSError: + if os.path.exists(old): + os.rename(old, entry) + raise + shutil.rmtree(old, ignore_errors=True) + write_root = root + break + except OSError as exc: + _log(f"L2 store failed for {root}: {exc}") + if tmp is not None: + shutil.rmtree(tmp, ignore_errors=True) + + if write_root is None: + _log("L2 store failed in all cache roots — no entry written") + return + + _touch_store_marker(write_root) + + # L1→L2 mapping — write to same root that accepted the L2 entry + try: + _write_l1_mapping(_l1_file(write_root, l1_key), l2_key, deps, incfp) + except OSError as exc: + _log(f"L1 map store failed: {exc}") + + +def _touch_store_marker(root): + """Record that this root received a write, so --evict knows to run.""" + try: + with open(os.path.join(root, "_last_store"), "w"): + pass + except OSError: + pass + + +def _store_l1_mapping(roots, l1_key, l2_key, deps, incfp): + """Write an L1→L2 mapping to the first writable root (no L2 write).""" + for root in roots: + try: + _write_l1_mapping(_l1_file(root, l1_key), l2_key, deps, incfp) + _touch_store_marker(root) + return + except OSError: + continue + + +def _run_castxml(castxml_bin, passthrough_flags): + result = subprocess.run([castxml_bin] + passthrough_flags) + return result.returncode + + +def main(): + argv = sys.argv[1:] + if not argv: + print(__doc__, file=sys.stderr) + return 1 + + # Stand-alone eviction subcommand: python3 itk-castxml-cache.py --evict DAYS + if argv[0] == "--evict": + import argparse + + p = argparse.ArgumentParser(prog="itk-castxml-cache.py --evict") + p.add_argument( + "days", + type=float, + help="Remove entries not used in this many days (e.g. 13.9)", + ) + p.add_argument( + "--max-size", + type=float, + default=None, + help="Max cache size in GB after time eviction (default: ITK_WRAP_CACHE_MAX_SIZE or 2.0)", + ) + p.add_argument( + "--cache-dir", default=None, help="Cache root (overrides ITK_WRAP_CACHE)" + ) + args = p.parse_args(argv[1:]) + max_size = args.max_size if args.max_size is not None else _max_cache_gb() + roots = [args.cache_dir] if args.cache_dir else _cache_roots() + _evict(roots[0], max_size=max_size, days=args.days) + return 0 + + castxml_bin, output_xml, inc_file, cxx_file, passthrough_flags, no_cache = ( + _parse_args(argv) + ) + + if castxml_bin is None: + print("itk-castxml-cache: no castxml binary specified", file=sys.stderr) + return 1 + if no_cache or _bypass_mode() or not output_xml or not cxx_file: + return _run_castxml(castxml_bin, passthrough_flags) + + roots = _cache_roots() + # primary_root is used for binary sidecar storage; writes go to first writable + primary_root = roots[0] + + bin_hash = _castxml_content_hash(castxml_bin, primary_root) + incfp = _incdirs_fingerprint(inc_file) if inc_file else "" + + # ── L1 check (fast, no subprocess) ────────────────────────────────────── + l1_key = _l1_key(bin_hash, inc_file, cxx_file, passthrough_flags) + + stored = None + stored_l1f = None + for root in roots: + l1f = _l1_file(root, l1_key) + if os.path.isfile(l1f): + try: + with open(l1f) as f: + entry = json.load(f) + except (OSError, ValueError): + continue + if isinstance(entry, dict) and entry.get("l2_key"): + stored = entry + stored_l1f = l1f + break + + # ── L1 HIT: skip castxml -E only when recorded headers are unchanged and + # no header has appeared in (or left) any include dir since the store ── + if ( + stored is not None + and stored.get("deps") + and stored.get("incfp") == incfp + and _deps_unchanged(stored["deps"]) + ): + hit_root = _restore_from_caches(roots, stored["l2_key"], output_xml) + if hit_root is not None: + try: + os.utime(stored_l1f, None) # mark used so age eviction keeps it + except OSError: + pass + _log(f"HIT {os.path.basename(cxx_file)} (l1→l2={stored['l2_key'][:8]})") + return 0 + # L2 entry missing or corrupt despite L1 hit — fall through to -E check. + _log(f"L2 entry missing for {cxx_file}, re-running castxml -E") + + # ── castxml -E to compute actual L2 key (L1 miss or header changed) ───── + actual_l2_key, deps = _compute_l2_key(castxml_bin, passthrough_flags, bin_hash) + + if actual_l2_key is None: + _log(f"preprocess failed for {cxx_file} — passing through") + return _run_castxml(castxml_bin, passthrough_flags) + + # ── L2 store lookup (handles cross-dir hits: L1 miss, L2 populated) ───── + hit_root = _restore_from_caches(roots, actual_l2_key, output_xml) + if hit_root is not None: + _log(f"HIT {os.path.basename(cxx_file)} (l2={actual_l2_key[:8]})") + # Populate L1 map so the next rebuild skips castxml -E + _store_l1_mapping(roots, l1_key, actual_l2_key, deps, incfp) + return 0 + + # ── Full castxml run ───────────────────────────────────────────────────── + _log(f"MISS {os.path.basename(cxx_file)}") + try: + os.unlink(output_xml) + except OSError: + pass + rc = _run_castxml(castxml_bin, passthrough_flags) + if rc == 0: + _store(roots, l1_key, actual_l2_key, output_xml, deps, incfp) + return rc + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/Wrapping/macro_files/itk_auto_load_submodules.cmake b/Wrapping/macro_files/itk_auto_load_submodules.cmake index 4ca246ebc1e..1a0edec86ba 100644 --- a/Wrapping/macro_files/itk_auto_load_submodules.cmake +++ b/Wrapping/macro_files/itk_auto_load_submodules.cmake @@ -199,12 +199,33 @@ function(generate_castxml_commandline_flags) endforeach() # ===== Run the castxml command + if( + ITK_WRAP_CASTXML_CACHE + AND + Python3_EXECUTABLE + AND + ITK_WRAP_CASTXML_CACHE_SCRIPT + ) + set( + _castxml_cmd + ${Python3_EXECUTABLE} + "${ITK_WRAP_CASTXML_CACHE_SCRIPT}" + ${CASTXML_EXECUTABLE} + ) + list(APPEND _castxml_depends "${ITK_WRAP_CASTXML_CACHE_SCRIPT}") + else() + set( + _castxml_cmd + ${_ccache_cmd} + ${CASTXML_EXECUTABLE} + ) + endif() add_custom_command( OUTPUT ${xml_file} COMMAND - ${_build_env} ${_ccache_cmd} ${CASTXML_EXECUTABLE} -o ${xml_file} - --castxml-output=1 ${_target} --castxml-start _wrapping_ ${_castxml_cc} -w + ${_build_env} ${_castxml_cmd} -o ${xml_file} --castxml-output=1 ${_target} + --castxml-start _wrapping_ ${_castxml_cc} -w -c # needed for ccache to think we are not calling for link @${castxml_inc_file} ${cxx_file} VERBATIM @@ -214,6 +235,7 @@ function(generate_castxml_commandline_flags) ${castxml_inc_file} ${_hdrs} ) + unset(_castxml_cmd) unset(cxx_file) unset(castxml_inc_file) unset(_build_env) diff --git a/pixi.lock b/pixi.lock index d476f4e4cce..0bea1db9e08 100644 --- a/pixi.lock +++ b/pixi.lock @@ -1592,6 +1592,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/linux-64/c-ares-1.34.5-hb9d3cd8_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/c-compiler-1.11.0-h4d9bdce_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/castxml-0.7.0-hde8d07d_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/ccache-4.13.6-hedf47ba_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/cmake-4.1.2-hc85cc9f_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/conda-gcc-specs-14.3.0-hb991d5c_7.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/cxx-compiler-1.11.0-hfcd1e18_0.conda @@ -1618,6 +1619,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/linux-64/libgfortran-15.2.0-h69a702a_7.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/libgfortran5-15.2.0-hcd61629_7.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/libgomp-15.2.0-h767d61c_7.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/libhiredis-1.3.0-h5888daf_1.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/libiconv-1.18-h3b78370_2.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/liblapack-3.9.0-38_h47877c9_openblas.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/libllvm20-20.1.8-hf7376ad_1.conda @@ -1645,6 +1647,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/linux-64/rhash-1.4.6-hb9d3cd8_1.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/swig-4.4.1-h7a96c5f_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/tk-8.6.13-noxft_hd72426e_102.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/xxhash-0.8.3-hb47aa4a_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/zstd-1.5.7-hb8e6e7a_2.conda - conda: https://conda.anaconda.org/conda-forge/noarch/ca-certificates-2025.10.5-hbd8a1cb_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/colorama-0.4.6-pyhd8ed1ab_1.conda @@ -1671,6 +1674,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/c-ares-1.34.5-h86ecc28_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/c-compiler-1.11.0-hdceaead_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/castxml-0.7.0-ha3e84ed_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/ccache-4.13.6-h185addb_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/cmake-4.1.2-hc9d863e_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/conda-gcc-specs-14.3.0-h92dcf8a_7.conda - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/cxx-compiler-1.11.0-h7b35c40_0.conda @@ -1697,6 +1701,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/libgfortran-15.2.0-he9431aa_7.conda - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/libgfortran5-15.2.0-h87db57e_7.conda - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/libgomp-15.2.0-he277a41_7.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/libhiredis-1.3.0-h5ad3122_1.conda - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/libiconv-1.18-h90929bb_2.conda - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/liblapack-3.9.0-38_h88aeb00_openblas.conda - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/libllvm20-20.1.8-hfd2ba90_1.conda @@ -1724,6 +1729,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/rhash-1.4.6-h86ecc28_1.conda - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/swig-4.4.1-h512d76c_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/tk-8.6.13-noxft_h5688188_102.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/xxhash-0.8.3-hd794028_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/zstd-1.5.7-hbcf94c1_2.conda - conda: https://conda.anaconda.org/conda-forge/noarch/ca-certificates-2025.10.5-hbd8a1cb_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/colorama-0.4.6-pyhd8ed1ab_1.conda @@ -1759,6 +1765,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/osx-64/c-ares-1.34.5-hf13058a_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-64/c-compiler-1.11.0-h7a00415_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-64/castxml-0.7.0-hb171174_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-64/ccache-4.13.6-h894318c_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-64/cctools-1024.3-h67a6458_9.conda - conda: https://conda.anaconda.org/conda-forge/osx-64/cctools_osx-64-1024.3-llvm19_1_h3b512aa_9.conda - conda: https://conda.anaconda.org/conda-forge/osx-64/clang-19-19.1.7-default_hc369343_5.conda @@ -1787,6 +1794,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/osx-64/libffi-3.5.2-h750e83c_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-64/libgfortran-15.2.0-h306097a_1.conda - conda: https://conda.anaconda.org/conda-forge/osx-64/libgfortran5-15.2.0-h336fb69_1.conda + - conda: https://conda.anaconda.org/conda-forge/osx-64/libhiredis-1.3.0-h240833e_1.conda - conda: https://conda.anaconda.org/conda-forge/osx-64/libiconv-1.18-h57a12c2_2.conda - conda: https://conda.anaconda.org/conda-forge/osx-64/liblapack-3.9.0-38_h859234e_openblas.conda - conda: https://conda.anaconda.org/conda-forge/osx-64/libllvm19-19.1.7-h56e7563_2.conda @@ -1816,6 +1824,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/osx-64/swig-4.4.1-hdac4ec2_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-64/tapi-1300.6.5-h390ca13_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-64/tk-8.6.13-hf689a15_2.conda + - conda: https://conda.anaconda.org/conda-forge/osx-64/xxhash-0.8.3-h13e91ac_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-64/zstd-1.5.7-h8210216_2.conda osx-arm64: - conda: https://conda.anaconda.org/conda-forge/noarch/ca-certificates-2025.10.5-hbd8a1cb_0.conda @@ -1835,6 +1844,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/osx-arm64/c-ares-1.34.5-h5505292_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/c-compiler-1.11.0-h61f9b84_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/castxml-0.7.0-hfc9ce51_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/ccache-4.13.6-h414bf82_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/cctools-1024.3-hd01ab73_9.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/cctools_osx-arm64-1024.3-llvm19_1_h8c76c84_9.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/clang-19-19.1.7-default_h73dfc95_5.conda @@ -1864,6 +1874,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libffi-3.5.2-he5f378a_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libgfortran-15.2.0-hfcf01ff_1.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libgfortran5-15.2.0-h742603c_1.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libhiredis-1.3.0-h286801f_1.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libiconv-1.18-h23cfdf5_2.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/liblapack-3.9.0-38_hd9741b5_openblas.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libllvm19-19.1.7-h8e0c9ce_2.conda @@ -1893,6 +1904,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/osx-arm64/swig-4.4.1-h4366dc5_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/tapi-1300.6.5-h03f4b80_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/tk-8.6.13-h892fb3f_2.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/xxhash-0.8.3-haa4e116_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/zstd-1.5.7-h6491c7d_2.conda win-64: - conda: https://conda.anaconda.org/conda-forge/noarch/ca-certificates-2025.10.5-h4c7d964_0.conda @@ -1911,6 +1923,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/win-64/_openmp_mutex-4.5-2_gnu.conda - conda: https://conda.anaconda.org/conda-forge/win-64/bzip2-1.0.8-h0ad9c76_8.conda - conda: https://conda.anaconda.org/conda-forge/win-64/castxml-0.7.0-ha22e26b_0.conda + - conda: https://conda.anaconda.org/conda-forge/win-64/ccache-4.13.6-h7fd822b_0.conda - conda: https://conda.anaconda.org/conda-forge/win-64/cmake-4.1.2-hdcbee5b_0.conda - conda: https://conda.anaconda.org/conda-forge/win-64/cxx-compiler-1.11.0-h1c1089f_0.conda - conda: https://conda.anaconda.org/conda-forge/win-64/krb5-1.21.3-hdf4eb48_0.conda @@ -1922,6 +1935,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/win-64/libgcc-15.2.0-h1383e82_7.conda - conda: https://conda.anaconda.org/conda-forge/win-64/libgfortran5-15.2.0-hf2bee02_7.conda - conda: https://conda.anaconda.org/conda-forge/win-64/libgomp-15.2.0-h1383e82_7.conda + - conda: https://conda.anaconda.org/conda-forge/win-64/libhiredis-1.3.0-he0c23c2_1.conda - conda: https://conda.anaconda.org/conda-forge/win-64/libhwloc-2.12.1-default_h64bd3f2_1002.conda - conda: https://conda.anaconda.org/conda-forge/win-64/libiconv-1.18-hc1393d2_2.conda - conda: https://conda.anaconda.org/conda-forge/win-64/liblapack-3.9.0-38_hf9ab0e9_mkl.conda @@ -1950,6 +1964,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/win-64/vc14_runtime-14.44.35208-h818238b_32.conda - conda: https://conda.anaconda.org/conda-forge/win-64/vcomp14-14.44.35208-h818238b_32.conda - conda: https://conda.anaconda.org/conda-forge/win-64/vs2022_win-64-19.44.35207-ha74f236_32.conda + - conda: https://conda.anaconda.org/conda-forge/win-64/xxhash-0.8.3-hbba6f48_0.conda - conda: https://conda.anaconda.org/conda-forge/win-64/zstd-1.5.7-hbeecb71_2.conda packages: - conda: https://conda.anaconda.org/conda-forge/linux-64/_libgcc_mutex-0.1-conda_forge.tar.bz2 @@ -2211,6 +2226,7 @@ packages: - zstd >=1.5.7,<1.6.0a0 license: GPL-3.0-only license_family: GPL + run_exports: {} size: 855698 timestamp: 1777926446042 - conda: https://conda.anaconda.org/conda-forge/linux-64/cffi-2.0.0-py313hf46b229_1.conda @@ -3206,6 +3222,9 @@ packages: - libstdcxx >=13 license: BSD-3-Clause license_family: BSD + run_exports: + weak: + - libhiredis >=1.3.0,<1.4.0a0 size: 140759 timestamp: 1748219397797 - conda: https://conda.anaconda.org/conda-forge/linux-64/libiconv-1.18-h3b78370_2.conda @@ -4283,6 +4302,9 @@ packages: - libgcc >=13 license: BSD-2-Clause license_family: BSD + run_exports: + weak: + - xxhash >=0.8.3,<0.8.4.0a0 size: 108219 timestamp: 1746457673761 - conda: https://conda.anaconda.org/conda-forge/linux-64/yaml-0.2.5-h280c20c_3.conda @@ -4560,6 +4582,7 @@ packages: - zstd >=1.5.7,<1.6.0a0 license: GPL-3.0-only license_family: GPL + run_exports: {} size: 843745 timestamp: 1777926452238 - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/cffi-2.0.0-py313h897158f_1.conda @@ -5499,6 +5522,9 @@ packages: - libstdcxx >=13 license: BSD-3-Clause license_family: BSD + run_exports: + weak: + - libhiredis >=1.3.0,<1.4.0a0 size: 140290 timestamp: 1748220539026 - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/libiconv-1.18-h90929bb_2.conda @@ -6499,6 +6525,9 @@ packages: - libgcc >=13 license: BSD-2-Clause license_family: BSD + run_exports: + weak: + - xxhash >=0.8.3,<0.8.4.0a0 size: 105762 timestamp: 1746457675564 - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/yaml-0.2.5-h80f16a2_3.conda @@ -7376,6 +7405,7 @@ packages: - libhiredis >=1.3.0,<1.4.0a0 license: GPL-3.0-only license_family: GPL + run_exports: {} size: 657026 timestamp: 1777926755291 - conda: https://conda.anaconda.org/conda-forge/osx-64/cctools-1024.3-h67a6458_9.conda @@ -8299,6 +8329,9 @@ packages: - libcxx >=18 license: BSD-3-Clause license_family: BSD + run_exports: + weak: + - libhiredis >=1.3.0,<1.4.0a0 size: 59830 timestamp: 1748219625377 - conda: https://conda.anaconda.org/conda-forge/osx-64/libiconv-1.18-h57a12c2_2.conda @@ -9048,6 +9081,9 @@ packages: - __osx >=10.13 license: BSD-2-Clause license_family: BSD + run_exports: + weak: + - xxhash >=0.8.3,<0.8.4.0a0 size: 108449 timestamp: 1746457796808 - conda: https://conda.anaconda.org/conda-forge/osx-64/yaml-0.2.5-h4132b18_3.conda @@ -9210,6 +9246,7 @@ packages: - libhiredis >=1.3.0,<1.4.0a0 license: GPL-3.0-only license_family: GPL + run_exports: {} size: 601285 timestamp: 1777926636412 - conda: https://conda.anaconda.org/conda-forge/osx-arm64/cctools-1024.3-hd01ab73_9.conda @@ -10143,6 +10180,9 @@ packages: - libcxx >=18 license: BSD-3-Clause license_family: BSD + run_exports: + weak: + - libhiredis >=1.3.0,<1.4.0a0 size: 56746 timestamp: 1748219528586 - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libiconv-1.18-h23cfdf5_2.conda @@ -10893,6 +10933,9 @@ packages: - __osx >=11.0 license: BSD-2-Clause license_family: BSD + run_exports: + weak: + - xxhash >=0.8.3,<0.8.4.0a0 size: 98913 timestamp: 1746457827085 - conda: https://conda.anaconda.org/conda-forge/osx-arm64/yaml-0.2.5-h925e9cb_3.conda @@ -11048,6 +11091,7 @@ packages: - xxhash >=0.8.3,<0.8.4.0a0 license: GPL-3.0-only license_family: GPL + run_exports: {} size: 692199 timestamp: 1777926529520 - conda: https://conda.anaconda.org/conda-forge/win-64/cffi-2.0.0-py313h5ea7bf4_1.conda @@ -11530,6 +11574,9 @@ packages: - vc14_runtime >=14.29.30139 license: BSD-3-Clause license_family: BSD + run_exports: + weak: + - libhiredis >=1.3.0,<1.4.0a0 size: 64205 timestamp: 1748219812303 - conda: https://conda.anaconda.org/conda-forge/win-64/libhwloc-2.12.1-default_h64bd3f2_1002.conda @@ -12394,6 +12441,9 @@ packages: - vc14_runtime >=14.29.30139 license: BSD-2-Clause license_family: BSD + run_exports: + weak: + - xxhash >=0.8.3,<0.8.4.0a0 size: 105768 timestamp: 1746458183583 - conda: https://conda.anaconda.org/conda-forge/win-64/yaml-0.2.5-h6a83c73_3.conda diff --git a/pyproject.toml b/pyproject.toml index 2fe4d3b2644..70b13a55ce0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -39,6 +39,7 @@ libopenblas = ">=0.3.30,<0.4" libgfortran5 = ">=15.2.0,<16" swig = ">=4.4.1" castxml = ">=0.7.0,<0.8" +ccache = ">=4.13.6,<5" [tool.pixi.feature.cxx.tasks.configure] cmd = '''cmake \ @@ -236,10 +237,37 @@ cmd = '''cmake \ -DITK_USE_SYSTEM_SWIG:BOOL=ON \ -DITK_USE_SYSTEM_CASTXML:BOOL=ON \ -DCMAKE_BUILD_TYPE:STRING=Release \ - -DBUILD_TESTING:BOOL=ON''' + -DBUILD_TESTING:BOOL=ON \ + -DCMAKE_C_COMPILER_LAUNCHER:STRING=ccache \ + -DCMAKE_CXX_COMPILER_LAUNCHER:STRING=ccache''' description = "Configure ITK Python" outputs = ["build-python/CMakeFiles/"] +[tool.pixi.feature.python.tasks.configure-python-ci] +cmd = '''cmake \ + -Bbuild-python \ + -S. \ + -GNinja \ + -DITK_WRAP_PYTHON:BOOL=ON \ + -DITK_WRAP_CASTXML_CACHE:BOOL=ON \ + -DCMAKE_BUILD_TYPE:STRING=Release \ + -DBUILD_TESTING:BOOL=ON \ + -DCMAKE_C_COMPILER_LAUNCHER:STRING=ccache \ + -DCMAKE_CXX_COMPILER_LAUNCHER:STRING=ccache''' +description = "Configure ITK Python for CI (castxml cache enabled; set ITK_WRAP_CACHE before building)" +outputs = ["build-python/CMakeFiles/"] + +[tool.pixi.feature.python.tasks.build-python-ci] +cmd = "cmake --build build-python" +description = "Build ITK Python for CI" +outputs = ["build-python/lib/**"] +depends-on = ["configure-python-ci"] + +[tool.pixi.feature.python.tasks.test-python-ci] +cmd = "ctest -j3 --test-dir build-python --output-on-failure" +description = "Test ITK Python for CI" +depends-on = ["build-python-ci"] + [tool.pixi.feature.python.tasks.build-python] cmd = "cmake --build build-python" description = "Build ITK Python" From f5026e9333e9fdf94ca62ab4075807df11bd3249 Mon Sep 17 00:00:00 2001 From: Hans Johnson Date: Tue, 30 Jun 2026 07:41:37 -0500 Subject: [PATCH 5/6] STYLE: Explicitly set ITK_WRAP_CASTXML_CACHE=ON in all Python CI configs Relies on CMake default (ON) elsewhere; explicit here avoids silent no-op if the option is ever overridden upstream in the dashboard script. --- .github/workflows/arm.yml | 1 + Testing/ContinuousIntegration/AzurePipelinesLinuxPython.yml | 1 + Testing/ContinuousIntegration/AzurePipelinesMacOSPython.yml | 1 + Testing/ContinuousIntegration/AzurePipelinesWindowsPython.yml | 1 + 4 files changed, 4 insertions(+) diff --git a/.github/workflows/arm.yml b/.github/workflows/arm.yml index 6c0d918d418..46a28b95647 100644 --- a/.github/workflows/arm.yml +++ b/.github/workflows/arm.yml @@ -80,6 +80,7 @@ jobs: BUILD_SHARED_LIBS:BOOL=OFF BUILD_EXAMPLES:BOOL=OFF ITK_WRAP_PYTHON:BOOL=ON + ITK_WRAP_CASTXML_CACHE:BOOL=ON ITK_USE_CLANG_FORMAT:BOOL=OFF ITK_COMPUTER_MEMORY_SIZE:STRING=11 ctest-options: "-E itkPyBufferMemoryLeakTest" diff --git a/Testing/ContinuousIntegration/AzurePipelinesLinuxPython.yml b/Testing/ContinuousIntegration/AzurePipelinesLinuxPython.yml index b4d162752f8..ed2e182fe58 100644 --- a/Testing/ContinuousIntegration/AzurePipelinesLinuxPython.yml +++ b/Testing/ContinuousIntegration/AzurePipelinesLinuxPython.yml @@ -133,6 +133,7 @@ jobs: BUILD_SHARED_LIBS:BOOL=OFF BUILD_EXAMPLES:BOOL=OFF ITK_WRAP_PYTHON:BOOL=ON + ITK_WRAP_CASTXML_CACHE:BOOL=ON CMAKE_C_COMPILER_LAUNCHER:STRING=ccache CMAKE_CXX_COMPILER_LAUNCHER:STRING=ccache ITK_COMPUTER_MEMORY_SIZE:STRING=4.5 diff --git a/Testing/ContinuousIntegration/AzurePipelinesMacOSPython.yml b/Testing/ContinuousIntegration/AzurePipelinesMacOSPython.yml index ee84c981241..5429d5f51bd 100644 --- a/Testing/ContinuousIntegration/AzurePipelinesMacOSPython.yml +++ b/Testing/ContinuousIntegration/AzurePipelinesMacOSPython.yml @@ -136,6 +136,7 @@ jobs: BUILD_SHARED_LIBS:BOOL=ON BUILD_EXAMPLES:BOOL=OFF ITK_WRAP_PYTHON:BOOL=ON + ITK_WRAP_CASTXML_CACHE:BOOL=ON CMAKE_C_COMPILER_LAUNCHER:STRING=ccache CMAKE_CXX_COMPILER_LAUNCHER:STRING=ccache ITK_COMPUTER_MEMORY_SIZE:STRING=11 diff --git a/Testing/ContinuousIntegration/AzurePipelinesWindowsPython.yml b/Testing/ContinuousIntegration/AzurePipelinesWindowsPython.yml index c021e87db2e..822ccb75d93 100644 --- a/Testing/ContinuousIntegration/AzurePipelinesWindowsPython.yml +++ b/Testing/ContinuousIntegration/AzurePipelinesWindowsPython.yml @@ -113,6 +113,7 @@ jobs: BUILD_SHARED_LIBS:BOOL=ON BUILD_EXAMPLES:BOOL=OFF ITK_WRAP_PYTHON:BOOL=ON + ITK_WRAP_CASTXML_CACHE:BOOL=ON ITK_BUILD_DEFAULT_MODULES:BOOL=OFF ITKGroup_Core:BOOL=ON Module_ITKThresholding:BOOL=ON From 54767e5523f29c13c36cd64b59b39c72ba209e01 Mon Sep 17 00:00:00 2001 From: Hans Johnson Date: Thu, 2 Jul 2026 12:20:49 -0500 Subject: [PATCH 6/6] ENH: Add validation suite for wrapping cache infrastructure Pytest suite for the CastXML two-level cache, SQLite pkl database, and pyi stub pipeline: key/manifest unit behavior, staleness (header edits, shadowing headers, castxml upgrades), corruption recovery, eviction, multi-root cascade, parallel-writer and racing-store concurrency, prune gating for external wrap projects, DB self-heal, and cache relocation matching CI cache-task transport. Runs in ~7s with no compiler or ITK build. Registered as the WrappingInfrastructure CTest when pytest is available in the build interpreter (pytest added to the Python CI package installs); also runs standalone. Verified on Linux, macOS, and Windows (Python 3.11 and 3.13). --- .github/workflows/arm.yml | 1 + .../AzurePipelinesLinuxPython.yml | 2 +- .../AzurePipelinesMacOSPython.yml | 2 +- .../AzurePipelinesWindowsPython.yml | 2 +- Wrapping/CMakeLists.txt | 34 ++++ Wrapping/Generators/Tests/README.md | 35 ++++ Wrapping/Generators/Tests/conftest.py | 134 ++++++++++++++ Wrapping/Generators/Tests/fake_castxml.py | 44 +++++ Wrapping/Generators/Tests/smoke_realbuild.sh | 34 ++++ .../Tests/tests/test_concurrency.py | 128 +++++++++++++ .../Tests/tests/test_integration.py | 173 ++++++++++++++++++ .../Tests/tests/test_pyi_generator.py | 137 ++++++++++++++ .../Generators/Tests/tests/test_relocate.py | 42 +++++ .../Generators/Tests/tests/test_unit_cache.py | 162 ++++++++++++++++ .../Tests/tests/test_unit_pkl_db.py | 58 ++++++ 15 files changed, 985 insertions(+), 3 deletions(-) create mode 100644 Wrapping/Generators/Tests/README.md create mode 100644 Wrapping/Generators/Tests/conftest.py create mode 100644 Wrapping/Generators/Tests/fake_castxml.py create mode 100755 Wrapping/Generators/Tests/smoke_realbuild.sh create mode 100644 Wrapping/Generators/Tests/tests/test_concurrency.py create mode 100644 Wrapping/Generators/Tests/tests/test_integration.py create mode 100644 Wrapping/Generators/Tests/tests/test_pyi_generator.py create mode 100644 Wrapping/Generators/Tests/tests/test_relocate.py create mode 100644 Wrapping/Generators/Tests/tests/test_unit_cache.py create mode 100644 Wrapping/Generators/Tests/tests/test_unit_pkl_db.py diff --git a/.github/workflows/arm.yml b/.github/workflows/arm.yml index 46a28b95647..e461edd8f56 100644 --- a/.github/workflows/arm.yml +++ b/.github/workflows/arm.yml @@ -161,6 +161,7 @@ jobs: run: | set -x python3 -m pip install --upgrade --pre numpy + python3 -m pip install pytest - name: Get specific version of CMake, latest Ninja uses: lukka/get-cmake@v4.0.1 diff --git a/Testing/ContinuousIntegration/AzurePipelinesLinuxPython.yml b/Testing/ContinuousIntegration/AzurePipelinesLinuxPython.yml index ed2e182fe58..fd0b9263aa0 100644 --- a/Testing/ContinuousIntegration/AzurePipelinesLinuxPython.yml +++ b/Testing/ContinuousIntegration/AzurePipelinesLinuxPython.yml @@ -77,7 +77,7 @@ jobs: sudo apt-get update sudo apt-get install -y ccache locales sudo locale-gen de_DE.UTF-8 - python3 -m pip install ninja typing-extensions + python3 -m pip install ninja typing-extensions pytest python3 -m pip install --upgrade --pre numpy python3 -m pip install lxml scikit-ci-addons dask distributed displayName: 'Install dependencies' diff --git a/Testing/ContinuousIntegration/AzurePipelinesMacOSPython.yml b/Testing/ContinuousIntegration/AzurePipelinesMacOSPython.yml index 5429d5f51bd..e6c9da98d0d 100644 --- a/Testing/ContinuousIntegration/AzurePipelinesMacOSPython.yml +++ b/Testing/ContinuousIntegration/AzurePipelinesMacOSPython.yml @@ -81,7 +81,7 @@ jobs: sudo pip3 install ninja numpy>=1.20 typing-extensions sudo python3 -m pip install --upgrade setuptools sudo python3 -m pip install scikit-ci-addons - sudo python3 -m pip install lxml dask distributed + sudo python3 -m pip install lxml dask distributed pytest brew install ccache displayName: 'Install dependencies' diff --git a/Testing/ContinuousIntegration/AzurePipelinesWindowsPython.yml b/Testing/ContinuousIntegration/AzurePipelinesWindowsPython.yml index 822ccb75d93..e4d1da9f0b4 100644 --- a/Testing/ContinuousIntegration/AzurePipelinesWindowsPython.yml +++ b/Testing/ContinuousIntegration/AzurePipelinesWindowsPython.yml @@ -58,7 +58,7 @@ jobs: architecture: 'x64' - script: | - python -m pip install ninja numpy>=1.20 typing-extensions + python -m pip install ninja numpy>=1.20 typing-extensions pytest python -m pip install --upgrade setuptools python -m pip install scikit-ci-addons dask distributed lxml choco install ccache -y diff --git a/Wrapping/CMakeLists.txt b/Wrapping/CMakeLists.txt index a7a9defb416..306eea179c4 100644 --- a/Wrapping/CMakeLists.txt +++ b/Wrapping/CMakeLists.txt @@ -293,4 +293,38 @@ if(ITK_WRAP_PYTHON) unset(ITK_STUB_DIR CACHE) unset(ITK_PKL_DIR CACHE) + + # Wrapping infrastructure suite (Generators/Tests): registered only when + # pytest is available in the build interpreter; needs no built ITK. + if(BUILD_TESTING AND NOT EXTERNAL_WRAP_ITK_PROJECT) + execute_process( + COMMAND + ${Python3_EXECUTABLE} -m pytest --version + RESULT_VARIABLE _pytest_missing + OUTPUT_QUIET + ERROR_QUIET + ) + if(_pytest_missing) + message( + STATUS + "pytest not found for ${Python3_EXECUTABLE}: WrappingInfrastructure test not registered" + ) + else() + add_test( + NAME WrappingInfrastructure + COMMAND + ${Python3_EXECUTABLE} -m pytest -q -p no:cacheprovider + ${CMAKE_CURRENT_SOURCE_DIR}/Generators/Tests + ) + set_tests_properties( + WrappingInfrastructure + PROPERTIES + TIMEOUT + 300 + LABELS + Python + ) + endif() + unset(_pytest_missing) + endif() endif() diff --git a/Wrapping/Generators/Tests/README.md b/Wrapping/Generators/Tests/README.md new file mode 100644 index 00000000000..849ef3595c7 --- /dev/null +++ b/Wrapping/Generators/Tests/README.md @@ -0,0 +1,35 @@ +# Wrapping infrastructure tests + +Validation suite for the CastXML two-level cache +(`../CastXML/itk-castxml-cache.py`), the SQLite pkl database +(`../pkl_db.py`), and the pyi stub pipeline +(`../Python/itk/pyi_generator.py`). + +Registered as the `WrappingInfrastructure` CTest when wrapping and +`BUILD_TESTING` are enabled and `pytest` is available in the build +interpreter (skipped otherwise). Also runs standalone with only +Python >= 3.11 and `pytest` — no compiler, CMake, or ITK build: + +```bash +python -m pytest Wrapping/Generators/Tests # from the ITK source root +ctest -R WrappingInfrastructure # from a configured build tree +``` + +Runs in ~7 s. `ITK_SOURCE_DIR` points the suite at a different checkout +(default: the enclosing tree). The fake castxml (`fake_castxml.py`) logs +every invocation so tests assert exact subprocess counts: cold = 2 +(`-E` + full), L2-hit = 1, L1-hit = 0. + +| File | Covers | +|---|---| +| `tests/test_unit_cache.py` | key/manifest/parse functions, Windows path unescaping, include-dir fingerprints | +| `tests/test_unit_pkl_db.py` | schema, WAL (+ NFS fallback), upsert, reader self-containment | +| `tests/test_integration.py` | cold/warm/staleness/corruption/eviction/bypass/multi-root | +| `tests/test_pyi_generator.py` | `--prune` gating (external-project safety), DB self-heal, guards | +| `tests/test_concurrency.py` | 32 parallel pkl writers, keyset-DELETE, racing same-key stores | +| `tests/test_relocate.py` | cache relocation (models `actions/cache` / Azure `Cache@2` transport) | +| `smoke_realbuild.sh` | optional real-castxml double build (Linux, needs a configured tree) | + +A cross-platform mirror with a {Linux, macOS, Windows} × {3.11, 3.13} +GitHub Actions matrix lives at +. diff --git a/Wrapping/Generators/Tests/conftest.py b/Wrapping/Generators/Tests/conftest.py new file mode 100644 index 00000000000..a62d5bc579f --- /dev/null +++ b/Wrapping/Generators/Tests/conftest.py @@ -0,0 +1,134 @@ +"""Shared fixtures: load ITK wrapping scripts by path, build a fake castxml.""" + +import importlib.util +import os +import subprocess +import sys +from pathlib import Path + +import pytest + +# Default to the enclosing ITK source tree; ITK_SOURCE_DIR overrides to +# point the suite at a different checkout. +ITK_SOURCE_DIR = Path( + os.environ.get("ITK_SOURCE_DIR", Path(__file__).resolve().parents[3]) +) +CACHE_SCRIPT = ITK_SOURCE_DIR / "Wrapping/Generators/CastXML/itk-castxml-cache.py" +PKL_DB = ITK_SOURCE_DIR / "Wrapping/Generators/pkl_db.py" +PYI_GENERATOR = ITK_SOURCE_DIR / "Wrapping/Generators/Python/itk/pyi_generator.py" + +FAKE_CASTXML = Path(__file__).parent / "fake_castxml.py" + + +def _load(name, path): + spec = importlib.util.spec_from_file_location(name, path) + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) + return mod + + +@pytest.fixture(scope="session") +def cache_mod(): + return _load("itk_castxml_cache", CACHE_SCRIPT) + + +@pytest.fixture(scope="session") +def pkl_db_mod(): + return _load("pkl_db", PKL_DB) + + +class FakeCastxml: + """A castxml stand-in whose invocations are observable. + + -E mode emits preprocessed output with `# N "path"` line markers naming + the header fixture files; full mode writes XML derived from the header + contents and FAKE_CASTXML_VERSION, so output provably tracks the inputs. + """ + + def __init__(self, tmp_path, monkeypatch): + self.dir = tmp_path + self.log = tmp_path / "invocations.log" + self.cache = tmp_path / "cache" + self.cache.mkdir() + self.incdir_a = tmp_path / "incdirA" + self.incdir_b = tmp_path / "incdirB" + self.incdir_a.mkdir() + self.incdir_b.mkdir() + self.build = tmp_path / "build" + self.build.mkdir() + + (self.incdir_a / "foo.h").write_text("content-A v1\n") + (self.incdir_b / "bar.h").write_text("content-B v1\n") + self.inc_file = self.build / "module.inc" + self.inc_file.write_text(f'"-I{self.incdir_a}"\n"-I{self.incdir_b}"\n') + self.cxx_file = self.build / "module.cxx" + self.cxx_file.write_text("// module source\n") + self.out_xml = self.build / "out.xml" + + if sys.platform == "win32": + self.bin = tmp_path / "castxml.cmd" + self.bin.write_text( + f'@echo off\r\n"{sys.executable}" "{FAKE_CASTXML}" %*\r\n' + ) + else: + self.bin = tmp_path / "castxml" + self.bin.write_text( + f'#!/usr/bin/env bash\nexec "{sys.executable}" "{FAKE_CASTXML}" "$@"\n' + ) + self.bin.chmod(0o755) + + monkeypatch.setenv("ITK_WRAP_CACHE", str(self.cache)) + monkeypatch.setenv("FAKE_LOG", str(self.log)) + monkeypatch.setenv( + "FAKE_HDRS", + os.pathsep.join( + [str(self.incdir_a / "foo.h"), str(self.incdir_b / "bar.h")] + ), + ) + monkeypatch.delenv("ITK_WRAP_CACHE_FORMAT", raising=False) + monkeypatch.delenv("ITK_WRAP_CACHE_BYPASS", raising=False) + + def invoke(self, extra_env=None, no_cache=False): + cmd = [sys.executable, str(CACHE_SCRIPT)] + if no_cache: + cmd.append("--no-cache") + cmd += [ + str(self.bin), + "--castxml-cc-gnu", + "g++", + f"@{self.inc_file}", + "-o", + str(self.out_xml), + str(self.cxx_file), + ] + env = dict(os.environ) + if extra_env: + env.update(extra_env) + return subprocess.run(cmd, env=env, capture_output=True, text=True) + + def invocations(self): + if not self.log.exists(): + return [] + return self.log.read_text().split() + + def reset_log(self): + self.log.write_text("") + + def evict(self, days="13.9"): + return subprocess.run( + [ + sys.executable, + str(CACHE_SCRIPT), + "--evict", + days, + "--cache-dir", + str(self.cache), + ], + capture_output=True, + text=True, + ) + + +@pytest.fixture +def fake(tmp_path, monkeypatch): + return FakeCastxml(tmp_path, monkeypatch) diff --git a/Wrapping/Generators/Tests/fake_castxml.py b/Wrapping/Generators/Tests/fake_castxml.py new file mode 100644 index 00000000000..2f774a50cd4 --- /dev/null +++ b/Wrapping/Generators/Tests/fake_castxml.py @@ -0,0 +1,44 @@ +"""Instrumented castxml stand-in driven by env vars (see conftest.FakeCastxml).""" + +import os +import sys + + +def main(): + argv = sys.argv[1:] + with open(os.environ["FAKE_LOG"], "a") as log: + log.write(("PRE" if "-E" in argv else "FULL") + "\n") + + if os.environ.get("FAKE_FAIL") == "1": + return 1 + + out = None + for i, a in enumerate(argv): + if a == "-o" and i + 1 < len(argv): + out = argv[i + 1] + + hdrs = [p for p in os.environ.get("FAKE_HDRS", "").split(os.pathsep) if p] + ver = os.environ.get("FAKE_CASTXML_VERSION", "1") + + if "-E" in argv: + with open(out, "wb") as f: + marker_root = os.environ.get("FAKE_MARKER_ROOT", "") + f.write(b'# 1 "' + (marker_root + "/module.cxx").encode() + b'"\n') + body = b"" + for h in hdrs: + f.write(f'# 1 "{h}"\n'.encode()) + with open(h, "rb") as hf: + body += hf.read() + f.write(body) + f.write(b"int payload;\n") + else: + with open(out, "wb") as f: + content = b"".join(open(h, "rb").read() for h in hdrs) + f.write(b"\n") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/Wrapping/Generators/Tests/smoke_realbuild.sh b/Wrapping/Generators/Tests/smoke_realbuild.sh new file mode 100755 index 00000000000..e4d092eac17 --- /dev/null +++ b/Wrapping/Generators/Tests/smoke_realbuild.sh @@ -0,0 +1,34 @@ +#!/usr/bin/env bash +# L3 smoke: two fresh ITK builds sharing one castxml cache; asserts the second +# issues zero real castxml runs. Linux only; expects a configured pixi ITK +# checkout. Run on demand — this is a ~2x full-build operation. +# +# Usage: ITK_SOURCE_DIR=/path/to/ITK ./smoke_realbuild.sh +set -euo pipefail +ITK=${ITK_SOURCE_DIR:?set ITK_SOURCE_DIR} +CACHE=$(mktemp -d)/wrap-cache +export ITK_WRAP_CACHE=$CACHE + +cd "$ITK" +for round in cold warm; do + rm -rf build-python + start=$(date +%s) + pixi run build-python-ci + echo "$round: $(($(date +%s) - start))s" +done + +entries=$(find "$CACHE/l2" -name _ok | wc -l) +echo "cache entries: $entries" +python3 - "$ITK/build-python/.ninja_log" <<'EOF' +import sys +rows = [ + int(p[1]) - int(p[0]) + for line in open(sys.argv[1]) + if (p := line.split("\t")) and len(p) >= 4 + and p[3].endswith(".xml") and "castxml_inputs" in p[3] +] +mean = sum(rows) / len(rows) +print(f"warm castxml steps: {len(rows)}, mean {mean:.0f}ms") +assert mean < 1000, "warm castxml steps too slow — cache hits not happening" +EOF +echo "SMOKE PASS" diff --git a/Wrapping/Generators/Tests/tests/test_concurrency.py b/Wrapping/Generators/Tests/tests/test_concurrency.py new file mode 100644 index 00000000000..ca01fed9249 --- /dev/null +++ b/Wrapping/Generators/Tests/tests/test_concurrency.py @@ -0,0 +1,128 @@ +"""L2: multiprocess stress — parallel pkl writers and racing castxml stores.""" + +import multiprocessing +import sqlite3 +import subprocess +import sys + +from conftest import PKL_DB + +UPSERT = ( + "INSERT INTO pkl_data(key, data) VALUES(?,?)" + " ON CONFLICT(key) DO UPDATE SET data=excluded.data" +) + +WRITER = f""" +import sys +sys.path.insert(0, {str(PKL_DB.parent)!r}) +from pkl_db import open_pkl_db +wid, pkl_dir = int(sys.argv[1]), sys.argv[2] +conn = open_pkl_db(pkl_dir) +with conn: + for i in range(50): + conn.execute({UPSERT!r}, (f"shared-{{i}}", b"w%d" % wid)) + conn.execute({UPSERT!r}, (f"w{{wid}}-{{i}}", b"x")) +conn.close() +""" + + +def test_parallel_writers_no_lock_errors(pkl_db_mod, tmp_path): + n = 32 + procs = [ + subprocess.Popen( + [sys.executable, "-c", WRITER, str(i), str(tmp_path)], + stderr=subprocess.PIPE, + ) + for i in range(n) + ] + errs = [p.communicate()[1] for p in procs] + assert all(p.returncode == 0 for p in procs), b"\n".join(errs)[:2000] + conn = sqlite3.connect(pkl_db_mod._pkl_db_path(str(tmp_path))) + count = conn.execute("SELECT count(*) FROM pkl_data").fetchone()[0] + assert count == 50 + n * 50 # 50 shared + 50 per writer + conn.close() + + +def test_keyset_delete_against_concurrent_upserts(pkl_db_mod, tmp_path): + conn = pkl_db_mod.open_pkl_db(str(tmp_path)) + with conn: + for i in range(200): + conn.execute(UPSERT, (f"live-{i}", b"x")) + for i in range(200): + conn.execute(UPSERT, (f"stale-{i}", b"x")) + with conn: + conn.execute("CREATE TEMP TABLE _live_keys (key TEXT PRIMARY KEY)") + conn.executemany( + "INSERT OR IGNORE INTO _live_keys VALUES(?)", + [(f"live-{i}",) for i in range(200)], + ) + conn.execute( + "DELETE FROM pkl_data WHERE key NOT IN (SELECT key FROM _live_keys)" + ) + conn.execute("DROP TABLE _live_keys") + keys = {r[0] for r in conn.execute("SELECT key FROM pkl_data")} + assert keys == {f"live-{i}" for i in range(200)} + conn.close() + + +def _invoke_worker(args): + cache_script, castxml_bin, inc, cxx, out, env_pairs = args + import os + import subprocess as sp + + env = dict(os.environ) + env.update(env_pairs) + r = sp.run( + [ + sys.executable, + cache_script, + castxml_bin, + "--castxml-cc-gnu", + "g++", + f"@{inc}", + "-o", + out, + cxx, + ], + env=env, + capture_output=True, + ) + return r.returncode + + +def test_racing_stores_same_key_leave_valid_entry(fake, tmp_path): + from conftest import CACHE_SCRIPT + + outs = [str(fake.build / f"out{i}.xml") for i in range(8)] + env_pairs = { + "ITK_WRAP_CACHE": str(fake.cache), + "FAKE_LOG": str(fake.log), + "FAKE_HDRS": str(fake.incdir_a / "foo.h"), + } + args = [ + ( + str(CACHE_SCRIPT), + str(fake.bin), + str(fake.inc_file), + str(fake.cxx_file), + o, + env_pairs, + ) + for o in outs + ] + with multiprocessing.Pool(8) as pool: + rcs = pool.map(_invoke_worker, args) + assert all(rc == 0 for rc in rcs) + oks = list(fake.cache.glob("l2/*/*/_ok")) + assert len(oks) == 1 + # the surviving entry must serve a valid warm hit + fake.reset_log() + fake.out_xml.unlink(missing_ok=True) + import os + + for k, v in env_pairs.items(): + os.environ[k] = v + r = fake.invoke(extra_env=env_pairs) + assert r.returncode == 0 + assert fake.out_xml.stat().st_size > 0 + assert "FULL" not in fake.invocations() diff --git a/Wrapping/Generators/Tests/tests/test_integration.py b/Wrapping/Generators/Tests/tests/test_integration.py new file mode 100644 index 00000000000..07d4ae69134 --- /dev/null +++ b/Wrapping/Generators/Tests/tests/test_integration.py @@ -0,0 +1,173 @@ +"""L1: hit/miss/staleness/corruption/eviction scenarios with the fake castxml.""" + +import gzip +import os +import time + + +def test_cold_miss_stores_and_marks(fake): + r = fake.invoke() + assert r.returncode == 0, r.stderr + assert fake.invocations() == ["PRE", "FULL"] + assert fake.out_xml.stat().st_size > 0 + assert (fake.cache / "_last_store").exists() + assert list(fake.cache.glob("l2/*/*/_ok")) + assert list(fake.cache.glob("l1/*/*/l2_key")) + + +def test_warm_l1_hit_no_subprocess(fake): + fake.invoke() + first = fake.out_xml.read_bytes() + fake.reset_log() + fake.out_xml.unlink() + r = fake.invoke() + assert r.returncode == 0 + assert fake.invocations() == [] + assert fake.out_xml.read_bytes() == first + + +def test_restored_output_mtime_is_fresh(fake): + fake.invoke() + fake.cxx_file.touch() + time.sleep(0.05) + fake.out_xml.unlink() + fake.invoke() + assert fake.out_xml.stat().st_mtime >= fake.cxx_file.stat().st_mtime + + +def test_header_content_edit_misses(fake): + fake.invoke() + (fake.incdir_a / "foo.h").write_text("content-A v2\n") + fake.reset_log() + fake.invoke() + assert fake.invocations() == ["PRE", "FULL"] + assert b"content-A v2" in fake.out_xml.read_bytes() + + +def test_new_shadowing_header_forces_recheck(fake): + fake.invoke() + fake.reset_log() + fake.invoke() + assert fake.invocations() == [] # warmed + (fake.incdir_a / "bar.h").write_text("shadow\n") # earlier dir, same name + fake.reset_log() + fake.invoke() + assert "PRE" in fake.invocations() # incdir fingerprint miss -> -E re-check + + +def test_castxml_upgrade_serves_new_output(fake, monkeypatch): + fake.invoke() + with open(fake.bin, "a") as f: + f.write("# upgraded\n") # binary content change + monkeypatch.setenv("FAKE_CASTXML_VERSION", "2") + fake.reset_log() + fake.out_xml.unlink() + fake.invoke(extra_env={"FAKE_CASTXML_VERSION": "2"}) + assert "FULL" in fake.invocations() + assert b"castxml-version='2'" in fake.out_xml.read_bytes() + + +def test_truncated_gzip_recovers(fake): + fake.invoke() + gz = next(fake.cache.glob("l2/*/*/output.xml.gz")) + gz.write_bytes(gz.read_bytes()[:20]) # truncate -> EOFError territory + fake.reset_log() + fake.out_xml.unlink() + r = fake.invoke() + assert r.returncode == 0 + assert fake.out_xml.stat().st_size > 0 + assert "FULL" in fake.invocations() # re-ran and re-stored + + +def test_empty_cached_xml_not_served(fake): + fake.invoke() + gz = next(fake.cache.glob("l2/*/*/output.xml.gz")) + gz.write_bytes(gzip.compress(b"")) + fake.reset_log() + fake.out_xml.unlink() + r = fake.invoke() + assert r.returncode == 0 + assert fake.out_xml.stat().st_size > 0 + + +def test_bypass_modes_run_castxml_directly(fake, monkeypatch): + r = fake.invoke(no_cache=True) + assert r.returncode == 0 + assert fake.invocations() == ["FULL"] + assert not list(fake.cache.glob("l2/*")) + + fake.reset_log() + r = fake.invoke(extra_env={"ITK_WRAP_CACHE_BYPASS": "1"}) + assert r.returncode == 0 + assert fake.invocations() == ["FULL"] + assert not list(fake.cache.glob("l2/*")) + + +def test_failed_castxml_stores_nothing(fake): + r = fake.invoke(extra_env={"FAKE_FAIL": "1"}) + assert r.returncode != 0 + assert not list(fake.cache.glob("l2/*/*/_ok")) + + +def test_multi_root_reads_secondary_writes_primary(fake, monkeypatch, tmp_path): + fake.invoke() # populate what will become the secondary root + secondary = fake.cache + primary = tmp_path / "primary" + primary.mkdir() + sep = ";" if os.name == "nt" else ":" + monkeypatch.setenv("ITK_WRAP_CACHE", f"{primary}{sep}{secondary}") + fake.reset_log() + fake.out_xml.unlink() + r = fake.invoke() + assert r.returncode == 0 + assert fake.out_xml.stat().st_size > 0 + assert "FULL" not in fake.invocations() # served from secondary + + +def test_uncompressed_format_roundtrip(fake, monkeypatch): + monkeypatch.setenv("ITK_WRAP_CACHE_FORMAT", "uncompressed") + fake.invoke(extra_env={"ITK_WRAP_CACHE_FORMAT": "uncompressed"}) + entry = next(fake.cache.glob("l2/*/*/output.xml")) + time.sleep(1.1) + fake.reset_log() + fake.out_xml.unlink() + fake.invoke(extra_env={"ITK_WRAP_CACHE_FORMAT": "uncompressed"}) + assert fake.invocations() == [] + assert fake.out_xml.stat().st_mtime > entry.stat().st_mtime + 0.5 + + +def test_evict_removes_dead_and_staging_dirs_after_grace(fake): + fake.invoke() + fake.evict() # establish _last_evict + dead = fake.cache / "l2" / "zz" / "deadentry" + dead.mkdir(parents=True) + (dead / "output.xml.gz").write_bytes(b"x") # no _ok + staging = fake.cache / "l2" / "zz" / "abc.tmpq1w2e3" + staging.mkdir() + past = time.time() - 7200 + os.utime(dead, (past, past)) + os.utime(staging, (past, past)) + (fake.cache / "_last_store").touch() # signal stores -> eviction must run + fake.evict() + assert not dead.exists() + assert not staging.exists() + + +def test_evict_early_exits_when_no_stores(fake): + fake.invoke() + fake.evict() + past = time.time() - 7200 + os.utime(fake.cache / "_last_store", (past, past)) + before = sorted(p.name for p in fake.cache.rglob("*")) + fake.evict() + assert sorted(p.name for p in fake.cache.rglob("*")) == before + + +def test_evict_age_removes_old_entries(fake): + fake.invoke() + ok = next(fake.cache.glob("l2/*/*/_ok")) + past = time.time() - 30 * 86400 + os.utime(ok, (past, past)) + (fake.cache / "_last_store").touch() + fake.evict(days="13.9") + assert not ok.exists() diff --git a/Wrapping/Generators/Tests/tests/test_pyi_generator.py b/Wrapping/Generators/Tests/tests/test_pyi_generator.py new file mode 100644 index 00000000000..d610d2d1282 --- /dev/null +++ b/Wrapping/Generators/Tests/tests/test_pyi_generator.py @@ -0,0 +1,137 @@ +"""L1: pyi_generator prune gating, self-heal, and guard behavior against a +synthetic pkl DB (no ITK build required).""" + +import pickle +import subprocess +import sys + +import pytest + +from conftest import PYI_GENERATOR, _load + +UPSERT = ( + "INSERT INTO pkl_data(key, data) VALUES(?,?)" + " ON CONFLICT(key) DO UPDATE SET data=excluded.data" +) + + +@pytest.fixture +def pkl_tree(pkl_db_mod, tmp_path): + """A pkl dir with 6 synthetic classes, index manifests, and stamp files.""" + gen = _load("pyi_generator_for_pickling", PYI_GENERATOR) + cls = gen.ITKClass + # igenerator pickles ITKClass from a script run as __main__; reproduce that + cls.__module__ = "__main__" + setattr(sys.modules["__main__"], "ITKClass", cls) + + pkl_dir = tmp_path / "itk-pkl" + pyi_dir = tmp_path / "pyi" + pkl_dir.mkdir() + pyi_dir.mkdir() + conn = pkl_db_mod.open_pkl_db(str(pkl_dir)) + keys = [] + for i in range(6): + name = f"FakeClass{i}" + key = f"{name}.itk{name}" + obj = cls(name) + with conn: + conn.execute(UPSERT, (key, pickle.dumps(obj))) + (pkl_dir / f"itk{name}.index.txt").write_text(key + "\n") + keys.append(key) + conn.close() + (pkl_dir / "ITKFake.stamp").write_text("") + (pkl_dir / "ITKFake2.stamp").write_text("") + + # CMake writes GlobalIdxFilesList.txt with forward-slash paths on every + # platform; pyi_generator normalizes its glob results to match. + index_files = sorted(p.as_posix() for p in pkl_dir.glob("*.index.txt")) + full_list = tmp_path / "full_list.txt" + full_list.write_text(";".join(index_files)) + partial_list = tmp_path / "partial_list.txt" + partial_list.write_text(index_files[0]) + return pkl_dir, pyi_dir, full_list, partial_list, keys + + +def run_pyi(pkl_dir, pyi_dir, list_file, *extra): + return subprocess.run( + [ + sys.executable, + str(PYI_GENERATOR), + "--pyi_dir", + str(pyi_dir), + "--pkl_dir", + str(pkl_dir), + "--index_list_file", + str(list_file), + *extra, + ], + capture_output=True, + text=True, + ) + + +def rowcount(pkl_db_mod, pkl_dir): + import sqlite3 + + conn = sqlite3.connect(pkl_db_mod._pkl_db_path(str(pkl_dir))) + n = conn.execute("SELECT count(*) FROM pkl_data").fetchone()[0] + conn.close() + return n + + +def test_healthy_run_generates_stubs(pkl_db_mod, pkl_tree): + pkl_dir, pyi_dir, full_list, _partial, _keys = pkl_tree + r = run_pyi(pkl_dir, pyi_dir, full_list, "--prune") + assert r.returncode == 0, r.stderr + assert (pyi_dir / "__init__.pyi").exists() + assert rowcount(pkl_db_mod, pkl_dir) == 6 + + +def test_partial_manifest_without_prune_preserves_rows(pkl_db_mod, pkl_tree): + """External-wrap-project simulation: shared DB must survive.""" + pkl_dir, pyi_dir, _full, partial_list, _keys = pkl_tree + r = run_pyi(pkl_dir, pyi_dir, partial_list) + assert r.returncode == 0, r.stderr + assert rowcount(pkl_db_mod, pkl_dir) == 6 + + +def test_partial_manifest_with_prune_deletes(pkl_db_mod, pkl_tree): + pkl_dir, pyi_dir, _full, partial_list, _keys = pkl_tree + r = run_pyi(pkl_dir, pyi_dir, partial_list, "--prune") + assert r.returncode == 0, r.stderr + assert rowcount(pkl_db_mod, pkl_dir) == 1 + + +def test_prune_skipped_when_manifest_missing(pkl_db_mod, pkl_tree, tmp_path): + pkl_dir, pyi_dir, full_list, _partial, _keys = pkl_tree + with_missing = tmp_path / "with_missing.txt" + with_missing.write_text( + full_list.read_text() + ";" + (pkl_dir / "DOES_NOT_EXIST.index.txt").as_posix() + ) + r = run_pyi(pkl_dir, pyi_dir, with_missing, "--prune") + assert r.returncode == 0, r.stderr + assert "skipping pkl DB pruning" in r.stdout + assert rowcount(pkl_db_mod, pkl_dir) == 6 + + +def test_self_heal_on_damaged_db(pkl_db_mod, pkl_tree): + """Deleted DB rows with fresh stamps -> stamps removed + actionable error.""" + import sqlite3 + + pkl_dir, pyi_dir, full_list, _partial, keys = pkl_tree + conn = sqlite3.connect(pkl_db_mod._pkl_db_path(str(pkl_dir))) + with conn: + conn.execute("DELETE FROM pkl_data WHERE key IN (?,?)", keys[:2]) + conn.close() + assert list(pkl_dir.glob("*.stamp")) + r = run_pyi(pkl_dir, pyi_dir, full_list, "--prune") + assert r.returncode != 0 + assert "missing from the pkl database" in r.stderr + assert not list(pkl_dir.glob("*.stamp")) + + +def test_bogus_pkl_dir_rejected(pkl_tree): + _pkl, pyi_dir, full_list, _partial, _keys = pkl_tree + r = run_pyi("/nonexistent/pkl-dir", pyi_dir, full_list) + assert r.returncode != 0 + assert "Invalid directory provided" in r.stderr diff --git a/Wrapping/Generators/Tests/tests/test_relocate.py b/Wrapping/Generators/Tests/tests/test_relocate.py new file mode 100644 index 00000000000..5868a41d44b --- /dev/null +++ b/Wrapping/Generators/Tests/tests/test_relocate.py @@ -0,0 +1,42 @@ +"""L4-sim: tar round-trip to a different absolute path — models what +actions/cache and Azure Cache@2 do to the cache directory between CI runs.""" + +import shutil + + +def test_cache_relocation_preserves_hits(fake, monkeypatch, tmp_path): + fake.invoke() + assert fake.invocations() == ["PRE", "FULL"] + + relocated = tmp_path / "relocated" / "deeper" / "cache" + relocated.parent.mkdir(parents=True) + shutil.make_archive(str(tmp_path / "ci-cache"), "tar", fake.cache) + shutil.unpack_archive(str(tmp_path / "ci-cache.tar"), relocated) + + monkeypatch.setenv("ITK_WRAP_CACHE", str(relocated)) + fake.reset_log() + fake.out_xml.unlink() + r = fake.invoke(extra_env={"ITK_WRAP_CACHE": str(relocated)}) + assert r.returncode == 0 + assert fake.invocations() == [] # pure L1 hit from the relocated cache + assert fake.out_xml.stat().st_size > 0 + + +def test_cross_branch_restore_selectively_misses(fake, monkeypatch, tmp_path): + """Restore-keys prefix fallback across branches is safe by construction: + unchanged modules hit, the changed module misses.""" + fake.invoke() # "branch A" populates the cache + + # "branch B": one header differs + (fake.incdir_a / "foo.h").write_text("branch-B content\n") + fake.reset_log() + fake.invoke() + assert fake.invocations() == ["PRE", "FULL"] # changed module misses + + # reverting to branch-A content: the L1 map now records branch-B deps, so + # the -E pass re-runs once and lands an L2 hit on the branch-A entry + (fake.incdir_a / "foo.h").write_text("content-A v1\n") + fake.reset_log() + fake.out_xml.unlink() + fake.invoke() + assert fake.invocations() == ["PRE"] # no FULL: branch-A XML still cached diff --git a/Wrapping/Generators/Tests/tests/test_unit_cache.py b/Wrapping/Generators/Tests/tests/test_unit_cache.py new file mode 100644 index 00000000000..0a9111876b3 --- /dev/null +++ b/Wrapping/Generators/Tests/tests/test_unit_cache.py @@ -0,0 +1,162 @@ +"""L0: pure functions of itk-castxml-cache.py — no subprocess, no ITK build.""" + +import os +import sys + +import pytest + + +def test_strip_line_markers(cache_mod): + data = b'# 1 "/a/b.h"\nint x;\n# 22 "/c/d.h" 2\nint y;\n' + assert cache_mod._strip_line_markers(data) == b"int x;\nint y;" + + +def test_extract_dep_paths_dedup_and_pseudo(cache_mod): + pre = b'# 1 "/a/b.h"\n# 5 "/a/b.h"\n# 1 ""\n# 1 ""\n' + assert cache_mod._extract_dep_paths(pre) == ["/a/b.h"] + + +def test_extract_dep_paths_unescapes_windows_backslashes(cache_mod): + pre = b'# 1 "C:\\\\src\\\\ITK\\\\itkImage.h"\n' + assert cache_mod._extract_dep_paths(pre) == ["C:\\src\\ITK\\itkImage.h"] + + +def test_deps_manifest_and_unchanged_roundtrip(cache_mod, tmp_path): + h = tmp_path / "a.h" + h.write_text("v1") + deps = cache_mod._deps_manifest([str(h)]) + assert cache_mod._deps_unchanged(deps) + h.write_text("v2") + assert not cache_mod._deps_unchanged(deps) + + +def test_deps_unchanged_deleted_header_misses(cache_mod, tmp_path): + h = tmp_path / "a.h" + h.write_text("v1") + deps = cache_mod._deps_manifest([str(h)]) + h.unlink() + assert not cache_mod._deps_unchanged(deps) + + +def test_deps_unreadable_at_store_and_check_is_consistent(cache_mod): + deps = cache_mod._deps_manifest(["/nonexistent/path.h"]) + assert deps == [["/nonexistent/path.h", ""]] + assert cache_mod._deps_unchanged(deps) + + +def test_deps_malformed_manifest_misses(cache_mod): + assert not cache_mod._deps_unchanged([["path-without-hash"]]) + assert not cache_mod._deps_unchanged(["not-a-list"]) + + +def test_parse_args(cache_mod): + binp, out, inc, cxx, flags, no_cache = cache_mod._parse_args( + ["/bin/castxml", "-o", "a.xml", "@inc.txt", "mod.cxx", "--flag"] + ) + assert (binp, out, inc, cxx, no_cache) == ( + "/bin/castxml", + "a.xml", + "inc.txt", + "mod.cxx", + False, + ) + assert flags == ["-o", "a.xml", "@inc.txt", "mod.cxx", "--flag"] + + +def test_parse_args_no_cache_flag(cache_mod): + binp, _out, _inc, _cxx, _flags, no_cache = cache_mod._parse_args( + ["--no-cache", "/bin/castxml", "mod.cxx"] + ) + assert binp == "/bin/castxml" and no_cache + + +def test_cache_roots_split(cache_mod, monkeypatch): + sep = ";" if sys.platform == "win32" else ":" + monkeypatch.setenv("ITK_WRAP_CACHE", f"/one{sep}{sep}/two") + assert cache_mod._cache_roots() == ["/one", "/two"] + monkeypatch.delenv("ITK_WRAP_CACHE") + assert len(cache_mod._cache_roots()) == 1 + + +def test_l1_key_includes_key_version(cache_mod): + k1 = cache_mod._l1_key("binhash", None, None, []) + orig = cache_mod._KEY_VERSION + try: + cache_mod._KEY_VERSION = b"vTEST\x00" + k2 = cache_mod._l1_key("binhash", None, None, []) + finally: + cache_mod._KEY_VERSION = orig + assert k1 != k2 + + +def test_l1_key_includes_bin_hash_and_flag_order(cache_mod): + assert cache_mod._l1_key("h1", None, None, ["-a", "-b"]) != cache_mod._l1_key( + "h2", None, None, ["-a", "-b"] + ) + assert cache_mod._l1_key("h1", None, None, ["-a", "-b"]) != cache_mod._l1_key( + "h1", None, None, ["-b", "-a"] + ) + + +def test_incdirs_from_inc_file(cache_mod, tmp_path): + inc = tmp_path / "m.inc" + inc.write_text('"-I/one"\n"-I/two"\n"-I/one"\n"-DFOO"\n') + assert cache_mod._incdirs_from_inc_file(str(inc)) == ["/one", "/two"] + + +def test_incdirs_from_inc_file_isystem(cache_mod, tmp_path): + inc = tmp_path / "m.inc" + inc.write_text( + '"-I/one"\n"-isystem" "/sys/a"\n"-isystem" "/sys/b"\n"-isystem/sys/c"\n"-isystem" "/sys/a"\n' + ) + assert cache_mod._incdirs_from_inc_file(str(inc)) == [ + "/one", + "/sys/a", + "/sys/b", + "/sys/c", + ] + + +def test_incdirs_fingerprint_covers_isystem_dirs(cache_mod, tmp_path): + d = tmp_path / "sysinc" + d.mkdir() + (d / "a.h").write_text("x") + inc = tmp_path / "m.inc" + inc.write_text(f'"-isystem" "{d}"\n') + fp1 = cache_mod._incdirs_fingerprint(str(inc)) + (d / "b.h").write_text("shadow") + assert cache_mod._incdirs_fingerprint(str(inc)) != fp1 + + +def test_incdirs_fingerprint_changes_on_new_header(cache_mod, tmp_path): + d = tmp_path / "inc" + d.mkdir() + (d / "a.h").write_text("x") + inc = tmp_path / "m.inc" + inc.write_text(f'"-I{d}"\n') + fp1 = cache_mod._incdirs_fingerprint(str(inc)) + (d / "b.h").write_text("shadow") + fp2 = cache_mod._incdirs_fingerprint(str(inc)) + assert fp1 != fp2 + (d / "b.h").unlink() + assert cache_mod._incdirs_fingerprint(str(inc)) == fp1 + + +def test_incdirs_fingerprint_ignores_content_edits(cache_mod, tmp_path): + d = tmp_path / "inc" + d.mkdir() + (d / "a.h").write_text("x") + inc = tmp_path / "m.inc" + inc.write_text(f'"-I{d}"\n') + fp1 = cache_mod._incdirs_fingerprint(str(inc)) + (d / "a.h").write_text("y") # content change is the deps manifest's job + assert cache_mod._incdirs_fingerprint(str(inc)) == fp1 + + +def test_max_cache_gb_bad_value_defaults(cache_mod, monkeypatch): + monkeypatch.setenv("ITK_WRAP_CACHE_MAX_SIZE", "not-a-number") + assert cache_mod._max_cache_gb() == 2.0 + + +def test_docstring_documents_no_phantom_env_vars(cache_mod): + assert "ITK_WRAP_CACHE_MAX_DAYS" not in cache_mod.__doc__ diff --git a/Wrapping/Generators/Tests/tests/test_unit_pkl_db.py b/Wrapping/Generators/Tests/tests/test_unit_pkl_db.py new file mode 100644 index 00000000000..5a86da5950e --- /dev/null +++ b/Wrapping/Generators/Tests/tests/test_unit_pkl_db.py @@ -0,0 +1,58 @@ +"""L0: pkl_db.py schema, journaling, and upsert semantics.""" + +import sqlite3 + + +def test_schema_created_idempotently(pkl_db_mod, tmp_path): + c1 = pkl_db_mod.open_pkl_db(str(tmp_path)) + c1.close() + c2 = pkl_db_mod.open_pkl_db(str(tmp_path)) + cols = {r[1] for r in c2.execute("PRAGMA table_info(pkl_data)")} + assert cols == {"key", "data"} + c2.close() + + +def test_db_filename_carries_schema_version(pkl_db_mod, tmp_path): + path = pkl_db_mod._pkl_db_path(str(tmp_path)) + assert f"v{pkl_db_mod.PKL_DB_SCHEMA_VERSION}" in path.name + + +def test_wal_mode_on_local_fs(pkl_db_mod, tmp_path): + conn = pkl_db_mod.open_pkl_db(str(tmp_path)) + mode = conn.execute("PRAGMA journal_mode").fetchone()[0] + assert mode in ("wal", "delete") # delete = documented NFS fallback + conn.close() + + +def test_upsert_overwrites(pkl_db_mod, tmp_path): + conn = pkl_db_mod.open_pkl_db(str(tmp_path)) + upsert = ( + "INSERT INTO pkl_data(key, data) VALUES(?,?)" + " ON CONFLICT(key) DO UPDATE SET data=excluded.data" + ) + with conn: + conn.execute(upsert, ("k", b"v1")) + conn.execute(upsert, ("k", b"v2")) + assert ( + conn.execute("SELECT data FROM pkl_data WHERE key='k'").fetchone()[0] == b"v2" + ) + conn.close() + + +def test_reader_side_is_self_contained(pkl_db_mod, tmp_path): + """Opening a fresh dir must not raise 'no such table' on first SELECT.""" + conn = pkl_db_mod.open_pkl_db(str(tmp_path / "fresh")) + assert conn.execute("SELECT count(*) FROM pkl_data").fetchone()[0] == 0 + conn.close() + + +def test_blob_not_null(pkl_db_mod, tmp_path): + conn = pkl_db_mod.open_pkl_db(str(tmp_path)) + try: + with conn: + conn.execute("INSERT INTO pkl_data(key, data) VALUES('k', NULL)") + raised = False + except sqlite3.IntegrityError: + raised = True + assert raised + conn.close()