diff --git a/.github/workflows/arm.yml b/.github/workflows/arm.yml index 5fa97d3b5e3..e461edd8f56 100644 --- a/.github/workflows/arm.yml +++ b/.github/workflows/arm.yml @@ -80,6 +80,7 @@ jobs: BUILD_SHARED_LIBS:BOOL=OFF BUILD_EXAMPLES:BOOL=OFF ITK_WRAP_PYTHON:BOOL=ON + ITK_WRAP_CASTXML_CACHE:BOOL=ON ITK_USE_CLANG_FORMAT:BOOL=OFF ITK_COMPUTER_MEMORY_SIZE:STRING=11 ctest-options: "-E itkPyBufferMemoryLeakTest" @@ -102,6 +103,7 @@ jobs: echo "CCACHE_SLOPPINESS=pch_defines,time_macros" >> "$GITHUB_ENV" echo "CCACHE_DIR=${{ runner.temp }}/ccache" >> "$GITHUB_ENV" echo "CCACHE_MAXSIZE=5G" >> "$GITHUB_ENV" + echo "ITK_WRAP_CACHE=${{ runner.temp }}/itk-castxml-cache" >> "$GITHUB_ENV" if [ "$RUNNER_OS" == "Linux" ]; then sudo apt-get update -qq && sudo apt-get install -y ccache locales sudo locale-gen de_DE.UTF-8 @@ -118,6 +120,16 @@ jobs: restore-keys: | ccache-v4-${{ runner.os }}-${{ matrix.name }}- + - name: Restore CastXML cache + if: matrix.python-version != '' + id: restore-castxml-cache + uses: actions/cache/restore@v5 + with: + path: ${{ runner.temp }}/itk-castxml-cache + key: castxml-v1-${{ runner.os }}-arm-python-${{ github.sha }} + restore-keys: | + castxml-v1-${{ runner.os }}-arm-python- + - name: Restore ExternalData object store id: restore-externaldata uses: actions/cache/restore@v5 @@ -149,6 +161,7 @@ jobs: run: | set -x python3 -m pip install --upgrade --pre numpy + python3 -m pip install pytest - name: Get specific version of CMake, latest Ninja uses: lukka/get-cmake@v4.0.1 @@ -202,6 +215,13 @@ jobs: path: ${{ runner.temp }}/ccache key: ccache-v4-${{ runner.os }}-${{ matrix.name }}-${{ github.sha }} + - name: Save CastXML cache + if: ${{ !cancelled() && matrix.python-version != '' }} + uses: actions/cache/save@v5 + with: + path: ${{ runner.temp }}/itk-castxml-cache + key: castxml-v1-${{ runner.os }}-arm-python-${{ github.sha }} + # ExternalData object store is populated by # .github/workflows/populate-externaldata-cache.yml — a dedicated # workflow whose only job is to prefetch every CID and write the diff --git a/CMake/itkWrapCastXMLCacheSupport.cmake b/CMake/itkWrapCastXMLCacheSupport.cmake new file mode 100644 index 00000000000..57a650acc22 --- /dev/null +++ b/CMake/itkWrapCastXMLCacheSupport.cmake @@ -0,0 +1,66 @@ +# Locate the wrapper script relative to this file so the default also +# resolves when ITK_SOURCE_DIR is not defined in the including scope. +get_filename_component( + _itk_wrap_castxml_cache_src_root + "${CMAKE_CURRENT_LIST_DIR}/.." + ABSOLUTE +) +set( + _ITK_WRAP_CASTXML_CACHE_SCRIPT_DEFAULT + "${_itk_wrap_castxml_cache_src_root}/Wrapping/Generators/CastXML/itk-castxml-cache.py" +) +unset(_itk_wrap_castxml_cache_src_root) + +option( + ITK_WRAP_CASTXML_CACHE + "Use a content-addressed two-level cache for CastXML wrapping steps." + ON +) +mark_as_advanced(ITK_WRAP_CASTXML_CACHE) + +if(ITK_WRAP_CASTXML_CACHE) + set( + ITK_WRAP_CASTXML_CACHE_SCRIPT + "${_ITK_WRAP_CASTXML_CACHE_SCRIPT_DEFAULT}" + CACHE FILEPATH + "Path to the CastXML content-addressed cache wrapper script" + ) + mark_as_advanced(ITK_WRAP_CASTXML_CACHE_SCRIPT) + + if(NOT EXISTS "${ITK_WRAP_CASTXML_CACHE_SCRIPT}") + message( + FATAL_ERROR + "ITK_WRAP_CASTXML_CACHE is ON but the wrapper script was not found:\n" + " ${ITK_WRAP_CASTXML_CACHE_SCRIPT}\n" + "Set ITK_WRAP_CASTXML_CACHE_SCRIPT to the correct path or turn off ITK_WRAP_CASTXML_CACHE." + ) + endif() + + if(NOT Python3_EXECUTABLE) + message( + FATAL_ERROR + "ITK_WRAP_CASTXML_CACHE requires Python3_EXECUTABLE to be set." + ) + endif() + + set( + ITK_WRAP_CASTXML_CACHE_MAX_DAYS + "13.9" + CACHE STRING + "Days before a CastXML cache entry is evicted by --evict after each build (default 13.9)" + ) + mark_as_advanced(ITK_WRAP_CASTXML_CACHE_MAX_DAYS) + + message(STATUS "CastXML content-addressed cache enabled") + message(STATUS " Script: ${ITK_WRAP_CASTXML_CACHE_SCRIPT}") + message( + STATUS + " Cache root: set ITK_WRAP_CACHE env var at build time (default: ~/.cache/itk-wrap)" + ) + message( + STATUS + " Eviction (after each build): entries older than ${ITK_WRAP_CASTXML_CACHE_MAX_DAYS} days, then oldest trimmed to ITK_WRAP_CACHE_MAX_SIZE GB (default 2.0)" + ) +endif() + +unset(_ITK_WRAP_CASTXML_CACHE_SCRIPT_DEFAULT) diff --git a/Documentation/docs/contributing/index.md b/Documentation/docs/contributing/index.md index bb36f8305b7..ca18462e249 100644 --- a/Documentation/docs/contributing/index.md +++ b/Documentation/docs/contributing/index.md @@ -469,6 +469,7 @@ CDash Dashboard dashboard.md updating_third_party.md python_packaging.md +wrapping_architecture.md ../README.md ``` diff --git a/Documentation/docs/contributing/wrapping_architecture.md b/Documentation/docs/contributing/wrapping_architecture.md new file mode 100644 index 00000000000..b030c6085a2 --- /dev/null +++ b/Documentation/docs/contributing/wrapping_architecture.md @@ -0,0 +1,222 @@ +Python Wrapping Architecture +============================ + +ITK's Python wrapping pipeline converts C++ template declarations into +importable Python extension modules (`.abi3.so`) and type-stub files +(`.pyi`). The pipeline runs in two distinct phases: a **configure phase** +driven by CMake and a **build phase** driven by the CMake-selected build +system (Ninja, Make, Xcode, Visual Studio, ...). + +## Configure phase: `.wrap` → `castxml_inputs/` + +Each ITK module that supports wrapping contains a `wrapping/` subdirectory +with one `.wrap` file per submodule. A `.wrap` file is a CMake script that +calls macros such as `itk_wrap_class()` and `itk_wrap_template()` to declare +which C++ template instantiations should be exposed to Python. + +CMake processes every `.wrap` file at configure time and writes three +files per submodule into `/Wrapping/castxml_inputs/`: + +| Generated file | How | Contents | +|---|---|---| +| `.cxx` | `configure_file` | `#include` directives + `_wrapping_` namespace with `using` aliases for every requested template instantiation | +| `.castxml.inc` | `file(GENERATE …)` | Compiler `-I` and `-D` flags needed to parse the `.cxx` file | +| `SwigInterface.h.in` | `configure_file` | `#include` list used by SWIG | + +CMake also registers one `add_custom_command` per submodule (for CastXML) +and one per ITK module (for `igenerator.py`). No compilation happens at +configure time; only the input files and build rules are written. + +## Build phase: CastXML → igenerator → SWIG → compile → link + +### Step 1 — CastXML (one independent job per submodule) + +Each submodule produces exactly one XML file: + +``` +castxml_inputs/.cxx +castxml_inputs/.castxml.inc ──▶ itk-castxml-cache.py ──▶ castxml_inputs/.xml +Modules/.../include/.h (many) +``` + +`itk-castxml-cache.py` wraps the CastXML binary with a two-level +content-addressed cache (`~/.cache/itk-wrap` or `$ITK_WRAP_CACHE`): + +- **L1** — hash of the `.cxx` file content → L2 key +- **L2** — `castxml -E` (preprocessor only) output hash → cached `output.xml.gz` + +A cache hit avoids running CastXML entirely. There is one CastXML job per +submodule (~800 in a full build; the exact count depends on the enabled +modules, pixel types, and dimensions), all independent and fully parallel. No CastXML job reads or modifies +another submodule's `.xml` output. + +### Step 2 — `igenerator.py` (one job per ITK module) + +Each ITK module (e.g. `ITKImageIntensity`) batches all of its submodules +into a single `igenerator.py` invocation: + +``` +castxml_inputs/itkAbsImageFilter.xml ──┐ +castxml_inputs/itkImage.xml ──┤ +castxml_inputs/itkOffset.xml ──┤ igenerator.py [ITKImageIntensity] +... (all N submodule XMLs) ──┘ --submodule-order "sub1;sub2;...;subN" + │ + ┌──────────────────────────┼─────────────────────────────┐ + │ per submodule (×N) │ │ + ▼ ▼ ▼ + Typedefs/.i itk-pkl/.index.txt itk-pkl-v3.db (SQLite) + Typedefs/.idx (lists DB keys; byproduct) (class metadata; WAL mode) + Typedefs/SwigInterface.h + + per module (×1): + itk-pkl/.stamp + itk/Configuration/_snake_case.py +``` + +`igenerator.py` uses `pygccxml` to parse each `.xml` file and emit SWIG +interface (`.i`) and index (`.idx`) files, class-metadata rows in a +build-local SQLite database consumed later by `pyi_generator.py`, and a +`.index.txt` manifest listing the DB keys for each submodule. + +The SQLite database (`itk-pkl-v3.db`) is written to the `itk-pkl/` directory +inside the build tree. It is intermediate handoff state keyed by bare class +name, so it is always build-tree-local and is never shared via +`ITK_WRAP_CACHE`. + +**Build-system scheduling**: an `igenerator.py` job for module A starts as soon +as all of A's CastXML jobs are complete, even while CastXML is still +running for module B. There is no global barrier between the CastXML and +`igenerator.py` layers. + +### Step 3 — SWIG, compile, link (per submodule) + +``` +Typedefs/.i +Typedefs/SwigInterface.h ──▶ swig ──▶ Modules/.../Python.cpp + Generators/Python/itk/Python.py + +Modules/.../Python.cpp ──▶ ccache + g++ ──▶ .o ──▶ link ──▶ _Python.abi3.so +``` + +### Step 4 — `pyi_generator.py` (one global job) + +After **all** `.index.txt` files exist, `pyi_generator.py` reads every +`.index.txt`, queries the SQLite database for each key, and writes the +type-stub files used by IDEs: + +``` +itk-pkl/.index.txt (all) ──▶ pyi_generator.py ──▶ _proxies.pyi +itk-pkl-v3.db (SQLite) (queries DB via keys in .index.txt) __init__.pyi +``` + +## Key file reference + +| Path pattern | Written by | Read by | +|---|---|---| +| `Wrapping/castxml_inputs/.cxx` | CMake `configure_file` | CastXML | +| `Wrapping/castxml_inputs/.castxml.inc` | CMake `file(GENERATE)` | `itk-castxml-cache.py` | +| `Wrapping/castxml_inputs/.xml` | `itk-castxml-cache.py` / CastXML | `igenerator.py` | +| `Wrapping/Typedefs/.i` | `igenerator.py` | SWIG | +| `Wrapping/Typedefs/.idx` | `igenerator.py` | SWIG | +| `Wrapping/Generators/Python/itk-pkl/.index.txt` | `igenerator.py` | `pyi_generator.py` | +| `Wrapping/Generators/Python/itk-pkl/itk-pkl-v3.db` | `igenerator.py` | `pyi_generator.py` | +| `Wrapping/Generators/Python/itk-pkl/.stamp` | `igenerator.py` | build system (tracks DB write completeness) | +| `Wrapping/Generators/Python/itk/_Python.abi3.so` | linker | Python `import itk` | +| `Wrapping/Generators/Python/itk/_proxies.pyi` | `pyi_generator.py` | IDEs | + +## Caches + +### CastXML cache (`itk-castxml-cache.py`) + +| Variable | Default | Purpose | +|---|---|---| +| `ITK_WRAP_CACHE` | `~/.cache/itk-wrap` | Cache root for CastXML `.xml.gz` files | +| `ITK_WRAP_CACHE_VERBOSE` | unset | Set to `1` to log HIT/MISS per file | + +The CastXML cache is content-addressed and generator-version-stamped +(`_KEY_VERSION` in `itk-castxml-cache.py`). It is shared across build +directories; a fresh configure reuses XML from a previous build on the same +machine. + +### pkl SQLite database (`igenerator.py` / `pyi_generator.py`) + +The pkl database (`itk-pkl-v3.db`) lives in the build tree's `itk-pkl/` +directory and is a build artifact, not a user-level cache. Unlike the +content-addressed CastXML cache, it is keyed by bare `class.submodule` +strings, so it is **always build-tree-local and never honors +`ITK_WRAP_CACHE`** — sharing it across build trees would collide same-named +classes from different ITK sources. + +`igenerator.py` writes the rows; `pyi_generator.py` reads them and, when +invoked with `--prune` (ITK's own build only), prunes any row whose key is +absent from the current build's `.index.txt` manifests (exact keyset +cleanup). External wrap projects share the ITK build tree's database but +their manifests cover only their own modules, so they never prune. The +schema version in the filename means a database from an older schema is +ignored rather than migrated. + +### ccache + +CastXML re-runs produce identical `.xml` files (the content is deterministic) +but are slow. The CastXML cache eliminates that cost. For the C++ compilation +steps (Step 3), ccache caches compiled `.o` files keyed on source content. +Both caches are independent and complement each other. + +### Build-phase timing on 2-core CI runners + +On a 2-core CI runner (cold caches throughout): + +| Phase | Approx. time | Notes | +|---|---|---| +| CastXML (~800 jobs, 2 cores) | ~32 min | Eliminated on warm-cache runs | +| igenerator + SWIG + C++ compile | ~225 min | Reduced by ccache on subsequent runs | +| Tests | ~44 min | | + +CastXML is ~10 % of the cold-cache total. The C++ compilation phase +dominates; `ccache` is the primary lever there. + +## Build dependency graph summary + +``` +.wrap files (configure time, not in graph) + │ cmake configure_file / file(GENERATE) + ▼ +castxml_inputs/.cxx + .castxml.inc + .h headers + │ itk-castxml-cache.py [one parallel job per submodule] + ▼ +castxml_inputs/.xml (write-once; never mutated after creation) + │ igenerator.py [one job per module; starts per-module, not globally gated] + ▼ +Typedefs/.i + .idx + SwigInterface.h +itk-pkl/.index.txt (DB keys) + itk-pkl-v3.db (SQLite) + .stamp + │ swig + ccache compile + link [parallel per submodule] + ▼ +_Python.abi3.so + │ pyi_generator.py [1 global job; needs all .index.txt] + ▼ +_proxies.pyi + __init__.pyi +``` + +## Troubleshooting + +**CastXML cache not being used** +: Set `ITK_WRAP_CACHE_VERBOSE=1` and rebuild one module to confirm HIT or + MISS log lines. Ensure `ITK_WRAP_CASTXML_CACHE=ON` is set in CMake. + A version bump to `_KEY_VERSION` in `itk-castxml-cache.py` forces a cold + cache for all entries. + +**`indexed pkl keys are missing from the pkl database`** +: The `.index.txt` manifests exist (so the build system considers `igenerator.py` + up-to-date) but the pkl database is absent or damaged. `pyi_generator.py` + detects this, deletes the stamp files itself, and fails the build once; + the next build re-runs `igenerator.py` and repopulates the DB. Manual + recovery is only needed if the automatic pass keeps failing: + ```bash + find /Wrapping/Generators/Python/itk-pkl -name "*.index.txt" -o -name "*.stamp" | xargs rm -f + cmake --build + ``` + +**Adding a new wrapped class** +: Add `itk_wrap_class()` / `itk_wrap_template()` calls to the relevant + `.wrap` file. Re-run CMake to regenerate the `.cxx` and `.castxml.inc` + files, then build normally. CMake will automatically include the new + submodule in the `--submodule-order` passed to `igenerator.py`. diff --git a/Testing/ContinuousIntegration/AzurePipelinesLinuxPython.yml b/Testing/ContinuousIntegration/AzurePipelinesLinuxPython.yml index 8cc9be6c2dc..fd0b9263aa0 100644 --- a/Testing/ContinuousIntegration/AzurePipelinesLinuxPython.yml +++ b/Testing/ContinuousIntegration/AzurePipelinesLinuxPython.yml @@ -36,6 +36,7 @@ variables: CCACHE_NOHASHDIR: 'true' CCACHE_SLOPPINESS: pch_defines,time_macros CCACHE_MAXSIZE: 8G + ITK_WRAP_CACHE: $(Pipeline.Workspace)/.castxml-cache jobs: - job: Linux timeoutInMinutes: 0 @@ -55,6 +56,7 @@ jobs: df -h / displayName: 'Free preinstalled software' + - checkout: self clean: true fetchDepth: 5 @@ -75,7 +77,7 @@ jobs: sudo apt-get update sudo apt-get install -y ccache locales sudo locale-gen de_DE.UTF-8 - python3 -m pip install ninja typing-extensions + python3 -m pip install ninja typing-extensions pytest python3 -m pip install --upgrade --pre numpy python3 -m pip install lxml scikit-ci-addons dask distributed displayName: 'Install dependencies' @@ -102,6 +104,14 @@ jobs: path: $(CCACHE_DIR) displayName: 'Restore ccache' + - task: Cache@2 + inputs: + key: '"castxml-v1" | "$(Agent.OS)" | "LinuxPython" | "$(Build.SourceVersion)"' + restoreKeys: | + "castxml-v1" | "$(Agent.OS)" | "LinuxPython" + path: $(ITK_WRAP_CACHE) + displayName: 'Restore CastXML cache' + - bash: | set -x ccache --zero-stats @@ -123,6 +133,7 @@ jobs: BUILD_SHARED_LIBS:BOOL=OFF BUILD_EXAMPLES:BOOL=OFF ITK_WRAP_PYTHON:BOOL=ON + ITK_WRAP_CASTXML_CACHE:BOOL=ON CMAKE_C_COMPILER_LAUNCHER:STRING=ccache CMAKE_CXX_COMPILER_LAUNCHER:STRING=ccache ITK_COMPUTER_MEMORY_SIZE:STRING=4.5 diff --git a/Testing/ContinuousIntegration/AzurePipelinesMacOSPython.yml b/Testing/ContinuousIntegration/AzurePipelinesMacOSPython.yml index 898ba9743cf..e6c9da98d0d 100644 --- a/Testing/ContinuousIntegration/AzurePipelinesMacOSPython.yml +++ b/Testing/ContinuousIntegration/AzurePipelinesMacOSPython.yml @@ -36,6 +36,7 @@ variables: CCACHE_NOHASHDIR: 'true' CCACHE_SLOPPINESS: pch_defines,time_macros CCACHE_MAXSIZE: 8G + ITK_WRAP_CACHE: $(Pipeline.Workspace)/.castxml-cache jobs: - job: macOS timeoutInMinutes: 0 @@ -80,7 +81,7 @@ jobs: sudo pip3 install ninja numpy>=1.20 typing-extensions sudo python3 -m pip install --upgrade setuptools sudo python3 -m pip install scikit-ci-addons - sudo python3 -m pip install lxml dask distributed + sudo python3 -m pip install lxml dask distributed pytest brew install ccache displayName: 'Install dependencies' @@ -106,6 +107,14 @@ jobs: path: $(CCACHE_DIR) displayName: 'Restore ccache' + - task: Cache@2 + inputs: + key: '"castxml-v1" | "$(Agent.OS)" | "macOSPython" | "$(Build.SourceVersion)"' + restoreKeys: | + "castxml-v1" | "$(Agent.OS)" | "macOSPython" + path: $(ITK_WRAP_CACHE) + displayName: 'Restore CastXML cache' + - bash: | set -x ccache --zero-stats @@ -127,6 +136,7 @@ jobs: BUILD_SHARED_LIBS:BOOL=ON BUILD_EXAMPLES:BOOL=OFF ITK_WRAP_PYTHON:BOOL=ON + ITK_WRAP_CASTXML_CACHE:BOOL=ON CMAKE_C_COMPILER_LAUNCHER:STRING=ccache CMAKE_CXX_COMPILER_LAUNCHER:STRING=ccache ITK_COMPUTER_MEMORY_SIZE:STRING=11 diff --git a/Testing/ContinuousIntegration/AzurePipelinesWindowsPython.yml b/Testing/ContinuousIntegration/AzurePipelinesWindowsPython.yml index 3e834a7dfda..e4d1da9f0b4 100644 --- a/Testing/ContinuousIntegration/AzurePipelinesWindowsPython.yml +++ b/Testing/ContinuousIntegration/AzurePipelinesWindowsPython.yml @@ -36,6 +36,7 @@ variables: CCACHE_NOHASHDIR: 'true' CCACHE_SLOPPINESS: pch_defines,time_macros CCACHE_MAXSIZE: 8G + ITK_WRAP_CACHE: $(Pipeline.Workspace)/.castxml-cache jobs: - job: Windows timeoutInMinutes: 0 @@ -57,7 +58,7 @@ jobs: architecture: 'x64' - script: | - python -m pip install ninja numpy>=1.20 typing-extensions + python -m pip install ninja numpy>=1.20 typing-extensions pytest python -m pip install --upgrade setuptools python -m pip install scikit-ci-addons dask distributed lxml choco install ccache -y @@ -84,6 +85,14 @@ jobs: path: $(CCACHE_DIR) displayName: 'Restore ccache' + - task: Cache@2 + inputs: + key: '"castxml-v1" | "$(Agent.OS)" | "WindowsPython" | "$(Build.SourceVersion)"' + restoreKeys: | + "castxml-v1" | "$(Agent.OS)" | "WindowsPython" + path: $(ITK_WRAP_CACHE) + displayName: 'Restore CastXML cache' + - bash: | set -x ccache --zero-stats @@ -104,6 +113,7 @@ jobs: BUILD_SHARED_LIBS:BOOL=ON BUILD_EXAMPLES:BOOL=OFF ITK_WRAP_PYTHON:BOOL=ON + ITK_WRAP_CASTXML_CACHE:BOOL=ON ITK_BUILD_DEFAULT_MODULES:BOOL=OFF ITKGroup_Core:BOOL=ON Module_ITKThresholding:BOOL=ON diff --git a/Wrapping/CMakeLists.txt b/Wrapping/CMakeLists.txt index 1f4d8c95cd3..306eea179c4 100644 --- a/Wrapping/CMakeLists.txt +++ b/Wrapping/CMakeLists.txt @@ -152,6 +152,12 @@ endif() include(ConfigureWrapping.cmake) +# The cache support module is not installed and its script lives in the ITK +# source tree, so external wrap projects build without the CastXML cache. +if(ITK_WRAP_PYTHON AND NOT EXTERNAL_WRAP_ITK_PROJECT) + include(itkWrapCastXMLCacheSupport) +endif() + ############################################################################### # Configure specific wrapper modules ############################################################################### @@ -233,6 +239,13 @@ if(ITK_WRAP_PYTHON) # ${GLOBAL_IdxFilesList}: is described within SwigInterface/CMakeLists.txt # The variable is passed to pyi_generator to make sure that the function only uses current index files # All index files found that are not in the given list are assumed to be outdated and are removed. + # Prune only for ITK's own build: external wrap projects share ITK_PKL_DIR + # but their manifests cover only their own modules, so pruning there would + # delete every ITK-core row. + set(_pyi_prune_arg) + if(NOT EXTERNAL_WRAP_ITK_PROJECT) + set(_pyi_prune_arg --prune) + endif() add_custom_target( itk-stub-files ALL @@ -242,13 +255,14 @@ if(ITK_WRAP_PYTHON) COMMAND ${Python3_EXECUTABLE} ${PYI_GENERATOR} --pyi_dir "${ITK_STUB_DIR}" --pkl_dir "${ITK_PKL_DIR}" --index_list_file - "${CMAKE_CURRENT_BINARY_DIR}/GlobalIdxFilesList.txt" + "${CMAKE_CURRENT_BINARY_DIR}/GlobalIdxFilesList.txt" ${_pyi_prune_arg} DEPENDS ${PYI_GENERATOR} ${GLOBAL_IdxFilesList} COMMENT "Generating .pyi files for Python wrapper interface" VERBATIM ) + unset(_pyi_prune_arg) # Add module dependencies to ensure .index.txt files have been generated unique(WRAPPED_MODULE_LIST "${WRAP_ITK_MODULES}") @@ -264,6 +278,53 @@ if(ITK_WRAP_PYTHON) endif() endforeach() + if(ITK_WRAP_CASTXML_CACHE) + add_custom_command( + TARGET itk-stub-files + POST_BUILD + COMMAND + ${Python3_EXECUTABLE} "${ITK_WRAP_CASTXML_CACHE_SCRIPT}" --evict + "${ITK_WRAP_CASTXML_CACHE_MAX_DAYS}" + COMMENT + "CastXML cache: evicting entries older than ${ITK_WRAP_CASTXML_CACHE_MAX_DAYS} days, then trimming to the size cap" + VERBATIM + ) + endif() + unset(ITK_STUB_DIR CACHE) unset(ITK_PKL_DIR CACHE) + + # Wrapping infrastructure suite (Generators/Tests): registered only when + # pytest is available in the build interpreter; needs no built ITK. + if(BUILD_TESTING AND NOT EXTERNAL_WRAP_ITK_PROJECT) + execute_process( + COMMAND + ${Python3_EXECUTABLE} -m pytest --version + RESULT_VARIABLE _pytest_missing + OUTPUT_QUIET + ERROR_QUIET + ) + if(_pytest_missing) + message( + STATUS + "pytest not found for ${Python3_EXECUTABLE}: WrappingInfrastructure test not registered" + ) + else() + add_test( + NAME WrappingInfrastructure + COMMAND + ${Python3_EXECUTABLE} -m pytest -q -p no:cacheprovider + ${CMAKE_CURRENT_SOURCE_DIR}/Generators/Tests + ) + set_tests_properties( + WrappingInfrastructure + PROPERTIES + TIMEOUT + 300 + LABELS + Python + ) + endif() + unset(_pytest_missing) + endif() endif() diff --git a/Wrapping/Generators/CastXML/itk-castxml-cache.py b/Wrapping/Generators/CastXML/itk-castxml-cache.py new file mode 100755 index 00000000000..bb565427a88 --- /dev/null +++ b/Wrapping/Generators/CastXML/itk-castxml-cache.py @@ -0,0 +1,843 @@ +#!/usr/bin/env python3 +""" +Two-level content-addressed cache wrapper for ITK's CastXML wrapping step. + +Invocation (replaces ccache + castxml in the cmake COMMAND): + python3 itk-castxml-cache.py /path/to/castxml [castxml args...] + python3 itk-castxml-cache.py --no-cache /path/to/castxml [castxml args...] + python3 itk-castxml-cache.py --evict DAYS [--max-size GB] # e.g. --evict 13.9 --max-size 2.0 + +Cache key hierarchy: + L1 (fast, no subprocess): sha256 of key version + castxml content-hash + + inc file + cxx file + flags. + The castxml binary is fingerprinted by SHA-256 of its content (not mtime), so + `ninja -t clean` re-links the binary without changing the L1 key. Each L1 + entry also records the `-E` header set with content hashes plus a listing + fingerprint of every -I include dir; an L1 hit is honored only when both + still match, so a changed header, or a new header appearing in any include + dir (shadowing), correctly misses instead of restoring stale XML. A new + header inside an existing subdirectory of an include dir is not detected + (same residual gap as compiler depfiles). + L2 (content-addressed): sha256 of key version + castxml content-hash + + normalised `castxml -E` preprocessed output. Preprocessor line markers + (# N "path") are stripped before hashing, making L2 keys path-independent + across build directories; the castxml content-hash ensures a castxml + upgrade never restores XML produced by the previous binary. + +Lookup: + L1 HIT (recorded header hashes still match) → restore L2 → DONE (no subprocess) + L1 miss / header changed → run castxml -E → compute L2 key + L2 HIT → restore; refresh L1 map + L2 miss → run full castxml; store; write L1 map + +Storage formats (ITK_WRAP_CACHE_FORMAT): + gzip (default): output.xml.gz, ~8x smaller than raw XML. Decompressed on + restore. 253 MB for a full 807-module ITK build; each build directory + gets its own decompressed copy (copy is nearly as fast as a hardlink). + uncompressed: output.xml, plain copy on restore. ~2.2 G per full build. + Set ITK_WRAP_CACHE_FORMAT=uncompressed to opt in. + +Multi-path cascade (ITK_WRAP_CACHE, colon-separated): + Reads search all paths in order, returning the first hit. Writes go to the + first path that accepts them (atomic rename succeeds). A read-only shared + NFS cache can be listed after the user's writable local SSD cache, e.g.: + export ITK_WRAP_CACHE=/local/ssd/cache:/nfs/lab/shared-cache + Students benefit from the lab cache for L2 hits (saving the full castxml + run) while writing L1 maps only to their own writable local cache. + +Eviction: run once after a full wrapping build via --evict DAYS. + Skipped immediately when nothing was stored since the previous eviction, so + null builds pay two stat calls instead of a cache-tree walk. Time pass + removes entries older than DAYS (the CMake ITK_WRAP_CASTXML_CACHE_MAX_DAYS + variable supplies DAYS in ITK builds); size pass then removes oldest entries + until total is under ITK_WRAP_CACHE_MAX_SIZE GB (default 2.0). Staging + leftovers and entries lacking their _ok marker are removed after a one-hour + grace period. --evict touches only the first writable cache in the list. + +Environment: + ITK_WRAP_CACHE colon-separated cache roots (default: ~/.cache/itk-wrap) + ITK_WRAP_CACHE_FORMAT storage format: gzip (default) or uncompressed + ITK_WRAP_CACHE_VERBOSE set to 1 for hit/miss logging to stderr + ITK_WRAP_CACHE_BYPASS set to 1 to bypass all caching (same as --no-cache) + ITK_WRAP_CACHE_MAX_SIZE max cache size in GB enforced by --evict (default: 2.0) +""" + +import gzip +import hashlib +import json +import os +import re +import shutil +import subprocess +import sys +import tempfile +import time +import zlib + +# Bump when the key algorithm changes; old entries orphan and age out. +# Mixed into both the L1 and L2 keys so a bump invalidates every lookup path. +_KEY_VERSION = b"v2\x00" + +# Strip "# N " preprocessor line markers so the L2 hash is path-independent. +_LINE_MARKER_RE = re.compile(rb"^# \d+ ", re.MULTILINE) + + +def _strip_line_markers(data: bytes) -> bytes: + return b"\n".join( + line for line in data.splitlines() if not _LINE_MARKER_RE.match(line) + ) + + +# Captures the file path from a preprocessor line marker: # N "path" flags +_DEP_MARKER_RE = re.compile(rb'^# \d+ "([^"]*)"', re.MULTILINE) + + +def _extract_dep_paths(preprocessed: bytes) -> list: + """Return sorted real file paths named in `castxml -E` line markers.""" + paths = set() + for m in _DEP_MARKER_RE.finditer(preprocessed): + p = m.group(1).decode("utf-8", "surrogateescape") + # Line markers escape backslashes on Windows (C:\\path\\file.h) + p = p.replace("\\\\", "\\") + if p and not p.startswith("<"): # skip , + paths.add(p) + return sorted(paths) + + +def _hash_file(path: str) -> str: + h = hashlib.sha256() + with open(path, "rb") as f: + for chunk in iter(lambda: f.read(1 << 16), b""): + h.update(chunk) + return h.hexdigest() + + +def _deps_manifest(paths: list) -> list: + """Return [[path, sha256], ...] for the preprocessed dependency set.""" + manifest = [] + for p in paths: + try: + manifest.append([p, _hash_file(p)]) + except OSError: + manifest.append([p, ""]) + return manifest + + +def _deps_unchanged(deps: list) -> bool: + """True when every recorded dependency still hashes to its stored value.""" + for dep in deps: + if not (isinstance(dep, list) and len(dep) == 2): + return False # malformed manifest entry: force a miss + path, expected = dep + try: + actual = _hash_file(path) + except OSError: + actual = "" # matches _deps_manifest's record for unreadable paths + if actual != expected: + return False + return True + + +def _incdirs_from_inc_file(inc_file: str) -> list: + """Return the ordered, deduplicated -I/-isystem directories from an @inc response file.""" + dirs = [] + seen = set() + + def _add(d: str) -> None: + d = d.strip() + if d and d not in seen: + seen.add(d) + dirs.append(d) + + try: + with open(inc_file, encoding="utf-8", errors="surrogateescape") as f: + for line in f: + tokens = re.findall(r'"([^"]*)"', line) or [line.strip()] + it = iter(tokens) + for tok in it: + if tok == "-isystem": + _add(next(it, "")) + elif tok.startswith("-isystem"): + _add(tok[len("-isystem") :]) + elif tok.startswith("-I"): + _add(tok[2:]) + except OSError: + pass + return dirs + + +def _incdirs_fingerprint(inc_file: str) -> str: + """Hash of the top-level entry names of every -I dir. + + A header appearing in (or vanishing from) any include dir changes the + fingerprint, so a newly-added shadowing header forces an L1 miss and the + `castxml -E` pass re-resolves includes against the real search path. + """ + h = hashlib.sha256(_KEY_VERSION) + for d in _incdirs_from_inc_file(inc_file): + h.update(d.encode("utf-8", "surrogateescape")) + h.update(b"\x00") + try: + names = sorted(os.listdir(d)) + except OSError: + names = [] + for name in names: + h.update(name.encode("utf-8", "surrogateescape")) + h.update(b"\x00") + h.update(b"\x01") + return h.hexdigest() + + +def _cache_roots(): + """Return ordered list of cache root directories from ITK_WRAP_CACHE. + + The env var is a colon-separated list (like PATH). Reads search all + roots in order; writes go to the first root that accepts them. + """ + raw = os.environ.get("ITK_WRAP_CACHE", "") + if not raw: + return [os.path.join(os.path.expanduser("~"), ".cache", "itk-wrap")] + sep = ";" if sys.platform == "win32" else ":" + return [p for p in raw.split(sep) if p] + + +def _verbose(): + return os.environ.get("ITK_WRAP_CACHE_VERBOSE", "") == "1" + + +def _log(msg): + if _verbose(): + print(f"itk-castxml-cache: {msg}", file=sys.stderr) + + +def _use_uncompressed(): + """Return True when uncompressed storage is explicitly requested via ITK_WRAP_CACHE_FORMAT.""" + return os.environ.get("ITK_WRAP_CACHE_FORMAT", "").lower() in ( + "uncompressed", + "raw", + "plain", + ) + + +def _max_cache_gb(): + """Parse ITK_WRAP_CACHE_MAX_SIZE (default 2.0) into GB as a float.""" + try: + return float(os.environ.get("ITK_WRAP_CACHE_MAX_SIZE", "2.0")) + except ValueError: + return 2.0 + + +def _remove_if_stale(path, grace_cutoff): + """Remove a dead dir (crashed staging / missing _ok) past the grace period.""" + try: + if os.stat(path).st_mtime < grace_cutoff: + shutil.rmtree(path, ignore_errors=True) + return 1 + except OSError: + pass + return 0 + + +def _evict(cache_root, max_size=2.0, days=13.9): + """Evict cache entries; run once after a full wrapping build via --evict. + + Exits immediately when nothing was stored since the previous eviction. + L2: remove entries whose _ok mtime is older than `days`, then remove the + oldest by mtime until the total is under `max_size` GB. + L1: remove mapping entries whose mtime is older than `days`. + Staging leftovers and entries without _ok are removed after a one-hour + grace period so crashed stores cannot grow the cache unbounded. + """ + l2_root = os.path.join(cache_root, "l2") + if not os.path.isdir(l2_root): + return + + last_store = os.path.join(cache_root, "_last_store") + last_evict = os.path.join(cache_root, "_last_evict") + try: + evict_mtime = os.stat(last_evict).st_mtime + except OSError: + evict_mtime = None + if evict_mtime is not None: + try: + store_mtime = os.stat(last_store).st_mtime + except OSError: + store_mtime = 0.0 + if evict_mtime >= store_mtime: + _log("evict: no stores since last eviction — skipping") + return + + max_bytes = int(max_size * (1 << 30)) + cutoff = time.time() - days * 86400 + grace_cutoff = time.time() - 3600 + kept = [] + total = 0 + removed = 0 + + for shard in os.listdir(l2_root): + shard_dir = os.path.join(l2_root, shard) + if not os.path.isdir(shard_dir): + continue + for key in os.listdir(shard_dir): + entry = os.path.join(shard_dir, key) + if ".tmp" in key or key.endswith(".old"): + removed += _remove_if_stale(entry, grace_cutoff) + continue + ok = os.path.join(entry, "_ok") + try: + mtime = os.stat(ok).st_mtime + except OSError: + removed += _remove_if_stale(entry, grace_cutoff) + continue + try: + entry_bytes = sum( + os.path.getsize(os.path.join(entry, fn)) for fn in os.listdir(entry) + ) + if mtime < cutoff: + shutil.rmtree(entry) + removed += 1 + else: + kept.append((mtime, entry_bytes, entry)) + total += entry_bytes + except OSError: + pass + + if total > max_bytes: + kept.sort(key=lambda x: x[0]) # oldest first + for _mtime, entry_bytes, entry in kept: + if total <= max_bytes: + break + try: + shutil.rmtree(entry) + total -= entry_bytes + removed += 1 + except OSError: + pass + + l1_root = os.path.join(cache_root, "l1") + if os.path.isdir(l1_root): + for shard in os.listdir(l1_root): + shard_dir = os.path.join(l1_root, shard) + if not os.path.isdir(shard_dir): + continue + for key_name in os.listdir(shard_dir): + key_dir = os.path.join(shard_dir, key_name) + l1f = os.path.join(key_dir, "l2_key") + try: + if os.stat(l1f).st_mtime < cutoff: + shutil.rmtree(key_dir) + removed += 1 + except OSError: + removed += _remove_if_stale(key_dir, grace_cutoff) + + try: + with open(last_evict, "w"): + pass + except OSError: + pass + + _log( + f"evict: removed {removed} entries" + f" (age>{days}d or size>{max_size:.1f}GB);" + f" {total / (1 << 30):.2f} GB remaining" + ) + + +def _bypass_mode(): + """Return True when caching should be skipped entirely. + + Controlled by --no-cache as the first positional arg (before castxml binary) + or by ITK_WRAP_CACHE_BYPASS=1 in the environment. Use for single-use builds + where the castxml -E overhead would cost more than the cache saves. + """ + return os.environ.get("ITK_WRAP_CACHE_BYPASS", "") == "1" + + +def _parse_args(argv): + """ + Parse a castxml command line into structured components. + + Strips a leading --no-cache flag (before the castxml binary) when present. + Returns (castxml_bin, output_xml, inc_file, cxx_file, passthrough_flags, no_cache) + where passthrough_flags preserves the original ordering for subprocess use. + """ + no_cache = False + if argv and argv[0] == "--no-cache": + no_cache = True + argv = argv[1:] + + if not argv: + return None, None, None, None, [], no_cache + + castxml_bin = argv[0] + output_xml = None + inc_file = None + cxx_file = None + passthrough_flags = [] + + i = 1 + while i < len(argv): + arg = argv[i] + if arg == "-o" and i + 1 < len(argv): + output_xml = argv[i + 1] + # Include in passthrough so castxml writes its output normally + passthrough_flags.extend([arg, argv[i + 1]]) + i += 2 + elif arg.startswith("@"): + inc_file = arg[1:] + passthrough_flags.append(arg) + i += 1 + elif arg.endswith(".cxx"): + cxx_file = arg + passthrough_flags.append(arg) + i += 1 + else: + passthrough_flags.append(arg) + i += 1 + + return castxml_bin, output_xml, inc_file, cxx_file, passthrough_flags, no_cache + + +def _castxml_content_hash(castxml_bin, primary_root): + """Return a stable SHA-256 of the castxml binary, cached on disk. + + Sidecar file stores "size mtime_ns sha256" so re-hashing only happens when + size or mtime changes. After `ninja -t clean`, castxml is re-linked with + the same content → same hash → stable L1 key → L1 hits on warm rebuilds. + Sidecar lives in the first (writable) cache root. + """ + try: + st = os.stat(castxml_bin) + except OSError: + return "missing" + + # One sidecar per binary path (path key avoids slashes in filename) + path_key = hashlib.sha256(castxml_bin.encode()).hexdigest()[:16] + sidecar = os.path.join(primary_root, "_binhash", path_key) + + try: + with open(sidecar) as f: + parts = f.read().split() + if ( + len(parts) == 3 + and int(parts[0]) == st.st_size + and int(parts[1]) == st.st_mtime_ns + ): + return parts[2] + except (OSError, ValueError): + pass + + # Sidecar miss or stale — hash the binary (one-time cost per unique binary) + h = hashlib.sha256() + try: + with open(castxml_bin, "rb") as f: + for chunk in iter(lambda: f.read(1 << 20), b""): + h.update(chunk) + except OSError: + return "unreadable" + + content_hash = h.hexdigest() + try: + os.makedirs(os.path.dirname(sidecar), exist_ok=True) + # Unique staging name + os.replace: concurrency-safe and refreshable on Windows + fd, tmp = tempfile.mkstemp(dir=os.path.dirname(sidecar), suffix=".tmp") + try: + with os.fdopen(fd, "w") as f: + f.write(f"{st.st_size} {st.st_mtime_ns} {content_hash}\n") + os.replace(tmp, sidecar) + except OSError: + os.unlink(tmp) + except OSError: + pass + return content_hash + + +def _l1_key(bin_hash, inc_file, cxx_file, passthrough_flags): + """Compute L1 cache key from direct inputs only (~0.2s, no subprocess).""" + h = hashlib.sha256(_KEY_VERSION) + + # Stable content fingerprint — survives re-link with unchanged binary. + h.update(f"castxml\x00{bin_hash}\x00".encode()) + + # Response file: include dirs + defines passed via @file + if inc_file: + try: + with open(inc_file, "rb") as f: + h.update(f.read()) + except OSError: + h.update(b"\x00inc_miss\x00") + h.update(b"\x00") + + if cxx_file: + try: + with open(cxx_file, "rb") as f: + h.update(f.read()) + except OSError: + h.update(b"\x00cxx_miss\x00") + h.update(b"\x00") + + # Remaining flags (--castxml-cc-gnu, compiler path, std flags, etc.) + # Skip -o and the output xml path — irrelevant to content. + skip_next = False + for flag in passthrough_flags: + if skip_next: + skip_next = False + continue + if flag == "-o": + skip_next = True + continue + if flag.endswith(".xml"): + continue + h.update(flag.encode()) + h.update(b"\x00") + + return h.hexdigest() + + +def _build_preprocess_cmd(castxml_bin, passthrough_flags, pre_output): + """ + Build a `castxml -E` command that preprocesses the same inputs. + + Strips --castxml-output, --castxml-start (XML-generation flags), + replaces -o with the temp preprocess output path, appends -E. + """ + cmd = [castxml_bin] + skip_next = False + for arg in passthrough_flags: + if skip_next: + skip_next = False + continue + if arg.startswith("--castxml-output"): + continue + if arg.startswith("--castxml-start"): + if "=" not in arg: + skip_next = True + continue + if arg == "-o": + skip_next = True # drop -o and its value (xml output path) + continue + if arg.endswith(".xml"): + continue + cmd.append(arg) + cmd.extend(["-E", "-o", pre_output]) + return cmd + + +def _compute_l2_key(castxml_bin, passthrough_flags, bin_hash): + """Run castxml -E; return (l2_key, deps_manifest) or (None, None) on failure.""" + pre_path = None + try: + with tempfile.NamedTemporaryFile(suffix=".i", delete=False) as tmp: + pre_path = tmp.name + cmd = _build_preprocess_cmd(castxml_bin, passthrough_flags, pre_path) + result = subprocess.run(cmd, capture_output=True) + if result.returncode != 0: + _log(f"castxml -E failed (exit {result.returncode})") + return None, None + with open(pre_path, "rb") as f: + pre = f.read() + h = hashlib.sha256(_KEY_VERSION) + # Same preprocessed text from a different castxml must never collide. + h.update(f"castxml\x00{bin_hash}\x00".encode()) + h.update(_strip_line_markers(pre)) + return h.hexdigest(), _deps_manifest(_extract_dep_paths(pre)) + except OSError: + return None, None + finally: + if pre_path is not None: + try: + os.unlink(pre_path) + except OSError: + pass + + +def _l1_file(cache_root, l1_key): + return os.path.join(cache_root, "l1", l1_key[:2], l1_key, "l2_key") + + +def _l2_dir(cache_root, l2_key): + return os.path.join(cache_root, "l2", l2_key[:2], l2_key) + + +def _restore_xml(cache_root, l2_key, output_xml): + """Restore cached XML to output_xml from one cache root. Returns True on success.""" + entry = _l2_dir(cache_root, l2_key) + ok_file = os.path.join(entry, "_ok") + xml_plain = os.path.join(entry, "output.xml") + xml_gz = os.path.join(entry, "output.xml.gz") + + if not os.path.isfile(ok_file): + return False + + try: + os.makedirs(os.path.dirname(os.path.abspath(output_xml)), exist_ok=True) + if os.path.isfile(xml_plain): + # copyfile (not copy2): a fresh mtime keeps ninja's edge up to date + shutil.copyfile(xml_plain, output_xml) + elif os.path.isfile(xml_gz): + with gzip.open(xml_gz, "rb") as src, open(output_xml, "wb") as dst: + shutil.copyfileobj(src, dst) + else: + return False + except (OSError, zlib.error, EOFError): + return False # truncated/corrupt cache entry: miss, fall through to -E + + try: + if os.path.getsize(output_xml) == 0: + return False # empty decompression: treat as corrupt + except OSError: + return False + + try: + os.utime(ok_file, None) + except OSError: + pass + + return True + + +def _restore_from_caches(roots, l2_key, output_xml): + """Search all cache roots for l2_key, restore on first hit.""" + for root in roots: + if _restore_xml(root, l2_key, output_xml): + return root # return the root that had the hit + return None + + +def _write_l1_mapping(l1f, l2_key, deps, incfp): + """Atomically write the L1 manifest (L2 key, dependency hashes, incdir fingerprint).""" + os.makedirs(os.path.dirname(l1f), exist_ok=True) + # Unique staging name + os.replace: concurrency-safe and refreshable on Windows + fd, tmp = tempfile.mkstemp(dir=os.path.dirname(l1f), suffix=".tmp") + try: + with os.fdopen(fd, "w") as f: + json.dump({"l2_key": l2_key, "deps": deps, "incfp": incfp}, f) + os.replace(tmp, l1f) + except OSError: + try: + os.unlink(tmp) + except OSError: + pass + raise + + +def _store(roots, l1_key, l2_key, output_xml, deps, incfp): + """Store output_xml to L2 cache and write L1→L2 mapping atomically. + + Tries each root in order and writes to the first that accepts an atomic + rename. Read-only roots (e.g. a shared NFS lab cache) are silently + skipped. + """ + uncompressed = _use_uncompressed() + + # L2 entry — write to first writable root + write_root = None + for root in roots: + entry = _l2_dir(root, l2_key) + tmp = None + try: + if not os.path.isfile(output_xml): + continue + os.makedirs(os.path.dirname(entry), exist_ok=True) + # Unique staging dir: concurrent stores of the same key cannot + # delete or interleave into each other's staging area. + tmp = tempfile.mkdtemp(dir=os.path.dirname(entry), prefix=l2_key + ".tmp") + # mkdtemp creates 0700; restore umask-derived perms for shared caches + umask = os.umask(0) + os.umask(umask) + os.chmod(tmp, 0o777 & ~umask) + + if uncompressed: + shutil.copy2(output_xml, os.path.join(tmp, "output.xml")) + else: + with ( + open(output_xml, "rb") as src, + gzip.open( + os.path.join(tmp, "output.xml.gz"), "wb", compresslevel=6 + ) as dst, + ): + shutil.copyfileobj(src, dst) + + with open(os.path.join(tmp, "_meta.json"), "w") as f: + json.dump( + { + "l1_key": l1_key, + "l2_key": l2_key, + "format": "uncompressed" if uncompressed else "gzip", + }, + f, + ) + open(os.path.join(tmp, "_ok"), "w").close() # noqa: WPS515 + + old = entry + ".old" + shutil.rmtree( + old, ignore_errors=True + ) # clear stale .old from a prior crash + if os.path.exists(entry): + os.rename(entry, old) + try: + os.rename(tmp, entry) + except OSError: + if os.path.exists(old): + os.rename(old, entry) + raise + shutil.rmtree(old, ignore_errors=True) + write_root = root + break + except OSError as exc: + _log(f"L2 store failed for {root}: {exc}") + if tmp is not None: + shutil.rmtree(tmp, ignore_errors=True) + + if write_root is None: + _log("L2 store failed in all cache roots — no entry written") + return + + _touch_store_marker(write_root) + + # L1→L2 mapping — write to same root that accepted the L2 entry + try: + _write_l1_mapping(_l1_file(write_root, l1_key), l2_key, deps, incfp) + except OSError as exc: + _log(f"L1 map store failed: {exc}") + + +def _touch_store_marker(root): + """Record that this root received a write, so --evict knows to run.""" + try: + with open(os.path.join(root, "_last_store"), "w"): + pass + except OSError: + pass + + +def _store_l1_mapping(roots, l1_key, l2_key, deps, incfp): + """Write an L1→L2 mapping to the first writable root (no L2 write).""" + for root in roots: + try: + _write_l1_mapping(_l1_file(root, l1_key), l2_key, deps, incfp) + _touch_store_marker(root) + return + except OSError: + continue + + +def _run_castxml(castxml_bin, passthrough_flags): + result = subprocess.run([castxml_bin] + passthrough_flags) + return result.returncode + + +def main(): + argv = sys.argv[1:] + if not argv: + print(__doc__, file=sys.stderr) + return 1 + + # Stand-alone eviction subcommand: python3 itk-castxml-cache.py --evict DAYS + if argv[0] == "--evict": + import argparse + + p = argparse.ArgumentParser(prog="itk-castxml-cache.py --evict") + p.add_argument( + "days", + type=float, + help="Remove entries not used in this many days (e.g. 13.9)", + ) + p.add_argument( + "--max-size", + type=float, + default=None, + help="Max cache size in GB after time eviction (default: ITK_WRAP_CACHE_MAX_SIZE or 2.0)", + ) + p.add_argument( + "--cache-dir", default=None, help="Cache root (overrides ITK_WRAP_CACHE)" + ) + args = p.parse_args(argv[1:]) + max_size = args.max_size if args.max_size is not None else _max_cache_gb() + roots = [args.cache_dir] if args.cache_dir else _cache_roots() + _evict(roots[0], max_size=max_size, days=args.days) + return 0 + + castxml_bin, output_xml, inc_file, cxx_file, passthrough_flags, no_cache = ( + _parse_args(argv) + ) + + if castxml_bin is None: + print("itk-castxml-cache: no castxml binary specified", file=sys.stderr) + return 1 + if no_cache or _bypass_mode() or not output_xml or not cxx_file: + return _run_castxml(castxml_bin, passthrough_flags) + + roots = _cache_roots() + # primary_root is used for binary sidecar storage; writes go to first writable + primary_root = roots[0] + + bin_hash = _castxml_content_hash(castxml_bin, primary_root) + incfp = _incdirs_fingerprint(inc_file) if inc_file else "" + + # ── L1 check (fast, no subprocess) ────────────────────────────────────── + l1_key = _l1_key(bin_hash, inc_file, cxx_file, passthrough_flags) + + stored = None + stored_l1f = None + for root in roots: + l1f = _l1_file(root, l1_key) + if os.path.isfile(l1f): + try: + with open(l1f) as f: + entry = json.load(f) + except (OSError, ValueError): + continue + if isinstance(entry, dict) and entry.get("l2_key"): + stored = entry + stored_l1f = l1f + break + + # ── L1 HIT: skip castxml -E only when recorded headers are unchanged and + # no header has appeared in (or left) any include dir since the store ── + if ( + stored is not None + and stored.get("deps") + and stored.get("incfp") == incfp + and _deps_unchanged(stored["deps"]) + ): + hit_root = _restore_from_caches(roots, stored["l2_key"], output_xml) + if hit_root is not None: + try: + os.utime(stored_l1f, None) # mark used so age eviction keeps it + except OSError: + pass + _log(f"HIT {os.path.basename(cxx_file)} (l1→l2={stored['l2_key'][:8]})") + return 0 + # L2 entry missing or corrupt despite L1 hit — fall through to -E check. + _log(f"L2 entry missing for {cxx_file}, re-running castxml -E") + + # ── castxml -E to compute actual L2 key (L1 miss or header changed) ───── + actual_l2_key, deps = _compute_l2_key(castxml_bin, passthrough_flags, bin_hash) + + if actual_l2_key is None: + _log(f"preprocess failed for {cxx_file} — passing through") + return _run_castxml(castxml_bin, passthrough_flags) + + # ── L2 store lookup (handles cross-dir hits: L1 miss, L2 populated) ───── + hit_root = _restore_from_caches(roots, actual_l2_key, output_xml) + if hit_root is not None: + _log(f"HIT {os.path.basename(cxx_file)} (l2={actual_l2_key[:8]})") + # Populate L1 map so the next rebuild skips castxml -E + _store_l1_mapping(roots, l1_key, actual_l2_key, deps, incfp) + return 0 + + # ── Full castxml run ───────────────────────────────────────────────────── + _log(f"MISS {os.path.basename(cxx_file)}") + try: + os.unlink(output_xml) + except OSError: + pass + rc = _run_castxml(castxml_bin, passthrough_flags) + if rc == 0: + _store(roots, l1_key, actual_l2_key, output_xml, deps, incfp) + return rc + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/Wrapping/Generators/Python/itk/pyi_generator.py b/Wrapping/Generators/Python/itk/pyi_generator.py index d8007fb6aed..ce2aa4787e5 100644 --- a/Wrapping/Generators/Python/itk/pyi_generator.py +++ b/Wrapping/Generators/Python/itk/pyi_generator.py @@ -5,17 +5,20 @@ """ import os +import sys +import sqlite3 from os import remove from argparse import ArgumentParser from io import StringIO from pathlib import Path, PurePath import pickle import glob -import re from collections import defaultdict +sys.path.insert(0, str(Path(__file__).parents[2])) +from pkl_db import open_pkl_db # noqa: E402 + -# The ITKClass is duplicated in igenerator.py class ITKClass: def __init__(self, l_class_name): # Structure @@ -379,13 +382,21 @@ def write_class_proxy_pyi( pyi_file.write(interfaces_code) -def unpack(file_names: [str], save_dir: str) -> str | None: - class_definitions = [] +def unpack(keys: list[str], db_conn: sqlite3.Connection, save_dir: str) -> str | None: + placeholders = ",".join("?" * len(keys)) + rows = { + k: data + for k, data in db_conn.execute( + f"SELECT key, data FROM pkl_data WHERE key IN ({placeholders})", keys + ) + } - for file_name in file_names: - with open(file_name, "rb") as pickled_file: - itk_class = pickle.load(pickled_file) - class_definitions.append(itk_class) + class_definitions = [] + for key in keys: + if key not in rows: + print(f"WARNING: pkl key {key!r} not found in database, skipping.") + continue + class_definitions.append(pickle.loads(rows[key])) base = merge(class_definitions) @@ -419,9 +430,9 @@ def unpack(file_names: [str], save_dir: str) -> str | None: return base.class_name else: - files = "\n\t".join(file_names) + files = "\n\t".join(keys) print( - f"WARNING: The following files do not contain a class definition for Python stub wrapping:\n" + f"WARNING: The following keys do not contain a class definition for Python stub wrapping:\n" f"\t{files}" ) @@ -522,6 +533,13 @@ def merge(class_definitions: []) -> ITKClass | None: type=str, help="Configured file listing the index files containing pickle file references", ) + cmdln_arg_parser.add_argument( + "--prune", + action="store_true", + dest="prune", + help="Delete pkl DB rows whose keys are absent from the index manifests. " + "Only safe when the manifests cover every module sharing the DB.", + ) cmdln_arg_parser.add_argument( "-d", action="store_true", @@ -530,7 +548,7 @@ def merge(class_definitions: []) -> ITKClass | None: ) options = cmdln_arg_parser.parse_args() - if not Path(options.pkl_dir).exists: + if not Path(options.pkl_dir).exists(): except_comment = f"Invalid directory provided '{options.pkl_dir}'" raise Exception(except_comment) @@ -556,7 +574,6 @@ def merge(class_definitions: []) -> ITKClass | None: remove(invalid_index_file) for missing_index_file in missing_index_files: - # continue on without missing file, display warning print( f"WARNING: index file {missing_index_file} is missing, " f"Python stub hints will not be generated for this file. " @@ -568,52 +585,84 @@ def merge(class_definitions: []) -> ITKClass | None: print(f"Exception: {except_comment}") raise Exception(except_comment) - indexed_pickled_files = set() + # Collect DB keys from all .index.txt manifests (keys, not file paths). + indexed_pkl_keys: set[str] = set() for index_file in sorted(index_files): with open(index_file) as file: for line in file: - indexed_pickled_files.add(line.strip()) + key = line.strip() + if key: + indexed_pkl_keys.add(key) - existing_pickled_files = { - filepath.replace(os.sep, "/") - for filepath in glob.glob(f"{options.pkl_dir}/*.pkl") - } - - invalid_pickled_files = existing_pickled_files - indexed_pickled_files - missing_pickled_files = indexed_pickled_files - existing_pickled_files - - if options.debug_code: - for invalid_pickle_file in invalid_pickled_files: - print( - f"WARNING: Outdated pickle file {invalid_pickle_file} has been removed" - ) - remove(invalid_pickle_file) + if len(indexed_pkl_keys) == 0: + raise Exception(f"No pkl keys were found in index files in '{options.pkl_dir}'") - for missing_file in missing_pickled_files: - # continue on without missing file, display warning - print( - f"WARNING: pickle file {missing_file} is missing, Python stub hints will not be generated for this file." - ) - indexed_pickled_files.remove(missing_file) - - if len(indexed_pickled_files) == 0: - raise Exception(f"No pickle files were found in directory {options.pkl_dir}") - - indexed_pickled_files = sorted(list(indexed_pickled_files)) + indexed_pkl_keys_sorted = sorted(indexed_pkl_keys) output_template_import_list: list[str] = [] output_proxy_import_list: list[str] = [] - class_name_dict = defaultdict(list) - for file in indexed_pickled_files: - current_class_name = re.split(r"\.", PurePath(file).parts[-1])[0] - class_name_dict[current_class_name].append(file) + class_name_dict: defaultdict[str, list[str]] = defaultdict(list) + for key in indexed_pkl_keys_sorted: + current_class_name = key.split(".")[0] + class_name_dict[current_class_name].append(key) + + db_conn = open_pkl_db(options.pkl_dir) + + # Indexed keys absent from the DB mean it was deleted or damaged while the + # stamp files stayed fresh, so ninja would never re-run igenerator; drop + # the stamps and fail so the next build regenerates the database. + present_keys: set[str] = set() + for chunk_start in range(0, len(indexed_pkl_keys_sorted), 500): + chunk = indexed_pkl_keys_sorted[chunk_start : chunk_start + 500] + placeholders = ",".join("?" * len(chunk)) + present_keys.update( + row[0] + for row in db_conn.execute( + f"SELECT key FROM pkl_data WHERE key IN ({placeholders})", chunk + ) + ) + absent_keys = set(indexed_pkl_keys_sorted) - present_keys + if absent_keys: + db_conn.close() + for stamp_file in glob.glob(f"{options.pkl_dir}/*.stamp"): + remove(stamp_file) + raise Exception( + f"{len(absent_keys)} indexed pkl keys are missing from the pkl " + f"database in '{options.pkl_dir}'. The igenerator stamp files " + "were removed; re-run the build to regenerate the database." + ) - for current_class_name, class_files in class_name_dict.items(): - class_name = unpack(class_files, options.pyi_dir) + for current_class_name, class_keys in class_name_dict.items(): + class_name = unpack(class_keys, db_conn, options.pyi_dir) if class_name is not None: output_template_import_list.append(f"from .{class_name}Template import *\n") output_proxy_import_list.append(f"from .{class_name}Proxy import *\n") + if not output_template_import_list: + raise Exception( + f"No Python stub classes were generated from pkl keys in '{options.pkl_dir}'" + ) + + if options.prune: + if missing_index_files: + print( + "WARNING: skipping pkl DB pruning: the index manifest set is " + "incomplete, so live keys cannot be determined safely." + ) + else: + with db_conn: + db_conn.execute("CREATE TEMP TABLE _live_keys (key TEXT PRIMARY KEY)") + db_conn.executemany( + "INSERT OR IGNORE INTO _live_keys VALUES(?)", + [(k,) for k in indexed_pkl_keys_sorted], + ) + db_conn.execute( + "DELETE FROM pkl_data WHERE key NOT IN (SELECT key FROM _live_keys)" + ) + db_conn.execute("DROP TABLE _live_keys") + + db_conn.close() + output_init_import_file = init_init_import_file() output_proxy_import_file = init_proxy_import_file() diff --git a/Wrapping/Generators/SwigInterface/CMakeLists.txt b/Wrapping/Generators/SwigInterface/CMakeLists.txt index af8526463b6..0f554984117 100644 --- a/Wrapping/Generators/SwigInterface/CMakeLists.txt +++ b/Wrapping/Generators/SwigInterface/CMakeLists.txt @@ -397,6 +397,13 @@ set( "igenerator.py path" FORCE ) +set( + IGENERATOR_PKL_DB + "${CMAKE_CURRENT_SOURCE_DIR}/../pkl_db.py" + CACHE INTERNAL + "pkl_db.py path (shared module imported by igenerator.py)" + FORCE +) macro(itk_wrap_simple_type_swig_interface wrap_class swig_name) string(APPEND SWIG_INTERFACE_TYPEDEFS "using ${swig_name} = ${wrap_class};\n") diff --git a/Wrapping/Generators/SwigInterface/igenerator.py b/Wrapping/Generators/SwigInterface/igenerator.py index bcda8695eab..f05465ed710 100755 --- a/Wrapping/Generators/SwigInterface/igenerator.py +++ b/Wrapping/Generators/SwigInterface/igenerator.py @@ -62,12 +62,14 @@ import re from argparse import ArgumentParser from io import StringIO -from os.path import exists from pathlib import Path from keyword import iskeyword from typing import Any import logging +sys.path.insert(0, str(Path(__file__).parents[1])) +from pkl_db import open_pkl_db # noqa: E402 + def argument_parser(): cmdln_arg_parser = ArgumentParser() @@ -205,7 +207,16 @@ def argument_parser(): dest="pkl_dir", default="", type=str, - help="The directory for .pyi files to be generated", + help="Directory for per-submodule .index.txt manifests and per-module stamp files.", + ) + cmdln_arg_parser.add_argument( + "--pkl_stamp", + action="store", + dest="pkl_stamp", + default="", + type=str, + help="Stamp file touched after the pkl DB transaction commits; " + "declared as a CMake OUTPUT so ninja re-runs igenerator when stale.", ) cmdln_arg_parser.add_argument( "-d", @@ -219,8 +230,16 @@ def argument_parser(): glb_options = argument_parser() -sys.path.insert(1, glb_options.pygccxml_path) -import pygccxml # noqa: E402 +pygccxml = None # populated lazily by _load_pygccxml() + + +def _load_pygccxml(): + """Import pygccxml into the global namespace; loaded lazily, not at import time.""" + global pygccxml + import importlib + + sys.path.insert(1, glb_options.pygccxml_path) + pygccxml = importlib.import_module("pygccxml") # Global debugging variables @@ -1901,7 +1920,7 @@ def get_submodule_namespace( def generate_swig_input( submodule_name, - pkl_dir, + pkl_db_rows, pygccxml_config, options, snake_case_process_object_functions, @@ -1924,45 +1943,23 @@ def generate_swig_input( swig_input_generator.snakeCaseProcessObjectFunctions ) - # Write index list of generated .pkl files index_file_contents: StringIO = StringIO() all_keys = swig_input_generator.classes.keys() if len(all_keys): for itk_class in all_keys: - # Future problem will be that a few files will be empty - # Can either somehow detect this or accept it - # pickle class here class_name: str = swig_input_generator.classes[itk_class].class_name submodule_name: str = swig_input_generator.classes[itk_class].submodule_name - pickled_filename: str = f"{pkl_dir}/{class_name}.{submodule_name}.pkl" - - # Only write to the pickle file if it does not match what is already saved. - overwrite: bool = False - pickle_exists: bool = exists(pickled_filename) - if pickle_exists: - with open(pickled_filename, "rb") as pickled_file: - existing_itk_class = pickle.load(pickled_file) - overwrite = not ( - existing_itk_class == swig_input_generator.classes[itk_class] - ) - if overwrite or not pickle_exists: - with open(pickled_filename, "wb") as pickled_file: - pickle.dump(swig_input_generator.classes[itk_class], pickled_file) - - index_file_contents.write(pickled_filename + "\n") + key: str = f"{class_name}.{submodule_name}" + pkl_db_rows.append( + (key, pickle.dumps(swig_input_generator.classes[itk_class])) + ) + index_file_contents.write(key + "\n") else: - # The following warning is useful for debugging, and eventually we - # may wish to find a way to remove modules that are not currently part - # of the build. For example, currently all *.wrap files are processed and listed - # as module dependencies. If FFTW is not enabled, that causes empty submodules - # to be created as dependencies unnecessarily. - # Changing that behavior will require structural code changes, or alternate - # mechanisms to be implemented. if glb_options.debug_code: print( f"WARNING: {submodule_name} has no classes identified, but was listed as a dependent submodule." ) - generate_pyi_index_files(submodule_name, index_file_contents, pkl_dir) + generate_pyi_index_files(submodule_name, index_file_contents, options.pkl_dir) def main(): @@ -1971,7 +1968,7 @@ def main(): raise ValueError(f"Required directory missing '{options.pyi_dir}'") if options.pkl_dir == "": - raise ValueError(f"Required directory missing '{options.pkl_dir}'") + raise ValueError("--pkl_dir is required for .index.txt manifests") # Ensure that the requested stub file directory exists if options.pyi_dir != "": @@ -1979,18 +1976,6 @@ def main(): if options.pkl_dir != "": Path(options.pkl_dir).mkdir(exist_ok=True, parents=True) - # init the pygccxml stuff - pygccxml.utils.loggers.cxx_parser.setLevel(logging.CRITICAL) - pygccxml.declarations.scopedef_t.RECURSIVE_DEFAULT = False - pygccxml.declarations.scopedef_t.ALLOW_EMPTY_MDECL_WRAPPER = True - - pygccxml_config = pygccxml.parser.config.xml_generator_configuration_t( - xml_generator_path=options.castxml_path, - xml_generator="castxml", - # Use castxml-output=1 to take advantage of the newer XML format - flags=["--castxml-output=1"], - ) - submodule_names_list: list[str] = [] # The first mdx file is the master index file for this module. master_mdx_filename: Path = Path(options.mdx[0]) @@ -2011,6 +1996,18 @@ def main(): if submodule_name not in submodule_names_list: submodule_names_list.append(submodule_name) + _load_pygccxml() + + pygccxml.utils.loggers.cxx_parser.setLevel(logging.CRITICAL) + pygccxml.declarations.scopedef_t.RECURSIVE_DEFAULT = False + pygccxml.declarations.scopedef_t.ALLOW_EMPTY_MDECL_WRAPPER = True + + pygccxml_config = pygccxml.parser.config.xml_generator_configuration_t( + xml_generator_path=options.castxml_path, + xml_generator="castxml", + flags=["--castxml-output=1"], + ) + for submodule_name in submodule_names_list: wrappers_namespace: Any = global_submodule_cache.get_submodule_namespace( submodule_name, options.library_output_dir, pygccxml_config @@ -2032,15 +2029,26 @@ def main(): ordered_submodule_list.append(submodule_name) del submodule_names_list + pkl_db_rows: list[tuple[str, bytes]] = [] for submodule_name in ordered_submodule_list: generate_swig_input( submodule_name, - options.pkl_dir, + pkl_db_rows, pygccxml_config, options, snake_case_process_object_functions, ) + if pkl_db_rows: + conn = open_pkl_db(options.pkl_dir) + with conn: + conn.executemany( + "INSERT INTO pkl_data(key, data) VALUES(?,?)" + " ON CONFLICT(key) DO UPDATE SET data=excluded.data", + pkl_db_rows, + ) + conn.close() + snake_case_file = options.snake_case_file if len(snake_case_file) > 1: with open(snake_case_file, "w") as ff: @@ -2054,6 +2062,9 @@ def main(): ff.write("'" + function + "', ") ff.write(")\n") + if options.pkl_stamp: + Path(options.pkl_stamp).touch() + if __name__ == "__main__": main() diff --git a/Wrapping/Generators/Tests/README.md b/Wrapping/Generators/Tests/README.md new file mode 100644 index 00000000000..849ef3595c7 --- /dev/null +++ b/Wrapping/Generators/Tests/README.md @@ -0,0 +1,35 @@ +# Wrapping infrastructure tests + +Validation suite for the CastXML two-level cache +(`../CastXML/itk-castxml-cache.py`), the SQLite pkl database +(`../pkl_db.py`), and the pyi stub pipeline +(`../Python/itk/pyi_generator.py`). + +Registered as the `WrappingInfrastructure` CTest when wrapping and +`BUILD_TESTING` are enabled and `pytest` is available in the build +interpreter (skipped otherwise). Also runs standalone with only +Python >= 3.11 and `pytest` — no compiler, CMake, or ITK build: + +```bash +python -m pytest Wrapping/Generators/Tests # from the ITK source root +ctest -R WrappingInfrastructure # from a configured build tree +``` + +Runs in ~7 s. `ITK_SOURCE_DIR` points the suite at a different checkout +(default: the enclosing tree). The fake castxml (`fake_castxml.py`) logs +every invocation so tests assert exact subprocess counts: cold = 2 +(`-E` + full), L2-hit = 1, L1-hit = 0. + +| File | Covers | +|---|---| +| `tests/test_unit_cache.py` | key/manifest/parse functions, Windows path unescaping, include-dir fingerprints | +| `tests/test_unit_pkl_db.py` | schema, WAL (+ NFS fallback), upsert, reader self-containment | +| `tests/test_integration.py` | cold/warm/staleness/corruption/eviction/bypass/multi-root | +| `tests/test_pyi_generator.py` | `--prune` gating (external-project safety), DB self-heal, guards | +| `tests/test_concurrency.py` | 32 parallel pkl writers, keyset-DELETE, racing same-key stores | +| `tests/test_relocate.py` | cache relocation (models `actions/cache` / Azure `Cache@2` transport) | +| `smoke_realbuild.sh` | optional real-castxml double build (Linux, needs a configured tree) | + +A cross-platform mirror with a {Linux, macOS, Windows} × {3.11, 3.13} +GitHub Actions matrix lives at +. diff --git a/Wrapping/Generators/Tests/conftest.py b/Wrapping/Generators/Tests/conftest.py new file mode 100644 index 00000000000..a62d5bc579f --- /dev/null +++ b/Wrapping/Generators/Tests/conftest.py @@ -0,0 +1,134 @@ +"""Shared fixtures: load ITK wrapping scripts by path, build a fake castxml.""" + +import importlib.util +import os +import subprocess +import sys +from pathlib import Path + +import pytest + +# Default to the enclosing ITK source tree; ITK_SOURCE_DIR overrides to +# point the suite at a different checkout. +ITK_SOURCE_DIR = Path( + os.environ.get("ITK_SOURCE_DIR", Path(__file__).resolve().parents[3]) +) +CACHE_SCRIPT = ITK_SOURCE_DIR / "Wrapping/Generators/CastXML/itk-castxml-cache.py" +PKL_DB = ITK_SOURCE_DIR / "Wrapping/Generators/pkl_db.py" +PYI_GENERATOR = ITK_SOURCE_DIR / "Wrapping/Generators/Python/itk/pyi_generator.py" + +FAKE_CASTXML = Path(__file__).parent / "fake_castxml.py" + + +def _load(name, path): + spec = importlib.util.spec_from_file_location(name, path) + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) + return mod + + +@pytest.fixture(scope="session") +def cache_mod(): + return _load("itk_castxml_cache", CACHE_SCRIPT) + + +@pytest.fixture(scope="session") +def pkl_db_mod(): + return _load("pkl_db", PKL_DB) + + +class FakeCastxml: + """A castxml stand-in whose invocations are observable. + + -E mode emits preprocessed output with `# N "path"` line markers naming + the header fixture files; full mode writes XML derived from the header + contents and FAKE_CASTXML_VERSION, so output provably tracks the inputs. + """ + + def __init__(self, tmp_path, monkeypatch): + self.dir = tmp_path + self.log = tmp_path / "invocations.log" + self.cache = tmp_path / "cache" + self.cache.mkdir() + self.incdir_a = tmp_path / "incdirA" + self.incdir_b = tmp_path / "incdirB" + self.incdir_a.mkdir() + self.incdir_b.mkdir() + self.build = tmp_path / "build" + self.build.mkdir() + + (self.incdir_a / "foo.h").write_text("content-A v1\n") + (self.incdir_b / "bar.h").write_text("content-B v1\n") + self.inc_file = self.build / "module.inc" + self.inc_file.write_text(f'"-I{self.incdir_a}"\n"-I{self.incdir_b}"\n') + self.cxx_file = self.build / "module.cxx" + self.cxx_file.write_text("// module source\n") + self.out_xml = self.build / "out.xml" + + if sys.platform == "win32": + self.bin = tmp_path / "castxml.cmd" + self.bin.write_text( + f'@echo off\r\n"{sys.executable}" "{FAKE_CASTXML}" %*\r\n' + ) + else: + self.bin = tmp_path / "castxml" + self.bin.write_text( + f'#!/usr/bin/env bash\nexec "{sys.executable}" "{FAKE_CASTXML}" "$@"\n' + ) + self.bin.chmod(0o755) + + monkeypatch.setenv("ITK_WRAP_CACHE", str(self.cache)) + monkeypatch.setenv("FAKE_LOG", str(self.log)) + monkeypatch.setenv( + "FAKE_HDRS", + os.pathsep.join( + [str(self.incdir_a / "foo.h"), str(self.incdir_b / "bar.h")] + ), + ) + monkeypatch.delenv("ITK_WRAP_CACHE_FORMAT", raising=False) + monkeypatch.delenv("ITK_WRAP_CACHE_BYPASS", raising=False) + + def invoke(self, extra_env=None, no_cache=False): + cmd = [sys.executable, str(CACHE_SCRIPT)] + if no_cache: + cmd.append("--no-cache") + cmd += [ + str(self.bin), + "--castxml-cc-gnu", + "g++", + f"@{self.inc_file}", + "-o", + str(self.out_xml), + str(self.cxx_file), + ] + env = dict(os.environ) + if extra_env: + env.update(extra_env) + return subprocess.run(cmd, env=env, capture_output=True, text=True) + + def invocations(self): + if not self.log.exists(): + return [] + return self.log.read_text().split() + + def reset_log(self): + self.log.write_text("") + + def evict(self, days="13.9"): + return subprocess.run( + [ + sys.executable, + str(CACHE_SCRIPT), + "--evict", + days, + "--cache-dir", + str(self.cache), + ], + capture_output=True, + text=True, + ) + + +@pytest.fixture +def fake(tmp_path, monkeypatch): + return FakeCastxml(tmp_path, monkeypatch) diff --git a/Wrapping/Generators/Tests/fake_castxml.py b/Wrapping/Generators/Tests/fake_castxml.py new file mode 100644 index 00000000000..2f774a50cd4 --- /dev/null +++ b/Wrapping/Generators/Tests/fake_castxml.py @@ -0,0 +1,44 @@ +"""Instrumented castxml stand-in driven by env vars (see conftest.FakeCastxml).""" + +import os +import sys + + +def main(): + argv = sys.argv[1:] + with open(os.environ["FAKE_LOG"], "a") as log: + log.write(("PRE" if "-E" in argv else "FULL") + "\n") + + if os.environ.get("FAKE_FAIL") == "1": + return 1 + + out = None + for i, a in enumerate(argv): + if a == "-o" and i + 1 < len(argv): + out = argv[i + 1] + + hdrs = [p for p in os.environ.get("FAKE_HDRS", "").split(os.pathsep) if p] + ver = os.environ.get("FAKE_CASTXML_VERSION", "1") + + if "-E" in argv: + with open(out, "wb") as f: + marker_root = os.environ.get("FAKE_MARKER_ROOT", "") + f.write(b'# 1 "' + (marker_root + "/module.cxx").encode() + b'"\n') + body = b"" + for h in hdrs: + f.write(f'# 1 "{h}"\n'.encode()) + with open(h, "rb") as hf: + body += hf.read() + f.write(body) + f.write(b"int payload;\n") + else: + with open(out, "wb") as f: + content = b"".join(open(h, "rb").read() for h in hdrs) + f.write(b"\n") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/Wrapping/Generators/Tests/smoke_realbuild.sh b/Wrapping/Generators/Tests/smoke_realbuild.sh new file mode 100755 index 00000000000..e4d092eac17 --- /dev/null +++ b/Wrapping/Generators/Tests/smoke_realbuild.sh @@ -0,0 +1,34 @@ +#!/usr/bin/env bash +# L3 smoke: two fresh ITK builds sharing one castxml cache; asserts the second +# issues zero real castxml runs. Linux only; expects a configured pixi ITK +# checkout. Run on demand — this is a ~2x full-build operation. +# +# Usage: ITK_SOURCE_DIR=/path/to/ITK ./smoke_realbuild.sh +set -euo pipefail +ITK=${ITK_SOURCE_DIR:?set ITK_SOURCE_DIR} +CACHE=$(mktemp -d)/wrap-cache +export ITK_WRAP_CACHE=$CACHE + +cd "$ITK" +for round in cold warm; do + rm -rf build-python + start=$(date +%s) + pixi run build-python-ci + echo "$round: $(($(date +%s) - start))s" +done + +entries=$(find "$CACHE/l2" -name _ok | wc -l) +echo "cache entries: $entries" +python3 - "$ITK/build-python/.ninja_log" <<'EOF' +import sys +rows = [ + int(p[1]) - int(p[0]) + for line in open(sys.argv[1]) + if (p := line.split("\t")) and len(p) >= 4 + and p[3].endswith(".xml") and "castxml_inputs" in p[3] +] +mean = sum(rows) / len(rows) +print(f"warm castxml steps: {len(rows)}, mean {mean:.0f}ms") +assert mean < 1000, "warm castxml steps too slow — cache hits not happening" +EOF +echo "SMOKE PASS" diff --git a/Wrapping/Generators/Tests/tests/test_concurrency.py b/Wrapping/Generators/Tests/tests/test_concurrency.py new file mode 100644 index 00000000000..ca01fed9249 --- /dev/null +++ b/Wrapping/Generators/Tests/tests/test_concurrency.py @@ -0,0 +1,128 @@ +"""L2: multiprocess stress — parallel pkl writers and racing castxml stores.""" + +import multiprocessing +import sqlite3 +import subprocess +import sys + +from conftest import PKL_DB + +UPSERT = ( + "INSERT INTO pkl_data(key, data) VALUES(?,?)" + " ON CONFLICT(key) DO UPDATE SET data=excluded.data" +) + +WRITER = f""" +import sys +sys.path.insert(0, {str(PKL_DB.parent)!r}) +from pkl_db import open_pkl_db +wid, pkl_dir = int(sys.argv[1]), sys.argv[2] +conn = open_pkl_db(pkl_dir) +with conn: + for i in range(50): + conn.execute({UPSERT!r}, (f"shared-{{i}}", b"w%d" % wid)) + conn.execute({UPSERT!r}, (f"w{{wid}}-{{i}}", b"x")) +conn.close() +""" + + +def test_parallel_writers_no_lock_errors(pkl_db_mod, tmp_path): + n = 32 + procs = [ + subprocess.Popen( + [sys.executable, "-c", WRITER, str(i), str(tmp_path)], + stderr=subprocess.PIPE, + ) + for i in range(n) + ] + errs = [p.communicate()[1] for p in procs] + assert all(p.returncode == 0 for p in procs), b"\n".join(errs)[:2000] + conn = sqlite3.connect(pkl_db_mod._pkl_db_path(str(tmp_path))) + count = conn.execute("SELECT count(*) FROM pkl_data").fetchone()[0] + assert count == 50 + n * 50 # 50 shared + 50 per writer + conn.close() + + +def test_keyset_delete_against_concurrent_upserts(pkl_db_mod, tmp_path): + conn = pkl_db_mod.open_pkl_db(str(tmp_path)) + with conn: + for i in range(200): + conn.execute(UPSERT, (f"live-{i}", b"x")) + for i in range(200): + conn.execute(UPSERT, (f"stale-{i}", b"x")) + with conn: + conn.execute("CREATE TEMP TABLE _live_keys (key TEXT PRIMARY KEY)") + conn.executemany( + "INSERT OR IGNORE INTO _live_keys VALUES(?)", + [(f"live-{i}",) for i in range(200)], + ) + conn.execute( + "DELETE FROM pkl_data WHERE key NOT IN (SELECT key FROM _live_keys)" + ) + conn.execute("DROP TABLE _live_keys") + keys = {r[0] for r in conn.execute("SELECT key FROM pkl_data")} + assert keys == {f"live-{i}" for i in range(200)} + conn.close() + + +def _invoke_worker(args): + cache_script, castxml_bin, inc, cxx, out, env_pairs = args + import os + import subprocess as sp + + env = dict(os.environ) + env.update(env_pairs) + r = sp.run( + [ + sys.executable, + cache_script, + castxml_bin, + "--castxml-cc-gnu", + "g++", + f"@{inc}", + "-o", + out, + cxx, + ], + env=env, + capture_output=True, + ) + return r.returncode + + +def test_racing_stores_same_key_leave_valid_entry(fake, tmp_path): + from conftest import CACHE_SCRIPT + + outs = [str(fake.build / f"out{i}.xml") for i in range(8)] + env_pairs = { + "ITK_WRAP_CACHE": str(fake.cache), + "FAKE_LOG": str(fake.log), + "FAKE_HDRS": str(fake.incdir_a / "foo.h"), + } + args = [ + ( + str(CACHE_SCRIPT), + str(fake.bin), + str(fake.inc_file), + str(fake.cxx_file), + o, + env_pairs, + ) + for o in outs + ] + with multiprocessing.Pool(8) as pool: + rcs = pool.map(_invoke_worker, args) + assert all(rc == 0 for rc in rcs) + oks = list(fake.cache.glob("l2/*/*/_ok")) + assert len(oks) == 1 + # the surviving entry must serve a valid warm hit + fake.reset_log() + fake.out_xml.unlink(missing_ok=True) + import os + + for k, v in env_pairs.items(): + os.environ[k] = v + r = fake.invoke(extra_env=env_pairs) + assert r.returncode == 0 + assert fake.out_xml.stat().st_size > 0 + assert "FULL" not in fake.invocations() diff --git a/Wrapping/Generators/Tests/tests/test_integration.py b/Wrapping/Generators/Tests/tests/test_integration.py new file mode 100644 index 00000000000..07d4ae69134 --- /dev/null +++ b/Wrapping/Generators/Tests/tests/test_integration.py @@ -0,0 +1,173 @@ +"""L1: hit/miss/staleness/corruption/eviction scenarios with the fake castxml.""" + +import gzip +import os +import time + + +def test_cold_miss_stores_and_marks(fake): + r = fake.invoke() + assert r.returncode == 0, r.stderr + assert fake.invocations() == ["PRE", "FULL"] + assert fake.out_xml.stat().st_size > 0 + assert (fake.cache / "_last_store").exists() + assert list(fake.cache.glob("l2/*/*/_ok")) + assert list(fake.cache.glob("l1/*/*/l2_key")) + + +def test_warm_l1_hit_no_subprocess(fake): + fake.invoke() + first = fake.out_xml.read_bytes() + fake.reset_log() + fake.out_xml.unlink() + r = fake.invoke() + assert r.returncode == 0 + assert fake.invocations() == [] + assert fake.out_xml.read_bytes() == first + + +def test_restored_output_mtime_is_fresh(fake): + fake.invoke() + fake.cxx_file.touch() + time.sleep(0.05) + fake.out_xml.unlink() + fake.invoke() + assert fake.out_xml.stat().st_mtime >= fake.cxx_file.stat().st_mtime + + +def test_header_content_edit_misses(fake): + fake.invoke() + (fake.incdir_a / "foo.h").write_text("content-A v2\n") + fake.reset_log() + fake.invoke() + assert fake.invocations() == ["PRE", "FULL"] + assert b"content-A v2" in fake.out_xml.read_bytes() + + +def test_new_shadowing_header_forces_recheck(fake): + fake.invoke() + fake.reset_log() + fake.invoke() + assert fake.invocations() == [] # warmed + (fake.incdir_a / "bar.h").write_text("shadow\n") # earlier dir, same name + fake.reset_log() + fake.invoke() + assert "PRE" in fake.invocations() # incdir fingerprint miss -> -E re-check + + +def test_castxml_upgrade_serves_new_output(fake, monkeypatch): + fake.invoke() + with open(fake.bin, "a") as f: + f.write("# upgraded\n") # binary content change + monkeypatch.setenv("FAKE_CASTXML_VERSION", "2") + fake.reset_log() + fake.out_xml.unlink() + fake.invoke(extra_env={"FAKE_CASTXML_VERSION": "2"}) + assert "FULL" in fake.invocations() + assert b"castxml-version='2'" in fake.out_xml.read_bytes() + + +def test_truncated_gzip_recovers(fake): + fake.invoke() + gz = next(fake.cache.glob("l2/*/*/output.xml.gz")) + gz.write_bytes(gz.read_bytes()[:20]) # truncate -> EOFError territory + fake.reset_log() + fake.out_xml.unlink() + r = fake.invoke() + assert r.returncode == 0 + assert fake.out_xml.stat().st_size > 0 + assert "FULL" in fake.invocations() # re-ran and re-stored + + +def test_empty_cached_xml_not_served(fake): + fake.invoke() + gz = next(fake.cache.glob("l2/*/*/output.xml.gz")) + gz.write_bytes(gzip.compress(b"")) + fake.reset_log() + fake.out_xml.unlink() + r = fake.invoke() + assert r.returncode == 0 + assert fake.out_xml.stat().st_size > 0 + + +def test_bypass_modes_run_castxml_directly(fake, monkeypatch): + r = fake.invoke(no_cache=True) + assert r.returncode == 0 + assert fake.invocations() == ["FULL"] + assert not list(fake.cache.glob("l2/*")) + + fake.reset_log() + r = fake.invoke(extra_env={"ITK_WRAP_CACHE_BYPASS": "1"}) + assert r.returncode == 0 + assert fake.invocations() == ["FULL"] + assert not list(fake.cache.glob("l2/*")) + + +def test_failed_castxml_stores_nothing(fake): + r = fake.invoke(extra_env={"FAKE_FAIL": "1"}) + assert r.returncode != 0 + assert not list(fake.cache.glob("l2/*/*/_ok")) + + +def test_multi_root_reads_secondary_writes_primary(fake, monkeypatch, tmp_path): + fake.invoke() # populate what will become the secondary root + secondary = fake.cache + primary = tmp_path / "primary" + primary.mkdir() + sep = ";" if os.name == "nt" else ":" + monkeypatch.setenv("ITK_WRAP_CACHE", f"{primary}{sep}{secondary}") + fake.reset_log() + fake.out_xml.unlink() + r = fake.invoke() + assert r.returncode == 0 + assert fake.out_xml.stat().st_size > 0 + assert "FULL" not in fake.invocations() # served from secondary + + +def test_uncompressed_format_roundtrip(fake, monkeypatch): + monkeypatch.setenv("ITK_WRAP_CACHE_FORMAT", "uncompressed") + fake.invoke(extra_env={"ITK_WRAP_CACHE_FORMAT": "uncompressed"}) + entry = next(fake.cache.glob("l2/*/*/output.xml")) + time.sleep(1.1) + fake.reset_log() + fake.out_xml.unlink() + fake.invoke(extra_env={"ITK_WRAP_CACHE_FORMAT": "uncompressed"}) + assert fake.invocations() == [] + assert fake.out_xml.stat().st_mtime > entry.stat().st_mtime + 0.5 + + +def test_evict_removes_dead_and_staging_dirs_after_grace(fake): + fake.invoke() + fake.evict() # establish _last_evict + dead = fake.cache / "l2" / "zz" / "deadentry" + dead.mkdir(parents=True) + (dead / "output.xml.gz").write_bytes(b"x") # no _ok + staging = fake.cache / "l2" / "zz" / "abc.tmpq1w2e3" + staging.mkdir() + past = time.time() - 7200 + os.utime(dead, (past, past)) + os.utime(staging, (past, past)) + (fake.cache / "_last_store").touch() # signal stores -> eviction must run + fake.evict() + assert not dead.exists() + assert not staging.exists() + + +def test_evict_early_exits_when_no_stores(fake): + fake.invoke() + fake.evict() + past = time.time() - 7200 + os.utime(fake.cache / "_last_store", (past, past)) + before = sorted(p.name for p in fake.cache.rglob("*")) + fake.evict() + assert sorted(p.name for p in fake.cache.rglob("*")) == before + + +def test_evict_age_removes_old_entries(fake): + fake.invoke() + ok = next(fake.cache.glob("l2/*/*/_ok")) + past = time.time() - 30 * 86400 + os.utime(ok, (past, past)) + (fake.cache / "_last_store").touch() + fake.evict(days="13.9") + assert not ok.exists() diff --git a/Wrapping/Generators/Tests/tests/test_pyi_generator.py b/Wrapping/Generators/Tests/tests/test_pyi_generator.py new file mode 100644 index 00000000000..d610d2d1282 --- /dev/null +++ b/Wrapping/Generators/Tests/tests/test_pyi_generator.py @@ -0,0 +1,137 @@ +"""L1: pyi_generator prune gating, self-heal, and guard behavior against a +synthetic pkl DB (no ITK build required).""" + +import pickle +import subprocess +import sys + +import pytest + +from conftest import PYI_GENERATOR, _load + +UPSERT = ( + "INSERT INTO pkl_data(key, data) VALUES(?,?)" + " ON CONFLICT(key) DO UPDATE SET data=excluded.data" +) + + +@pytest.fixture +def pkl_tree(pkl_db_mod, tmp_path): + """A pkl dir with 6 synthetic classes, index manifests, and stamp files.""" + gen = _load("pyi_generator_for_pickling", PYI_GENERATOR) + cls = gen.ITKClass + # igenerator pickles ITKClass from a script run as __main__; reproduce that + cls.__module__ = "__main__" + setattr(sys.modules["__main__"], "ITKClass", cls) + + pkl_dir = tmp_path / "itk-pkl" + pyi_dir = tmp_path / "pyi" + pkl_dir.mkdir() + pyi_dir.mkdir() + conn = pkl_db_mod.open_pkl_db(str(pkl_dir)) + keys = [] + for i in range(6): + name = f"FakeClass{i}" + key = f"{name}.itk{name}" + obj = cls(name) + with conn: + conn.execute(UPSERT, (key, pickle.dumps(obj))) + (pkl_dir / f"itk{name}.index.txt").write_text(key + "\n") + keys.append(key) + conn.close() + (pkl_dir / "ITKFake.stamp").write_text("") + (pkl_dir / "ITKFake2.stamp").write_text("") + + # CMake writes GlobalIdxFilesList.txt with forward-slash paths on every + # platform; pyi_generator normalizes its glob results to match. + index_files = sorted(p.as_posix() for p in pkl_dir.glob("*.index.txt")) + full_list = tmp_path / "full_list.txt" + full_list.write_text(";".join(index_files)) + partial_list = tmp_path / "partial_list.txt" + partial_list.write_text(index_files[0]) + return pkl_dir, pyi_dir, full_list, partial_list, keys + + +def run_pyi(pkl_dir, pyi_dir, list_file, *extra): + return subprocess.run( + [ + sys.executable, + str(PYI_GENERATOR), + "--pyi_dir", + str(pyi_dir), + "--pkl_dir", + str(pkl_dir), + "--index_list_file", + str(list_file), + *extra, + ], + capture_output=True, + text=True, + ) + + +def rowcount(pkl_db_mod, pkl_dir): + import sqlite3 + + conn = sqlite3.connect(pkl_db_mod._pkl_db_path(str(pkl_dir))) + n = conn.execute("SELECT count(*) FROM pkl_data").fetchone()[0] + conn.close() + return n + + +def test_healthy_run_generates_stubs(pkl_db_mod, pkl_tree): + pkl_dir, pyi_dir, full_list, _partial, _keys = pkl_tree + r = run_pyi(pkl_dir, pyi_dir, full_list, "--prune") + assert r.returncode == 0, r.stderr + assert (pyi_dir / "__init__.pyi").exists() + assert rowcount(pkl_db_mod, pkl_dir) == 6 + + +def test_partial_manifest_without_prune_preserves_rows(pkl_db_mod, pkl_tree): + """External-wrap-project simulation: shared DB must survive.""" + pkl_dir, pyi_dir, _full, partial_list, _keys = pkl_tree + r = run_pyi(pkl_dir, pyi_dir, partial_list) + assert r.returncode == 0, r.stderr + assert rowcount(pkl_db_mod, pkl_dir) == 6 + + +def test_partial_manifest_with_prune_deletes(pkl_db_mod, pkl_tree): + pkl_dir, pyi_dir, _full, partial_list, _keys = pkl_tree + r = run_pyi(pkl_dir, pyi_dir, partial_list, "--prune") + assert r.returncode == 0, r.stderr + assert rowcount(pkl_db_mod, pkl_dir) == 1 + + +def test_prune_skipped_when_manifest_missing(pkl_db_mod, pkl_tree, tmp_path): + pkl_dir, pyi_dir, full_list, _partial, _keys = pkl_tree + with_missing = tmp_path / "with_missing.txt" + with_missing.write_text( + full_list.read_text() + ";" + (pkl_dir / "DOES_NOT_EXIST.index.txt").as_posix() + ) + r = run_pyi(pkl_dir, pyi_dir, with_missing, "--prune") + assert r.returncode == 0, r.stderr + assert "skipping pkl DB pruning" in r.stdout + assert rowcount(pkl_db_mod, pkl_dir) == 6 + + +def test_self_heal_on_damaged_db(pkl_db_mod, pkl_tree): + """Deleted DB rows with fresh stamps -> stamps removed + actionable error.""" + import sqlite3 + + pkl_dir, pyi_dir, full_list, _partial, keys = pkl_tree + conn = sqlite3.connect(pkl_db_mod._pkl_db_path(str(pkl_dir))) + with conn: + conn.execute("DELETE FROM pkl_data WHERE key IN (?,?)", keys[:2]) + conn.close() + assert list(pkl_dir.glob("*.stamp")) + r = run_pyi(pkl_dir, pyi_dir, full_list, "--prune") + assert r.returncode != 0 + assert "missing from the pkl database" in r.stderr + assert not list(pkl_dir.glob("*.stamp")) + + +def test_bogus_pkl_dir_rejected(pkl_tree): + _pkl, pyi_dir, full_list, _partial, _keys = pkl_tree + r = run_pyi("/nonexistent/pkl-dir", pyi_dir, full_list) + assert r.returncode != 0 + assert "Invalid directory provided" in r.stderr diff --git a/Wrapping/Generators/Tests/tests/test_relocate.py b/Wrapping/Generators/Tests/tests/test_relocate.py new file mode 100644 index 00000000000..5868a41d44b --- /dev/null +++ b/Wrapping/Generators/Tests/tests/test_relocate.py @@ -0,0 +1,42 @@ +"""L4-sim: tar round-trip to a different absolute path — models what +actions/cache and Azure Cache@2 do to the cache directory between CI runs.""" + +import shutil + + +def test_cache_relocation_preserves_hits(fake, monkeypatch, tmp_path): + fake.invoke() + assert fake.invocations() == ["PRE", "FULL"] + + relocated = tmp_path / "relocated" / "deeper" / "cache" + relocated.parent.mkdir(parents=True) + shutil.make_archive(str(tmp_path / "ci-cache"), "tar", fake.cache) + shutil.unpack_archive(str(tmp_path / "ci-cache.tar"), relocated) + + monkeypatch.setenv("ITK_WRAP_CACHE", str(relocated)) + fake.reset_log() + fake.out_xml.unlink() + r = fake.invoke(extra_env={"ITK_WRAP_CACHE": str(relocated)}) + assert r.returncode == 0 + assert fake.invocations() == [] # pure L1 hit from the relocated cache + assert fake.out_xml.stat().st_size > 0 + + +def test_cross_branch_restore_selectively_misses(fake, monkeypatch, tmp_path): + """Restore-keys prefix fallback across branches is safe by construction: + unchanged modules hit, the changed module misses.""" + fake.invoke() # "branch A" populates the cache + + # "branch B": one header differs + (fake.incdir_a / "foo.h").write_text("branch-B content\n") + fake.reset_log() + fake.invoke() + assert fake.invocations() == ["PRE", "FULL"] # changed module misses + + # reverting to branch-A content: the L1 map now records branch-B deps, so + # the -E pass re-runs once and lands an L2 hit on the branch-A entry + (fake.incdir_a / "foo.h").write_text("content-A v1\n") + fake.reset_log() + fake.out_xml.unlink() + fake.invoke() + assert fake.invocations() == ["PRE"] # no FULL: branch-A XML still cached diff --git a/Wrapping/Generators/Tests/tests/test_unit_cache.py b/Wrapping/Generators/Tests/tests/test_unit_cache.py new file mode 100644 index 00000000000..0a9111876b3 --- /dev/null +++ b/Wrapping/Generators/Tests/tests/test_unit_cache.py @@ -0,0 +1,162 @@ +"""L0: pure functions of itk-castxml-cache.py — no subprocess, no ITK build.""" + +import os +import sys + +import pytest + + +def test_strip_line_markers(cache_mod): + data = b'# 1 "/a/b.h"\nint x;\n# 22 "/c/d.h" 2\nint y;\n' + assert cache_mod._strip_line_markers(data) == b"int x;\nint y;" + + +def test_extract_dep_paths_dedup_and_pseudo(cache_mod): + pre = b'# 1 "/a/b.h"\n# 5 "/a/b.h"\n# 1 ""\n# 1 ""\n' + assert cache_mod._extract_dep_paths(pre) == ["/a/b.h"] + + +def test_extract_dep_paths_unescapes_windows_backslashes(cache_mod): + pre = b'# 1 "C:\\\\src\\\\ITK\\\\itkImage.h"\n' + assert cache_mod._extract_dep_paths(pre) == ["C:\\src\\ITK\\itkImage.h"] + + +def test_deps_manifest_and_unchanged_roundtrip(cache_mod, tmp_path): + h = tmp_path / "a.h" + h.write_text("v1") + deps = cache_mod._deps_manifest([str(h)]) + assert cache_mod._deps_unchanged(deps) + h.write_text("v2") + assert not cache_mod._deps_unchanged(deps) + + +def test_deps_unchanged_deleted_header_misses(cache_mod, tmp_path): + h = tmp_path / "a.h" + h.write_text("v1") + deps = cache_mod._deps_manifest([str(h)]) + h.unlink() + assert not cache_mod._deps_unchanged(deps) + + +def test_deps_unreadable_at_store_and_check_is_consistent(cache_mod): + deps = cache_mod._deps_manifest(["/nonexistent/path.h"]) + assert deps == [["/nonexistent/path.h", ""]] + assert cache_mod._deps_unchanged(deps) + + +def test_deps_malformed_manifest_misses(cache_mod): + assert not cache_mod._deps_unchanged([["path-without-hash"]]) + assert not cache_mod._deps_unchanged(["not-a-list"]) + + +def test_parse_args(cache_mod): + binp, out, inc, cxx, flags, no_cache = cache_mod._parse_args( + ["/bin/castxml", "-o", "a.xml", "@inc.txt", "mod.cxx", "--flag"] + ) + assert (binp, out, inc, cxx, no_cache) == ( + "/bin/castxml", + "a.xml", + "inc.txt", + "mod.cxx", + False, + ) + assert flags == ["-o", "a.xml", "@inc.txt", "mod.cxx", "--flag"] + + +def test_parse_args_no_cache_flag(cache_mod): + binp, _out, _inc, _cxx, _flags, no_cache = cache_mod._parse_args( + ["--no-cache", "/bin/castxml", "mod.cxx"] + ) + assert binp == "/bin/castxml" and no_cache + + +def test_cache_roots_split(cache_mod, monkeypatch): + sep = ";" if sys.platform == "win32" else ":" + monkeypatch.setenv("ITK_WRAP_CACHE", f"/one{sep}{sep}/two") + assert cache_mod._cache_roots() == ["/one", "/two"] + monkeypatch.delenv("ITK_WRAP_CACHE") + assert len(cache_mod._cache_roots()) == 1 + + +def test_l1_key_includes_key_version(cache_mod): + k1 = cache_mod._l1_key("binhash", None, None, []) + orig = cache_mod._KEY_VERSION + try: + cache_mod._KEY_VERSION = b"vTEST\x00" + k2 = cache_mod._l1_key("binhash", None, None, []) + finally: + cache_mod._KEY_VERSION = orig + assert k1 != k2 + + +def test_l1_key_includes_bin_hash_and_flag_order(cache_mod): + assert cache_mod._l1_key("h1", None, None, ["-a", "-b"]) != cache_mod._l1_key( + "h2", None, None, ["-a", "-b"] + ) + assert cache_mod._l1_key("h1", None, None, ["-a", "-b"]) != cache_mod._l1_key( + "h1", None, None, ["-b", "-a"] + ) + + +def test_incdirs_from_inc_file(cache_mod, tmp_path): + inc = tmp_path / "m.inc" + inc.write_text('"-I/one"\n"-I/two"\n"-I/one"\n"-DFOO"\n') + assert cache_mod._incdirs_from_inc_file(str(inc)) == ["/one", "/two"] + + +def test_incdirs_from_inc_file_isystem(cache_mod, tmp_path): + inc = tmp_path / "m.inc" + inc.write_text( + '"-I/one"\n"-isystem" "/sys/a"\n"-isystem" "/sys/b"\n"-isystem/sys/c"\n"-isystem" "/sys/a"\n' + ) + assert cache_mod._incdirs_from_inc_file(str(inc)) == [ + "/one", + "/sys/a", + "/sys/b", + "/sys/c", + ] + + +def test_incdirs_fingerprint_covers_isystem_dirs(cache_mod, tmp_path): + d = tmp_path / "sysinc" + d.mkdir() + (d / "a.h").write_text("x") + inc = tmp_path / "m.inc" + inc.write_text(f'"-isystem" "{d}"\n') + fp1 = cache_mod._incdirs_fingerprint(str(inc)) + (d / "b.h").write_text("shadow") + assert cache_mod._incdirs_fingerprint(str(inc)) != fp1 + + +def test_incdirs_fingerprint_changes_on_new_header(cache_mod, tmp_path): + d = tmp_path / "inc" + d.mkdir() + (d / "a.h").write_text("x") + inc = tmp_path / "m.inc" + inc.write_text(f'"-I{d}"\n') + fp1 = cache_mod._incdirs_fingerprint(str(inc)) + (d / "b.h").write_text("shadow") + fp2 = cache_mod._incdirs_fingerprint(str(inc)) + assert fp1 != fp2 + (d / "b.h").unlink() + assert cache_mod._incdirs_fingerprint(str(inc)) == fp1 + + +def test_incdirs_fingerprint_ignores_content_edits(cache_mod, tmp_path): + d = tmp_path / "inc" + d.mkdir() + (d / "a.h").write_text("x") + inc = tmp_path / "m.inc" + inc.write_text(f'"-I{d}"\n') + fp1 = cache_mod._incdirs_fingerprint(str(inc)) + (d / "a.h").write_text("y") # content change is the deps manifest's job + assert cache_mod._incdirs_fingerprint(str(inc)) == fp1 + + +def test_max_cache_gb_bad_value_defaults(cache_mod, monkeypatch): + monkeypatch.setenv("ITK_WRAP_CACHE_MAX_SIZE", "not-a-number") + assert cache_mod._max_cache_gb() == 2.0 + + +def test_docstring_documents_no_phantom_env_vars(cache_mod): + assert "ITK_WRAP_CACHE_MAX_DAYS" not in cache_mod.__doc__ diff --git a/Wrapping/Generators/Tests/tests/test_unit_pkl_db.py b/Wrapping/Generators/Tests/tests/test_unit_pkl_db.py new file mode 100644 index 00000000000..5a86da5950e --- /dev/null +++ b/Wrapping/Generators/Tests/tests/test_unit_pkl_db.py @@ -0,0 +1,58 @@ +"""L0: pkl_db.py schema, journaling, and upsert semantics.""" + +import sqlite3 + + +def test_schema_created_idempotently(pkl_db_mod, tmp_path): + c1 = pkl_db_mod.open_pkl_db(str(tmp_path)) + c1.close() + c2 = pkl_db_mod.open_pkl_db(str(tmp_path)) + cols = {r[1] for r in c2.execute("PRAGMA table_info(pkl_data)")} + assert cols == {"key", "data"} + c2.close() + + +def test_db_filename_carries_schema_version(pkl_db_mod, tmp_path): + path = pkl_db_mod._pkl_db_path(str(tmp_path)) + assert f"v{pkl_db_mod.PKL_DB_SCHEMA_VERSION}" in path.name + + +def test_wal_mode_on_local_fs(pkl_db_mod, tmp_path): + conn = pkl_db_mod.open_pkl_db(str(tmp_path)) + mode = conn.execute("PRAGMA journal_mode").fetchone()[0] + assert mode in ("wal", "delete") # delete = documented NFS fallback + conn.close() + + +def test_upsert_overwrites(pkl_db_mod, tmp_path): + conn = pkl_db_mod.open_pkl_db(str(tmp_path)) + upsert = ( + "INSERT INTO pkl_data(key, data) VALUES(?,?)" + " ON CONFLICT(key) DO UPDATE SET data=excluded.data" + ) + with conn: + conn.execute(upsert, ("k", b"v1")) + conn.execute(upsert, ("k", b"v2")) + assert ( + conn.execute("SELECT data FROM pkl_data WHERE key='k'").fetchone()[0] == b"v2" + ) + conn.close() + + +def test_reader_side_is_self_contained(pkl_db_mod, tmp_path): + """Opening a fresh dir must not raise 'no such table' on first SELECT.""" + conn = pkl_db_mod.open_pkl_db(str(tmp_path / "fresh")) + assert conn.execute("SELECT count(*) FROM pkl_data").fetchone()[0] == 0 + conn.close() + + +def test_blob_not_null(pkl_db_mod, tmp_path): + conn = pkl_db_mod.open_pkl_db(str(tmp_path)) + try: + with conn: + conn.execute("INSERT INTO pkl_data(key, data) VALUES('k', NULL)") + raised = False + except sqlite3.IntegrityError: + raised = True + assert raised + conn.close() diff --git a/Wrapping/Generators/pkl_db.py b/Wrapping/Generators/pkl_db.py new file mode 100644 index 00000000000..bdc58801ce9 --- /dev/null +++ b/Wrapping/Generators/pkl_db.py @@ -0,0 +1,31 @@ +import sqlite3 +from pathlib import Path + +PKL_DB_SCHEMA_VERSION = 3 + + +def _pkl_db_path(pkl_dir: str) -> Path: + """Return the build-tree-local pkl SQLite DB path. + + The DB is intermediate handoff state keyed by bare class name, so it must + stay local to one build tree; it is never shared via ITK_WRAP_CACHE. + """ + return Path(pkl_dir) / f"itk-pkl-v{PKL_DB_SCHEMA_VERSION}.db" + + +def open_pkl_db(pkl_dir: str) -> sqlite3.Connection: + """Open (creating if absent) the build-tree-local pkl DB.""" + db_path = _pkl_db_path(pkl_dir) + db_path.parent.mkdir(parents=True, exist_ok=True) + conn = sqlite3.connect(db_path, timeout=30) + try: + # WAL needs shared memory; on filesystems without it (e.g. NFS) fall + # back to the default rollback journal, serialized by the busy timeout. + conn.execute("PRAGMA journal_mode=WAL") + except sqlite3.OperationalError: + pass + conn.execute( + "CREATE TABLE IF NOT EXISTS pkl_data (key TEXT PRIMARY KEY, data BLOB NOT NULL)" + ) + conn.commit() + return conn diff --git a/Wrapping/macro_files/itk_auto_load_submodules.cmake b/Wrapping/macro_files/itk_auto_load_submodules.cmake index 4ca246ebc1e..1a0edec86ba 100644 --- a/Wrapping/macro_files/itk_auto_load_submodules.cmake +++ b/Wrapping/macro_files/itk_auto_load_submodules.cmake @@ -199,12 +199,33 @@ function(generate_castxml_commandline_flags) endforeach() # ===== Run the castxml command + if( + ITK_WRAP_CASTXML_CACHE + AND + Python3_EXECUTABLE + AND + ITK_WRAP_CASTXML_CACHE_SCRIPT + ) + set( + _castxml_cmd + ${Python3_EXECUTABLE} + "${ITK_WRAP_CASTXML_CACHE_SCRIPT}" + ${CASTXML_EXECUTABLE} + ) + list(APPEND _castxml_depends "${ITK_WRAP_CASTXML_CACHE_SCRIPT}") + else() + set( + _castxml_cmd + ${_ccache_cmd} + ${CASTXML_EXECUTABLE} + ) + endif() add_custom_command( OUTPUT ${xml_file} COMMAND - ${_build_env} ${_ccache_cmd} ${CASTXML_EXECUTABLE} -o ${xml_file} - --castxml-output=1 ${_target} --castxml-start _wrapping_ ${_castxml_cc} -w + ${_build_env} ${_castxml_cmd} -o ${xml_file} --castxml-output=1 ${_target} + --castxml-start _wrapping_ ${_castxml_cc} -w -c # needed for ccache to think we are not calling for link @${castxml_inc_file} ${cxx_file} VERBATIM @@ -214,6 +235,7 @@ function(generate_castxml_commandline_flags) ${castxml_inc_file} ${_hdrs} ) + unset(_castxml_cmd) unset(cxx_file) unset(castxml_inc_file) unset(_build_env) diff --git a/Wrapping/macro_files/itk_end_wrap_module.cmake b/Wrapping/macro_files/itk_end_wrap_module.cmake index 04bde56dc4b..fd44a660e18 100644 --- a/Wrapping/macro_files/itk_end_wrap_module.cmake +++ b/Wrapping/macro_files/itk_end_wrap_module.cmake @@ -161,11 +161,13 @@ macro(itk_end_wrap_module) # Set up outputs and byproducts for custom command set(igenerator_outputs "") set(igenerator_byproducts "") + set(pkl_stamp_file "${ITK_PKL_DIR}/${WRAPPER_LIBRARY_NAME}.stamp") list(APPEND igenerator_outputs "${i_files}") # Typedefs/.i list(APPEND igenerator_outputs "${typedef_files}") # Typedefs/SwigInterface.h list(APPEND igenerator_outputs "${idx_files}") # Typedefs/.idx list(APPEND igenerator_outputs "${snake_case_config_file}") # Generators/Python/itk/Configuration/_snake_case.py + list(APPEND igenerator_outputs "${pkl_stamp_file}") # itk-pkl/.stamp if(CMAKE_GENERATOR STREQUAL "Ninja") # Ninja generator requires byproduct for correct dependency handling # See https://cmake.org/cmake/help/latest/policy/CMP0058.html @@ -188,9 +190,10 @@ macro(itk_end_wrap_module) --library-output-dir "${WRAPPER_LIBRARY_OUTPUT_DIR}" --submodule-order "${THIS_MODULE_SUBMODULE_ORDER}" --pyi_index_list "${ITK_PYI_INDEX_FILES}" --pyi_dir "${ITK_STUB_DIR}" --pkl_dir - "${ITK_PKL_DIR}" + "${ITK_PKL_DIR}" --pkl_stamp "${pkl_stamp_file}" DEPENDS ${IGENERATOR} + ${IGENERATOR_PKL_DB} ${ITK_WRAP_DOC_DOCSTRING_FILES} ${CastXML_OUTPUT_FILES} ${typedef_in_files} @@ -202,6 +205,7 @@ macro(itk_end_wrap_module) VERBATIM ) + unset(pkl_stamp_file) unset(snake_case_config_file) else() #message(FATAL_ERROR "Number of interface files is 0 :${WRAPPER_LIBRARY_NAME}:") diff --git a/pixi.lock b/pixi.lock index d476f4e4cce..0bea1db9e08 100644 --- a/pixi.lock +++ b/pixi.lock @@ -1592,6 +1592,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/linux-64/c-ares-1.34.5-hb9d3cd8_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/c-compiler-1.11.0-h4d9bdce_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/castxml-0.7.0-hde8d07d_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/ccache-4.13.6-hedf47ba_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/cmake-4.1.2-hc85cc9f_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/conda-gcc-specs-14.3.0-hb991d5c_7.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/cxx-compiler-1.11.0-hfcd1e18_0.conda @@ -1618,6 +1619,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/linux-64/libgfortran-15.2.0-h69a702a_7.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/libgfortran5-15.2.0-hcd61629_7.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/libgomp-15.2.0-h767d61c_7.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/libhiredis-1.3.0-h5888daf_1.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/libiconv-1.18-h3b78370_2.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/liblapack-3.9.0-38_h47877c9_openblas.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/libllvm20-20.1.8-hf7376ad_1.conda @@ -1645,6 +1647,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/linux-64/rhash-1.4.6-hb9d3cd8_1.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/swig-4.4.1-h7a96c5f_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/tk-8.6.13-noxft_hd72426e_102.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/xxhash-0.8.3-hb47aa4a_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/zstd-1.5.7-hb8e6e7a_2.conda - conda: https://conda.anaconda.org/conda-forge/noarch/ca-certificates-2025.10.5-hbd8a1cb_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/colorama-0.4.6-pyhd8ed1ab_1.conda @@ -1671,6 +1674,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/c-ares-1.34.5-h86ecc28_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/c-compiler-1.11.0-hdceaead_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/castxml-0.7.0-ha3e84ed_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/ccache-4.13.6-h185addb_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/cmake-4.1.2-hc9d863e_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/conda-gcc-specs-14.3.0-h92dcf8a_7.conda - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/cxx-compiler-1.11.0-h7b35c40_0.conda @@ -1697,6 +1701,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/libgfortran-15.2.0-he9431aa_7.conda - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/libgfortran5-15.2.0-h87db57e_7.conda - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/libgomp-15.2.0-he277a41_7.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/libhiredis-1.3.0-h5ad3122_1.conda - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/libiconv-1.18-h90929bb_2.conda - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/liblapack-3.9.0-38_h88aeb00_openblas.conda - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/libllvm20-20.1.8-hfd2ba90_1.conda @@ -1724,6 +1729,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/rhash-1.4.6-h86ecc28_1.conda - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/swig-4.4.1-h512d76c_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/tk-8.6.13-noxft_h5688188_102.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/xxhash-0.8.3-hd794028_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/zstd-1.5.7-hbcf94c1_2.conda - conda: https://conda.anaconda.org/conda-forge/noarch/ca-certificates-2025.10.5-hbd8a1cb_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/colorama-0.4.6-pyhd8ed1ab_1.conda @@ -1759,6 +1765,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/osx-64/c-ares-1.34.5-hf13058a_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-64/c-compiler-1.11.0-h7a00415_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-64/castxml-0.7.0-hb171174_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-64/ccache-4.13.6-h894318c_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-64/cctools-1024.3-h67a6458_9.conda - conda: https://conda.anaconda.org/conda-forge/osx-64/cctools_osx-64-1024.3-llvm19_1_h3b512aa_9.conda - conda: https://conda.anaconda.org/conda-forge/osx-64/clang-19-19.1.7-default_hc369343_5.conda @@ -1787,6 +1794,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/osx-64/libffi-3.5.2-h750e83c_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-64/libgfortran-15.2.0-h306097a_1.conda - conda: https://conda.anaconda.org/conda-forge/osx-64/libgfortran5-15.2.0-h336fb69_1.conda + - conda: https://conda.anaconda.org/conda-forge/osx-64/libhiredis-1.3.0-h240833e_1.conda - conda: https://conda.anaconda.org/conda-forge/osx-64/libiconv-1.18-h57a12c2_2.conda - conda: https://conda.anaconda.org/conda-forge/osx-64/liblapack-3.9.0-38_h859234e_openblas.conda - conda: https://conda.anaconda.org/conda-forge/osx-64/libllvm19-19.1.7-h56e7563_2.conda @@ -1816,6 +1824,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/osx-64/swig-4.4.1-hdac4ec2_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-64/tapi-1300.6.5-h390ca13_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-64/tk-8.6.13-hf689a15_2.conda + - conda: https://conda.anaconda.org/conda-forge/osx-64/xxhash-0.8.3-h13e91ac_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-64/zstd-1.5.7-h8210216_2.conda osx-arm64: - conda: https://conda.anaconda.org/conda-forge/noarch/ca-certificates-2025.10.5-hbd8a1cb_0.conda @@ -1835,6 +1844,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/osx-arm64/c-ares-1.34.5-h5505292_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/c-compiler-1.11.0-h61f9b84_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/castxml-0.7.0-hfc9ce51_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/ccache-4.13.6-h414bf82_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/cctools-1024.3-hd01ab73_9.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/cctools_osx-arm64-1024.3-llvm19_1_h8c76c84_9.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/clang-19-19.1.7-default_h73dfc95_5.conda @@ -1864,6 +1874,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libffi-3.5.2-he5f378a_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libgfortran-15.2.0-hfcf01ff_1.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libgfortran5-15.2.0-h742603c_1.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libhiredis-1.3.0-h286801f_1.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libiconv-1.18-h23cfdf5_2.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/liblapack-3.9.0-38_hd9741b5_openblas.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libllvm19-19.1.7-h8e0c9ce_2.conda @@ -1893,6 +1904,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/osx-arm64/swig-4.4.1-h4366dc5_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/tapi-1300.6.5-h03f4b80_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/tk-8.6.13-h892fb3f_2.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/xxhash-0.8.3-haa4e116_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/zstd-1.5.7-h6491c7d_2.conda win-64: - conda: https://conda.anaconda.org/conda-forge/noarch/ca-certificates-2025.10.5-h4c7d964_0.conda @@ -1911,6 +1923,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/win-64/_openmp_mutex-4.5-2_gnu.conda - conda: https://conda.anaconda.org/conda-forge/win-64/bzip2-1.0.8-h0ad9c76_8.conda - conda: https://conda.anaconda.org/conda-forge/win-64/castxml-0.7.0-ha22e26b_0.conda + - conda: https://conda.anaconda.org/conda-forge/win-64/ccache-4.13.6-h7fd822b_0.conda - conda: https://conda.anaconda.org/conda-forge/win-64/cmake-4.1.2-hdcbee5b_0.conda - conda: https://conda.anaconda.org/conda-forge/win-64/cxx-compiler-1.11.0-h1c1089f_0.conda - conda: https://conda.anaconda.org/conda-forge/win-64/krb5-1.21.3-hdf4eb48_0.conda @@ -1922,6 +1935,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/win-64/libgcc-15.2.0-h1383e82_7.conda - conda: https://conda.anaconda.org/conda-forge/win-64/libgfortran5-15.2.0-hf2bee02_7.conda - conda: https://conda.anaconda.org/conda-forge/win-64/libgomp-15.2.0-h1383e82_7.conda + - conda: https://conda.anaconda.org/conda-forge/win-64/libhiredis-1.3.0-he0c23c2_1.conda - conda: https://conda.anaconda.org/conda-forge/win-64/libhwloc-2.12.1-default_h64bd3f2_1002.conda - conda: https://conda.anaconda.org/conda-forge/win-64/libiconv-1.18-hc1393d2_2.conda - conda: https://conda.anaconda.org/conda-forge/win-64/liblapack-3.9.0-38_hf9ab0e9_mkl.conda @@ -1950,6 +1964,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/win-64/vc14_runtime-14.44.35208-h818238b_32.conda - conda: https://conda.anaconda.org/conda-forge/win-64/vcomp14-14.44.35208-h818238b_32.conda - conda: https://conda.anaconda.org/conda-forge/win-64/vs2022_win-64-19.44.35207-ha74f236_32.conda + - conda: https://conda.anaconda.org/conda-forge/win-64/xxhash-0.8.3-hbba6f48_0.conda - conda: https://conda.anaconda.org/conda-forge/win-64/zstd-1.5.7-hbeecb71_2.conda packages: - conda: https://conda.anaconda.org/conda-forge/linux-64/_libgcc_mutex-0.1-conda_forge.tar.bz2 @@ -2211,6 +2226,7 @@ packages: - zstd >=1.5.7,<1.6.0a0 license: GPL-3.0-only license_family: GPL + run_exports: {} size: 855698 timestamp: 1777926446042 - conda: https://conda.anaconda.org/conda-forge/linux-64/cffi-2.0.0-py313hf46b229_1.conda @@ -3206,6 +3222,9 @@ packages: - libstdcxx >=13 license: BSD-3-Clause license_family: BSD + run_exports: + weak: + - libhiredis >=1.3.0,<1.4.0a0 size: 140759 timestamp: 1748219397797 - conda: https://conda.anaconda.org/conda-forge/linux-64/libiconv-1.18-h3b78370_2.conda @@ -4283,6 +4302,9 @@ packages: - libgcc >=13 license: BSD-2-Clause license_family: BSD + run_exports: + weak: + - xxhash >=0.8.3,<0.8.4.0a0 size: 108219 timestamp: 1746457673761 - conda: https://conda.anaconda.org/conda-forge/linux-64/yaml-0.2.5-h280c20c_3.conda @@ -4560,6 +4582,7 @@ packages: - zstd >=1.5.7,<1.6.0a0 license: GPL-3.0-only license_family: GPL + run_exports: {} size: 843745 timestamp: 1777926452238 - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/cffi-2.0.0-py313h897158f_1.conda @@ -5499,6 +5522,9 @@ packages: - libstdcxx >=13 license: BSD-3-Clause license_family: BSD + run_exports: + weak: + - libhiredis >=1.3.0,<1.4.0a0 size: 140290 timestamp: 1748220539026 - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/libiconv-1.18-h90929bb_2.conda @@ -6499,6 +6525,9 @@ packages: - libgcc >=13 license: BSD-2-Clause license_family: BSD + run_exports: + weak: + - xxhash >=0.8.3,<0.8.4.0a0 size: 105762 timestamp: 1746457675564 - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/yaml-0.2.5-h80f16a2_3.conda @@ -7376,6 +7405,7 @@ packages: - libhiredis >=1.3.0,<1.4.0a0 license: GPL-3.0-only license_family: GPL + run_exports: {} size: 657026 timestamp: 1777926755291 - conda: https://conda.anaconda.org/conda-forge/osx-64/cctools-1024.3-h67a6458_9.conda @@ -8299,6 +8329,9 @@ packages: - libcxx >=18 license: BSD-3-Clause license_family: BSD + run_exports: + weak: + - libhiredis >=1.3.0,<1.4.0a0 size: 59830 timestamp: 1748219625377 - conda: https://conda.anaconda.org/conda-forge/osx-64/libiconv-1.18-h57a12c2_2.conda @@ -9048,6 +9081,9 @@ packages: - __osx >=10.13 license: BSD-2-Clause license_family: BSD + run_exports: + weak: + - xxhash >=0.8.3,<0.8.4.0a0 size: 108449 timestamp: 1746457796808 - conda: https://conda.anaconda.org/conda-forge/osx-64/yaml-0.2.5-h4132b18_3.conda @@ -9210,6 +9246,7 @@ packages: - libhiredis >=1.3.0,<1.4.0a0 license: GPL-3.0-only license_family: GPL + run_exports: {} size: 601285 timestamp: 1777926636412 - conda: https://conda.anaconda.org/conda-forge/osx-arm64/cctools-1024.3-hd01ab73_9.conda @@ -10143,6 +10180,9 @@ packages: - libcxx >=18 license: BSD-3-Clause license_family: BSD + run_exports: + weak: + - libhiredis >=1.3.0,<1.4.0a0 size: 56746 timestamp: 1748219528586 - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libiconv-1.18-h23cfdf5_2.conda @@ -10893,6 +10933,9 @@ packages: - __osx >=11.0 license: BSD-2-Clause license_family: BSD + run_exports: + weak: + - xxhash >=0.8.3,<0.8.4.0a0 size: 98913 timestamp: 1746457827085 - conda: https://conda.anaconda.org/conda-forge/osx-arm64/yaml-0.2.5-h925e9cb_3.conda @@ -11048,6 +11091,7 @@ packages: - xxhash >=0.8.3,<0.8.4.0a0 license: GPL-3.0-only license_family: GPL + run_exports: {} size: 692199 timestamp: 1777926529520 - conda: https://conda.anaconda.org/conda-forge/win-64/cffi-2.0.0-py313h5ea7bf4_1.conda @@ -11530,6 +11574,9 @@ packages: - vc14_runtime >=14.29.30139 license: BSD-3-Clause license_family: BSD + run_exports: + weak: + - libhiredis >=1.3.0,<1.4.0a0 size: 64205 timestamp: 1748219812303 - conda: https://conda.anaconda.org/conda-forge/win-64/libhwloc-2.12.1-default_h64bd3f2_1002.conda @@ -12394,6 +12441,9 @@ packages: - vc14_runtime >=14.29.30139 license: BSD-2-Clause license_family: BSD + run_exports: + weak: + - xxhash >=0.8.3,<0.8.4.0a0 size: 105768 timestamp: 1746458183583 - conda: https://conda.anaconda.org/conda-forge/win-64/yaml-0.2.5-h6a83c73_3.conda diff --git a/pyproject.toml b/pyproject.toml index 2fe4d3b2644..70b13a55ce0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -39,6 +39,7 @@ libopenblas = ">=0.3.30,<0.4" libgfortran5 = ">=15.2.0,<16" swig = ">=4.4.1" castxml = ">=0.7.0,<0.8" +ccache = ">=4.13.6,<5" [tool.pixi.feature.cxx.tasks.configure] cmd = '''cmake \ @@ -236,10 +237,37 @@ cmd = '''cmake \ -DITK_USE_SYSTEM_SWIG:BOOL=ON \ -DITK_USE_SYSTEM_CASTXML:BOOL=ON \ -DCMAKE_BUILD_TYPE:STRING=Release \ - -DBUILD_TESTING:BOOL=ON''' + -DBUILD_TESTING:BOOL=ON \ + -DCMAKE_C_COMPILER_LAUNCHER:STRING=ccache \ + -DCMAKE_CXX_COMPILER_LAUNCHER:STRING=ccache''' description = "Configure ITK Python" outputs = ["build-python/CMakeFiles/"] +[tool.pixi.feature.python.tasks.configure-python-ci] +cmd = '''cmake \ + -Bbuild-python \ + -S. \ + -GNinja \ + -DITK_WRAP_PYTHON:BOOL=ON \ + -DITK_WRAP_CASTXML_CACHE:BOOL=ON \ + -DCMAKE_BUILD_TYPE:STRING=Release \ + -DBUILD_TESTING:BOOL=ON \ + -DCMAKE_C_COMPILER_LAUNCHER:STRING=ccache \ + -DCMAKE_CXX_COMPILER_LAUNCHER:STRING=ccache''' +description = "Configure ITK Python for CI (castxml cache enabled; set ITK_WRAP_CACHE before building)" +outputs = ["build-python/CMakeFiles/"] + +[tool.pixi.feature.python.tasks.build-python-ci] +cmd = "cmake --build build-python" +description = "Build ITK Python for CI" +outputs = ["build-python/lib/**"] +depends-on = ["configure-python-ci"] + +[tool.pixi.feature.python.tasks.test-python-ci] +cmd = "ctest -j3 --test-dir build-python --output-on-failure" +description = "Test ITK Python for CI" +depends-on = ["build-python-ci"] + [tool.pixi.feature.python.tasks.build-python] cmd = "cmake --build build-python" description = "Build ITK Python"