diff --git a/CMakeLists.txt b/CMakeLists.txt index 8ca97ec..0f1047d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -18,6 +18,7 @@ nanobind_add_module(_core src/bindings/filters.cxx src/bindings/flow.cxx src/bindings/graph.cxx + src/bindings/graph_contraction.cxx src/bindings/ground_truth.cxx src/bindings/label_multiset.cxx src/bindings/mesh.cxx @@ -68,6 +69,46 @@ if(BIOIMAGE_FLOW_FMA_DISPATCH) endif() endif() +option( + BIOIMAGE_FILTERS_AVX2_DISPATCH + "Build x86 AVX2/FMA filter kernels with runtime CPU dispatch" + ON +) +if(BIOIMAGE_FILTERS_AVX2_DISPATCH) + string(TOLOWER "${CMAKE_SYSTEM_PROCESSOR}" _bioimage_filters_processor) + if(_bioimage_filters_processor MATCHES "^(x86_64|amd64|i[3-6]86)$") + if(MSVC) + target_sources( + _core PRIVATE + src/cpp/filters/dispatch.cxx + src/cpp/filters/convolve_avx2.cxx + src/cpp/filters/eigenvalues_avx2.cxx + ) + set_property( + SOURCE + src/cpp/filters/convolve_avx2.cxx + src/cpp/filters/eigenvalues_avx2.cxx + APPEND PROPERTY COMPILE_OPTIONS /arch:AVX2 + ) + target_compile_definitions(_core PRIVATE BIOIMAGE_FILTERS_AVX2_DISPATCH=1) + elseif(CMAKE_CXX_COMPILER_ID MATCHES "GNU|Clang") + target_sources( + _core PRIVATE + src/cpp/filters/dispatch.cxx + src/cpp/filters/convolve_avx2.cxx + src/cpp/filters/eigenvalues_avx2.cxx + ) + set_property( + SOURCE + src/cpp/filters/convolve_avx2.cxx + src/cpp/filters/eigenvalues_avx2.cxx + APPEND PROPERTY COMPILE_OPTIONS -mavx2 -mfma + ) + target_compile_definitions(_core PRIVATE BIOIMAGE_FILTERS_AVX2_DISPATCH=1) + endif() + endif() +endif() + option(BIOIMAGE_PROFILE "Enable per-phase profiling instrumentation (development only)" OFF) if(BIOIMAGE_PROFILE) target_compile_definitions(_core PRIVATE BIOIMAGE_PROFILE) diff --git a/MIGRATION_GUIDE.md b/MIGRATION_GUIDE.md index adf74b7..5092fd1 100644 --- a/MIGRATION_GUIDE.md +++ b/MIGRATION_GUIDE.md @@ -204,6 +204,7 @@ Notes: wrapper dispatches to a templated C++ instantiation per dtype; other floating dtypes are cast to `float32`. If the two arrays' dtypes do not match, both are promoted to `float64` rather than silently downcast. +- Both weight arrays must contain only finite values. - Higher weights are processed first (in descending order) — the same convention affogato uses. - The implementation reuses the union-find and per-root mutex-set helpers @@ -331,6 +332,7 @@ Notes: - All three weight arrays (`weights`, `mutex_weights`, `semantic_weights`) must have the same floating dtype, or all three are promoted to `float64`. +- All three weight arrays must contain only finite values. - Output `labels` are dense `uint64` ids in `0 .. number_of_clusters - 1` (first-occurrence order, matching the regular graph mutex watershed — *not* the 1-based foreground labels produced by the grid variant). @@ -460,6 +462,75 @@ Common method/property mapping: | `extractSubgraphFromNodes` | `extract_subgraph_from_nodes` | | `edgesFromNodeList` | `edges_from_node_list` | +`node_degrees()` returns the degree of every node as a `uint64` array. + +### Mutable Graph Contractions + +`ContractionGraph` supports custom graph postprocessing with stable ids and +named value maps: + +```python +import numpy as np +import bioimage_cpp as bic + +graph = bic.graph.UndirectedGraph.from_edges( + 3, + [[0, 1], [1, 2], [0, 2]], +) +work = bic.graph.ContractionGraph(graph) +work.add_node_values( + "position", + np.array([[0.0], [2.0], [10.0]], dtype=np.float32), + reduction="mean", +) +work.add_edge_values( + "weight", + np.array([100.0, 2.0, 3.0], dtype=np.float64), + reduction="sum", +) + +work.contract_edge(0, keep_node=0) +result = work.materialize() + +contracted = result.graph +positions = result.node_values["position"] +weights = result.edge_values["weight"] +``` + +Register all value maps before the first mutation. Maps can use `float32` or +`float64` independently. The supported reductions are `sum`, `mean`, `min`, +and `max`. + +Node values reduce when an edge contraction merges two nodes. Edge values +reduce when parallel edges merge. The contracted edge becomes internal and has +no output value. `suppress_node` removes a degree-2 node and reduces its two +incident edge values into the replacement edge. + +Use `can_suppress_node(node)` before suppression when the mutable graph can +contain parallel edges. The method also rejects a suppression that would create +a self-edge. + +The input must be a simple `UndirectedGraph`. Use +`parallel_edges="keep"` to keep parallel edges that mutations create: + +```python +work = bic.graph.ContractionGraph(graph, parallel_edges="keep") +``` + +The default `parallel_edges="merge"` folds these edges immediately. In keep +mode, `find_edges(u, v)` returns all matching active edge ids. + +`materialize()` always returns a simple-graph snapshot. In keep mode, it folds +parallel edges in the snapshot with each edge map's reduction. It does not +change the mutable graph. Rows in `node_values` and `edge_values` match the ids +in the materialized graph. `node_mapping` and `edge_mapping` map input ids to +materialized ids. Multiple input edges can map to one output edge. The mappings +use `-1` when no output item represents an input item. + +The mutable object is not an `UndirectedGraph`. Active ids stay stable and can +have gaps after mutations. Materialization creates the dense +`UndirectedGraph`. + ### Region Adjacency Graphs Nifty: @@ -877,7 +948,7 @@ Notes: - `edge_weights` must be 1D with length `graph.number_of_edges`. Supported dtypes are `float32` and `float64`; other floating dtypes are promoted to `float32` (matching nifty, whose Python binding is `float32`-only). Non-float - dtypes raise `TypeError`. + dtypes raise `TypeError`. All weights must be finite. - `seeds` must be 1D with length `graph.number_of_nodes`. Supported dtypes are `uint32`, `uint64`, `int32`, `int64`. The value `0` marks unlabeled nodes; non-zero ids are propagated along low-weight paths. Signed seed arrays must @@ -1084,6 +1155,12 @@ Notes: `downsampleMultiset` returns only four of them and leaves the caller to reconstruct `entry_offsets` / `entry_sizes`; `bioimage-cpp` returns them directly so multi-level downsample chains compose without bookkeeping. +- `offsets` and `unique_offsets` are element offsets into `ids` and `counts`. + They are not byte offsets. +- Construction checks integer ranges and the complete flat-storage structure. + Native operations repeat the checks because `LabelMultiset` is mutable. +- Stored entries must contain at least one `(id, count)` pair. A completely + empty top-level `LabelMultiset` is valid. - `multiset_from_labels(labels, block_shape)` builds the level-0 multiset from a `uint32` or `uint64` label volume in one call. There is no nifty equivalent. @@ -1141,10 +1218,10 @@ energy = objective.energy(labels) - `graph` — an `UndirectedGraph` or `RegionAdjacencyGraph`. The constructor copies the topology, so further mutations on the input graph do not affect the objective. -- `edge_costs` — 1D `float64` array of length `graph.number_of_edges`. +- `edge_costs` — 1D array of finite values with length + `graph.number_of_edges`; compatible inputs are converted to `float64`. - `lifted_uvs` / `lifted_costs` — optional `(n_lifted, 2)` uint64 array and 1D - float64 array of equal length, listing the additional lifted edges and - their weights. + float64 array of equal length. Lifted costs must be finite. - `bfs_distance` — optional positive integer. Adds a zero-weight lifted edge for every pair of nodes within this many base-graph hops of each other (excluding nodes already connected by a base edge). Pairs with both @@ -1325,10 +1402,11 @@ labels = bic.graph.multicut.GreedyAdditiveMulticut().optimize(objective) energy = objective.energy(labels) ``` -`MulticutObjective` accepts an `UndirectedGraph` or a `RegionAdjacencyGraph` and -a 1D `edge_costs` array of length `graph.number_of_edges`. The objective owns -the current best `labels`; `optimize` updates them in place and also returns -the new array. +`MulticutObjective` accepts an `UndirectedGraph` or a `RegionAdjacencyGraph`. +It converts a 1D `edge_costs` array to `float64`. The array length must equal +`graph.number_of_edges`, and all costs must be finite. The objective owns the +current best `labels`; `optimize` updates them in place and also returns the +new array. Available solvers: @@ -1390,12 +1468,24 @@ labels = solver.optimize(objective) Notes: -- `edge_costs` must be `float64` and 1D with length `graph.number_of_edges`. +- `edge_costs` must be finite and 1D with length `graph.number_of_edges`. - Output labels are dense `uint64` ids in `0 .. number_of_clusters - 1`. - `MulticutObjective.energy(labels)` is the multicut energy used internally; it matches `nmc.multicutObjective(...).evalNodeLabels(labels)`. - `objective.reset_labels()` restores the per-node initial labeling, useful when re-running solvers from a clean state. +- `number_of_threads=0` on `MulticutDecomposer` uses available hardware + concurrency. A positive value sets the maximum worker count. +- `MulticutDecomposer` runs decomposition and component solvers in C++. It + accepts `GreedyAdditiveMulticut`, `GreedyFixationMulticut`, + `KernighanLinMulticut`, and chains composed only of these solvers. +- `MulticutDecomposer` does not accept custom Python solvers, + `FusionMoveMulticut`, or another `MulticutDecomposer` as a component or + fallthrough solver. Use one of the supported native solvers instead. +- Multicut solvers no longer expose `clone()`. Native decomposition creates + independent C++ solver instances for its workers. +- Components can finish in any order. The decomposer applies labels in + component order, so scheduling does not change output labels. Intentional differences vs. nifty: @@ -1407,6 +1497,9 @@ Intentional differences vs. nifty: - `MulticutDecomposer` short-circuits the trivial case where the sub-solver is `GreedyAdditiveMulticut` and no fallthrough is given — the greedy solver already operates on each connected component internally. +- Unlike nifty's factory-based decomposer, the native decomposer supports only + the solver types listed above. This keeps worker execution outside the GIL + and avoids Python callbacks from native threads. #### Fusion Moves @@ -1546,6 +1639,13 @@ Differences from nifty: underlying loop is the same. - Both `float32` and `float64` inputs are accepted; computation runs in `float64` internally. +- All indicators, weights, sizes, and node features that affect priorities + must contain finite values. +- `MalaClusterPolicy.num_bins` must be an integer greater than or equal to `2`. +- `MalaClusterPolicy.num_edges_stop` counts all active edges after each + contraction. This includes the contracted edge and folded parallel edges. + Earlier bioimage-cpp versions counted only folds and could stop too late on + sparse graphs. The new behavior matches nifty's contraction graph. - Tie-breaks follow the deterministic order of edge ids returned by `UndirectedGraph`, which may differ from nifty's. On inputs where many edges share the same indicator value, this combines with the @@ -2606,6 +2706,22 @@ Important differences and current scope: heap. Compact IDs avoid full-volume shortest-path fields; the public Dijkstra functions remain dense and exact FP64. +### Skeleton graph postprocessing + +`remove_ticks(vertices, edges, tick_length, radii=None)` removes short terminal +branches from a skeleton graph. It compresses degree-2 paths, sums their +physical lengths, and repeatedly removes the shortest terminal branch below +the strict threshold. + +The mutable topology and length reduction run in the C++ contraction backend. +The Python wrapper selects branches and compacts the output arrays. Distinct +compressed paths between the same branch nodes remain separate, so cyclic and +theta-shaped skeletons keep their degree semantics. + +Standalone paths and components that contain only a cycle remain unchanged. +The result keeps the existing array contract: vertices are `float64`, edges are +`int64`, and optional radii preserve their dtype. + ### Blockwise skeletonization and exact stitching `bioimage_cpp.skeleton.distributed` provides the dependency-light primitives diff --git a/development/filters/PERFORMANCE_NOTES.md b/development/filters/PERFORMANCE_NOTES.md index 6dddeb4..42b469e 100644 --- a/development/filters/PERFORMANCE_NOTES.md +++ b/development/filters/PERFORMANCE_NOTES.md @@ -1,512 +1,198 @@ # Filter benchmark — performance notes -Results from `development/filters/benchmark.py` for the six Tier-1 filters -on 2D and 3D test data from `skimage.data`. Re-run with: +This document records the P2 filter optimization from `REVIEW.md` and the +follow-up vector-math work. + +Run the parity gate before the benchmark: ```bash -python development/filters/check_parity.py # parity gate, must PASS first -python development/filters/benchmark.py # headline numbers +python development/filters/check_parity.py +python development/filters/check_parity.py --force-scalar +python development/filters/validate_eigenvalue_approximation.py +python development/filters/benchmark_eigenvalues.py +python development/filters/benchmark.py --repeats 5 ``` -All four libraries produce the same response (verified by `check_parity.py` -on the image interior, dropping a `window_size * sigma` margin on each side; -or `inner + outer` for the structure tensor). Reported numbers are -**median wall-clock per call** across 5 timed repeats, after one untimed -warmup, with calls interleaved round-robin between libraries to share cache -state fairly. - -## Setup - -- CPU: Intel Core i7-1185G7 (Tiger Lake, 4C/8T, AVX2 + AVX-512) -- Compiler: gcc 15.2.0 (conda-forge), `-O3`, no `-march=native` -- Python 3.11.15 on Linux x86_64 -- `numpy 2.4.5`, `scipy 1.17.1`, `vigra 1.12.3`, `fastfilters 0.3-4-g87f08b5`, - `bioimage_cpp 0.1.0` -- All libraries called single-threaded; `bioimage_cpp` has no SIMD intrinsics - (Tier-1: scalar C++20 + compiler auto-vectorisation). - -## Benchmark configuration - -| parameter | value | notes | -|---|---|---| -| sigma | 1.5 | smoothing / derivative / LoG / gradient / Hessian | -| inner_sigma | 1.0 | structure-tensor gradient scale | -| outer_sigma | 2.0 | structure-tensor smoothing scale | -| window_size | 3.0 | kernel half-width / sigma; matched across libraries | -| repeats | 5 | timed, interleaved round-robin | -| 2D image | `skimage.data.camera()` | 512×512 → float32, normalised to [0, 1] | -| 3D volume | `skimage.data.cells3d()[:, 1]` | 60×256×256 nuclei channel → float32 | - -`gaussian_derivative` uses order = 1 along the trailing axis only. `fastfilters` -only supports uniform per-axis order, so its row is `n/a`. - -## Headline ratios - -Geometric mean of `bioimage_cpp.median / other.median` across all benched -(filter, dim) combinations. **>1.0 means `bioimage_cpp` is slower** than the -other library; **<1.0 means faster**. - -| comparison | geomean ratio | n | -|---|---|---| -| `bioimage_cpp` / `fastfilters` | **2.00** | 10 | -| `bioimage_cpp` / `vigra` | **0.18** | 12 | -| `bioimage_cpp` / `scipy` | **0.20** | 12 | - -Interpretation: `fastfilters` (hand-AVX2) is 2× faster than ours on average; -ours is ~5× faster than vigra and ~5× faster than scipy/numpy. - -## 2D results — `camera()` 512×512 - -Times in milliseconds (median of 5 repeats). The `x ours` column is -`bioimage_cpp.median / this_lib.median`; **values >1.0 mean the library is -faster than `bioimage_cpp`**. - -| filter | bioimage_cpp ms | fastfilters ms | x ours | vigra ms | x ours | scipy ms | x ours | -|---|---:|---:|---:|---:|---:|---:|---:| -| gaussian_smoothing | 0.82 | 0.49 | 1.67 | 5.18 | 0.16 | 3.59 | 0.23 | -| gaussian_derivative | 0.83 | n/a | n/a | 5.15 | 0.16 | 3.85 | 0.21 | -| gaussian_gradient_magnitude | 2.11 | 0.91 | 2.32 | 13.85 | 0.15 | 7.77 | 0.27 | -| laplacian_of_gaussian | 1.92 | 0.96 | 2.00 | 8.88 | 0.22 | 7.50 | 0.26 | -| hessian_of_gaussian_eigenvalues | 3.02 | 1.67 | 1.81 | 19.55 | 0.15 | 77.79 | 0.04 | -| structure_tensor_eigenvalues | 5.16 | 2.86 | 1.80 | 26.22 | 0.20 | 82.99 | 0.06 | - -## 3D results — `cells3d()[:, 1]` 60×256×256 - -| filter | bioimage_cpp ms | fastfilters ms | x ours | vigra ms | x ours | scipy ms | x ours | -|---|---:|---:|---:|---:|---:|---:|---:| -| gaussian_smoothing | 33.27 | 15.02 | 2.22 | 192.29 | 0.17 | 85.37 | 0.39 | -| gaussian_derivative | 33.05 | n/a | n/a | 195.27 | 0.17 | 86.19 | 0.38 | -| gaussian_gradient_magnitude | 106.42 | 61.91 | 1.72 | 640.83 | 0.17 | 276.64 | 0.38 | -| laplacian_of_gaussian | 103.69 | 61.93 | 1.67 | 584.71 | 0.18 | 266.87 | 0.39 | -| hessian_of_gaussian_eigenvalues | 486.38 | 175.75 | 2.77 | 2195.57 | 0.22 | 4031.79 | 0.12 | -| structure_tensor_eigenvalues | 599.90 | 263.14 | 2.28 | 1938.37 | 0.31 | 4301.12 | 0.14 | - -## Per-filter throughput vs `bioimage_cpp` (megapixels / second) - -Throughput = `image_size / median_time`. Useful for cross-shape comparison. - -### 2D (512×512 = 0.262 megapixels per output) - -| filter | bioimage_cpp | fastfilters | vigra | scipy | -|---|---:|---:|---:|---:| -| gaussian_smoothing | 320 | 535 | 51 | 73 | -| gaussian_gradient_magnitude | 124 | 288 | 19 | 34 | -| laplacian_of_gaussian | 136 | 273 | 30 | 35 | -| hessian_of_gaussian_eigenvalues | 87 | 157 | 13 | 3 | -| structure_tensor_eigenvalues | 51 | 92 | 10 | 3 | - -### 3D (60×256×256 = 3.93 megavoxels per output) - -| filter | bioimage_cpp | fastfilters | vigra | scipy | -|---|---:|---:|---:|---:| -| gaussian_smoothing | 118 | 262 | 20 | 46 | -| gaussian_gradient_magnitude | 37 | 64 | 6 | 14 | -| laplacian_of_gaussian | 38 | 64 | 7 | 15 | -| hessian_of_gaussian_eigenvalues | 8 | 22 | 2 | 1 | -| structure_tensor_eigenvalues | 7 | 15 | 2 | 1 | - -## Takeaways - -- **vs `fastfilters`** (the hand-AVX2 target). We sit at 1.67× – 2.77× - slower across the board, exactly in the 1.0×–2.0× band the Tier-1 plan - predicted (a touch worse on Hessian/structure-tensor 3D where eigenvalue - arithmetic dominates and benefits less from auto-vectorisation). This is - the gap a Tier-2 manual-AVX2 path would aim to close. -- **vs `vigra`**. ~5× faster across the board. Same algorithmic family, - scalar code in both cases; the win comes from fewer abstraction layers - and tighter kernels (constant-bound inner loops, half-kernel storage, - X-strip pattern for the strided pass). -- **vs `scipy.ndimage` (+ `numpy.linalg.eigvalsh` for the eigenvalue - filters)**. ~5× faster on simple filters and ~10–25× faster on the - eigenvalue paths. `scipy.ndimage` itself is competitive on plain - convolution; the eigenvalue gap is almost entirely the per-pixel numpy - `eigvalsh` cost — exactly what scipy-only users currently pay. - -## Where the Tier-2 SIMD work would help most - -Largest absolute gaps to `fastfilters` (3D, in ms per call): - -| filter | ours | ff | absolute gap | gap × 5 calls/sec | -|---|---:|---:|---:|---:| -| hessian_of_gaussian_eigenvalues | 486 | 176 | 311 ms | 1.55 s/sec saved | -| structure_tensor_eigenvalues | 600 | 263 | 337 ms | 1.69 s/sec saved | -| gaussian_gradient_magnitude | 106 | 62 | 44 ms | 0.22 s/sec saved | - -If/when Tier-2 lands, instrument the Hessian and structure-tensor 3D paths -first — they share the same separable-FIR primitives plus the 3×3 trig -eigensolver, so closing the gap on those carries the gradient/magnitude / -LoG paths along for free. - -## Reproducibility notes - -- Run after `pip install -e . --no-build-isolation` so the C++ extension - matches the source tree. -- For absolute-time comparisons across machines, also report CPU model, - compiler version, and whether `-march=native` was set (we do NOT set it - in normal builds; this benchmark used the default `-O3`). -- For a quick re-check use `--small` (crops to 128×128 and 32×64×64); - finishes in a few seconds. -- For raw per-(filter, library) timings (useful for plotting), pass - `--csv path.csv`. - -## Tier 2 SIMD — design notes (deferred) - -**Status (2026-05-17): not pursued for now.** Tier 1 sits within ~2× of -fastfilters' hand-AVX2 on the headline benchmark while being ~5× faster -than `vigra` and `scipy.ndimage`. The marginal user value of closing that -2× gap doesn't yet justify the extra build complexity and dual-path -maintenance burden. This section captures the design so a future coding -agent (or future-us) can pick it up without re-deriving the choices. - -### Trigger conditions — when to revisit - -Open this section again when **at least one** is true: - -1. Real users are hitting the Hessian-3D / structure-tensor-3D paths on - volumes large enough that ~300 ms vs ~150 ms per call materially - matters in their pipeline (typically batch feature extraction over - many large 3D blocks). -2. "Performance parity with fastfilters" becomes a stated project goal - (e.g. for a migration story or comparison documentation). -3. Profiling on a real downstream workflow shows `bioimage_cpp.filters` - is the bottleneck and the gap to fastfilters is the dominant slice. - -If none of those is true: stay on Tier 1. - -### Scope — what to ship, what to keep out - -**In scope** (the whole Tier 2 delivery): - -- Hand-written AVX2 + FMA implementations of exactly two inner kernels: - - `convolve_x_radius` — the X (innermost contiguous) pass. - - `convolve_strided_radius` — the Y/Z (strided) pass. - - Both currently live in - `include/bioimage_cpp/filters/convolve.hxx::detail`. -- One-time CPUID dispatch at module load that picks scalar vs AVX2 - function pointers for those two kernels. - -**Out of scope** (do NOT add any of these as part of Tier 2): - -- AVX-512 path. The win over AVX2 is small on memory-bound separable FIR - and doubles the kernel binary footprint; revisit only if a user with a - Sapphire Rapids / Zen 5 workload asks specifically. -- NEON / arm64 hand-tuning. Tier 1 auto-vectorization on Apple Clang is - already competitive; this would be a separate project with its own - trigger conditions. -- Replacing `std::acos` / `std::cos` in `eigenvalues.hxx` with a - vectorized math library (this is what `fastfilters` vendors as - `avx_mathfun.h`, 924 lines). The Tier-1 plan explicitly rejected - vendoring it; revisit only if eigenvalue profiling shows the trig - calls dominate the remaining gap. Don't bundle this into Tier 2. -- Any change to `kernel.hxx`, `eigenvalues.hxx`, `gaussian.hxx`, the - binding layer, or the Python wrapper. Tier 2 is a *drop-in* speedup of - two leaf functions; if you find yourself changing anything else, - something is wrong. - -### File layout +The benchmark reports the median wall time across five interleaved calls after +one warmup call. -``` -include/bioimage_cpp/filters/ - convolve.hxx # existing scalar; renamed entry points - # to point at function pointers (see below) - convolve_dispatch.hxx # NEW — function-pointer table + CPUID - -src/cpp/filters/ - convolve_avx2.cxx # NEW — AVX2+FMA kernels (compiled with - # per-file -mavx2 -mfma / /arch:AVX2) - convolve_dispatch.cxx # NEW — one-time init of the pointers -``` +## Environment -The existing `convolve_axis_x` / `convolve_axis_strided` entry points in -`convolve.hxx` keep their signatures. Their bodies switch from "directly -call `detail::convolve_x_radius`" to "call -`bioimage_cpp::filters::dispatch::convolve_x_table[R][Sym]`". Higher -levels (`gaussian.hxx`, the six composite filters, the binding layer) are -unchanged. +- CPU: Intel Core i7-1185G7, 4 cores and 8 threads, AVX2 and FMA +- Compiler: conda-forge GCC 14.3.0, `-O3`, no `-march=native` +- Python 3.13.13 on Linux x86-64 +- `bioimage_cpp 0.8.0` +- `fastfilters 0.3-5-ge484a99` +- `numpy 2.4.6` +- `scipy 1.17.1` +- `scikit-image 0.26.0` -### CMake wiring +All filter calls are single-threaded. -Add to the `nanobind_add_module(_core ...)` source list: +## Configuration -```cmake -src/cpp/filters/convolve_avx2.cxx -src/cpp/filters/convolve_dispatch.cxx -``` +| Parameter | Value | +| --- | ---: | +| sigma | 1.5 | +| inner sigma | 1.0 | +| outer sigma | 2.0 | +| window size | 3.0 | +| 2D input | `camera()`, 512 × 512, float32 | +| 3D input | `cells3d()[:, 1]`, 60 × 256 × 256, float32 | -Then attach per-file flags so only the AVX2 TU gets AVX2 instructions -(the rest of the wheel stays at the manylinux SSE2 baseline): - -```cmake -if(MSVC) - set_source_files_properties( - src/cpp/filters/convolve_avx2.cxx - PROPERTIES COMPILE_OPTIONS "/arch:AVX2" - ) -else() - set_source_files_properties( - src/cpp/filters/convolve_avx2.cxx - PROPERTIES COMPILE_OPTIONS "-mavx2;-mfma" - ) -endif() -``` +## Result -Do **not** add `-march=native` or change the global `-O3`. The wheel -must keep installing on any pre-Haswell x86_64 machine that -manylinux2014 supports; the AVX2 instructions only execute behind the -CPUID check. +The matched `bioimage_cpp / fastfilters` geometric-mean ratio improved from +`2.194` to `1.498`. Values above one mean that bioimage-cpp is slower. -### Runtime dispatch pattern +| Filter | Dim. | Before, ms | After, ms | fastfilters, ms | Final ratio | +| --- | ---: | ---: | ---: | ---: | ---: | +| Gaussian smoothing | 2D | 0.81 | 0.88 | 0.60 | 1.47 | +| Gradient magnitude | 2D | 2.06 | 1.67 | 1.03 | 1.63 | +| Laplacian of Gaussian | 2D | 1.78 | 1.23 | 0.88 | 1.40 | +| Hessian eigenvalues | 2D | 3.01 | 2.77 | 1.70 | 1.63 | +| Structure-tensor eigenvalues | 2D | 5.13 | 4.11 | 3.00 | 1.37 | +| Gaussian smoothing | 3D | 36.86 | 32.24 | 14.29 | 2.26 | +| Gradient magnitude | 3D | 108.44 | 72.44 | 53.07 | 1.37 | +| Laplacian of Gaussian | 3D | 104.82 | 68.93 | 47.29 | 1.46 | +| Hessian eigenvalues | 3D | 492.18 | 207.25 | 148.64 | 1.39 | +| Structure-tensor eigenvalues | 3D | 621.41 | 293.79 | 242.84 | 1.21 | -In `convolve_dispatch.hxx`: +The final geometric-mean ratios against the other references are: -```cpp -namespace bioimage_cpp::filters::dispatch { +| Comparison | Ratio | Cases | +| --- | ---: | ---: | +| bioimage-cpp / fastfilters | 1.498 | 10 | +| bioimage-cpp / vigra | 0.136 | 12 | +| bioimage-cpp / scipy | 0.124 | 12 | -using ConvolveXFn = void (*)( - const float*, float*, std::ptrdiff_t, std::ptrdiff_t, const float* -); -using ConvolveStridedFn = void (*)( - const float*, float*, std::ptrdiff_t, std::ptrdiff_t, std::ptrdiff_t, - const float* -); +The result does not reach the `1.2` fastfilters stretch target. The current +implementation remains about seven times faster than vigra and eight times +faster than SciPy on this matrix. -// One entry per (radius R in 1..kMaxSpecialisedRadius, Symmetric in {0,1}). -// Filled at module load by init(). -extern ConvolveXFn convolve_x_table[kMaxSpecialisedRadius + 1][2]; -extern ConvolveStridedFn convolve_strided_table[kMaxSpecialisedRadius + 1][2]; +## Implemented changes -void init(); // called once from bind_filters() +### Scratch storage -} -``` +Each filter call now owns one uninitialized scratch allocation. It divides the +allocation into full-volume slots. -In `convolve_dispatch.cxx`: - -```cpp -namespace { -bool detect_avx2_fma() { -#if defined(__GNUC__) || defined(__clang__) - __builtin_cpu_init(); - return __builtin_cpu_supports("avx2") && __builtin_cpu_supports("fma"); -#elif defined(_MSC_VER) - int regs1[4]; __cpuid(regs1, 1); - const bool fma = (regs1[2] & (1 << 12)) != 0; - int regs7[4]; __cpuidex(regs7, 7, 0); - const bool avx2 = (regs7[1] & (1 << 5)) != 0; - // Also OSXSAVE + XGETBV to confirm OS-saved YMM state. - ... - return avx2 && fma; -#else - return false; -#endif -} -} - -void init() { - const bool use_avx2 = detect_avx2_fma(); - // Macros generate the per-R table entries to avoid 24 hand-written - // lines (the same boost-preprocessor-style explosion fastfilters - // does, kept tiny with simple X-macros). - #define BIO_FILL(R) \ - if (use_avx2) { \ - convolve_x_table[R][0] = &avx2::convolve_x_radius_sym; \ - convolve_x_table[R][1] = &avx2::convolve_x_radius_anti; \ - convolve_strided_table[R][0] = &avx2::convolve_strided_radius_sym; \ - convolve_strided_table[R][1] = &avx2::convolve_strided_radius_anti; \ - } else { \ - convolve_x_table[R][0] = &detail::convolve_x_radius_sym; \ - convolve_x_table[R][1] = &detail::convolve_x_radius_anti; \ - convolve_strided_table[R][0] = &detail::convolve_strided_radius_sym; \ - convolve_strided_table[R][1] = &detail::convolve_strided_radius_anti; \ - } - BIO_FILL(1) BIO_FILL(2) ... BIO_FILL(12) - #undef BIO_FILL -} -``` +- Hessian uses four slots in 2D and seven slots in 3D. +- Structure tensor uses six slots in 2D and ten slots in 3D. +- The previous structure-tensor implementation used seven and eleven slots. -(Internally split each existing `template ` into -two non-templated-on-`Symmetric` aliases — `_sym` and `_anti` — so the -function-pointer types are concrete and the table is plain data.) - -Call `dispatch::init()` from `bind_filters()` in `src/bindings/filters.cxx` -(once, before any kernel binding can be invoked). Use a -`static std::once_flag` guard so re-imports don't double-initialise. - -### AVX2 kernel skeleton - -The X-pass kernel in `convolve_avx2.cxx` is essentially the scalar main -loop with explicit `__m256` registers: - -```cpp -namespace bioimage_cpp::filters::avx2 { - -template -void convolve_x_radius_sym( - const float* __restrict in, - float* __restrict out, - std::ptrdiff_t n_rows, - std::ptrdiff_t n_cols, - const float* __restrict h -) { - const std::ptrdiff_t prologue_end = std::min(R, n_cols); - const std::ptrdiff_t epilogue_start = std::max(prologue_end, n_cols - R); - - for (std::ptrdiff_t row = 0; row < n_rows; ++row) { - const float* __restrict in_row = in + row * n_cols; - float* __restrict out_row = out + row * n_cols; - - // --- border prologue: reuse scalar mirror code unchanged --- - scalar_border_sym(in_row, out_row, 0, prologue_end, n_cols, h); - - // --- main AVX2 loop --- - std::ptrdiff_t x = prologue_end; - const __m256 h0 = _mm256_set1_ps(h[0]); - for (; x + 8 <= epilogue_start; x += 8) { - __m256 acc = _mm256_mul_ps(_mm256_loadu_ps(in_row + x), h0); - for (int k = 1; k <= R; ++k) { - const __m256 hk = _mm256_set1_ps(h[k]); - const __m256 sum = _mm256_add_ps( - _mm256_loadu_ps(in_row + x + k), - _mm256_loadu_ps(in_row + x - k) - ); - acc = _mm256_fmadd_ps(hk, sum, acc); - } - _mm256_storeu_ps(out_row + x, acc); - } - // --- scalar tail (0..7 floats) --- - scalar_main_sym(in_row, out_row, x, epilogue_start, h); - - // --- border epilogue --- - scalar_border_sym(in_row, out_row, epilogue_start, n_cols, n_cols, h); - } -} - -template -void convolve_x_radius_anti(...) { /* same shape, _mm256_sub_ps instead of _add_ps */ } - -} -``` +The implementation does not retain scratch memory between calls. Concurrent +calls do not share writable state. + +### Shared 3D passes + +Composite filters reuse common Z derivatives. + +- Gradient magnitude and Laplacian of Gaussian use two Z passes instead of + three. +- Hessian uses one Z pass for each derivative order, for three passes instead + of six. +- Structure-tensor gradient calculation uses two Z passes instead of three. + +This scalar stage reduced the 3D gradient-magnitude time from `108.44 ms` to +about `77.49 ms`. It reduced the 3D Laplacian time from `104.82 ms` to about +`76.34 ms`. + +### Fused structure-tensor products + +A fused pass forms all unique gradient products and applies the first outer +Gaussian axis in one traversal. It removes the temporary product volume. + +The AVX2 version reduced the profiled 3D outer-product phase from `120.5 ms` to +`85.5 ms`. Scalar and AVX2 results differ by less than `1.3e-8` in the direct +backend comparison. + +### AVX2 and FMA dispatch + +The build option `BIOIMAGE_FILTERS_AVX2_DISPATCH` is enabled by default on +supported x86 builds. Only `src/cpp/filters/convolve_avx2.cxx` and +`src/cpp/filters/eigenvalues_avx2.cxx` receive AVX2 and FMA compiler flags. + +Runtime dispatch checks CPU and operating-system AVX support. It falls back to +the scalar kernels in these cases: + +- the CPU does not support AVX2 and FMA; +- the build target is not supported x86; +- a convolution kernel radius is above 12; +- `BIOIMAGE_CPP_FILTERS_FORCE_SCALAR` is set to a nonzero value. + +The specialized kernels use unaligned loads. They share scalar mirror-border +helpers with the portable implementation. + +### 3 × 3 eigenvalues + +Profiling before the final eigensolver change showed: + +| Filter | Convolution and products | Eigenvalues | +| --- | ---: | ---: | +| 3D Hessian | 34.7% | 65.3% | +| 3D structure tensor | 47.8% | 52.2% | + +The AVX2 eigensolver processes eight symmetric matrices per batch. It reads six +structure-of-arrays component streams and writes interleaved triples. The +scalar implementation handles the tail and unsupported targets. + +The vector path uses fixed float32 polynomials: + +- degree-7 Chebyshev for `acos(abs(r)) / sqrt(1 - abs(r))`; +- degree-4 power polynomials for `cos(phi)` and `sin(phi) / phi`. + +The `acos` implementation reflects negative inputs. The direct sine polynomial +avoids cancellation near repeated roots. The implementation clamps the +invariant to `[-1, 1]` and handles zero, isotropic, and subnormal matrices. + +The approximation validator reports: + +| Measurement | Maximum error | +| --- | ---: | +| `acos` absolute error | `4.223e-7` | +| `cos` absolute error | `8.511e-8` | +| `sin` absolute error | `8.277e-8` | +| random eigenvalue scaled error | `1.363e-5` | +| repeated-root scaled error | `1.982e-4` | +| AVX2 versus scalar scaled error | `2.038e-5` | + +The direct benchmark uses `3,932,160` matrices. The AVX2 median is `49.871 ms`. +The scalar median is `246.645 ms`. The AVX2 path is `4.946` times faster. + +A paired end-to-end benchmark captured immediately before this change used the +same environment and input: + +| Filter | Before, ms | After, ms | Reduction | +| --- | ---: | ---: | ---: | +| 3D Hessian eigenvalues | 404.57 | 207.25 | 48.8% | +| 3D structure-tensor eigenvalues | 480.03 | 293.79 | 38.8% | + +Both reductions exceed the 15% acceptance threshold. The unaffected 3D +smoothing, gradient-magnitude, and Laplacian times changed by at most 3.5% +relative to the preceding P2 benchmark. + +The change adds no external dependency. The portable build continues to use +the scalar standard-library implementation. + +## Correctness and portability + +- The full parity matrix passes on scalar and AVX2 paths with the original + tolerances. +- The automatic filter test run passes 91 tests. +- The full repository test suite passes 1378 tests. +- A build with `BIOIMAGE_FILTERS_AVX2_DISPATCH=OFF` passes 77 filter tests. It + skips 14 AVX2-only checks. +- Tests cover vector boundaries, random scales, repeated roots, subnormal + inputs, short image axes, radii above 12, concurrent calls, and + scalar-to-AVX2 agreement. + +## Known reference differences -The strided kernel follows the same pattern but loops over `kStripBlock` -in steps of 8, using `__m256` for the accumulator strip. Crucially the -strip stays at 64 floats (`kStripBlock` is already a multiple of 8), so -no new tiling decision is needed. - -`scalar_border_sym` / `scalar_main_sym` are just the existing scalar -loop bodies hoisted into small inline helpers callable from both the -AVX2 and the scalar TU. **The mirror-boundary handling code must not be -duplicated between the two TUs** — that's where divergence bugs would -hide. Make the helpers `inline` in a shared header. - -### What stays byte-for-byte identical - -- Kernel-coefficient generation in `kernel.hxx`. -- Eigenvalue solvers in `eigenvalues.hxx`. -- Composite filters in `gaussian.hxx`. -- Binding layer in `src/bindings/filters.cxx`. -- Python wrapper in `src/bioimage_cpp/filters/_filters.py`. -- The public `convolve_axis_x` / `convolve_axis_strided` signatures in - `convolve.hxx`. -- The mirror-index function `detail::mirror_index` and the border - prologue/epilogue logic. - -If a Tier-2 change is touching any of these, stop and re-read the scope -section — it's almost certainly not what Tier 2 is for. - -### Expected speedup - -Based on the gap to `fastfilters` in this benchmark -(`bioimage_cpp / fastfilters` geomean = 2.00): - -- Simple filters (smoothing, derivative, gradient_magnitude, LoG): - realistic post-Tier-2 ratio **1.0 – 1.3×** fastfilters (essentially - tied to slightly behind). -- 3D Hessian / structure-tensor eigenvalues: realistic post-Tier-2 - ratio **1.4 – 1.7×** fastfilters. The remaining gap is in `acos`/`cos` - inside the 3×3 trig eigensolver, which intrinsics alone do not help - with. - -Do not expect to *match* fastfilters exactly without also vendoring a -vectorized math library and per-radius file-copy specialisation — that's -the next-tier-after-Tier-2 work, deliberately out of scope here. - -### Verification - -1. **Parity gate stays green**: - `python development/filters/check_parity.py` on a machine with AVX2 - support must still PASS at the same tolerances. If it doesn't, the - AVX2 kernel disagrees with the scalar kernel — that's the most - likely failure mode and is almost always an off-by-one in the - prologue / main / epilogue boundary handling. -2. **Scalar path stays green**: re-run with the AVX2 path forced off - (set the function-pointer table to the scalar entries unconditionally - in a debug build, or guard the dispatch decision with an environment - variable like `BIOIMAGE_FORCE_SCALAR=1`). The full pytest suite must - still pass; this catches scalar-only regressions introduced when - refactoring shared helpers. -3. **Benchmark**: - `python development/filters/benchmark.py` should show the - `bioimage_cpp / fastfilters` geomean drop from ~2.00 toward ~1.2. - Update this file with the new numbers. -4. **Pre-Haswell smoke**: the dispatch must take the scalar path on a - machine without AVX2. Easiest local check: temporarily make - `detect_avx2_fma()` return `false` and confirm correctness + - performance fall back to today's Tier-1 numbers. - -### Smallest first step - -Don't ship both kernels at once. The recommended sequence is: - -1. Land the dispatch scaffolding (`convolve_dispatch.hxx` / - `.cxx`, function-pointer tables, CMake wiring) **with both pointers - still pointing at the existing scalar kernels**. No behavior change. - Tests stay green. This isolates the build-system part of the work. -2. Add `convolve_x_radius_avx2` only. Re-run parity + benchmark. - Expect the simple filters to move; the Y/Z-bound filters - (gradient_magnitude, LoG, Hessian) move proportionally less. -3. Add `convolve_strided_radius_avx2`. Re-run parity + benchmark. - Expect the 3D filters to move significantly. - -If after step 2 the speedup is smaller than expected, stop and profile -before continuing — it usually means the autovectorizer was already -doing better than this section assumes, and the marginal value of -step 3 is lower than it appears here. - -### Watch-outs - -- **Boundary-mode divergence** between scalar and AVX2 paths is the - single most likely correctness bug. Share the prologue/epilogue - helpers via an `inline` header; don't copy-paste. -- **Unaligned loads only.** Use `_mm256_loadu_ps` / `_mm256_storeu_ps`, - not the aligned variants. The bench inputs are not guaranteed to be - 32-byte aligned, and on modern Intel/AMD the unaligned-load - performance penalty is essentially zero. Trying to force alignment in - the binding layer is more complexity than the win. -- **MSVC AVX2 detection.** `__builtin_cpu_supports` is GCC/Clang only. - Use raw `__cpuid` / `__cpuidex` + an `_xgetbv` check (the OS must - have saved the YMM state for AVX to be safe to use). There's example - code in `fastfilters/src/library/cpu_intel.c` if you need a - reference; do not vendor it, just write the small bit you need. -- **Don't introduce OpenMP, std::thread, or any threading primitive in - this work.** Threading is a separate follow-up that should layer on - top of the dispatch scheme via `parallel_for_chunks` (see - `include/bioimage_cpp/detail/threading.hxx`). Mixing the two changes - is asking for trouble. -- **Don't add AVX-512 "while we're here."** It is a separate trigger - decision with separate trade-offs (frequency throttling on older - Xeons, larger binary, marginal win on memory-bound separable FIR). - -## Known caveats reflected in the adapters - -- `fastfilters.gaussianDerivative` only accepts a uniform per-axis order; - the bench cell is `n/a` rather than running an unequal operation. -- `fastfilters.structureTensorEigenvalues` swaps `innerScale` / - `outerScale` at the Python boundary (its wrapper calls the C function - with the args in the opposite order — `src/python/core.cxx:328` vs - `src/library/fir_filters.c:156` in the fastfilters source). The adapter - swaps them back so the bench compares the same operation as `vigra` / - `bioimage_cpp`. -- `scipy` and `bioimage_cpp` use scipy-style `mirror` (reflect without - edge-pixel repeat); `vigra` / `fastfilters` use reflect with edge - repeat. Parity is checked on the image interior to absorb the - difference. +- fastfilters does not support the benchmark's per-axis derivative order. The + derivative row is excluded from the fastfilters geometric mean. +- The fastfilters Python structure-tensor wrapper swaps its inner and outer + scales. The adapter swaps them back before comparison. +- SciPy and bioimage-cpp use mirror reflection without an edge-pixel repeat. + Vigra and fastfilters use reflection with an edge-pixel repeat. Parity checks + compare only the image interior. diff --git a/development/filters/benchmark.py b/development/filters/benchmark.py index 2b5416a..6c12848 100644 --- a/development/filters/benchmark.py +++ b/development/filters/benchmark.py @@ -11,10 +11,12 @@ import argparse import csv -import math +import os import sys from statistics import geometric_mean +import numpy as np + from _bench_utils import ( BenchConfig, FILTERS, @@ -40,6 +42,11 @@ def parse_args() -> argparse.Namespace: help="Crop to fast sizes for a smoke run.") parser.add_argument("--no-3d", action="store_true") parser.add_argument("--no-2d", action="store_true") + parser.add_argument( + "--force-scalar", + action="store_true", + help="Disable the runtime AVX2 filter backend.", + ) parser.add_argument( "--filters", default=",".join(FILTERS), help="Comma-separated subset of filters to benchmark.", @@ -52,7 +59,7 @@ def parse_args() -> argparse.Namespace: return parser.parse_args() -def _load_test_data(args) -> list[tuple[str, "np.ndarray"]]: +def _load_test_data(args) -> list[tuple[str, np.ndarray]]: crop_2d = (128, 128) if args.small else None crop_3d = (32, 64, 64) if args.small else None targets = [] @@ -65,6 +72,11 @@ def _load_test_data(args) -> list[tuple[str, "np.ndarray"]]: def main() -> int: args = parse_args() + if args.force_scalar: + os.environ["BIOIMAGE_CPP_FILTERS_FORCE_SCALAR"] = "1" + + from bioimage_cpp import _core + cfg = BenchConfig( sigma=args.sigma, inner_sigma=args.inner_sigma, @@ -81,7 +93,9 @@ def main() -> int: print( f"sigma={cfg.sigma}, inner={cfg.inner_sigma}, outer={cfg.outer_sigma}, " - f"window_size={cfg.window_size}, repeats={args.repeats}" + f"window_size={cfg.window_size}, repeats={args.repeats}, " + f"convolution_backend={_core._filters_convolution_backend()}, " + f"eigenvalue_backend={_core._filters_eigenvalue_backend()}" ) rows = [] diff --git a/development/filters/benchmark_eigenvalues.py b/development/filters/benchmark_eigenvalues.py new file mode 100644 index 0000000..df94abf --- /dev/null +++ b/development/filters/benchmark_eigenvalues.py @@ -0,0 +1,159 @@ +"""Benchmark the scalar and automatic 3 x 3 eigensolver backends. + +Run this script after an editable build: + + python development/filters/benchmark_eigenvalues.py +""" + +from __future__ import annotations + +import argparse +import csv +import os +from statistics import median +from time import perf_counter + +import numpy as np + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser( + description="Benchmark the internal batch eigensolver." + ) + parser.add_argument("--matrices", type=int, default=60 * 256 * 256) + parser.add_argument("--repeats", type=int, default=7) + parser.add_argument("--seed", type=int, default=17) + parser.add_argument("--minimum-speedup", type=float, default=3.0) + parser.add_argument("--csv", default=None) + return parser.parse_args() + + +def set_scalar_backend(force_scalar: bool) -> None: + if force_scalar: + os.environ["BIOIMAGE_CPP_FILTERS_FORCE_SCALAR"] = "1" + else: + os.environ.pop("BIOIMAGE_CPP_FILTERS_FORCE_SCALAR", None) + + +def run_once(components: np.ndarray, out: np.ndarray) -> None: + from bioimage_cpp import _core + + _core._filters_ev3_symmetric_float32(components, out) + + +def measure( + components: np.ndarray, out: np.ndarray, repeats: int, force_scalar: bool +) -> tuple[str, list[float], np.ndarray]: + from bioimage_cpp import _core + + set_scalar_backend(force_scalar) + backend = _core._filters_eigenvalue_backend() + run_once(components, out) + times = [] + for _ in range(repeats): + start = perf_counter() + run_once(components, out) + times.append(perf_counter() - start) + return backend, times, out.copy() + + +def main() -> int: + args = parse_args() + if args.matrices < 8: + raise ValueError("--matrices must be at least 8") + if args.repeats < 1: + raise ValueError("--repeats must be positive") + + rng = np.random.default_rng(args.seed) + components = rng.normal(size=(6, args.matrices)).astype(np.float32) + out = np.empty((args.matrices, 3), dtype=np.float32) + + original_force_scalar = os.environ.get("BIOIMAGE_CPP_FILTERS_FORCE_SCALAR") + try: + automatic_backend, automatic_times, automatic = measure( + components, out, args.repeats, False + ) + scalar_backend, scalar_times, scalar = measure( + components, out, args.repeats, True + ) + finally: + if original_force_scalar is None: + os.environ.pop("BIOIMAGE_CPP_FILTERS_FORCE_SCALAR", None) + else: + os.environ["BIOIMAGE_CPP_FILTERS_FORCE_SCALAR"] = original_force_scalar + + automatic_median = median(automatic_times) + scalar_median = median(scalar_times) + speedup = scalar_median / automatic_median + max_difference = float(np.max(np.abs(automatic - scalar))) + scale = np.maximum( + np.max(np.abs(scalar), axis=1), np.finfo(np.float32).tiny + ) + max_scaled_difference = float( + np.max(np.max(np.abs(automatic - scalar), axis=1) / scale) + ) + + print( + f"matrices={args.matrices}, repeats={args.repeats}, " + f"automatic_backend={automatic_backend}" + ) + print( + f"automatic: median={automatic_median * 1e3:.3f} ms, " + f"min={min(automatic_times) * 1e3:.3f} ms" + ) + print( + f"{scalar_backend}: median={scalar_median * 1e3:.3f} ms, " + f"min={min(scalar_times) * 1e3:.3f} ms" + ) + print(f"scalar / automatic speedup: {speedup:.3f}x") + print(f"maximum absolute difference: {max_difference:.3e}") + print(f"maximum scaled difference: {max_scaled_difference:.3e}") + + if args.csv is not None: + with open(args.csv, "w", newline="") as file: + writer = csv.DictWriter( + file, + fieldnames=[ + "backend", + "matrices", + "median_s", + "min_s", + "repeats", + ], + ) + writer.writeheader() + writer.writerow( + { + "backend": automatic_backend, + "matrices": args.matrices, + "median_s": automatic_median, + "min_s": min(automatic_times), + "repeats": args.repeats, + } + ) + writer.writerow( + { + "backend": scalar_backend, + "matrices": args.matrices, + "median_s": scalar_median, + "min_s": min(scalar_times), + "repeats": args.repeats, + } + ) + print(f"wrote {args.csv}") + + passed = ( + np.isfinite(automatic).all() + and np.isfinite(scalar).all() + and max_scaled_difference < 5e-5 + ) + if automatic_backend == "avx2": + passed = passed and speedup >= args.minimum_speedup + else: + print("The automatic build does not provide the AVX2 backend.") + print("PASS" if passed else "FAIL") + return 0 if passed else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/development/filters/check_parity.py b/development/filters/check_parity.py index 2a9450f..c56900f 100644 --- a/development/filters/check_parity.py +++ b/development/filters/check_parity.py @@ -14,13 +14,12 @@ from __future__ import annotations import argparse -import math +import os import sys import numpy as np from _bench_utils import ( - ADAPTERS, BenchConfig, FILTERS, LIBRARIES, @@ -45,6 +44,11 @@ def parse_args() -> argparse.Namespace: help="Override per-filter tolerance.") parser.add_argument("--no-3d", action="store_true") parser.add_argument("--no-2d", action="store_true") + parser.add_argument( + "--force-scalar", + action="store_true", + help="Disable the runtime AVX2 filter backend.", + ) parser.add_argument( "--filters", default=",".join(FILTERS), help="Comma-separated subset of filters to check.", @@ -99,6 +103,11 @@ def _run_one(filter_name: str, image: np.ndarray, cfg: BenchConfig, atol: float) def main() -> int: args = parse_args() + if args.force_scalar: + os.environ["BIOIMAGE_CPP_FILTERS_FORCE_SCALAR"] = "1" + + from bioimage_cpp import _core + cfg = BenchConfig( sigma=args.sigma, inner_sigma=args.inner_sigma, @@ -118,6 +127,10 @@ def main() -> int: targets.append(("3D", load_3d())) any_failure = False + print( + f"convolution_backend={_core._filters_convolution_backend()}, " + f"eigenvalue_backend={_core._filters_eigenvalue_backend()}" + ) for dim_label, image in targets: print(f"\n== {dim_label} parity (shape={image.shape}, dtype={image.dtype}) ==") for filter_name in requested: diff --git a/development/filters/validate_eigenvalue_approximation.py b/development/filters/validate_eigenvalue_approximation.py new file mode 100644 index 0000000..4f28687 --- /dev/null +++ b/development/filters/validate_eigenvalue_approximation.py @@ -0,0 +1,250 @@ +"""Validate the internal 3 x 3 eigenvalue approximations. + +Run this script after an editable build: + + python development/filters/validate_eigenvalue_approximation.py +""" + +from __future__ import annotations + +import argparse +import os + +import numpy as np + + +ACOS_COEFFICIENTS = np.array( + [ + 1.4866664409637451, + -0.07770606875419617, + 0.005770874209702015, + -0.0005768424598500133, + 0.00006643808592343703, + -0.000008315640116052236, + 0.0000010878551393034286, + -0.00000014935945102934056, + ], + dtype=np.float32, +) +COS_COEFFICIENTS = np.array( + [ + 1.0, + -0.4999999701976776, + 0.041666433215141296, + -0.0013882536441087723, + 0.000024095119442790747, + ], + dtype=np.float32, +) +SINC_COEFFICIENTS = np.array( + [ + 1.0, + -0.1666666567325592, + 0.008333305828273296, + -0.0001983467664103955, + 0.0000026878346943703946, + ], + dtype=np.float32, +) + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser( + description="Validate the internal vector-math approximations." + ) + parser.add_argument("--samples", type=int, default=200_000) + parser.add_argument("--seed", type=int, default=91) + return parser.parse_args() + + +def approximate_acos(values: np.ndarray) -> np.ndarray: + values = np.asarray(values, dtype=np.float32) + magnitude = np.abs(values) + z = magnitude + magnitude - np.float32(1.0) + two_z = z + z + b1 = np.zeros_like(z) + b2 = np.zeros_like(z) + for coefficient in ACOS_COEFFICIENTS[:0:-1]: + b0 = two_z * b1 + (coefficient - b2) + b2 = b1 + b1 = b0 + q = z * b1 + (ACOS_COEFFICIENTS[0] - b2) + base = np.sqrt(np.maximum(np.float32(0.0), np.float32(1.0) - magnitude)) * q + return np.where(values < 0.0, np.float32(np.pi) - base, base) + + +def approximate_cos(phi: np.ndarray) -> np.ndarray: + phi = np.asarray(phi, dtype=np.float32) + u = phi * phi + result = np.full_like(u, COS_COEFFICIENTS[-1]) + for coefficient in COS_COEFFICIENTS[-2::-1]: + result = result * u + coefficient + return result + + +def approximate_sin(phi: np.ndarray) -> np.ndarray: + phi = np.asarray(phi, dtype=np.float32) + u = phi * phi + sinc = np.full_like(u, SINC_COEFFICIENTS[-1]) + for coefficient in SINC_COEFFICIENTS[-2::-1]: + sinc = sinc * u + coefficient + return phi * sinc + + +def components_from_matrices(matrices: np.ndarray) -> np.ndarray: + return np.ascontiguousarray( + np.stack( + [ + matrices[:, 0, 0], + matrices[:, 0, 1], + matrices[:, 0, 2], + matrices[:, 1, 1], + matrices[:, 1, 2], + matrices[:, 2, 2], + ] + ), + dtype=np.float32, + ) + + +def direct_eigenvalues(components: np.ndarray) -> np.ndarray: + from bioimage_cpp import _core + + out = np.empty((components.shape[1], 3), dtype=np.float32) + _core._filters_ev3_symmetric_float32(components, out) + return out + + +def maximum_scaled_error(got: np.ndarray, expected: np.ndarray) -> float: + scale = np.maximum( + np.max(np.abs(expected), axis=1), np.finfo(np.float32).tiny + ) + return float(np.max(np.max(np.abs(got - expected), axis=1) / scale)) + + +def random_matrices(samples: int, seed: int) -> np.ndarray: + rng = np.random.default_rng(seed) + components = rng.normal(size=(6, samples)).astype(np.float32) + scales = np.power( + 10.0, rng.uniform(-15.0, 15.0, size=samples) + ).astype(np.float32) + components *= scales + + matrices = np.empty((samples, 3, 3), dtype=np.float32) + matrices[:, 0, 0] = components[0] + matrices[:, 0, 1] = matrices[:, 1, 0] = components[1] + matrices[:, 0, 2] = matrices[:, 2, 0] = components[2] + matrices[:, 1, 1] = components[3] + matrices[:, 1, 2] = matrices[:, 2, 1] = components[4] + matrices[:, 2, 2] = components[5] + return matrices + + +def adversarial_matrices(seed: int) -> np.ndarray: + rng = np.random.default_rng(seed) + spectra = [] + smallest_subnormal = float( + np.nextafter(np.float32(0.0), np.float32(1.0)) + ) + for scale in ( + smallest_subnormal, + np.finfo(np.float32).tiny, + 1e-20, + 1e-8, + 1.0, + 1e8, + 1e20, + ): + spectra.extend( + [ + [0.0, 0.0, 0.0], + [scale, scale, scale], + [3.0 * scale, 3.0 * scale, -2.0 * scale], + [3.0 * scale, -2.0 * scale, -2.0 * scale], + [scale, scale * (1.0 + 2e-6), -0.5 * scale], + [scale, scale * (1.0 - 2e-6), -0.5 * scale], + ] + ) + + matrices = [] + for eigenvalues in spectra: + q, _ = np.linalg.qr(rng.normal(size=(3, 3))) + matrices.append( + (q @ np.diag(eigenvalues) @ q.T).astype(np.float32) + ) + return np.stack(matrices) + + +def main() -> int: + args = parse_args() + if args.samples < 8: + raise ValueError("--samples must be at least 8") + + domain = np.linspace(-1.0, 1.0, args.samples + 1, dtype=np.float32) + acos_error = float( + np.max( + np.abs( + approximate_acos(domain).astype(np.float64) + - np.arccos(domain.astype(np.float64)) + ) + ) + ) + phi = np.linspace(0.0, np.pi / 3.0, args.samples + 1, dtype=np.float32) + phi64 = phi.astype(np.float64) + cos_error = float( + np.max(np.abs(approximate_cos(phi).astype(np.float64) - np.cos(phi64))) + ) + sin_error = float( + np.max(np.abs(approximate_sin(phi).astype(np.float64) - np.sin(phi64))) + ) + + random = random_matrices(args.samples, args.seed) + random_components = components_from_matrices(random) + random_ref = np.linalg.eigvalsh(random)[..., ::-1].astype(np.float32) + automatic = direct_eigenvalues(random_components) + random_error = maximum_scaled_error(automatic, random_ref) + + adversarial = adversarial_matrices(args.seed + 1) + adversarial_ref = np.linalg.eigvalsh(adversarial)[..., ::-1].astype(np.float32) + adversarial_got = direct_eigenvalues(components_from_matrices(adversarial)) + adversarial_error = maximum_scaled_error(adversarial_got, adversarial_ref) + + from bioimage_cpp import _core + + scalar_error = 0.0 + original_force_scalar = os.environ.get("BIOIMAGE_CPP_FILTERS_FORCE_SCALAR") + try: + if _core._filters_eigenvalue_backend() == "avx2": + os.environ["BIOIMAGE_CPP_FILTERS_FORCE_SCALAR"] = "1" + scalar = direct_eigenvalues(random_components) + scalar_error = maximum_scaled_error(automatic, scalar) + finally: + if original_force_scalar is None: + os.environ.pop("BIOIMAGE_CPP_FILTERS_FORCE_SCALAR", None) + else: + os.environ["BIOIMAGE_CPP_FILTERS_FORCE_SCALAR"] = original_force_scalar + + print(f"acos max absolute error: {acos_error:.3e}") + print(f"cos max absolute error: {cos_error:.3e}") + print(f"sin max absolute error: {sin_error:.3e}") + print(f"random eigen max scaled error: {random_error:.3e}") + print(f"tail eigen max scaled error: {adversarial_error:.3e}") + if scalar_error: + print(f"AVX2/scalar max scaled error: {scalar_error:.3e}") + + passed = ( + acos_error < 6e-7 + and cos_error < 1e-7 + and sin_error < 1e-7 + and random_error < 2e-5 + and adversarial_error < 3e-4 + and scalar_error < 3e-5 + and np.isfinite(automatic).all() + and np.isfinite(adversarial_got).all() + ) + print("PASS" if passed else "FAIL") + return 0 if passed else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/development/graph/agglomeration/PERFORMANCE_NOTES.md b/development/graph/agglomeration/PERFORMANCE_NOTES.md index e58ca19..d277e88 100644 --- a/development/graph/agglomeration/PERFORMANCE_NOTES.md +++ b/development/graph/agglomeration/PERFORMANCE_NOTES.md @@ -4,6 +4,17 @@ State of `bioimage_cpp.graph.agglomeration` vs `nifty.graph.agglo` on the external multicut problem set (samples A, B, C × sizes small, medium), and notes on the remaining algorithmic differences and possible optimisations. +## Shared contraction topology + +The agglomeration driver now uses `graph::detail::ContractionTopology`. +Policies keep only objective-specific state. Mutation observers update policy +state, the heap, and the union-find as each topology event occurs. Profile +builds report `topology_reset`, `contract`, `initialize`, and +`labels_from_sets`. + +`MalaClusterPolicy.num_edges_stop` now reads the topology's active-edge +count. This fixes the previous fold-only counter. + ## Current benchmark matrix Produced 2026-05-24 with the per-policy `check_*.py` scripts, each run with @@ -138,7 +149,7 @@ all policies: `O(log N)` and is called once per fold and once per neighbour in the final `contract_edge_done` sweep. For the largest problems we do ~10⁸ such updates; the `change` cycle dominates wall time. -2. **Adjacency restructuring inside `agglo_merge_dynamic_nodes`.** Per +2. **Adjacency restructuring inside `ContractionTopology`.** Per contraction we walk the removed node's adjacency, do an O(degree) `erase_by_neighbor` per fold, and write back into the survivor's adjacency. Memory-bound on large medium problems. @@ -217,11 +228,11 @@ current behaviour as functionally complete and correct. heap top — empirically this is rare, so the trade should be favourable. -6. **Profile-guided focus on `gasp_max` C/medium.** Add `BIOIMAGE_PROFILE` - scopes for `agglo_merge_dynamic_nodes` (broken into "fold loop", - "rekey loop", "contract_edge_done") and rerun. The 164 s vs 105 s - gap should be attributable to one of these phases; once it's - visible, the right primitive to optimise becomes obvious. +6. **Profile-guided focus on `gasp_max` C/medium.** Use the existing + `BIOIMAGE_PROFILE` scopes to separate initialization, contraction, and + label materialization. The 164 s vs 105 s gap should be attributable to + one of these phases. Add a focused scope inside the largest phase before + changing the algorithm. 7. **Sparse `MutexStorage` representation.** Each `std::unordered_set` carries ~50 B of overhead and a hash diff --git a/development/graph/agglomeration/_compatibility.py b/development/graph/agglomeration/_compatibility.py index 6575d64..2285daa 100644 --- a/development/graph/agglomeration/_compatibility.py +++ b/development/graph/agglomeration/_compatibility.py @@ -108,26 +108,39 @@ def time_call(function: Callable[[], np.ndarray], repeats: int): def variation_of_information(labels_a: np.ndarray, labels_b: np.ndarray) -> float: - labels_a = np.asarray(labels_a).astype(np.int64) - labels_b = np.asarray(labels_b).astype(np.int64) + labels_a = np.asarray(labels_a).reshape(-1) + labels_b = np.asarray(labels_b).reshape(-1) + if labels_a.size != labels_b.size: + raise ValueError( + "labels_a and labels_b must have the same number of elements, got " + f"{labels_a.size} and {labels_b.size}" + ) n = labels_a.size if n == 0: return 0.0 _, a_inv, a_counts = np.unique(labels_a, return_inverse=True, return_counts=True) _, b_inv, b_counts = np.unique(labels_b, return_inverse=True, return_counts=True) - pa = a_counts / n - pb = b_counts / n - contingency = np.zeros((a_counts.size, b_counts.size), dtype=np.float64) - np.add.at(contingency, (a_inv, b_inv), 1.0) - contingency /= n - with np.errstate(divide="ignore", invalid="ignore"): - ha = -np.sum(pa * np.log(pa, where=pa > 0)) - hb = -np.sum(pb * np.log(pb, where=pb > 0)) - joint = -np.sum( - contingency * np.log(contingency, where=contingency > 0) + pairs = np.stack([a_inv, b_inv], axis=1) + _, joint_counts = np.unique(pairs, axis=0, return_counts=True) + + def entropy(counts: np.ndarray) -> float: + probabilities = counts.astype(np.float64) / float(n) + return float(-np.sum(probabilities * np.log(probabilities))) + + ha = entropy(a_counts) + hb = entropy(b_counts) + joint = entropy(joint_counts) + result = 2.0 * joint - ha - hb + tolerance = ( + 64.0 + * np.finfo(np.float64).eps + * max(1.0, abs(ha), abs(hb), abs(joint)) + ) + if result < -tolerance: + raise RuntimeError( + "variation of information is negative beyond floating-point round-off" ) - mutual_info = ha + hb - joint - return float(2.0 * joint - ha - hb - 2.0 * mutual_info + ha + hb) + return max(0.0, float(result)) def adjusted_rand(labels_a: np.ndarray, labels_b: np.ndarray) -> float: diff --git a/development/graph/agglomeration/test_compatibility_metric.py b/development/graph/agglomeration/test_compatibility_metric.py new file mode 100644 index 0000000..7dad274 --- /dev/null +++ b/development/graph/agglomeration/test_compatibility_metric.py @@ -0,0 +1,75 @@ +from __future__ import annotations + +import numpy as np +import pytest + +import _compatibility + + +def test_variation_of_information_identical_partitions(): + labels = np.array([10, 10, 4, 4, 9]) + assert _compatibility.variation_of_information(labels, labels) == pytest.approx(0.0) + + +def test_variation_of_information_one_cluster_equals_other_entropy(): + one_cluster = np.zeros(4, dtype=np.int64) + balanced = np.array([0, 0, 1, 1]) + assert _compatibility.variation_of_information( + one_cluster, balanced + ) == pytest.approx(np.log(2.0)) + + +def test_variation_of_information_independent_binary_partitions(): + first = np.array([0, 0, 1, 1]) + second = np.array([0, 1, 0, 1]) + assert _compatibility.variation_of_information(first, second) == pytest.approx( + 2.0 * np.log(2.0) + ) + + +def test_variation_of_information_is_invariant_to_label_values(): + first = np.array([0, 0, 1, 2, 2, 2]) + second = np.array([4, 4, 8, 8, 9, 9]) + renumbered_first = np.array([30, 30, 10, 90, 90, 90]) + renumbered_second = np.array([2, 2, 7, 7, 5, 5]) + expected = _compatibility.variation_of_information(first, second) + assert _compatibility.variation_of_information( + renumbered_first, renumbered_second + ) == pytest.approx(expected) + + +def test_variation_of_information_empty_and_mismatched_inputs(): + assert _compatibility.variation_of_information([], []) == 0.0 + with pytest.raises(ValueError, match="same number of elements"): + _compatibility.variation_of_information([0], [0, 1]) + + +def test_variation_of_information_does_not_allocate_dense_contingency(monkeypatch): + original_zeros = np.zeros + + def reject_matrix(shape, *args, **kwargs): + if isinstance(shape, tuple) and len(shape) == 2: + raise AssertionError("dense contingency allocation") + return original_zeros(shape, *args, **kwargs) + + monkeypatch.setattr(_compatibility.np, "zeros", reject_matrix) + first = np.arange(5000, dtype=np.int64) + second = first[::-1] + assert _compatibility.variation_of_information(first, second) == pytest.approx(0.0) + + +def test_variation_of_information_matches_sklearn_when_available(): + sklearn_metrics = pytest.importorskip("sklearn.metrics") + rng = np.random.default_rng(42) + for _ in range(5): + first = rng.integers(0, 6, size=100) + second = rng.integers(0, 8, size=100) + first_probabilities = np.unique(first, return_counts=True)[1] / first.size + second_probabilities = np.unique(second, return_counts=True)[1] / second.size + ha = -np.sum(first_probabilities * np.log(first_probabilities)) + hb = -np.sum(second_probabilities * np.log(second_probabilities)) + mutual_information = sklearn_metrics.mutual_info_score(first, second) + expected = ha + hb - 2.0 * mutual_information + assert _compatibility.variation_of_information( + first, second + ) == pytest.approx(expected) diff --git a/development/graph/lifted_multicut/PERFORMANCE_NOTES.md b/development/graph/lifted_multicut/PERFORMANCE_NOTES.md index 823d176..ba5ee69 100644 --- a/development/graph/lifted_multicut/PERFORMANCE_NOTES.md +++ b/development/graph/lifted_multicut/PERFORMANCE_NOTES.md @@ -4,6 +4,69 @@ State of the lifted-multicut solvers vs nifty on the standard benchmark problems and notes on remaining optimization headroom. Read this before the next round of perf work. +## Shared contraction topology + +Lifted greedy additive now uses +`graph::detail::BasicContractionTopology`. An edge payload keeps each weight +and lifted-edge flag next to its endpoints. Mutation observers update the +heap and union-find without an intermediate event buffer. Folded edges remain +lifted only when both inputs are lifted. + +## Lifted KL optimization (2026-07-27) + +The P4 work kept the solver API and output unchanged. It made three changes: + +- Store the move chain in `ChainScratch` so pair chains reuse its allocation. +- Reset `moved` only for nodes that the chain popped. Do not reset + `cross_count`, because gain initialization overwrites it before each read. +- Remove nested profiler scopes. The reported phase total now matches the + measured solver work. + +The cleanup phase decreased from approximately 1.6 ms to 1.1 ms on the 3D +problem. The allocation reuse reduces work outside the previous nested +profile scopes. + +The final comparison used separate production wheels for commit `19ca22d` and +the candidate. Each row used 30 repeats and alternated bioimage-cpp with +nifty. The second comparison reversed the wheel order. + +| Order | Problem | Baseline | Candidate | Change | Nifty | Nifty / candidate | +| --- | --- | ---: | ---: | ---: | ---: | ---: | +| baseline, candidate | 2D | 5.467 ms | 5.175 ms | -5.3% | 5.226 ms | 1.010 | +| baseline, candidate | 3D | 98.330 ms | 92.393 ms | -6.0% | 115.965 ms | 1.255 | +| candidate, baseline | 2D | 5.540 ms | 5.172 ms | -6.6% | 5.245 ms | 1.014 | +| candidate, baseline | 3D | 95.142 ms | 89.787 ms | -5.6% | 114.645 ms | 1.277 | + +Both comparison orders passed the executable acceptance gate: + +```bash +python development/graph/lifted_multicut/check_benchmark_acceptance.py \ + BASELINE.jsonl CANDIDATE.jsonl \ + --expected-repeats 30 \ + --min-improvement 0.03 \ + --reference-parity 3d +``` + +Every repeat preserved the baseline energy and label SHA-256 digest. The 3D +energy difference from nifty remains `+0.069589`. + +A final production grid run preserved energy `-690544`. It took 569.5 s and +used 917,608 KiB peak RSS. + +The benchmark harness now records raw runtimes, medians, minima, energies, +label digests, source and build metadata, package paths, compiler details, +CPU details, and thread-related environment variables. It can run either +implementation independently and can require nifty. + +The following experiments did not meet the retention criteria: + +- Deriving heap membership from `cross_count` made the 3D median 2.7% slower. +- A solver-local adjacency-weight snapshot improved one grid run by 2.9% but + added 6.8% peak RSS. Its 2D and 3D gain did not reproduce in isolated-wheel + comparisons. +- A compact filtered-adjacency layout, a separate `in_heap` byte, and a sorted + cluster-pair pass did not reduce their profiled phases. + ## Current benchmark matrix Produced by `python evaluate_solvers.py` (2026-05-17). All runs @@ -317,12 +380,13 @@ The KL solver now beats nifty by ~17% on 3D and matches it on 2D. Further optimization is not currently a priority. Sketched levers, in case the workload changes: -### CSR adjacency layout (estimated: 10–15% off `pair_chains`) +### Solver-local adjacency snapshot -`UndirectedGraph` stores adjacency as `vector>` — one -heap allocation per node. A flat CSR built once at the start of -`kernighan_lin` would reduce per-node cache-line misses in -`chain_gain_init` and `chain_loop`. Doesn't change algorithm output. +`UndirectedGraph` already stores adjacency as contiguous CSR. The 2026-07-27 +work tested a solver-local array that aligned each directed adjacency with its +weight. The cache did not produce a stable 2D or 3D gain and increased grid +peak RSS. Do not repeat this experiment without a workload that shows a +larger weight-lookup cost. ### Skip pair-chains where heap stays empty (estimated: <5 ms) @@ -370,8 +434,9 @@ Unchanged from before; useful for sanity-checking future estimates: 1. Re-run `cd development/graph/lifted_multicut && python check_kernighan_lin.py --size 3d --repeats 5` to confirm the baseline hasn't drifted. 2. Build with `pip install -e . --no-build-isolation -C cmake.define.BIOIMAGE_PROFILE=ON` for the profile breakdown. -3. The next-most-attractive lever is CSR adjacency layout (10–15%), - but only worth doing if a heavier workload reveals the need. +3. Optimize another chain-loop operation only when a new profile identifies + a stable cost. The current heap-state and adjacency-layout experiments did + not improve the matched benchmark. ## Files that matter diff --git a/development/graph/lifted_multicut/check_benchmark_acceptance.py b/development/graph/lifted_multicut/check_benchmark_acceptance.py new file mode 100644 index 0000000..5c794af --- /dev/null +++ b/development/graph/lifted_multicut/check_benchmark_acceptance.py @@ -0,0 +1,230 @@ +from __future__ import annotations + +import argparse +import json +import math +from pathlib import Path +from statistics import median + + +PROBLEMS = ("2d", "3d", "grid") + + +def load_results( + path: Path, + *, + latest_run_only: bool, +) -> dict[tuple[str, str], dict]: + rows = [] + with path.open(encoding="utf-8") as file: + for line in file: + row = json.loads(line) + if row.get("record_type") == "result": + rows.append(row) + if latest_run_only and rows: + latest_run_id = rows[-1]["run_id"] + rows = [row for row in rows if row["run_id"] == latest_run_id] + return {(row["problem"], row["solver"]): row for row in rows} + + +def check_raw_measurements( + row: dict, + backend: str, + expected_repeats: int, +) -> list[str]: + errors = [] + runtimes = row[f"{backend}_runtimes_s"] + energies = row[f"{backend}_energies"] + digests = row[f"{backend}_label_digests"] + if len(runtimes) != expected_repeats: + errors.append( + f"{backend} has {len(runtimes)} runtimes; expected {expected_repeats}" + ) + if len(energies) != expected_repeats: + errors.append( + f"{backend} has {len(energies)} energies; expected {expected_repeats}" + ) + if len(digests) != expected_repeats: + errors.append( + f"{backend} has {len(digests)} digests; expected {expected_repeats}" + ) + if any(not math.isfinite(value) or value <= 0.0 for value in runtimes): + errors.append(f"{backend} runtimes must be finite and positive") + if any(not math.isfinite(value) for value in energies): + errors.append(f"{backend} energies must be finite") + if energies and len(set(energies)) != 1: + errors.append(f"{backend} energies differ between repeats") + if digests and len(set(digests)) != 1: + errors.append(f"{backend} label digests differ between repeats") + if runtimes: + if median(runtimes) != row[f"{backend}_runtime_median_s"]: + errors.append(f"{backend} runtime median does not match raw runtimes") + if min(runtimes) != row[f"{backend}_runtime_min_s"]: + errors.append(f"{backend} runtime minimum does not match raw runtimes") + if energies and median(energies) != row[f"{backend}_energy"]: + errors.append(f"{backend} energy does not match raw energies") + return errors + + +def check_problem( + baseline: dict, + candidate: dict, + problem: str, + solver: str, + expected_repeats: int, + max_regression: float, + min_improvement: float, + require_improvement: bool, + require_reference_parity: bool, +) -> tuple[dict, list[str]]: + key = (problem, solver) + errors = [] + if key not in baseline: + return {}, [f"baseline has no row for {problem} / {solver}"] + if key not in candidate: + return {}, [f"candidate has no row for {problem} / {solver}"] + + baseline_row = baseline[key] + candidate_row = candidate[key] + errors.extend( + f"baseline {error}" + for error in check_raw_measurements( + baseline_row, "bic", expected_repeats + ) + ) + errors.extend( + f"candidate {error}" + for error in check_raw_measurements( + candidate_row, "bic", expected_repeats + ) + ) + errors.extend( + f"candidate {error}" + for error in check_raw_measurements( + candidate_row, "nifty", expected_repeats + ) + ) + + if candidate_row["bic_energies"] != baseline_row["bic_energies"]: + errors.append("candidate energies differ from the baseline") + if candidate_row["bic_label_digests"] != baseline_row["bic_label_digests"]: + errors.append("candidate label digests differ from the baseline") + + baseline_runtime = baseline_row["bic_runtime_median_s"] + candidate_runtime = candidate_row["bic_runtime_median_s"] + reference_runtime = candidate_row["nifty_runtime_median_s"] + baseline_ratio = candidate_runtime / baseline_runtime + reference_ratio = reference_runtime / candidate_runtime + if baseline_ratio > 1.0 + max_regression: + errors.append( + f"runtime ratio {baseline_ratio:.6f} exceeds " + f"{1.0 + max_regression:.6f}" + ) + if require_improvement and baseline_ratio > 1.0 - min_improvement: + errors.append( + f"runtime ratio {baseline_ratio:.6f} exceeds improvement target " + f"{1.0 - min_improvement:.6f}" + ) + if require_reference_parity and candidate_runtime > reference_runtime: + errors.append( + f"candidate runtime {candidate_runtime:.6f}s exceeds " + f"reference runtime {reference_runtime:.6f}s" + ) + + return { + "problem": problem, + "baseline_s": baseline_runtime, + "candidate_s": candidate_runtime, + "candidate_over_baseline": baseline_ratio, + "reference_s": reference_runtime, + "reference_over_candidate": reference_ratio, + "status": "PASS" if not errors else "FAIL", + }, errors + + +def print_table(rows: list[dict]) -> None: + print( + "| problem | baseline_s | candidate_s | candidate / baseline | " + "nifty_s | nifty / candidate | status |" + ) + print("| --- | ---: | ---: | ---: | ---: | ---: | --- |") + for row in rows: + print( + f"| {row['problem']} | {row['baseline_s']:.6f} | " + f"{row['candidate_s']:.6f} | " + f"{row['candidate_over_baseline']:.6f} | " + f"{row['reference_s']:.6f} | " + f"{row['reference_over_candidate']:.6f} | {row['status']} |" + ) + + +def main() -> None: + parser = argparse.ArgumentParser( + description="Check lifted-KL benchmark results against a saved baseline." + ) + parser.add_argument("baseline", type=Path) + parser.add_argument("candidate", type=Path) + parser.add_argument("--solver", default="lifted_kernighan_lin") + parser.add_argument( + "--problems", + nargs="+", + choices=PROBLEMS, + default=("2d", "3d"), + ) + parser.add_argument("--expected-repeats", type=int, default=10) + parser.add_argument("--max-regression", type=float, default=0.03) + parser.add_argument("--min-improvement", type=float, default=0.03) + parser.add_argument( + "--improvement-problems", + nargs="*", + choices=PROBLEMS, + default=("3d",), + metavar="PROBLEM", + ) + parser.add_argument( + "--reference-parity", + nargs="*", + choices=PROBLEMS, + default=("3d",), + metavar="PROBLEM", + ) + args = parser.parse_args() + + if args.expected_repeats < 1: + raise ValueError("--expected-repeats must be at least 1") + if args.max_regression < 0.0: + raise ValueError("--max-regression must be non-negative") + if not 0.0 <= args.min_improvement < 1.0: + raise ValueError("--min-improvement must be in [0, 1)") + + baseline = load_results(args.baseline, latest_run_only=False) + candidate = load_results(args.candidate, latest_run_only=True) + rows = [] + failures = [] + improvement_problems = set(args.improvement_problems) + parity_problems = set(args.reference_parity) + for problem in args.problems: + row, errors = check_problem( + baseline, + candidate, + problem, + args.solver, + args.expected_repeats, + args.max_regression, + args.min_improvement, + problem in improvement_problems, + problem in parity_problems, + ) + if row: + rows.append(row) + failures.extend( + f"{problem} / {args.solver}: {error}" for error in errors + ) + + print_table(rows) + if failures: + raise SystemExit("Acceptance failed:\n- " + "\n- ".join(failures)) + + +if __name__ == "__main__": + main() diff --git a/development/graph/lifted_multicut/evaluate_solvers.py b/development/graph/lifted_multicut/evaluate_solvers.py index 09f47ec..f389a37 100644 --- a/development/graph/lifted_multicut/evaluate_solvers.py +++ b/development/graph/lifted_multicut/evaluate_solvers.py @@ -1,9 +1,19 @@ from __future__ import annotations import argparse +from datetime import datetime, timezone +import hashlib +import importlib.metadata +import importlib.util +import json +import os +import platform +import shlex +import subprocess import sys from dataclasses import dataclass -from statistics import mean +from pathlib import Path +from statistics import median from time import perf_counter from typing import Callable @@ -15,69 +25,88 @@ @dataclass(frozen=True) class SolverConfig: - make_bic_solver: Callable[[], object] - make_nifty_factory: Callable[[object], object] + make_bic_solver: Callable[[int], object] + make_nifty_factory: Callable[[object, int], object] def solver_configs(): import bioimage_cpp as bic - # Fair single-threaded comparison: nifty's lifted fusion-move backend is - # single-threaded, so we pin the bic side to threads=1 and one parallel - # proposal per iteration (matching nifty's "generate one, fuse" loop) and - # chain greedy-additive in front of nifty to mirror bic's auto warm-start. - # Watershed seeding strategy is forced to SEED_FROM_LOCAL on the nifty - # side because the bic proposal generator only sees the base graph. + # Nifty runs lifted fusion move with one thread. Keep its side at one + # thread when the bioimage-cpp benchmark requests more threads. return { "lifted_greedy_additive": SolverConfig( - make_bic_solver=lambda: bic.graph.lifted_multicut.LiftedGreedyAdditiveMulticut(), - make_nifty_factory=lambda objective: objective.liftedMulticutGreedyAdditiveFactory(), + make_bic_solver=lambda threads: ( + bic.graph.lifted_multicut.LiftedGreedyAdditiveMulticut() + ), + make_nifty_factory=lambda objective, threads: ( + objective.liftedMulticutGreedyAdditiveFactory() + ), ), "lifted_kernighan_lin": SolverConfig( - make_bic_solver=lambda: bic.graph.lifted_multicut.LiftedKernighanLinMulticut( - number_of_outer_iterations=10 + make_bic_solver=lambda threads: ( + bic.graph.lifted_multicut.LiftedKernighanLinMulticut( + number_of_outer_iterations=10 + ) ), - make_nifty_factory=lambda objective: objective.chainedSolversFactory( - [ - objective.liftedMulticutGreedyAdditiveFactory(), - objective.liftedMulticutKernighanLinFactory( - numberOfOuterIterations=10 - ), - ] + make_nifty_factory=lambda objective, threads: ( + objective.chainedSolversFactory( + [ + objective.liftedMulticutGreedyAdditiveFactory(), + objective.liftedMulticutKernighanLinFactory( + numberOfOuterIterations=10 + ), + ] + ) ), ), "lifted_fusion_move": SolverConfig( - make_bic_solver=lambda: bic.graph.lifted_multicut.FusionMoveLiftedMulticut( - proposal_generator=bic.graph.lifted_multicut.WatershedProposalGenerator(), - number_of_iterations=10, - stop_if_no_improvement=4, - number_of_threads=1, - number_of_parallel_proposals=1, + make_bic_solver=lambda threads: ( + bic.graph.lifted_multicut.FusionMoveLiftedMulticut( + proposal_generator=( + bic.graph.lifted_multicut.WatershedProposalGenerator() + ), + number_of_iterations=10, + stop_if_no_improvement=4, + number_of_threads=threads, + number_of_parallel_proposals=threads, + ) ), - make_nifty_factory=lambda objective: objective.chainedSolversFactory( - [ - objective.liftedMulticutGreedyAdditiveFactory(), - objective.fusionMoveBasedFactory( - proposalGenerator=objective.watershedProposalGenerator( - seedingStrategy="SEED_FROM_LOCAL", + make_nifty_factory=lambda objective, threads: ( + objective.chainedSolversFactory( + [ + objective.liftedMulticutGreedyAdditiveFactory(), + objective.fusionMoveBasedFactory( + proposalGenerator=objective.watershedProposalGenerator( + seedingStrategy="SEED_FROM_LOCAL", + ), + numberOfIterations=10, + stopIfNoImprovement=4, + numberOfThreads=1, ), - numberOfIterations=10, - stopIfNoImprovement=4, - numberOfThreads=1, - ), - ] + ] + ) ), ), } -def load_problem(size: str, *, timeout: float): +def load_problem(size: str, *, timeout: float, load_nifty: bool): import bioimage_cpp as bic + + problem = bic.graph.lifted_multicut.load_lifted_multicut_problem( + size, timeout=timeout + ) + bic_graph = bic.graph.UndirectedGraph.from_edges( + problem.n_nodes, problem.local_uvs + ) + + if not load_nifty: + return bic_graph, None, problem + import nifty import nifty.graph.opt.lifted_multicut as nlmc - problem = bic.graph.lifted_multicut.load_lifted_multicut_problem(size, timeout=timeout) - bic_graph = bic.graph.UndirectedGraph.from_edges(problem.n_nodes, problem.local_uvs) nifty_graph = nifty.graph.undirectedGraph(int(problem.n_nodes)) nifty_graph.insertEdges(problem.local_uvs.astype(np.uint64, copy=False)) @@ -91,7 +120,7 @@ def make_nifty_objective(): ) return objective - return bic_graph, nifty_graph, make_nifty_objective, problem + return bic_graph, make_nifty_objective, problem def make_bic_objective(bic_graph, problem): @@ -105,58 +134,188 @@ def make_bic_objective(bic_graph, problem): ) -def bic_energy(bic_graph, problem, labels: np.ndarray) -> float: - return float(make_bic_objective(bic_graph, problem).energy(labels)) +def label_digest(labels: np.ndarray) -> str: + contiguous = np.ascontiguousarray(labels) + digest = hashlib.sha256() + digest.update(str(contiguous.dtype).encode("ascii")) + digest.update(np.asarray(contiguous.shape, dtype=np.uint64).tobytes()) + digest.update(contiguous.tobytes()) + return digest.hexdigest() -def nifty_energy(nifty_objective, labels: np.ndarray) -> float: - return float(nifty_objective.evalNodeLabels(labels.astype(np.uint64, copy=False))) +def package_version(name: str) -> str | None: + try: + return importlib.metadata.version(name) + except importlib.metadata.PackageNotFoundError: + return None + + +def command_output(command: list[str]) -> str | None: + try: + completed = subprocess.run( + command, + check=False, + capture_output=True, + text=True, + timeout=10, + ) + except (OSError, subprocess.SubprocessError): + return None + output = completed.stdout.strip() or completed.stderr.strip() + return output or None + + +def cpu_model() -> str | None: + try: + with open("/proc/cpuinfo", encoding="utf-8") as file: + for line in file: + if line.startswith("model name"): + return line.split(":", maxsplit=1)[1].strip() + except OSError: + pass + return platform.processor() or None + + +def collect_environment(build_command: str | None) -> dict: + import bioimage_cpp as bic + + compiler = shlex.split(os.environ.get("CXX", "c++")) + compiler_version = command_output([*compiler, "--version"]) if compiler else None + tracked_status = command_output( + ["git", "status", "--short", "--untracked-files=no"] + ) + return { + "command": shlex.join([sys.executable, *sys.argv]), + "build_command": build_command, + "source_commit": command_output(["git", "rev-parse", "HEAD"]), + "tracked_dirty": bool(tracked_status), + "tracked_status": tracked_status, + "python": sys.version, + "platform": platform.platform(), + "cpu": cpu_model(), + "logical_cpu_count": os.cpu_count(), + "compiler_command": compiler, + "compiler_version": compiler_version, + "packages": { + "bioimage-cpp": package_version("bioimage-cpp"), + "numpy": np.__version__, + "nifty": package_version("nifty"), + }, + "module_paths": { + "bioimage_cpp": str(Path(bic.__file__).resolve()), + "nifty": ( + importlib.util.find_spec("nifty").origin + if importlib.util.find_spec("nifty") is not None + else None + ), + }, + "environment": { + key: os.environ.get(key) + for key in ( + "CXX", + "CXXFLAGS", + "CMAKE_ARGS", + "SKBUILD_CMAKE_ARGS", + "OMP_NUM_THREADS", + "OPENBLAS_NUM_THREADS", + "MKL_NUM_THREADS", + "NUMEXPR_NUM_THREADS", + ) + }, + } -def evaluate(problem_name: str, solver_name: str, config: SolverConfig, args): - bic_graph, _, make_nifty_objective, problem = load_problem( - problem_name, timeout=args.timeout +def evaluate( + problem_name: str, + solver_name: str, + config: SolverConfig, + args, + *, + run_bic: bool, + run_nifty: bool, +): + bic_graph, make_nifty_objective, problem = load_problem( + problem_name, + timeout=args.timeout, + load_nifty=run_nifty, ) bic_energies = [] nifty_energies = [] bic_runtimes = [] nifty_runtimes = [] - - for _ in range(args.n_repeats): - bic_objective = make_bic_objective(bic_graph, problem) - start = perf_counter() - bic_labels = config.make_bic_solver().optimize(bic_objective) - bic_runtimes.append(perf_counter() - start) - bic_energies.append(bic_energy(bic_graph, problem, bic_labels)) - - nifty_objective = make_nifty_objective() - start = perf_counter() - nifty_labels = ( - config.make_nifty_factory(nifty_objective) - .create(nifty_objective) - .optimize() - ) - nifty_runtimes.append(perf_counter() - start) - nifty_energies.append(nifty_energy(nifty_objective, np.asarray(nifty_labels))) - - bic_runtime = mean(bic_runtimes) - nifty_runtime = mean(nifty_runtimes) + bic_label_digests = [] + nifty_label_digests = [] + + for repeat in range(args.n_repeats): + order = ("bic", "nifty") if repeat % 2 == 0 else ("nifty", "bic") + for backend in order: + if backend == "bic" and run_bic: + objective = make_bic_objective(bic_graph, problem) + start = perf_counter() + labels = config.make_bic_solver(args.threads).optimize(objective) + bic_runtimes.append(perf_counter() - start) + bic_energies.append(float(objective.energy(labels))) + bic_label_digests.append(label_digest(labels)) + elif backend == "nifty" and run_nifty: + objective = make_nifty_objective() + start = perf_counter() + labels = ( + config.make_nifty_factory(objective, args.threads) + .create(objective) + .optimize() + ) + nifty_runtimes.append(perf_counter() - start) + labels = np.asarray(labels) + nifty_energies.append( + float( + objective.evalNodeLabels( + labels.astype(np.uint64, copy=False) + ) + ) + ) + nifty_label_digests.append(label_digest(labels)) + + bic_runtime = median(bic_runtimes) if bic_runtimes else None + nifty_runtime = median(nifty_runtimes) if nifty_runtimes else None + bic_energy = median(bic_energies) if bic_energies else None + nifty_energy = median(nifty_energies) if nifty_energies else None + energy_diff = ( + bic_energy - nifty_energy + if bic_energy is not None and nifty_energy is not None + else None + ) return { "problem": problem_name, "solver": solver_name, "nodes": int(problem.n_nodes), "local_edges": int(problem.local_uvs.shape[0]), "lifted_edges": int(problem.lifted_uvs.shape[0]), - "bic_energy": mean(bic_energies), - "nifty_energy": mean(nifty_energies), - "energy_diff": mean(bic_energies) - mean(nifty_energies), - "bic_runtime_s": bic_runtime, - "nifty_runtime_s": nifty_runtime, - "runtime_ratio": nifty_runtime / bic_runtime if bic_runtime > 0 else float("inf"), + "bic_energy": bic_energy, + "nifty_energy": nifty_energy, + "energy_diff": energy_diff, + "bic_energies": bic_energies, + "nifty_energies": nifty_energies, + "bic_label_digests": bic_label_digests, + "nifty_label_digests": nifty_label_digests, + "bic_runtimes_s": bic_runtimes, + "nifty_runtimes_s": nifty_runtimes, + "bic_runtime_median_s": bic_runtime, + "nifty_runtime_median_s": nifty_runtime, + "bic_runtime_min_s": min(bic_runtimes) if bic_runtimes else None, + "nifty_runtime_min_s": min(nifty_runtimes) if nifty_runtimes else None, + "runtime_ratio_median": ( + nifty_runtime / bic_runtime + if bic_runtime is not None + and nifty_runtime is not None + and bic_runtime > 0 + else None + ), } -def format_float(value: float) -> str: +def format_float(value: float | None) -> str: + if value is None: + return "" return f"{value:.6g}" @@ -167,15 +326,24 @@ def print_progress(row: dict) -> None: f"bic_energy={format_float(row['bic_energy'])}, " f"nifty_energy={format_float(row['nifty_energy'])}, " f"delta={format_float(row['energy_diff'])}, " - f"bic_runtime={format_float(row['bic_runtime_s'])}s, " - f"nifty_runtime={format_float(row['nifty_runtime_s'])}s, " - f"ratio={format_float(row['runtime_ratio'])}" + f"bic_median={format_float(row['bic_runtime_median_s'])}s, " + f"nifty_median={format_float(row['nifty_runtime_median_s'])}s, " + f"ratio={format_float(row['runtime_ratio_median'])}" ), file=sys.stderr, flush=True, ) +def append_jsonl(path: str, row: dict) -> None: + output_path = Path(path) + output_path.parent.mkdir(parents=True, exist_ok=True) + with output_path.open("a", encoding="utf-8") as file: + json.dump(row, file, sort_keys=True) + file.write("\n") + file.flush() + + def print_markdown_table(rows: list[dict]) -> None: headers = [ "problem", @@ -186,9 +354,11 @@ def print_markdown_table(rows: list[dict]) -> None: "bic_energy", "nifty_energy", "energy_diff", - "bic_runtime_s", - "nifty_runtime_s", - "runtime_ratio_nifty_over_bic", + "bic_runtime_median_s", + "bic_runtime_min_s", + "nifty_runtime_median_s", + "nifty_runtime_min_s", + "runtime_ratio_median_nifty_over_bic", ] print("| " + " | ".join(headers) + " |") print("| " + " | ".join(["---"] * len(headers)) + " |") @@ -202,9 +372,11 @@ def print_markdown_table(rows: list[dict]) -> None: format_float(row["bic_energy"]), format_float(row["nifty_energy"]), format_float(row["energy_diff"]), - format_float(row["bic_runtime_s"]), - format_float(row["nifty_runtime_s"]), - format_float(row["runtime_ratio"]), + format_float(row["bic_runtime_median_s"]), + format_float(row["bic_runtime_min_s"]), + format_float(row["nifty_runtime_median_s"]), + format_float(row["nifty_runtime_min_s"]), + format_float(row["runtime_ratio_median"]), ] print("| " + " | ".join(values) + " |") @@ -213,8 +385,7 @@ def main() -> None: configs = solver_configs() parser = argparse.ArgumentParser( description=( - "Evaluate matched bioimage-cpp and nifty lifted multicut solvers " - "on registered lifted multicut problems." + "Evaluate matched bioimage-cpp and nifty lifted multicut solvers." ) ) parser.add_argument( @@ -232,18 +403,82 @@ def main() -> None: help="Problems to evaluate. Defaults to all.", ) parser.add_argument("--n-repeats", type=int, default=1) + parser.add_argument("--threads", type=int, default=1) parser.add_argument("--timeout", type=float, default=60.0) + parser.add_argument( + "--backend", + choices=("both", "bic", "nifty"), + default="both", + help="Implementation to run. Defaults to both.", + ) + parser.add_argument( + "--results-jsonl", + help="Append metadata and completed rows to this JSONL file.", + ) + parser.add_argument( + "--build-command", + help="Build command to store with the benchmark metadata.", + ) + parser.add_argument( + "--require-reference", + action="append", + choices=("nifty",), + default=[], + help="Fail if the named reference is unavailable.", + ) args = parser.parse_args() if args.n_repeats < 1: raise ValueError("--n-repeats must be at least 1") + if args.threads < 1: + raise ValueError("--threads must be at least 1") + nifty_available = importlib.util.find_spec("nifty") is not None + if "nifty" in args.require_reference and not nifty_available: + raise RuntimeError("required reference 'nifty' is not available") + run_bic = args.backend in ("both", "bic") + run_nifty = args.backend in ("both", "nifty") and nifty_available + reference_status = ( + "OK" + if run_nifty + else "NO-REF" + if args.backend in ("both", "nifty") + else "NOT_REQUESTED" + ) + + run_id = datetime.now(timezone.utc).isoformat() + if args.results_jsonl is not None: + append_jsonl( + args.results_jsonl, + { + "record_type": "metadata", + "schema_version": 2, + "run_id": run_id, + "created_utc": run_id, + "reference_status": {"nifty": reference_status}, + "arguments": vars(args), + "environment": collect_environment(args.build_command), + }, + ) rows = [] for problem_name in args.problems: for solver_name in args.solvers: - row = evaluate(problem_name, solver_name, configs[solver_name], args) + row = evaluate( + problem_name, + solver_name, + configs[solver_name], + args, + run_bic=run_bic, + run_nifty=run_nifty, + ) + row["record_type"] = "result" + row["schema_version"] = 2 + row["run_id"] = run_id + row["reference_status"] = reference_status rows.append(row) print_progress(row) + if args.results_jsonl is not None: + append_jsonl(args.results_jsonl, row) print_markdown_table(rows) diff --git a/development/graph/multicut/PERFORMANCE_NOTES.md b/development/graph/multicut/PERFORMANCE_NOTES.md index 54a990a..1849ffb 100644 --- a/development/graph/multicut/PERFORMANCE_NOTES.md +++ b/development/graph/multicut/PERFORMANCE_NOTES.md @@ -4,7 +4,95 @@ State of the multicut solvers vs nifty on the standard benchmark problems and notes on remaining optimization headroom. Read this before the next round of perf work. -## Current benchmark matrix +## Shared contraction topology + +Greedy additive and greedy fixation now use +`graph::detail::BasicContractionTopology`. An edge payload keeps each weight +and constraint flag next to its endpoints. Mutation observers update the heap +and union-find without an intermediate event buffer. The solver workspace +retains all scratch allocations across calls. + +Profile builds report reset, initialization, contraction, and label +materialization separately. + +The refactor keeps the previous merge direction: the larger-degree endpoint +survives, and the edge's first endpoint survives a degree tie. + +## Kernighan-Lin optimization + +The P1 optimization ran on 2026-07-27. The acceptance run used a production +build, one thread, five repeats, and alternating backend order: + +```bash +python development/graph/multicut/evaluate_solvers.py \ + --solvers kernighan_lin \ + --problems A_small B_small C_small A_medium B_medium C_medium \ + --n-repeats 5 --backend both --require-reference nifty \ + --results-jsonl /tmp/kl_p1_final.jsonl \ + --build-command 'pip install -e . --no-build-isolation' +``` + +The acceptance criteria were: + +- Each bioimage-cpp median must be at most 1.03 times its baseline median. +- `B_medium` must be no slower than nifty. +- Every bioimage-cpp repeat must reproduce its baseline energy and label + SHA-256 digest. + +Run the executable gate with: + +```bash +python development/graph/multicut/check_benchmark_acceptance.py \ + BASELINE.jsonl CANDIDATE.jsonl \ + --reference-parity B_medium +``` + +All six problems passed. This includes `A_medium` and `C_medium`. + +| Problem | Baseline bic | Final bic | Final / baseline | Final nifty | Nifty / bic | Exact output | +|---|---:|---:|---:|---:|---:|---| +| A_small | 1.974 s | 1.691 s | 0.857 | 2.533 s | 1.498 | yes | +| B_small | 4.343 s | 3.759 s | 0.865 | 5.308 s | 1.412 | yes | +| C_small | 6.496 s | 6.126 s | 0.943 | 7.856 s | 1.282 | yes | +| A_medium | 125.723 s | 60.070 s | 0.478 | 134.766 s | 2.243 | yes | +| B_medium | 275.021 s | 156.324 s | 0.568 | 165.150 s | 1.056 | yes | +| C_medium | 220.694 s | 130.206 s | 0.590 | 234.075 s | 1.798 | yes | + +The implementation skips pair chains and split checks for clusters that did +not change in the previous iteration. The last configured iteration does not +use this gate. This final pass preserves the previous fixed output when the +solver reaches it. + +Each chain stores only adjacency entries whose other endpoint is in the active +cluster pair. The move loop reuses this filtered adjacency. It also derives +heap membership from the maintained cross-border count and resets scratch +state only for nodes that the chain touched. + +The ordinary and lifted implementations now share the cluster-change helper in +`detail/relabel.hxx`. Phase instrumentation uses `detail/profile.hxx` and has no +production-build cost. + +The `B_medium` profile changed as follows: + +| Scope | Before | After | +|---|---:|---:| +| End-to-end bic runtime | 267.927 s | 153.626 s | +| `chain_gain_init` | 110.342 s | 84.403 s | +| `chain_loop` | 134.652 s | 54.715 s | +| `chain_cleanup` | 7.520 s | 1.918 s | + +The P1 profile used a nested `pair_chains` aggregate. The current +instrumentation uses non-overlapping phase scopes, so the reported total does +not count any phase twice. Raw benchmark artifacts are environment-specific +and are not versioned. + +The benchmark harness now records raw runtimes, medians, minima, energies, +label digests, commands, build commands, source state, package paths, compiler +details, CPU details, and thread-related environment variables. It loads nifty +only when requested and can require the reference with +`--require-reference nifty`. + +## Previous benchmark matrix Produced by `python evaluate_solvers.py` (2026-05-17). Small problems were run with both implementations in one pass: @@ -74,7 +162,46 @@ positive means nifty did. The only nonzero differences are KL/chained rows, and they are tiny relative to the objective scale. Fusion-move, greedy-additive, greedy-fixation, and decomposer match nifty energies on every benchmark row. -## Current read +The decomposer rows above are legacy measurements. The old bioimage-cpp +configuration used `GreedyAdditiveMulticut` without a fallthrough solver, so +it selected the documented greedy-additive fast path and did not run +decomposition. The benchmark now supplies matching greedy-additive sub- and +fallthrough solvers to both implementations, passes `number_of_threads`, and +requires multiple non-singleton positive-cost components. Do not use the +legacy decomposer runtimes for performance comparisons. + +### Corrected decomposer smoke benchmark + +The corrected benchmark ran on `A_small` on 2026-07-26. Each row used one +repeat. Both implementations used greedy-additive sub- and fallthrough +solvers. The benchmark verified that the graph had at least two non-singleton +positive-cost components. + +| Threads | bic energy | nifty energy | bic runtime | nifty runtime | +|---|---|---|---|---| +| 1 | -76 914.5 | -76 914.5 | 0.469 s | 0.335 s | +| 4 | -76 914.5 | -76 914.5 | 0.429 s | 0.378 s | + +These single-repeat rows confirm matched objective values and exercise the +real decomposer path. They are smoke measurements, not stable performance +estimates. + +### Native decomposer benchmark + +The decomposer moved from Python orchestration to C++ on 2026-07-26. Matched +`A_small` runs used five repeats with greedy-additive sub- and fallthrough +solvers. + +| Threads | bic energy | nifty energy | bic runtime | nifty runtime | +|---|---|---|---|---| +| 1 | -76 914.5 | -76 914.5 | 0.240 s | 0.354 s | +| 4 | -76 914.5 | -76 914.5 | 0.221 s | 0.359 s | + +The native implementation matched the reference energy. It was 1.48 times +faster than nifty with one thread and 1.62 times faster with four threads on +this problem. + +## Previous read `KernighanLinMulticut` is the dominant runtime target. It is faster than nifty on most rows, but `B_medium` is the clear exception: bic KL/chained takes about diff --git a/development/graph/multicut/check_benchmark_acceptance.py b/development/graph/multicut/check_benchmark_acceptance.py new file mode 100644 index 0000000..a6de9e9 --- /dev/null +++ b/development/graph/multicut/check_benchmark_acceptance.py @@ -0,0 +1,229 @@ +from __future__ import annotations + +import argparse +import json +import math +from pathlib import Path +from statistics import median + + +PROBLEMS = tuple( + f"{sample}_{size}" + for sample in ("A", "B", "C") + for size in ("small", "medium") +) + + +def load_results( + path: Path, + *, + latest_run_only: bool, +) -> dict[tuple[str, str], dict]: + rows = [] + with path.open(encoding="utf-8") as file: + for line in file: + row = json.loads(line) + if row.get("record_type") == "result": + rows.append(row) + if latest_run_only and rows: + latest_run_id = rows[-1]["run_id"] + rows = [row for row in rows if row["run_id"] == latest_run_id] + return { + (row["problem"], row["solver"]): row + for row in rows + } + + +def check_raw_measurements( + row: dict, + backend: str, + expected_repeats: int, +) -> list[str]: + errors = [] + runtimes = row[f"{backend}_runtimes_s"] + energies = row[f"{backend}_energies"] + digests = row[f"{backend}_label_digests"] + if len(runtimes) != expected_repeats: + errors.append( + f"{backend} has {len(runtimes)} runtimes; expected {expected_repeats}" + ) + if len(energies) != expected_repeats: + errors.append( + f"{backend} has {len(energies)} energies; expected {expected_repeats}" + ) + if len(digests) != expected_repeats: + errors.append( + f"{backend} has {len(digests)} digests; expected {expected_repeats}" + ) + if any(not math.isfinite(value) or value <= 0.0 for value in runtimes): + errors.append(f"{backend} runtimes must be finite and positive") + if any(not math.isfinite(value) for value in energies): + errors.append(f"{backend} energies must be finite") + if energies and len(set(energies)) != 1: + errors.append(f"{backend} energies differ between repeats") + if digests and len(set(digests)) != 1: + errors.append(f"{backend} label digests differ between repeats") + if runtimes: + if median(runtimes) != row[f"{backend}_runtime_median_s"]: + errors.append(f"{backend} runtime median does not match raw runtimes") + if min(runtimes) != row[f"{backend}_runtime_min_s"]: + errors.append(f"{backend} runtime minimum does not match raw runtimes") + if energies and median(energies) != row[f"{backend}_energy"]: + errors.append(f"{backend} energy does not match raw energies") + return errors + + +def check_problem( + baseline: dict, + candidate: dict, + problem: str, + solver: str, + expected_repeats: int, + max_regression: float, + require_reference_parity: bool, +) -> tuple[dict, list[str]]: + key = (problem, solver) + errors = [] + if key not in baseline: + return {}, [f"baseline has no row for {problem} / {solver}"] + if key not in candidate: + return {}, [f"candidate has no row for {problem} / {solver}"] + + baseline_row = baseline[key] + candidate_row = candidate[key] + errors.extend( + f"baseline {error}" + for error in check_raw_measurements( + baseline_row, "bic", expected_repeats + ) + ) + errors.extend( + f"candidate {error}" + for error in check_raw_measurements( + candidate_row, "bic", expected_repeats + ) + ) + errors.extend( + f"candidate {error}" + for error in check_raw_measurements( + candidate_row, "nifty", expected_repeats + ) + ) + + baseline_energies = baseline_row["bic_energies"] + candidate_energies = candidate_row["bic_energies"] + baseline_digests = baseline_row["bic_label_digests"] + candidate_digests = candidate_row["bic_label_digests"] + if ( + baseline_energies + and candidate_energies + and candidate_energies[0] != baseline_energies[0] + ): + errors.append("candidate energy differs from the baseline") + if ( + baseline_digests + and candidate_digests + and candidate_digests[0] != baseline_digests[0] + ): + errors.append("candidate label digest differs from the baseline") + + baseline_runtime = baseline_row["bic_runtime_median_s"] + candidate_runtime = candidate_row["bic_runtime_median_s"] + reference_runtime = candidate_row["nifty_runtime_median_s"] + baseline_ratio = candidate_runtime / baseline_runtime + reference_ratio = reference_runtime / candidate_runtime + if baseline_ratio > 1.0 + max_regression: + errors.append( + f"runtime ratio {baseline_ratio:.6f} exceeds " + f"{1.0 + max_regression:.6f}" + ) + if require_reference_parity and candidate_runtime > reference_runtime: + errors.append( + f"candidate runtime {candidate_runtime:.6f}s exceeds " + f"reference runtime {reference_runtime:.6f}s" + ) + + return { + "problem": problem, + "baseline_s": baseline_runtime, + "candidate_s": candidate_runtime, + "candidate_over_baseline": baseline_ratio, + "reference_s": reference_runtime, + "reference_over_candidate": reference_ratio, + "status": "PASS" if not errors else "FAIL", + }, errors + + +def print_table(rows: list[dict]) -> None: + print( + "| problem | baseline_s | candidate_s | candidate / baseline | " + "nifty_s | nifty / candidate | status |" + ) + print("| --- | ---: | ---: | ---: | ---: | ---: | --- |") + for row in rows: + print( + f"| {row['problem']} | {row['baseline_s']:.6f} | " + f"{row['candidate_s']:.6f} | " + f"{row['candidate_over_baseline']:.6f} | " + f"{row['reference_s']:.6f} | " + f"{row['reference_over_candidate']:.6f} | {row['status']} |" + ) + + +def main() -> None: + parser = argparse.ArgumentParser( + description="Check multicut benchmark results against a saved baseline." + ) + parser.add_argument("baseline", type=Path) + parser.add_argument("candidate", type=Path) + parser.add_argument("--solver", default="kernighan_lin") + parser.add_argument( + "--problems", + nargs="+", + choices=PROBLEMS, + default=PROBLEMS, + ) + parser.add_argument("--expected-repeats", type=int, default=5) + parser.add_argument("--max-regression", type=float, default=0.03) + parser.add_argument( + "--reference-parity", + nargs="*", + choices=PROBLEMS, + default=[], + metavar="PROBLEM", + ) + args = parser.parse_args() + + if args.expected_repeats < 1: + raise ValueError("--expected-repeats must be at least 1") + if args.max_regression < 0.0: + raise ValueError("--max-regression must be non-negative") + + baseline = load_results(args.baseline, latest_run_only=False) + candidate = load_results(args.candidate, latest_run_only=True) + rows = [] + failures = [] + parity_problems = set(args.reference_parity) + for problem in args.problems: + row, errors = check_problem( + baseline, + candidate, + problem, + args.solver, + args.expected_repeats, + args.max_regression, + problem in parity_problems, + ) + if row: + rows.append(row) + failures.extend( + f"{problem} / {args.solver}: {error}" for error in errors + ) + + print_table(rows) + if failures: + raise SystemExit("Acceptance failed:\n- " + "\n- ".join(failures)) + + +if __name__ == "__main__": + main() diff --git a/development/graph/multicut/check_decomposer.py b/development/graph/multicut/check_decomposer.py index b7f38ba..ebccb74 100644 --- a/development/graph/multicut/check_decomposer.py +++ b/development/graph/multicut/check_decomposer.py @@ -9,7 +9,11 @@ def main() -> None: args = parser("Compare bioimage-cpp and nifty decomposer multicut.").parse_args() run_comparison( "decomposer", - lambda: bic.graph.multicut.MulticutDecomposer(bic.graph.multicut.GreedyAdditiveMulticut()), + lambda: bic.graph.multicut.MulticutDecomposer( + bic.graph.multicut.GreedyAdditiveMulticut(), + fallthrough_solver=bic.graph.multicut.GreedyAdditiveMulticut(), + number_of_threads=args.threads, + ), lambda objective: objective.multicutDecomposerFactory( submodelFactory=objective.greedyAdditiveFactory(), fallthroughFactory=objective.greedyAdditiveFactory(), diff --git a/development/graph/multicut/evaluate_solvers.py b/development/graph/multicut/evaluate_solvers.py index ef97083..cd60839 100644 --- a/development/graph/multicut/evaluate_solvers.py +++ b/development/graph/multicut/evaluate_solvers.py @@ -1,10 +1,19 @@ from __future__ import annotations import argparse +from datetime import datetime, timezone +import hashlib +import importlib.metadata +import importlib.util import json +import os +import platform +import shlex +import subprocess import sys from dataclasses import dataclass -from statistics import mean +from pathlib import Path +from statistics import median from time import perf_counter from typing import Callable @@ -61,7 +70,9 @@ def solver_configs(): ), "decomposer": SolverConfig( make_bic_solver=lambda threads: bic.graph.multicut.MulticutDecomposer( - bic.graph.multicut.GreedyAdditiveMulticut() + bic.graph.multicut.GreedyAdditiveMulticut(), + fallthrough_solver=bic.graph.multicut.GreedyAdditiveMulticut(), + number_of_threads=threads, ), make_nifty_factory=lambda objective, threads: objective.multicutDecomposerFactory( submodelFactory=objective.greedyAdditiveFactory(), @@ -103,9 +114,8 @@ def parse_problem_name(name: str) -> tuple[str, str]: return sample, size -def load_problem(problem_name: str, *, timeout: float): +def load_problem(problem_name: str, *, timeout: float, load_nifty: bool): import bioimage_cpp as bic - import nifty sample, size = parse_problem_name(problem_name) uv_ids, costs = bic.graph.multicut.load_multicut_problem_data( @@ -115,8 +125,12 @@ def load_problem(problem_name: str, *, timeout: float): ) n_nodes = int(uv_ids.max()) + 1 bic_graph = bic.graph.UndirectedGraph.from_edges(n_nodes, uv_ids) - nifty_graph = nifty.graph.undirectedGraph(n_nodes) - nifty_graph.insertEdges(uv_ids.astype(np.uint64, copy=False)) + nifty_graph = None + if load_nifty: + import nifty + + nifty_graph = nifty.graph.undirectedGraph(n_nodes) + nifty_graph.insertEdges(uv_ids.astype(np.uint64, copy=False)) return bic_graph, nifty_graph, costs @@ -132,39 +146,161 @@ def nifty_energy(graph, costs: np.ndarray, labels: np.ndarray) -> float: return float(nmc.multicutObjective(graph, costs).evalNodeLabels(labels)) -def evaluate(problem_name: str, solver_name: str, config: SolverConfig, args): +def label_digest(labels: np.ndarray) -> str: + contiguous = np.ascontiguousarray(labels) + digest = hashlib.sha256() + digest.update(str(contiguous.dtype).encode("ascii")) + digest.update(np.asarray(contiguous.shape, dtype=np.uint64).tobytes()) + digest.update(contiguous.tobytes()) + return digest.hexdigest() + + +def package_version(name: str) -> str | None: + try: + return importlib.metadata.version(name) + except importlib.metadata.PackageNotFoundError: + return None + + +def command_output(command: list[str]) -> str | None: + try: + completed = subprocess.run( + command, + check=False, + capture_output=True, + text=True, + timeout=10, + ) + except (OSError, subprocess.SubprocessError): + return None + output = completed.stdout.strip() or completed.stderr.strip() + return output or None + + +def cpu_model() -> str | None: + try: + with open("/proc/cpuinfo", encoding="utf-8") as file: + for line in file: + if line.startswith("model name"): + return line.split(":", maxsplit=1)[1].strip() + except OSError: + pass + return platform.processor() or None + + +def collect_environment(build_command: str | None) -> dict: import bioimage_cpp as bic - import nifty.graph.opt.multicut as nmc - bic_graph, nifty_graph, costs = load_problem(problem_name, timeout=args.timeout) + compiler = shlex.split(os.environ.get("CXX", "c++")) + compiler_version = command_output([*compiler, "--version"]) if compiler else None + tracked_status = command_output( + ["git", "status", "--short", "--untracked-files=no"] + ) + return { + "command": shlex.join([sys.executable, *sys.argv]), + "build_command": build_command, + "source_commit": command_output(["git", "rev-parse", "HEAD"]), + "tracked_dirty": bool(tracked_status), + "tracked_status": tracked_status, + "python": sys.version, + "platform": platform.platform(), + "cpu": cpu_model(), + "logical_cpu_count": os.cpu_count(), + "compiler_command": compiler, + "compiler_version": compiler_version, + "packages": { + "bioimage-cpp": package_version("bioimage-cpp"), + "numpy": np.__version__, + "nifty": package_version("nifty"), + }, + "module_paths": { + "bioimage_cpp": str(Path(bic.__file__).resolve()), + "nifty": ( + importlib.util.find_spec("nifty").origin + if importlib.util.find_spec("nifty") is not None + else None + ), + }, + "environment": { + key: os.environ.get(key) + for key in ( + "CXX", + "CXXFLAGS", + "CMAKE_ARGS", + "SKBUILD_CMAKE_ARGS", + "OMP_NUM_THREADS", + "OPENBLAS_NUM_THREADS", + "MKL_NUM_THREADS", + "NUMEXPR_NUM_THREADS", + ) + }, + } + + +def evaluate( + problem_name: str, + solver_name: str, + config: SolverConfig, + args, + *, + run_bic: bool, + run_nifty: bool, +): + import bioimage_cpp as bic + + nmc = None + if run_nifty: + import nifty.graph.opt.multicut as nmc + + bic_graph, nifty_graph, costs = load_problem( + problem_name, + timeout=args.timeout, + load_nifty=run_nifty, + ) + if solver_name == "decomposer": + component_labels = bic.graph.connected_components( + bic_graph, + edge_mask=costs > 0.0, + ) + component_sizes = np.bincount(component_labels.astype(np.intp, copy=False)) + if np.count_nonzero(component_sizes > 1) < 2: + raise ValueError( + "decomposer benchmark requires at least two non-singleton " + "positive-cost components" + ) bic_energies = [] nifty_energies = [] bic_runtimes = [] nifty_runtimes = [] - - for _ in range(args.n_repeats): - if args.backend in ("both", "bic"): - bic_objective = bic.graph.multicut.MulticutObjective(bic_graph, costs) - start = perf_counter() - bic_labels = config.make_bic_solver(args.threads).optimize(bic_objective) - bic_runtimes.append(perf_counter() - start) - bic_energies.append(bic_energy(bic_graph, costs, bic_labels)) - - if args.backend in ("both", "nifty"): - nifty_objective = nmc.multicutObjective(nifty_graph, costs) - start = perf_counter() - nifty_labels = ( - config.make_nifty_factory(nifty_objective, args.threads) - .create(nifty_objective) - .optimize() - ) - nifty_runtimes.append(perf_counter() - start) - nifty_energies.append(nifty_energy(nifty_graph, costs, nifty_labels)) - - bic_runtime = mean(bic_runtimes) if bic_runtimes else None - nifty_runtime = mean(nifty_runtimes) if nifty_runtimes else None - bic_energy_value = mean(bic_energies) if bic_energies else None - nifty_energy_value = mean(nifty_energies) if nifty_energies else None + bic_label_digests = [] + nifty_label_digests = [] + + for repeat in range(args.n_repeats): + order = ("bic", "nifty") if repeat % 2 == 0 else ("nifty", "bic") + for backend in order: + if backend == "bic" and run_bic: + bic_objective = bic.graph.multicut.MulticutObjective(bic_graph, costs) + start = perf_counter() + bic_labels = config.make_bic_solver(args.threads).optimize(bic_objective) + bic_runtimes.append(perf_counter() - start) + bic_energies.append(bic_energy(bic_graph, costs, bic_labels)) + bic_label_digests.append(label_digest(bic_labels)) + elif backend == "nifty" and run_nifty: + nifty_objective = nmc.multicutObjective(nifty_graph, costs) + start = perf_counter() + nifty_labels = ( + config.make_nifty_factory(nifty_objective, args.threads) + .create(nifty_objective) + .optimize() + ) + nifty_runtimes.append(perf_counter() - start) + nifty_energies.append(nifty_energy(nifty_graph, costs, nifty_labels)) + nifty_label_digests.append(label_digest(nifty_labels)) + + bic_runtime = median(bic_runtimes) if bic_runtimes else None + nifty_runtime = median(nifty_runtimes) if nifty_runtimes else None + bic_energy_value = median(bic_energies) if bic_energies else None + nifty_energy_value = median(nifty_energies) if nifty_energies else None energy_diff = ( bic_energy_value - nifty_energy_value if bic_energy_value is not None and nifty_energy_value is not None @@ -178,9 +314,17 @@ def evaluate(problem_name: str, solver_name: str, config: SolverConfig, args): "bic_energy": bic_energy_value, "nifty_energy": nifty_energy_value, "energy_diff": energy_diff, - "bic_runtime_s": bic_runtime, - "nifty_runtime_s": nifty_runtime, - "runtime_ratio": ( + "bic_energies": bic_energies, + "nifty_energies": nifty_energies, + "bic_label_digests": bic_label_digests, + "nifty_label_digests": nifty_label_digests, + "bic_runtimes_s": bic_runtimes, + "nifty_runtimes_s": nifty_runtimes, + "bic_runtime_median_s": bic_runtime, + "nifty_runtime_median_s": nifty_runtime, + "bic_runtime_min_s": min(bic_runtimes) if bic_runtimes else None, + "nifty_runtime_min_s": min(nifty_runtimes) if nifty_runtimes else None, + "runtime_ratio_median": ( nifty_runtime / bic_runtime if bic_runtime is not None and nifty_runtime is not None and bic_runtime > 0 else None @@ -188,7 +332,7 @@ def evaluate(problem_name: str, solver_name: str, config: SolverConfig, args): } -def format_float(value: float) -> str: +def format_float(value: float | None) -> str: if value is None: return "" return f"{value:.6g}" @@ -201,9 +345,9 @@ def print_progress(row: dict) -> None: f"bic_energy={format_float(row['bic_energy'])}, " f"nifty_energy={format_float(row['nifty_energy'])}, " f"delta={format_float(row['energy_diff'])}, " - f"bic_runtime={format_float(row['bic_runtime_s'])}s, " - f"nifty_runtime={format_float(row['nifty_runtime_s'])}s, " - f"ratio={format_float(row['runtime_ratio'])}" + f"bic_median={format_float(row['bic_runtime_median_s'])}s, " + f"nifty_median={format_float(row['nifty_runtime_median_s'])}s, " + f"ratio={format_float(row['runtime_ratio_median'])}" ), file=sys.stderr, flush=True, @@ -211,7 +355,9 @@ def print_progress(row: dict) -> None: def append_jsonl(path: str, row: dict) -> None: - with open(path, "a", encoding="utf-8") as f: + output_path = Path(path) + output_path.parent.mkdir(parents=True, exist_ok=True) + with output_path.open("a", encoding="utf-8") as f: json.dump(row, f, sort_keys=True) f.write("\n") f.flush() @@ -226,9 +372,11 @@ def print_markdown_table(rows: list[dict]) -> None: "bic_energy", "nifty_energy", "energy_diff", - "bic_runtime_s", - "nifty_runtime_s", - "runtime_ratio_nifty_over_bic", + "bic_runtime_median_s", + "bic_runtime_min_s", + "nifty_runtime_median_s", + "nifty_runtime_min_s", + "runtime_ratio_median_nifty_over_bic", ] print("| " + " | ".join(headers) + " |") print("| " + " | ".join(["---"] * len(headers)) + " |") @@ -241,9 +389,11 @@ def print_markdown_table(rows: list[dict]) -> None: format_float(row["bic_energy"]), format_float(row["nifty_energy"]), format_float(row["energy_diff"]), - format_float(row["bic_runtime_s"]), - format_float(row["nifty_runtime_s"]), - format_float(row["runtime_ratio"]), + format_float(row["bic_runtime_median_s"]), + format_float(row["bic_runtime_min_s"]), + format_float(row["nifty_runtime_median_s"]), + format_float(row["nifty_runtime_min_s"]), + format_float(row["runtime_ratio_median"]), ] print("| " + " | ".join(values) + " |") @@ -283,17 +433,66 @@ def main() -> None: "--results-jsonl", help="Append each completed row to this JSONL file.", ) + parser.add_argument( + "--build-command", + help="Build command to store with a JSONL benchmark run.", + ) + parser.add_argument( + "--require-reference", + action="append", + choices=("nifty",), + default=[], + help="Fail if the named reference is unavailable.", + ) args = parser.parse_args() if args.n_repeats < 1: raise ValueError("--n-repeats must be at least 1") if args.threads < 1: raise ValueError("--threads must be at least 1") + nifty_available = importlib.util.find_spec("nifty") is not None + if "nifty" in args.require_reference and not nifty_available: + raise RuntimeError("required reference 'nifty' is not available") + run_bic = args.backend in ("both", "bic") + run_nifty = args.backend in ("both", "nifty") and nifty_available + reference_status = ( + "OK" + if run_nifty + else "NO-REF" + if args.backend in ("both", "nifty") + else "NOT_REQUESTED" + ) + + run_id = datetime.now(timezone.utc).isoformat() + if args.results_jsonl is not None: + append_jsonl( + args.results_jsonl, + { + "record_type": "metadata", + "schema_version": 2, + "run_id": run_id, + "created_utc": run_id, + "reference_status": {"nifty": reference_status}, + "arguments": vars(args), + "environment": collect_environment(args.build_command), + }, + ) rows = [] for problem_name in args.problems: for solver_name in args.solvers: - row = evaluate(problem_name, solver_name, configs[solver_name], args) + row = evaluate( + problem_name, + solver_name, + configs[solver_name], + args, + run_bic=run_bic, + run_nifty=run_nifty, + ) + row["record_type"] = "result" + row["schema_version"] = 2 + row["run_id"] = run_id + row["reference_status"] = reference_status rows.append(row) print_progress(row) if args.results_jsonl is not None: diff --git a/include/bioimage_cpp/detail/relabel.hxx b/include/bioimage_cpp/detail/relabel.hxx index 7edd3d1..f8ff728 100644 --- a/include/bioimage_cpp/detail/relabel.hxx +++ b/include/bioimage_cpp/detail/relabel.hxx @@ -1,7 +1,9 @@ #pragma once +#include #include #include +#include #include #include @@ -22,4 +24,68 @@ inline std::vector dense_relabel(const std::vector return result; } +// Mark each current cluster whose node set differs from the previous +// partition. An unchanged cluster contains all nodes from exactly one previous +// cluster. `labels` must use the dense range [0, number_of_clusters). +inline std::vector changed_clusters( + const std::vector &labels, + const std::vector &previous_labels, + const std::uint64_t number_of_clusters +) { + std::vector changed( + static_cast(number_of_clusters), 0 + ); + if (number_of_clusters == 0) { + return changed; + } + if (previous_labels.size() != labels.size()) { + std::fill(changed.begin(), changed.end(), 1); + return changed; + } + + const auto max_previous = + *std::max_element(previous_labels.begin(), previous_labels.end()); + std::vector previous_sizes( + static_cast(max_previous) + 1, 0 + ); + for (const auto label : previous_labels) { + ++previous_sizes[static_cast(label)]; + } + + constexpr auto unmapped = std::numeric_limits::max(); + std::vector previous_of_current( + static_cast(number_of_clusters), unmapped + ); + std::vector current_sizes( + static_cast(number_of_clusters), 0 + ); + + for (std::size_t node = 0; node < labels.size(); ++node) { + const auto current = static_cast(labels[node]); + const auto previous = previous_labels[node]; + ++current_sizes[current]; + if (changed[current]) { + continue; + } + if (previous_of_current[current] == unmapped) { + previous_of_current[current] = previous; + } else if (previous_of_current[current] != previous) { + changed[current] = 1; + } + } + + for (std::size_t current = 0; current < changed.size(); ++current) { + if (changed[current]) { + continue; + } + const auto previous = previous_of_current[current]; + if (previous == unmapped + || previous_sizes[static_cast(previous)] + != current_sizes[current]) { + changed[current] = 1; + } + } + return changed; +} + } // namespace bioimage_cpp::detail diff --git a/include/bioimage_cpp/filters/convolve.hxx b/include/bioimage_cpp/filters/convolve.hxx index c25a4ee..5977841 100644 --- a/include/bioimage_cpp/filters/convolve.hxx +++ b/include/bioimage_cpp/filters/convolve.hxx @@ -2,7 +2,12 @@ #include "bioimage_cpp/filters/kernel.hxx" +#if defined(BIOIMAGE_FILTERS_AVX2_DISPATCH) +#include "bioimage_cpp/filters/dispatch.hxx" +#endif + #include +#include #include namespace bioimage_cpp::filters { @@ -20,6 +25,63 @@ inline std::ptrdiff_t mirror_index(std::ptrdiff_t x, std::ptrdiff_t n) { return r; } +template +inline void convolve_x_border_range( + const float *__restrict in_row, + float *__restrict out_row, + const std::ptrdiff_t begin, + const std::ptrdiff_t end, + const std::ptrdiff_t n_cols, + const float *__restrict h +) { + for (std::ptrdiff_t x = begin; x < end; ++x) { + float acc; + if constexpr (Symmetric) { + acc = h[0] * in_row[x]; + for (int k = 1; k <= R; ++k) { + const float left = in_row[mirror_index(x - k, n_cols)]; + const float right = in_row[mirror_index(x + k, n_cols)]; + acc += h[k] * (left + right); + } + } else { + acc = 0.0f; + for (int k = 1; k <= R; ++k) { + const float left = in_row[mirror_index(x - k, n_cols)]; + const float right = in_row[mirror_index(x + k, n_cols)]; + acc += h[k] * (right - left); + } + } + out_row[x] = acc; + } +} + +template +inline void convolve_x_main_range( + const float *__restrict in_row, + float *__restrict out_row, + const std::ptrdiff_t begin, + const std::ptrdiff_t end, + const float *__restrict h +) { + if constexpr (Symmetric) { + for (std::ptrdiff_t x = begin; x < end; ++x) { + float acc = h[0] * in_row[x]; + for (int k = 1; k <= R; ++k) { + acc += h[k] * (in_row[x + k] + in_row[x - k]); + } + out_row[x] = acc; + } + } else { + for (std::ptrdiff_t x = begin; x < end; ++x) { + float acc = 0.0f; + for (int k = 1; k <= R; ++k) { + acc += h[k] * (in_row[x + k] - in_row[x - k]); + } + out_row[x] = acc; + } + } +} + // Convolve along the contiguous (innermost) axis. Specialised for compile-time // radius R and symmetry. template @@ -39,66 +101,15 @@ void convolve_x_radius( const float *__restrict in_row = in + row * n_cols; float *__restrict out_row = out + row * n_cols; - // Border prologue - for (std::ptrdiff_t x = 0; x < prologue_end; ++x) { - float acc; - if constexpr (Symmetric) { - acc = h[0] * in_row[x]; - for (int k = 1; k <= R; ++k) { - const float left = in_row[mirror_index(x - k, n_cols)]; - const float right = in_row[mirror_index(x + k, n_cols)]; - acc += h[k] * (left + right); - } - } else { - acc = 0.0f; - for (int k = 1; k <= R; ++k) { - const float left = in_row[mirror_index(x - k, n_cols)]; - const float right = in_row[mirror_index(x + k, n_cols)]; - acc += h[k] * (right - left); - } - } - out_row[x] = acc; - } - - // Main loop (branchless): the compiler vectorizes the outer x loop. - if constexpr (Symmetric) { - for (std::ptrdiff_t x = prologue_end; x < epilogue_start; ++x) { - float acc = h[0] * in_row[x]; - for (int k = 1; k <= R; ++k) { - acc += h[k] * (in_row[x + k] + in_row[x - k]); - } - out_row[x] = acc; - } - } else { - for (std::ptrdiff_t x = prologue_end; x < epilogue_start; ++x) { - float acc = 0.0f; - for (int k = 1; k <= R; ++k) { - acc += h[k] * (in_row[x + k] - in_row[x - k]); - } - out_row[x] = acc; - } - } - - // Border epilogue - for (std::ptrdiff_t x = epilogue_start; x < n_cols; ++x) { - float acc; - if constexpr (Symmetric) { - acc = h[0] * in_row[x]; - for (int k = 1; k <= R; ++k) { - const float left = in_row[mirror_index(x - k, n_cols)]; - const float right = in_row[mirror_index(x + k, n_cols)]; - acc += h[k] * (left + right); - } - } else { - acc = 0.0f; - for (int k = 1; k <= R; ++k) { - const float left = in_row[mirror_index(x - k, n_cols)]; - const float right = in_row[mirror_index(x + k, n_cols)]; - acc += h[k] * (right - left); - } - } - out_row[x] = acc; - } + convolve_x_border_range( + in_row, out_row, 0, prologue_end, n_cols, h + ); + convolve_x_main_range( + in_row, out_row, prologue_end, epilogue_start, h + ); + convolve_x_border_range( + in_row, out_row, epilogue_start, n_cols, n_cols, h + ); } } @@ -182,6 +193,56 @@ void convolve_x_runtime( // and large enough to amortise the kernel-coefficient broadcast. inline constexpr std::ptrdiff_t kStripBlock = 64; +template +inline void convolve_strided_border_row( + const float *__restrict in_o, + float *__restrict out_o, + const std::ptrdiff_t y, + const std::ptrdiff_t n_axis, + const std::ptrdiff_t n_inner, + const float *__restrict h +) { + float *__restrict out_row = out_o + y * n_inner; + const float *__restrict center = in_o + y * n_inner; + + for (std::ptrdiff_t xb = 0; xb < n_inner; xb += kStripBlock) { + const std::ptrdiff_t strip = std::min(kStripBlock, n_inner - xb); + float acc[kStripBlock]; + + if constexpr (Symmetric) { + const float h0 = h[0]; + for (std::ptrdiff_t i = 0; i < strip; ++i) { + acc[i] = h0 * center[xb + i]; + } + } else { + for (std::ptrdiff_t i = 0; i < strip; ++i) { + acc[i] = 0.0f; + } + } + + for (int k = 1; k <= R; ++k) { + const float hk = h[k]; + const std::ptrdiff_t y_up = mirror_index(y - k, n_axis); + const std::ptrdiff_t y_dn = mirror_index(y + k, n_axis); + const float *__restrict up = in_o + y_up * n_inner + xb; + const float *__restrict dn = in_o + y_dn * n_inner + xb; + if constexpr (Symmetric) { + for (std::ptrdiff_t i = 0; i < strip; ++i) { + acc[i] += hk * (up[i] + dn[i]); + } + } else { + for (std::ptrdiff_t i = 0; i < strip; ++i) { + acc[i] += hk * (dn[i] - up[i]); + } + } + } + + for (std::ptrdiff_t i = 0; i < strip; ++i) { + out_row[xb + i] = acc[i]; + } + } +} + // Convolve along a strided (non-innermost) axis. Logical shape // (n_outer, n_axis, n_inner) in C-order; we accumulate into strips of // kStripBlock contiguous columns of the innermost axis, which gives the @@ -246,51 +307,16 @@ void convolve_strided_radius( } } - // Border rows (axis prologue + epilogue), with mirror on y±k. - auto run_border_row = [&](std::ptrdiff_t y) { - float *__restrict out_row = out_o + y * n_inner; - const float *__restrict center = in_o + y * n_inner; - - for (std::ptrdiff_t xb = 0; xb < n_inner; xb += kStripBlock) { - const std::ptrdiff_t strip = std::min(kStripBlock, n_inner - xb); - float acc[kStripBlock]; - - if constexpr (Symmetric) { - const float h0 = h[0]; - for (std::ptrdiff_t i = 0; i < strip; ++i) { - acc[i] = h0 * center[xb + i]; - } - } else { - for (std::ptrdiff_t i = 0; i < strip; ++i) { - acc[i] = 0.0f; - } - } - - for (int k = 1; k <= R; ++k) { - const float hk = h[k]; - const std::ptrdiff_t y_up = mirror_index(y - k, n_axis); - const std::ptrdiff_t y_dn = mirror_index(y + k, n_axis); - const float *__restrict up = in_o + y_up * n_inner + xb; - const float *__restrict dn = in_o + y_dn * n_inner + xb; - if constexpr (Symmetric) { - for (std::ptrdiff_t i = 0; i < strip; ++i) { - acc[i] += hk * (up[i] + dn[i]); - } - } else { - for (std::ptrdiff_t i = 0; i < strip; ++i) { - acc[i] += hk * (dn[i] - up[i]); - } - } - } - - for (std::ptrdiff_t i = 0; i < strip; ++i) { - out_row[xb + i] = acc[i]; - } - } - }; - - for (std::ptrdiff_t y = 0; y < prologue_end; ++y) run_border_row(y); - for (std::ptrdiff_t y = epilogue_start; y < n_axis; ++y) run_border_row(y); + for (std::ptrdiff_t y = 0; y < prologue_end; ++y) { + convolve_strided_border_row( + in_o, out_o, y, n_axis, n_inner, h + ); + } + for (std::ptrdiff_t y = epilogue_start; y < n_axis; ++y) { + convolve_strided_border_row( + in_o, out_o, y, n_axis, n_inner, h + ); + } } } @@ -357,6 +383,199 @@ void convolve_strided_runtime( } } +template +inline constexpr std::size_t kSymmetricComponents = D * (D + 1) / 2; + +template +void convolve_strided_outer_products_radius( + const std::array &gradients, + const std::array> &out, + std::ptrdiff_t n_outer, + std::ptrdiff_t n_axis, + std::ptrdiff_t n_inner, + const float *__restrict h +) { + static_assert(D == 2 || D == 3); + if (n_axis <= 0 || n_inner <= 0 || n_outer <= 0) return; + + constexpr std::size_t C = kSymmetricComponents; + const std::ptrdiff_t outer_stride = n_axis * n_inner; + const std::ptrdiff_t prologue_end = std::min(R, n_axis); + const std::ptrdiff_t epilogue_start = + std::max(prologue_end, n_axis - R); + + for (std::ptrdiff_t o = 0; o < n_outer; ++o) { + std::array in_o{}; + std::array out_o{}; + for (std::size_t d = 0; d < D; ++d) { + in_o[d] = gradients[d] + o * outer_stride; + } + for (std::size_t c = 0; c < C; ++c) { + out_o[c] = out[c] + o * outer_stride; + } + + auto run_row = [&](const std::ptrdiff_t y) { + for (std::ptrdiff_t xb = 0; xb < n_inner; xb += kStripBlock) { + const std::ptrdiff_t strip = std::min(kStripBlock, n_inner - xb); + float acc[C][kStripBlock]; + + for (std::ptrdiff_t i = 0; i < strip; ++i) { + const std::ptrdiff_t index = y * n_inner + xb + i; + const float g0 = in_o[0][index]; + const float g1 = in_o[1][index]; + acc[0][i] = h[0] * g0 * g0; + acc[1][i] = h[0] * g0 * g1; + if constexpr (D == 2) { + acc[2][i] = h[0] * g1 * g1; + } else { + const float g2 = in_o[2][index]; + acc[2][i] = h[0] * g0 * g2; + acc[3][i] = h[0] * g1 * g1; + acc[4][i] = h[0] * g1 * g2; + acc[5][i] = h[0] * g2 * g2; + } + } + + for (int k = 1; k <= R; ++k) { + const float hk = h[k]; + const std::ptrdiff_t y_up = + Mirror ? mirror_index(y - k, n_axis) : (y - k); + const std::ptrdiff_t y_dn = + Mirror ? mirror_index(y + k, n_axis) : (y + k); + const std::ptrdiff_t up_base = y_up * n_inner + xb; + const std::ptrdiff_t dn_base = y_dn * n_inner + xb; + + for (std::ptrdiff_t i = 0; i < strip; ++i) { + const float g0u = in_o[0][up_base + i]; + const float g0d = in_o[0][dn_base + i]; + const float g1u = in_o[1][up_base + i]; + const float g1d = in_o[1][dn_base + i]; + acc[0][i] += hk * (g0u * g0u + g0d * g0d); + acc[1][i] += hk * (g0u * g1u + g0d * g1d); + if constexpr (D == 2) { + acc[2][i] += hk * (g1u * g1u + g1d * g1d); + } else { + const float g2u = in_o[2][up_base + i]; + const float g2d = in_o[2][dn_base + i]; + acc[2][i] += hk * (g0u * g2u + g0d * g2d); + acc[3][i] += hk * (g1u * g1u + g1d * g1d); + acc[4][i] += hk * (g1u * g2u + g1d * g2d); + acc[5][i] += hk * (g2u * g2u + g2d * g2d); + } + } + } + + for (std::size_t c = 0; c < C; ++c) { + float *__restrict out_row = out_o[c] + y * n_inner + xb; + for (std::ptrdiff_t i = 0; i < strip; ++i) { + out_row[i] = acc[c][i]; + } + } + } + }; + + for (std::ptrdiff_t y = prologue_end; y < epilogue_start; ++y) { + run_row.template operator()(y); + } + for (std::ptrdiff_t y = 0; y < prologue_end; ++y) { + run_row.template operator()(y); + } + for (std::ptrdiff_t y = epilogue_start; y < n_axis; ++y) { + run_row.template operator()(y); + } + } +} + +template +void convolve_strided_outer_products_runtime( + const std::array &gradients, + const std::array> &out, + std::ptrdiff_t n_outer, + std::ptrdiff_t n_axis, + std::ptrdiff_t n_inner, + int radius, + const float *__restrict h +) { + static_assert(D == 2 || D == 3); + if (n_axis <= 0 || n_inner <= 0 || n_outer <= 0) return; + + constexpr std::size_t C = kSymmetricComponents; + const std::ptrdiff_t outer_stride = n_axis * n_inner; + const std::ptrdiff_t R = radius; + + for (std::ptrdiff_t o = 0; o < n_outer; ++o) { + std::array in_o{}; + std::array out_o{}; + for (std::size_t d = 0; d < D; ++d) { + in_o[d] = gradients[d] + o * outer_stride; + } + for (std::size_t c = 0; c < C; ++c) { + out_o[c] = out[c] + o * outer_stride; + } + + for (std::ptrdiff_t y = 0; y < n_axis; ++y) { + const bool needs_mirror = y < R || y >= n_axis - R; + for (std::ptrdiff_t xb = 0; xb < n_inner; xb += kStripBlock) { + const std::ptrdiff_t strip = std::min(kStripBlock, n_inner - xb); + float acc[C][kStripBlock]; + + for (std::ptrdiff_t i = 0; i < strip; ++i) { + const std::ptrdiff_t index = y * n_inner + xb + i; + const float g0 = in_o[0][index]; + const float g1 = in_o[1][index]; + acc[0][i] = h[0] * g0 * g0; + acc[1][i] = h[0] * g0 * g1; + if constexpr (D == 2) { + acc[2][i] = h[0] * g1 * g1; + } else { + const float g2 = in_o[2][index]; + acc[2][i] = h[0] * g0 * g2; + acc[3][i] = h[0] * g1 * g1; + acc[4][i] = h[0] * g1 * g2; + acc[5][i] = h[0] * g2 * g2; + } + } + + for (int k = 1; k <= R; ++k) { + const float hk = h[k]; + const std::ptrdiff_t y_up = + needs_mirror ? mirror_index(y - k, n_axis) : (y - k); + const std::ptrdiff_t y_dn = + needs_mirror ? mirror_index(y + k, n_axis) : (y + k); + const std::ptrdiff_t up_base = y_up * n_inner + xb; + const std::ptrdiff_t dn_base = y_dn * n_inner + xb; + + for (std::ptrdiff_t i = 0; i < strip; ++i) { + const float g0u = in_o[0][up_base + i]; + const float g0d = in_o[0][dn_base + i]; + const float g1u = in_o[1][up_base + i]; + const float g1d = in_o[1][dn_base + i]; + acc[0][i] += hk * (g0u * g0u + g0d * g0d); + acc[1][i] += hk * (g0u * g1u + g0d * g1d); + if constexpr (D == 2) { + acc[2][i] += hk * (g1u * g1u + g1d * g1d); + } else { + const float g2u = in_o[2][up_base + i]; + const float g2d = in_o[2][dn_base + i]; + acc[2][i] += hk * (g0u * g2u + g0d * g2d); + acc[3][i] += hk * (g1u * g1u + g1d * g1d); + acc[4][i] += hk * (g1u * g2u + g1d * g2d); + acc[5][i] += hk * (g2u * g2u + g2d * g2d); + } + } + } + + for (std::size_t c = 0; c < C; ++c) { + float *__restrict out_row = out_o[c] + y * n_inner + xb; + for (std::ptrdiff_t i = 0; i < strip; ++i) { + out_row[i] = acc[c][i]; + } + } + } + } + } +} + } // namespace detail // Maximum compile-time-specialised kernel radius. Kernels with radius > this @@ -376,6 +595,12 @@ inline void convolve_axis_x( const float *h = kernel.half_coefs.data(); const bool sym = kernel.is_symmetric; +#if defined(BIOIMAGE_FILTERS_AVX2_DISPATCH) + if (dispatch::try_convolve_axis_x(in, out, n_rows, n_cols, r, sym, h)) { + return; + } +#endif + #define BIOIMAGE_FILTERS_DISPATCH_RADIUS_X(R) \ case R: \ if (sym) detail::convolve_x_radius(in, out, n_rows, n_cols, h); \ @@ -417,6 +642,14 @@ inline void convolve_axis_strided( const float *h = kernel.half_coefs.data(); const bool sym = kernel.is_symmetric; +#if defined(BIOIMAGE_FILTERS_AVX2_DISPATCH) + if (dispatch::try_convolve_axis_strided( + in, out, n_outer, n_axis, n_inner, r, sym, h + )) { + return; + } +#endif + #define BIOIMAGE_FILTERS_DISPATCH_RADIUS_S(R) \ case R: \ if (sym) \ @@ -448,4 +681,69 @@ inline void convolve_axis_strided( #undef BIOIMAGE_FILTERS_DISPATCH_RADIUS_S } +template +inline void convolve_axis_strided_outer_products( + const std::array &gradients, + const std::array> &out, + std::ptrdiff_t n_outer, + std::ptrdiff_t n_axis, + std::ptrdiff_t n_inner, + const Kernel1D &kernel +) { + const int r = kernel.radius; + const float *h = kernel.half_coefs.data(); + +#if defined(BIOIMAGE_FILTERS_AVX2_DISPATCH) + if constexpr (D == 2) { + if (dispatch::try_convolve_outer_products_2d( + gradients, out, n_outer, n_axis, n_inner, r, h + )) { + return; + } + } else { + if (dispatch::try_convolve_outer_products_3d( + gradients, out, n_outer, n_axis, n_inner, r, h + )) { + return; + } + } +#endif + +#define BIOIMAGE_FILTERS_DISPATCH_RADIUS_OP(R) \ + case R: \ + detail::convolve_strided_outer_products_radius( \ + gradients, out, n_outer, n_axis, n_inner, h \ + ); \ + return; + + switch (r) { + BIOIMAGE_FILTERS_DISPATCH_RADIUS_OP(1) + BIOIMAGE_FILTERS_DISPATCH_RADIUS_OP(2) + BIOIMAGE_FILTERS_DISPATCH_RADIUS_OP(3) + BIOIMAGE_FILTERS_DISPATCH_RADIUS_OP(4) + BIOIMAGE_FILTERS_DISPATCH_RADIUS_OP(5) + BIOIMAGE_FILTERS_DISPATCH_RADIUS_OP(6) + BIOIMAGE_FILTERS_DISPATCH_RADIUS_OP(7) + BIOIMAGE_FILTERS_DISPATCH_RADIUS_OP(8) + BIOIMAGE_FILTERS_DISPATCH_RADIUS_OP(9) + BIOIMAGE_FILTERS_DISPATCH_RADIUS_OP(10) + BIOIMAGE_FILTERS_DISPATCH_RADIUS_OP(11) + BIOIMAGE_FILTERS_DISPATCH_RADIUS_OP(12) + default: + detail::convolve_strided_outer_products_runtime( + gradients, out, n_outer, n_axis, n_inner, r, h + ); + return; + } +#undef BIOIMAGE_FILTERS_DISPATCH_RADIUS_OP +} + +inline const char *convolution_backend() { +#if defined(BIOIMAGE_FILTERS_AVX2_DISPATCH) + return dispatch::convolution_backend(); +#else + return "scalar"; +#endif +} + } // namespace bioimage_cpp::filters diff --git a/include/bioimage_cpp/filters/dispatch.hxx b/include/bioimage_cpp/filters/dispatch.hxx new file mode 100644 index 0000000..0539f8b --- /dev/null +++ b/include/bioimage_cpp/filters/dispatch.hxx @@ -0,0 +1,63 @@ +#pragma once + +#include +#include + +namespace bioimage_cpp::filters::dispatch { + +bool try_convolve_axis_x( + const float *in, + float *out, + std::ptrdiff_t n_rows, + std::ptrdiff_t n_cols, + int radius, + bool symmetric, + const float *half_coefs +); + +bool try_convolve_axis_strided( + const float *in, + float *out, + std::ptrdiff_t n_outer, + std::ptrdiff_t n_axis, + std::ptrdiff_t n_inner, + int radius, + bool symmetric, + const float *half_coefs +); + +bool try_convolve_outer_products_2d( + const std::array &gradients, + const std::array &out, + std::ptrdiff_t n_outer, + std::ptrdiff_t n_axis, + std::ptrdiff_t n_inner, + int radius, + const float *half_coefs +); + +bool try_convolve_outer_products_3d( + const std::array &gradients, + const std::array &out, + std::ptrdiff_t n_outer, + std::ptrdiff_t n_axis, + std::ptrdiff_t n_inner, + int radius, + const float *half_coefs +); + +bool try_ev3_symmetric_descending_interleaved( + const float *a00, + const float *a01, + const float *a02, + const float *a11, + const float *a12, + const float *a22, + float *out, + std::ptrdiff_t n +); + +const char *convolution_backend(); +const char *eigenvalue_backend(); + +} // namespace bioimage_cpp::filters::dispatch diff --git a/include/bioimage_cpp/filters/eigenvalues.hxx b/include/bioimage_cpp/filters/eigenvalues.hxx index 0bece2f..5c624ba 100644 --- a/include/bioimage_cpp/filters/eigenvalues.hxx +++ b/include/bioimage_cpp/filters/eigenvalues.hxx @@ -1,8 +1,13 @@ #pragma once +#if defined(BIOIMAGE_FILTERS_AVX2_DISPATCH) +#include "bioimage_cpp/filters/dispatch.hxx" +#endif + #include #include #include +#include namespace bioimage_cpp::filters { @@ -55,13 +60,22 @@ inline void ev3_one_descending( return; } - const float inv_m = 1.0f / m; - a00 *= inv_m; - a01 *= inv_m; - a02 *= inv_m; - a11 *= inv_m; - a12 *= inv_m; - a22 *= inv_m; + if (m < std::numeric_limits::min()) { + a00 /= m; + a01 /= m; + a02 /= m; + a11 /= m; + a12 /= m; + a22 /= m; + } else { + const float inv_m = 1.0f / m; + a00 *= inv_m; + a01 *= inv_m; + a02 *= inv_m; + a11 *= inv_m; + a12 *= inv_m; + a22 *= inv_m; + } const float a = (a00 + a11 + a22) * (1.0f / 3.0f); const float b00 = a00 - a; @@ -91,17 +105,26 @@ inline void ev3_one_descending( // otherwise produce NaN from acos. r = std::max(-1.0f, std::min(1.0f, r)); - constexpr float kTwoPiOver3 = 2.0943951023931953f; + constexpr float kSqrtThreeOverTwo = 0.8660254037844386f; const float phi = std::acos(r) * (1.0f / 3.0f); const float two_p = 2.0f * p; + const float cos_phi = std::cos(phi); + const float sin_phi = std::sqrt(std::max(0.0f, 1.0f - cos_phi * cos_phi)); - const float root_large = a + two_p * std::cos(phi); - const float root_small = a + two_p * std::cos(phi + kTwoPiOver3); + const float root_large = a + two_p * cos_phi; + const float root_small = + a + two_p * (-0.5f * cos_phi - kSqrtThreeOverTwo * sin_phi); const float root_mid = 3.0f * a - root_large - root_small; - e0 = m * root_large; - e1 = m * root_mid; - e2 = m * root_small; + const float value_large = m * root_large; + const float value_mid = m * root_mid; + const float value_small = m * root_small; + const float high01 = std::max(value_large, value_mid); + const float low01 = std::min(value_large, value_mid); + e0 = std::max(high01, value_small); + const float middle_candidate = std::min(high01, value_small); + e1 = std::max(low01, middle_candidate); + e2 = std::min(low01, middle_candidate); } } // namespace detail @@ -131,4 +154,38 @@ inline void ev3_symmetric_descending( } } +inline void ev3_symmetric_descending_interleaved( + const float *__restrict a00, + const float *__restrict a01, + const float *__restrict a02, + const float *__restrict a11, + const float *__restrict a12, + const float *__restrict a22, + float *__restrict out, + const std::ptrdiff_t n +) { +#if defined(BIOIMAGE_FILTERS_AVX2_DISPATCH) + if (dispatch::try_ev3_symmetric_descending_interleaved( + a00, a01, a02, a11, a12, a22, out, n + )) { + return; + } +#endif + + for (std::ptrdiff_t i = 0; i < n; ++i) { + detail::ev3_one_descending( + a00[i], a01[i], a02[i], a11[i], a12[i], a22[i], + out[3 * i], out[3 * i + 1], out[3 * i + 2] + ); + } +} + +inline const char *eigenvalue_backend() { +#if defined(BIOIMAGE_FILTERS_AVX2_DISPATCH) + return dispatch::eigenvalue_backend(); +#else + return "scalar"; +#endif +} + } // namespace bioimage_cpp::filters diff --git a/include/bioimage_cpp/filters/gaussian.hxx b/include/bioimage_cpp/filters/gaussian.hxx index 0893c7a..a23a43d 100644 --- a/include/bioimage_cpp/filters/gaussian.hxx +++ b/include/bioimage_cpp/filters/gaussian.hxx @@ -1,26 +1,122 @@ #pragma once +#include "bioimage_cpp/detail/profile.hxx" #include "bioimage_cpp/filters/convolve.hxx" #include "bioimage_cpp/filters/eigenvalues.hxx" #include "bioimage_cpp/filters/kernel.hxx" +#include #include #include +#include +#include #include #include -#include namespace bioimage_cpp::filters { -// --------------------------------------------------------------------------- -// Separable Gaussian (derivative) along each axis. -// --------------------------------------------------------------------------- -// -// `in`, `out`, `workspace` are C-contiguous buffers of the same total size. -// `out` and `workspace` must NOT alias `in` (the binding ensures this by -// allocating fresh output and scratch). `out` and `workspace` may not alias -// each other. After the call, `out` holds the filter response and `workspace` -// is left in an unspecified state. +namespace detail { + +inline std::unique_ptr allocate_scratch( + const std::ptrdiff_t n, + const std::size_t slots +) { + if (n < 0) { + throw std::invalid_argument("filter scratch size must be non-negative"); + } + const auto size = static_cast(n); + if (slots != 0 && size > std::numeric_limits::max() / slots) { + throw std::length_error("filter scratch size overflow"); + } + return std::make_unique_for_overwrite(size * slots); +} + +inline float *scratch_slot( + const std::unique_ptr &scratch, + const std::ptrdiff_t n, + const std::size_t slot +) { + if (n == 0) { + return scratch.get(); + } + return scratch.get() + static_cast(n) * slot; +} + +template +inline void gaussian_separable_2d_profiled( + const float *in, + float *out, + float *workspace, + const std::ptrdiff_t ny, + const std::ptrdiff_t nx, + const Kernel1D &ky, + const Kernel1D &kx, + Profiler &profile +) { + { + BIOIMAGE_PROFILE_SCOPE(profile, "axis_y"); + convolve_axis_strided(in, workspace, 1, ny, nx, ky); + } + { + BIOIMAGE_PROFILE_SCOPE(profile, "axis_x"); + convolve_axis_x(workspace, out, ny, nx, kx); + } +} + +template +inline void gaussian_first_axis_3d_profiled( + const float *in, + float *partial, + const std::ptrdiff_t nz, + const std::ptrdiff_t ny, + const std::ptrdiff_t nx, + const Kernel1D &kz, + Profiler &profile +) { + BIOIMAGE_PROFILE_SCOPE(profile, "axis_z"); + convolve_axis_strided(in, partial, 1, nz, ny * nx, kz); +} + +template +inline void gaussian_remaining_axes_3d_profiled( + const float *partial, + float *out, + float *workspace, + const std::ptrdiff_t nz, + const std::ptrdiff_t ny, + const std::ptrdiff_t nx, + const Kernel1D &ky, + const Kernel1D &kx, + Profiler &profile +) { + { + BIOIMAGE_PROFILE_SCOPE(profile, "axis_y"); + convolve_axis_strided(partial, workspace, nz, ny, nx, ky); + } + { + BIOIMAGE_PROFILE_SCOPE(profile, "axis_x"); + convolve_axis_x(workspace, out, nz * ny, nx, kx); + } +} + +template +inline void gaussian_separable_3d_profiled( + const float *in, + float *out, + float *workspace, + const std::ptrdiff_t nz, + const std::ptrdiff_t ny, + const std::ptrdiff_t nx, + const Kernel1D &kz, + const Kernel1D &ky, + const Kernel1D &kx, + Profiler &profile +) { + gaussian_first_axis_3d_profiled(in, out, nz, ny, nx, kz, profile); + gaussian_remaining_axes_3d_profiled(out, out, workspace, nz, ny, nx, ky, kx, profile); +} + +} // namespace detail inline void gaussian_separable_2d( const float *in, @@ -31,10 +127,10 @@ inline void gaussian_separable_2d( const Kernel1D &ky, const Kernel1D &kx ) { - // Axis 0 (Y), strided: (n_outer=1, n_axis=ny, n_inner=nx). - convolve_axis_strided(in, workspace, 1, ny, nx, ky); - // Axis 1 (X), contiguous: (n_rows=ny, n_cols=nx). - convolve_axis_x(workspace, out, ny, nx, kx); + bioimage_cpp::detail::NullProfiler profile; + detail::gaussian_separable_2d_profiled( + in, out, workspace, ny, nx, ky, kx, profile + ); } inline void gaussian_separable_3d( @@ -48,12 +144,10 @@ inline void gaussian_separable_3d( const Kernel1D &ky, const Kernel1D &kx ) { - // Axis 0 (Z), strided: (1, nz, ny*nx). in -> out. - convolve_axis_strided(in, out, 1, nz, ny * nx, kz); - // Axis 1 (Y), strided: (nz, ny, nx). out -> workspace. - convolve_axis_strided(out, workspace, nz, ny, nx, ky); - // Axis 2 (X), contiguous: (nz*ny, nx). workspace -> out. - convolve_axis_x(workspace, out, nz * ny, nx, kx); + bioimage_cpp::detail::NullProfiler profile; + detail::gaussian_separable_3d_profiled( + in, out, workspace, nz, ny, nx, kz, ky, kx, profile + ); } // --------------------------------------------------------------------------- @@ -72,10 +166,19 @@ inline void gaussian_smoothing_2d( double sigma_x, double window_ratio ) { + BIOIMAGE_PROFILE_INIT(profile); + const std::ptrdiff_t n = ny * nx; const auto ky = gaussian_kernel(sigma_y, 0, window_ratio); const auto kx = gaussian_kernel(sigma_x, 0, window_ratio); - std::vector workspace(static_cast(ny * nx)); - gaussian_separable_2d(in, out, workspace.data(), ny, nx, ky, kx); + std::unique_ptr scratch; + { + BIOIMAGE_PROFILE_SCOPE(profile, "scratch_alloc"); + scratch = detail::allocate_scratch(n, 1); + } + detail::gaussian_separable_2d_profiled( + in, out, scratch.get(), ny, nx, ky, kx, profile + ); + BIOIMAGE_PROFILE_REPORT(profile); } inline void gaussian_smoothing_3d( @@ -89,11 +192,20 @@ inline void gaussian_smoothing_3d( double sigma_x, double window_ratio ) { + BIOIMAGE_PROFILE_INIT(profile); + const std::ptrdiff_t n = nz * ny * nx; const auto kz = gaussian_kernel(sigma_z, 0, window_ratio); const auto ky = gaussian_kernel(sigma_y, 0, window_ratio); const auto kx = gaussian_kernel(sigma_x, 0, window_ratio); - std::vector workspace(static_cast(nz * ny * nx)); - gaussian_separable_3d(in, out, workspace.data(), nz, ny, nx, kz, ky, kx); + std::unique_ptr scratch; + { + BIOIMAGE_PROFILE_SCOPE(profile, "scratch_alloc"); + scratch = detail::allocate_scratch(n, 1); + } + detail::gaussian_separable_3d_profiled( + in, out, scratch.get(), nz, ny, nx, kz, ky, kx, profile + ); + BIOIMAGE_PROFILE_REPORT(profile); } inline void gaussian_derivative_2d( @@ -107,10 +219,19 @@ inline void gaussian_derivative_2d( int order_x, double window_ratio ) { + BIOIMAGE_PROFILE_INIT(profile); + const std::ptrdiff_t n = ny * nx; const auto ky = gaussian_kernel(sigma_y, order_y, window_ratio); const auto kx = gaussian_kernel(sigma_x, order_x, window_ratio); - std::vector workspace(static_cast(ny * nx)); - gaussian_separable_2d(in, out, workspace.data(), ny, nx, ky, kx); + std::unique_ptr scratch; + { + BIOIMAGE_PROFILE_SCOPE(profile, "scratch_alloc"); + scratch = detail::allocate_scratch(n, 1); + } + detail::gaussian_separable_2d_profiled( + in, out, scratch.get(), ny, nx, ky, kx, profile + ); + BIOIMAGE_PROFILE_REPORT(profile); } inline void gaussian_derivative_3d( @@ -127,11 +248,20 @@ inline void gaussian_derivative_3d( int order_x, double window_ratio ) { + BIOIMAGE_PROFILE_INIT(profile); + const std::ptrdiff_t n = nz * ny * nx; const auto kz = gaussian_kernel(sigma_z, order_z, window_ratio); const auto ky = gaussian_kernel(sigma_y, order_y, window_ratio); const auto kx = gaussian_kernel(sigma_x, order_x, window_ratio); - std::vector workspace(static_cast(nz * ny * nx)); - gaussian_separable_3d(in, out, workspace.data(), nz, ny, nx, kz, ky, kx); + std::unique_ptr scratch; + { + BIOIMAGE_PROFILE_SCOPE(profile, "scratch_alloc"); + scratch = detail::allocate_scratch(n, 1); + } + detail::gaussian_separable_3d_profiled( + in, out, scratch.get(), nz, ny, nx, kz, ky, kx, profile + ); + BIOIMAGE_PROFILE_REPORT(profile); } inline void gaussian_gradient_magnitude_2d( @@ -143,24 +273,39 @@ inline void gaussian_gradient_magnitude_2d( double sigma_x, double window_ratio ) { + BIOIMAGE_PROFILE_INIT(profile); const std::ptrdiff_t n = ny * nx; const auto ky0 = gaussian_kernel(sigma_y, 0, window_ratio); const auto kx0 = gaussian_kernel(sigma_x, 0, window_ratio); const auto ky1 = gaussian_kernel(sigma_y, 1, window_ratio); const auto kx1 = gaussian_kernel(sigma_x, 1, window_ratio); - std::vector work1(static_cast(n)); - std::vector work2(static_cast(n)); - - // d/dy - gaussian_separable_2d(in, work1.data(), work2.data(), ny, nx, ky1, kx0); - for (std::ptrdiff_t i = 0; i < n; ++i) out[i] = work1[i] * work1[i]; - - // d/dx - gaussian_separable_2d(in, work1.data(), work2.data(), ny, nx, ky0, kx1); - for (std::ptrdiff_t i = 0; i < n; ++i) out[i] += work1[i] * work1[i]; + std::unique_ptr scratch; + { + BIOIMAGE_PROFILE_SCOPE(profile, "scratch_alloc"); + scratch = detail::allocate_scratch(n, 2); + } + float *work1 = detail::scratch_slot(scratch, n, 0); + float *work2 = detail::scratch_slot(scratch, n, 1); + + detail::gaussian_separable_2d_profiled( + in, work1, work2, ny, nx, ky1, kx0, profile + ); + { + BIOIMAGE_PROFILE_SCOPE(profile, "combine"); + for (std::ptrdiff_t i = 0; i < n; ++i) out[i] = work1[i] * work1[i]; + } - for (std::ptrdiff_t i = 0; i < n; ++i) out[i] = std::sqrt(out[i]); + detail::gaussian_separable_2d_profiled( + in, work1, work2, ny, nx, ky0, kx1, profile + ); + { + BIOIMAGE_PROFILE_SCOPE(profile, "combine"); + for (std::ptrdiff_t i = 0; i < n; ++i) { + out[i] = std::sqrt(out[i] + work1[i] * work1[i]); + } + } + BIOIMAGE_PROFILE_REPORT(profile); } inline void gaussian_gradient_magnitude_3d( @@ -174,6 +319,7 @@ inline void gaussian_gradient_magnitude_3d( double sigma_x, double window_ratio ) { + BIOIMAGE_PROFILE_INIT(profile); const std::ptrdiff_t n = nz * ny * nx; const auto kz0 = gaussian_kernel(sigma_z, 0, window_ratio); const auto ky0 = gaussian_kernel(sigma_y, 0, window_ratio); @@ -182,19 +328,44 @@ inline void gaussian_gradient_magnitude_3d( const auto ky1 = gaussian_kernel(sigma_y, 1, window_ratio); const auto kx1 = gaussian_kernel(sigma_x, 1, window_ratio); - std::vector work1(static_cast(n)); - std::vector work2(static_cast(n)); - - gaussian_separable_3d(in, work1.data(), work2.data(), nz, ny, nx, kz1, ky0, kx0); - for (std::ptrdiff_t i = 0; i < n; ++i) out[i] = work1[i] * work1[i]; - - gaussian_separable_3d(in, work1.data(), work2.data(), nz, ny, nx, kz0, ky1, kx0); - for (std::ptrdiff_t i = 0; i < n; ++i) out[i] += work1[i] * work1[i]; + std::unique_ptr scratch; + { + BIOIMAGE_PROFILE_SCOPE(profile, "scratch_alloc"); + scratch = detail::allocate_scratch(n, 2); + } + float *work1 = detail::scratch_slot(scratch, n, 0); + float *work2 = detail::scratch_slot(scratch, n, 1); + + detail::gaussian_first_axis_3d_profiled(in, out, nz, ny, nx, kz0, profile); + detail::gaussian_remaining_axes_3d_profiled( + out, work2, work1, nz, ny, nx, ky1, kx0, profile + ); + { + BIOIMAGE_PROFILE_SCOPE(profile, "combine"); + for (std::ptrdiff_t i = 0; i < n; ++i) work1[i] = work2[i] * work2[i]; + } - gaussian_separable_3d(in, work1.data(), work2.data(), nz, ny, nx, kz0, ky0, kx1); - for (std::ptrdiff_t i = 0; i < n; ++i) out[i] += work1[i] * work1[i]; + detail::gaussian_remaining_axes_3d_profiled( + out, out, work2, nz, ny, nx, ky0, kx1, profile + ); + { + BIOIMAGE_PROFILE_SCOPE(profile, "combine"); + for (std::ptrdiff_t i = 0; i < n; ++i) { + work1[i] += out[i] * out[i]; + } + } - for (std::ptrdiff_t i = 0; i < n; ++i) out[i] = std::sqrt(out[i]); + detail::gaussian_first_axis_3d_profiled(in, work2, nz, ny, nx, kz1, profile); + detail::gaussian_remaining_axes_3d_profiled( + work2, work2, out, nz, ny, nx, ky0, kx0, profile + ); + { + BIOIMAGE_PROFILE_SCOPE(profile, "combine"); + for (std::ptrdiff_t i = 0; i < n; ++i) { + out[i] = std::sqrt(work1[i] + work2[i] * work2[i]); + } + } + BIOIMAGE_PROFILE_REPORT(profile); } inline void laplacian_of_gaussian_2d( @@ -206,22 +377,37 @@ inline void laplacian_of_gaussian_2d( double sigma_x, double window_ratio ) { + BIOIMAGE_PROFILE_INIT(profile); const std::ptrdiff_t n = ny * nx; const auto ky0 = gaussian_kernel(sigma_y, 0, window_ratio); const auto kx0 = gaussian_kernel(sigma_x, 0, window_ratio); const auto ky2 = gaussian_kernel(sigma_y, 2, window_ratio); const auto kx2 = gaussian_kernel(sigma_x, 2, window_ratio); - std::vector work1(static_cast(n)); - std::vector work2(static_cast(n)); - - // d²/dy² - gaussian_separable_2d(in, work1.data(), work2.data(), ny, nx, ky2, kx0); - for (std::ptrdiff_t i = 0; i < n; ++i) out[i] = work1[i]; + std::unique_ptr scratch; + { + BIOIMAGE_PROFILE_SCOPE(profile, "scratch_alloc"); + scratch = detail::allocate_scratch(n, 2); + } + float *work1 = detail::scratch_slot(scratch, n, 0); + float *work2 = detail::scratch_slot(scratch, n, 1); + + detail::gaussian_separable_2d_profiled( + in, work1, work2, ny, nx, ky2, kx0, profile + ); + { + BIOIMAGE_PROFILE_SCOPE(profile, "combine"); + for (std::ptrdiff_t i = 0; i < n; ++i) out[i] = work1[i]; + } - // d²/dx² - gaussian_separable_2d(in, work1.data(), work2.data(), ny, nx, ky0, kx2); - for (std::ptrdiff_t i = 0; i < n; ++i) out[i] += work1[i]; + detail::gaussian_separable_2d_profiled( + in, work1, work2, ny, nx, ky0, kx2, profile + ); + { + BIOIMAGE_PROFILE_SCOPE(profile, "combine"); + for (std::ptrdiff_t i = 0; i < n; ++i) out[i] += work1[i]; + } + BIOIMAGE_PROFILE_REPORT(profile); } inline void laplacian_of_gaussian_3d( @@ -235,6 +421,7 @@ inline void laplacian_of_gaussian_3d( double sigma_x, double window_ratio ) { + BIOIMAGE_PROFILE_INIT(profile); const std::ptrdiff_t n = nz * ny * nx; const auto kz0 = gaussian_kernel(sigma_z, 0, window_ratio); const auto ky0 = gaussian_kernel(sigma_y, 0, window_ratio); @@ -243,17 +430,40 @@ inline void laplacian_of_gaussian_3d( const auto ky2 = gaussian_kernel(sigma_y, 2, window_ratio); const auto kx2 = gaussian_kernel(sigma_x, 2, window_ratio); - std::vector work1(static_cast(n)); - std::vector work2(static_cast(n)); - - gaussian_separable_3d(in, work1.data(), work2.data(), nz, ny, nx, kz2, ky0, kx0); - for (std::ptrdiff_t i = 0; i < n; ++i) out[i] = work1[i]; + std::unique_ptr scratch; + { + BIOIMAGE_PROFILE_SCOPE(profile, "scratch_alloc"); + scratch = detail::allocate_scratch(n, 2); + } + float *work1 = detail::scratch_slot(scratch, n, 0); + float *work2 = detail::scratch_slot(scratch, n, 1); + + detail::gaussian_first_axis_3d_profiled(in, out, nz, ny, nx, kz0, profile); + detail::gaussian_remaining_axes_3d_profiled( + out, work2, work1, nz, ny, nx, ky2, kx0, profile + ); + { + BIOIMAGE_PROFILE_SCOPE(profile, "combine"); + for (std::ptrdiff_t i = 0; i < n; ++i) work1[i] = work2[i]; + } - gaussian_separable_3d(in, work1.data(), work2.data(), nz, ny, nx, kz0, ky2, kx0); - for (std::ptrdiff_t i = 0; i < n; ++i) out[i] += work1[i]; + detail::gaussian_remaining_axes_3d_profiled( + out, out, work2, nz, ny, nx, ky0, kx2, profile + ); + { + BIOIMAGE_PROFILE_SCOPE(profile, "combine"); + for (std::ptrdiff_t i = 0; i < n; ++i) out[i] += work1[i]; + } - gaussian_separable_3d(in, work1.data(), work2.data(), nz, ny, nx, kz0, ky0, kx2); - for (std::ptrdiff_t i = 0; i < n; ++i) out[i] += work1[i]; + detail::gaussian_first_axis_3d_profiled(in, work2, nz, ny, nx, kz2, profile); + detail::gaussian_remaining_axes_3d_profiled( + work2, work2, work1, nz, ny, nx, ky0, kx0, profile + ); + { + BIOIMAGE_PROFILE_SCOPE(profile, "combine"); + for (std::ptrdiff_t i = 0; i < n; ++i) out[i] += work2[i]; + } + BIOIMAGE_PROFILE_REPORT(profile); } // --------------------------------------------------------------------------- @@ -271,6 +481,7 @@ inline void hessian_of_gaussian_eigenvalues_2d( double sigma_x, double window_ratio ) { + BIOIMAGE_PROFILE_INIT(profile); const std::ptrdiff_t n = ny * nx; const auto ky0 = gaussian_kernel(sigma_y, 0, window_ratio); const auto kx0 = gaussian_kernel(sigma_x, 0, window_ratio); @@ -279,29 +490,34 @@ inline void hessian_of_gaussian_eigenvalues_2d( const auto ky2 = gaussian_kernel(sigma_y, 2, window_ratio); const auto kx2 = gaussian_kernel(sigma_x, 2, window_ratio); - std::vector work(static_cast(n)); - std::vector hyy(static_cast(n)); - std::vector hyx(static_cast(n)); - std::vector hxx(static_cast(n)); - - // d²/dy² - gaussian_separable_2d(in, hyy.data(), work.data(), ny, nx, ky2, kx0); - // d²/(dy dx) - gaussian_separable_2d(in, hyx.data(), work.data(), ny, nx, ky1, kx1); - // d²/dx² - gaussian_separable_2d(in, hxx.data(), work.data(), ny, nx, ky0, kx2); - - // ev2 sorted descending into interleaved output. - for (std::ptrdiff_t i = 0; i < n; ++i) { - const float a = hyy[i]; - const float b = hyx[i]; - const float c = hxx[i]; - const float half_tr = 0.5f * (a + c); - const float half_diff = 0.5f * (a - c); - const float disc = std::sqrt(half_diff * half_diff + b * b); - out[2 * i + 0] = half_tr + disc; - out[2 * i + 1] = half_tr - disc; + std::unique_ptr scratch; + { + BIOIMAGE_PROFILE_SCOPE(profile, "scratch_alloc"); + scratch = detail::allocate_scratch(n, 4); } + float *work = detail::scratch_slot(scratch, n, 0); + float *hyy = detail::scratch_slot(scratch, n, 1); + float *hyx = detail::scratch_slot(scratch, n, 2); + float *hxx = detail::scratch_slot(scratch, n, 3); + + detail::gaussian_separable_2d_profiled(in, hyy, work, ny, nx, ky2, kx0, profile); + detail::gaussian_separable_2d_profiled(in, hyx, work, ny, nx, ky1, kx1, profile); + detail::gaussian_separable_2d_profiled(in, hxx, work, ny, nx, ky0, kx2, profile); + + { + BIOIMAGE_PROFILE_SCOPE(profile, "eigenvalues"); + for (std::ptrdiff_t i = 0; i < n; ++i) { + const float a = hyy[i]; + const float b = hyx[i]; + const float c = hxx[i]; + const float half_tr = 0.5f * (a + c); + const float half_diff = 0.5f * (a - c); + const float disc = std::sqrt(half_diff * half_diff + b * b); + out[2 * i + 0] = half_tr + disc; + out[2 * i + 1] = half_tr - disc; + } + } + BIOIMAGE_PROFILE_REPORT(profile); } inline void hessian_of_gaussian_eigenvalues_3d( @@ -315,6 +531,7 @@ inline void hessian_of_gaussian_eigenvalues_3d( double sigma_x, double window_ratio ) { + BIOIMAGE_PROFILE_INIT(profile); const std::ptrdiff_t n = nz * ny * nx; const auto kz0 = gaussian_kernel(sigma_z, 0, window_ratio); const auto ky0 = gaussian_kernel(sigma_y, 0, window_ratio); @@ -326,34 +543,50 @@ inline void hessian_of_gaussian_eigenvalues_3d( const auto ky2 = gaussian_kernel(sigma_y, 2, window_ratio); const auto kx2 = gaussian_kernel(sigma_x, 2, window_ratio); - std::vector work(static_cast(n)); - std::vector hzz(static_cast(n)); - std::vector hzy(static_cast(n)); - std::vector hzx(static_cast(n)); - std::vector hyy(static_cast(n)); - std::vector hyx(static_cast(n)); - std::vector hxx(static_cast(n)); - - gaussian_separable_3d(in, hzz.data(), work.data(), nz, ny, nx, kz2, ky0, kx0); - gaussian_separable_3d(in, hzy.data(), work.data(), nz, ny, nx, kz1, ky1, kx0); - gaussian_separable_3d(in, hzx.data(), work.data(), nz, ny, nx, kz1, ky0, kx1); - gaussian_separable_3d(in, hyy.data(), work.data(), nz, ny, nx, kz0, ky2, kx0); - gaussian_separable_3d(in, hyx.data(), work.data(), nz, ny, nx, kz0, ky1, kx1); - gaussian_separable_3d(in, hxx.data(), work.data(), nz, ny, nx, kz0, ky0, kx2); - - // ev3 sorted descending into interleaved output. - for (std::ptrdiff_t i = 0; i < n; ++i) { - float e0; - float e1; - float e2; - detail::ev3_one_descending( - hzz[i], hzy[i], hzx[i], hyy[i], hyx[i], hxx[i], - e0, e1, e2 + std::unique_ptr scratch; + { + BIOIMAGE_PROFILE_SCOPE(profile, "scratch_alloc"); + scratch = detail::allocate_scratch(n, 7); + } + float *work = detail::scratch_slot(scratch, n, 0); + float *hzz = detail::scratch_slot(scratch, n, 1); + float *hzy = detail::scratch_slot(scratch, n, 2); + float *hzx = detail::scratch_slot(scratch, n, 3); + float *hyy = detail::scratch_slot(scratch, n, 4); + float *hyx = detail::scratch_slot(scratch, n, 5); + float *hxx = detail::scratch_slot(scratch, n, 6); + + detail::gaussian_first_axis_3d_profiled(in, hxx, nz, ny, nx, kz0, profile); + detail::gaussian_remaining_axes_3d_profiled( + hxx, hyy, work, nz, ny, nx, ky2, kx0, profile + ); + detail::gaussian_remaining_axes_3d_profiled( + hxx, hyx, work, nz, ny, nx, ky1, kx1, profile + ); + detail::gaussian_remaining_axes_3d_profiled( + hxx, hxx, work, nz, ny, nx, ky0, kx2, profile + ); + + detail::gaussian_first_axis_3d_profiled(in, hzx, nz, ny, nx, kz1, profile); + detail::gaussian_remaining_axes_3d_profiled( + hzx, hzy, work, nz, ny, nx, ky1, kx0, profile + ); + detail::gaussian_remaining_axes_3d_profiled( + hzx, hzx, work, nz, ny, nx, ky0, kx1, profile + ); + + detail::gaussian_first_axis_3d_profiled(in, hzz, nz, ny, nx, kz2, profile); + detail::gaussian_remaining_axes_3d_profiled( + hzz, hzz, work, nz, ny, nx, ky0, kx0, profile + ); + + { + BIOIMAGE_PROFILE_SCOPE(profile, "eigenvalues"); + ev3_symmetric_descending_interleaved( + hzz, hzy, hzx, hyy, hyx, hxx, out, n ); - out[3 * i + 0] = e0; - out[3 * i + 1] = e1; - out[3 * i + 2] = e2; } + BIOIMAGE_PROFILE_REPORT(profile); } // --------------------------------------------------------------------------- @@ -374,6 +607,7 @@ inline void structure_tensor_eigenvalues_2d( double sigma_outer_x, double window_ratio ) { + BIOIMAGE_PROFILE_INIT(profile); const std::ptrdiff_t n = ny * nx; const auto kiy0 = gaussian_kernel(sigma_inner_y, 0, window_ratio); @@ -383,38 +617,51 @@ inline void structure_tensor_eigenvalues_2d( const auto koy0 = gaussian_kernel(sigma_outer_y, 0, window_ratio); const auto kox0 = gaussian_kernel(sigma_outer_x, 0, window_ratio); - std::vector work(static_cast(n)); - std::vector gy(static_cast(n)); - std::vector gx(static_cast(n)); - std::vector tmp(static_cast(n)); - std::vector syy(static_cast(n)); - std::vector syx(static_cast(n)); - std::vector sxx(static_cast(n)); - - // First-order partials at sigma_inner. - gaussian_separable_2d(in, gy.data(), work.data(), ny, nx, kiy1, kix0); - gaussian_separable_2d(in, gx.data(), work.data(), ny, nx, kiy0, kix1); - - // Outer products, smoothed with sigma_outer. - for (std::ptrdiff_t i = 0; i < n; ++i) tmp[i] = gy[i] * gy[i]; - gaussian_separable_2d(tmp.data(), syy.data(), work.data(), ny, nx, koy0, kox0); - - for (std::ptrdiff_t i = 0; i < n; ++i) tmp[i] = gy[i] * gx[i]; - gaussian_separable_2d(tmp.data(), syx.data(), work.data(), ny, nx, koy0, kox0); - - for (std::ptrdiff_t i = 0; i < n; ++i) tmp[i] = gx[i] * gx[i]; - gaussian_separable_2d(tmp.data(), sxx.data(), work.data(), ny, nx, koy0, kox0); - - for (std::ptrdiff_t i = 0; i < n; ++i) { - const float a = syy[i]; - const float b = syx[i]; - const float c = sxx[i]; - const float half_tr = 0.5f * (a + c); - const float half_diff = 0.5f * (a - c); - const float disc = std::sqrt(half_diff * half_diff + b * b); - out[2 * i + 0] = half_tr + disc; - out[2 * i + 1] = half_tr - disc; + std::unique_ptr scratch; + { + BIOIMAGE_PROFILE_SCOPE(profile, "scratch_alloc"); + scratch = detail::allocate_scratch(n, 6); } + float *work = detail::scratch_slot(scratch, n, 0); + float *gy = detail::scratch_slot(scratch, n, 1); + float *gx = detail::scratch_slot(scratch, n, 2); + float *syy_y = detail::scratch_slot(scratch, n, 3); + float *syx_y = detail::scratch_slot(scratch, n, 4); + float *sxx_y = detail::scratch_slot(scratch, n, 5); + + detail::gaussian_separable_2d_profiled(in, gy, work, ny, nx, kiy1, kix0, profile); + detail::gaussian_separable_2d_profiled(in, gx, work, ny, nx, kiy0, kix1, profile); + + { + BIOIMAGE_PROFILE_SCOPE(profile, "outer_products"); + const std::array gradients{gy, gx}; + const std::array components{syy_y, syx_y, sxx_y}; + convolve_axis_strided_outer_products<2>( + gradients, components, 1, ny, nx, koy0 + ); + } + + { + BIOIMAGE_PROFILE_SCOPE(profile, "axis_x"); + convolve_axis_x(syy_y, gy, ny, nx, kox0); + convolve_axis_x(syx_y, gx, ny, nx, kox0); + convolve_axis_x(sxx_y, work, ny, nx, kox0); + } + + { + BIOIMAGE_PROFILE_SCOPE(profile, "eigenvalues"); + for (std::ptrdiff_t i = 0; i < n; ++i) { + const float a = gy[i]; + const float b = gx[i]; + const float c = work[i]; + const float half_tr = 0.5f * (a + c); + const float half_diff = 0.5f * (a - c); + const float disc = std::sqrt(half_diff * half_diff + b * b); + out[2 * i + 0] = half_tr + disc; + out[2 * i + 1] = half_tr - disc; + } + } + BIOIMAGE_PROFILE_REPORT(profile); } inline void structure_tensor_eigenvalues_3d( @@ -431,6 +678,7 @@ inline void structure_tensor_eigenvalues_3d( double sigma_outer_x, double window_ratio ) { + BIOIMAGE_PROFILE_INIT(profile); const std::ptrdiff_t n = nz * ny * nx; const auto kiz0 = gaussian_kernel(sigma_inner_z, 0, window_ratio); @@ -443,52 +691,69 @@ inline void structure_tensor_eigenvalues_3d( const auto koy0 = gaussian_kernel(sigma_outer_y, 0, window_ratio); const auto kox0 = gaussian_kernel(sigma_outer_x, 0, window_ratio); - std::vector work(static_cast(n)); - std::vector gz(static_cast(n)); - std::vector gy(static_cast(n)); - std::vector gx(static_cast(n)); - std::vector tmp(static_cast(n)); - std::vector szz(static_cast(n)); - std::vector szy(static_cast(n)); - std::vector szx(static_cast(n)); - std::vector syy(static_cast(n)); - std::vector syx(static_cast(n)); - std::vector sxx(static_cast(n)); - - gaussian_separable_3d(in, gz.data(), work.data(), nz, ny, nx, kiz1, kiy0, kix0); - gaussian_separable_3d(in, gy.data(), work.data(), nz, ny, nx, kiz0, kiy1, kix0); - gaussian_separable_3d(in, gx.data(), work.data(), nz, ny, nx, kiz0, kiy0, kix1); - - for (std::ptrdiff_t i = 0; i < n; ++i) tmp[i] = gz[i] * gz[i]; - gaussian_separable_3d(tmp.data(), szz.data(), work.data(), nz, ny, nx, koz0, koy0, kox0); - - for (std::ptrdiff_t i = 0; i < n; ++i) tmp[i] = gz[i] * gy[i]; - gaussian_separable_3d(tmp.data(), szy.data(), work.data(), nz, ny, nx, koz0, koy0, kox0); - - for (std::ptrdiff_t i = 0; i < n; ++i) tmp[i] = gz[i] * gx[i]; - gaussian_separable_3d(tmp.data(), szx.data(), work.data(), nz, ny, nx, koz0, koy0, kox0); - - for (std::ptrdiff_t i = 0; i < n; ++i) tmp[i] = gy[i] * gy[i]; - gaussian_separable_3d(tmp.data(), syy.data(), work.data(), nz, ny, nx, koz0, koy0, kox0); - - for (std::ptrdiff_t i = 0; i < n; ++i) tmp[i] = gy[i] * gx[i]; - gaussian_separable_3d(tmp.data(), syx.data(), work.data(), nz, ny, nx, koz0, koy0, kox0); - - for (std::ptrdiff_t i = 0; i < n; ++i) tmp[i] = gx[i] * gx[i]; - gaussian_separable_3d(tmp.data(), sxx.data(), work.data(), nz, ny, nx, koz0, koy0, kox0); - - for (std::ptrdiff_t i = 0; i < n; ++i) { - float e0; - float e1; - float e2; - detail::ev3_one_descending( - szz[i], szy[i], szx[i], syy[i], syx[i], sxx[i], - e0, e1, e2 + std::unique_ptr scratch; + { + BIOIMAGE_PROFILE_SCOPE(profile, "scratch_alloc"); + scratch = detail::allocate_scratch(n, 10); + } + float *work = detail::scratch_slot(scratch, n, 0); + float *gz = detail::scratch_slot(scratch, n, 1); + float *gy = detail::scratch_slot(scratch, n, 2); + float *gx = detail::scratch_slot(scratch, n, 3); + float *szz = detail::scratch_slot(scratch, n, 4); + float *szy = detail::scratch_slot(scratch, n, 5); + float *szx = detail::scratch_slot(scratch, n, 6); + float *syy = detail::scratch_slot(scratch, n, 7); + float *syx = detail::scratch_slot(scratch, n, 8); + float *sxx = detail::scratch_slot(scratch, n, 9); + + detail::gaussian_first_axis_3d_profiled(in, gx, nz, ny, nx, kiz0, profile); + detail::gaussian_remaining_axes_3d_profiled( + gx, gy, work, nz, ny, nx, kiy1, kix0, profile + ); + detail::gaussian_remaining_axes_3d_profiled( + gx, gx, work, nz, ny, nx, kiy0, kix1, profile + ); + detail::gaussian_first_axis_3d_profiled(in, gz, nz, ny, nx, kiz1, profile); + detail::gaussian_remaining_axes_3d_profiled( + gz, gz, work, nz, ny, nx, kiy0, kix0, profile + ); + + { + BIOIMAGE_PROFILE_SCOPE(profile, "outer_products"); + const std::array gradients{gz, gy, gx}; + const std::array components{szz, szy, szx, syy, syx, sxx}; + convolve_axis_strided_outer_products<3>( + gradients, components, 1, nz, ny * nx, koz0 + ); + } + + detail::gaussian_remaining_axes_3d_profiled( + szz, szz, work, nz, ny, nx, koy0, kox0, profile + ); + detail::gaussian_remaining_axes_3d_profiled( + szy, szy, work, nz, ny, nx, koy0, kox0, profile + ); + detail::gaussian_remaining_axes_3d_profiled( + szx, szx, work, nz, ny, nx, koy0, kox0, profile + ); + detail::gaussian_remaining_axes_3d_profiled( + syy, syy, work, nz, ny, nx, koy0, kox0, profile + ); + detail::gaussian_remaining_axes_3d_profiled( + syx, syx, work, nz, ny, nx, koy0, kox0, profile + ); + detail::gaussian_remaining_axes_3d_profiled( + sxx, sxx, work, nz, ny, nx, koy0, kox0, profile + ); + + { + BIOIMAGE_PROFILE_SCOPE(profile, "eigenvalues"); + ev3_symmetric_descending_interleaved( + szz, szy, szx, syy, syx, sxx, out, n ); - out[3 * i + 0] = e0; - out[3 * i + 1] = e1; - out[3 * i + 2] = e2; } + BIOIMAGE_PROFILE_REPORT(profile); } } // namespace bioimage_cpp::filters diff --git a/include/bioimage_cpp/graph/agglomeration/agglomerative_clustering.hxx b/include/bioimage_cpp/graph/agglomeration/agglomerative_clustering.hxx index d5b9b23..c3e5106 100644 --- a/include/bioimage_cpp/graph/agglomeration/agglomerative_clustering.hxx +++ b/include/bioimage_cpp/graph/agglomeration/agglomerative_clustering.hxx @@ -1,8 +1,9 @@ #pragma once +#include "bioimage_cpp/detail/profile.hxx" #include "bioimage_cpp/graph/agglomeration/cluster_policy_base.hxx" #include "bioimage_cpp/graph/agglomeration/detail.hxx" -#include "bioimage_cpp/graph/multicut/detail.hxx" +#include "bioimage_cpp/graph/connected_components.hxx" #include "bioimage_cpp/graph/undirected_graph.hxx" #include "bioimage_cpp/util/union_find.hxx" @@ -14,26 +15,34 @@ namespace bioimage_cpp::graph::agglomeration { // Hierarchical agglomerative clustering driven by a `ClusterPolicyBase`. // -// The driver owns the dynamic graph, union-find and heap. The policy carries -// its own per-edge / per-node state (sizes, histograms, features, cannot-link -// constraints, ...) and decides per iteration whether to merge, skip, or -// stop. Returns dense node labels in `[0, k)` via the union-find roots. +// The policy owns objective-specific state and controls merge decisions. inline std::vector agglomerative_clustering( const UndirectedGraph &graph, ClusterPolicyBase &policy ) { - multicut::detail::DynamicGraph dynamic_graph(graph); + BIOIMAGE_PROFILE_INIT(profile); + ClusterPolicyBase::Topology topology; + { + BIOIMAGE_PROFILE_SCOPE(profile, "topology_reset"); + topology.reset(graph); + } util::UnionFind sets(static_cast(graph.number_of_nodes())); ClusterPolicyBase::EdgeHeap heap; heap.reset_capacity(static_cast(graph.number_of_edges())); - policy.initialize(graph, dynamic_graph, heap); - - while (!heap.empty() && dynamic_graph.alive_count > 1) { - if (policy.is_done(dynamic_graph)) { + { + BIOIMAGE_PROFILE_SCOPE(profile, "initialize"); + policy.initialize(graph, heap); + } + while (!heap.empty() && topology.number_of_nodes() > 1) { + if (policy.is_done(topology)) { break; } const auto top = heap.top(); - const auto action = policy.next_action(top.key, top.priority, dynamic_graph); + const auto action = policy.next_action( + top.key, + top.priority, + topology + ); if (action == ClusterPolicyBase::Action::kStop) { break; } @@ -41,12 +50,25 @@ inline std::vector agglomerative_clustering( heap.pop(); continue; } - const auto &edge = dynamic_graph.edges[top.key]; - detail::agglo_merge_dynamic_nodes( - dynamic_graph, sets, heap, edge.u, edge.v, policy + detail::contract_edge( + topology, + sets, + heap, + top.key, + policy, + profile + ); + } + std::vector labels; + { + BIOIMAGE_PROFILE_SCOPE(profile, "labels_from_sets"); + labels = dense_labels_from_union_find( + sets, + graph.number_of_nodes() ); } - return multicut::detail::labels_from_sets(sets, graph); + BIOIMAGE_PROFILE_REPORT(profile); + return labels; } } // namespace bioimage_cpp::graph::agglomeration diff --git a/include/bioimage_cpp/graph/agglomeration/cluster_policy_base.hxx b/include/bioimage_cpp/graph/agglomeration/cluster_policy_base.hxx index 57b78d1..c66f349 100644 --- a/include/bioimage_cpp/graph/agglomeration/cluster_policy_base.hxx +++ b/include/bioimage_cpp/graph/agglomeration/cluster_policy_base.hxx @@ -1,7 +1,7 @@ #pragma once #include "bioimage_cpp/detail/indexed_heap.hxx" -#include "bioimage_cpp/graph/multicut/detail.hxx" +#include "bioimage_cpp/graph/detail/contraction_topology.hxx" #include "bioimage_cpp/graph/undirected_graph.hxx" #include @@ -13,11 +13,8 @@ namespace bioimage_cpp::graph::agglomeration { // // A cluster policy carries all per-edge / per-node auxiliary state required // to compute heap priorities (edge sizes, node sizes, histograms, features, -// signed weights, cannot-link masks, ...). The driver -// (`agglomerative_clustering`) owns the `DynamicGraph`, `UnionFind` and -// `EdgeHeap` and delegates merge-rule decisions and weight updates to the -// policy. Implementations are typically constructed once per problem and -// passed by reference to the driver. +// signed weights, and cannot-link masks). The driver owns the contraction +// topology, union-find, and heap. // // The agglo heap is a min-heap (smallest priority pops first), matching // nifty's convention: edge indicators in the edge-weighted / node+edge- @@ -27,7 +24,7 @@ namespace bioimage_cpp::graph::agglomeration { // top of the same min-heap container. class ClusterPolicyBase { public: - using DynamicGraph = multicut::detail::DynamicGraph; + using Topology = graph::detail::ContractionTopology; using EdgeHeap = bioimage_cpp::detail::DenseIndexedHeap>; @@ -39,19 +36,14 @@ public: virtual ~ClusterPolicyBase() = default; - // Seed the heap with initial priorities and any per-edge / per-node - // policy state derived from `graph` / `dynamic_graph`. Called once at the - // start of `agglomerative_clustering` after `dynamic_graph` has been - // initialised. + // Seed the heap and initialize policy state. virtual void initialize( const UndirectedGraph &graph, - DynamicGraph &dynamic_graph, EdgeHeap &heap ) = 0; - // Iteration-level stop check, independent of the heap top. Typically - // checks `alive_count <= num_clusters_stop` or similar. - virtual bool is_done(const DynamicGraph &dynamic_graph) const = 0; + // Check termination independently of the heap top. + virtual bool is_done(const Topology &topology) const = 0; // Heap-top-dependent action. Called after `is_done` returns false and // before any contraction is attempted. `edge_id` is the heap top key @@ -59,7 +51,7 @@ public: virtual Action next_action( std::size_t edge_id, double priority, - const DynamicGraph &dynamic_graph + const Topology &topology ) = 0; // Called once per contraction, before the per-fold loop, to let the @@ -104,11 +96,11 @@ public: // changed. Default: no-op. virtual void contract_edge_done( std::size_t stable, - DynamicGraph &dynamic_graph, + const Topology &topology, EdgeHeap &heap ) { (void)stable; - (void)dynamic_graph; + (void)topology; (void)heap; } }; diff --git a/include/bioimage_cpp/graph/agglomeration/detail.hxx b/include/bioimage_cpp/graph/agglomeration/detail.hxx index 92c97f3..186ee06 100644 --- a/include/bioimage_cpp/graph/agglomeration/detail.hxx +++ b/include/bioimage_cpp/graph/agglomeration/detail.hxx @@ -1,133 +1,121 @@ #pragma once +#include "bioimage_cpp/detail/profile.hxx" #include "bioimage_cpp/graph/agglomeration/cluster_policy_base.hxx" -#include "bioimage_cpp/graph/multicut/detail.hxx" #include "bioimage_cpp/util/union_find.hxx" #include -#include +#include namespace bioimage_cpp::graph::agglomeration::detail { -// Contract the edge between super-nodes `u` and `v`. Structurally a clone of -// `multicut::detail::merge_dynamic_nodes`, but delegates the per-fold weight -// update (and the rekey-priority recomputation) to a policy. -// -// On each fold of two edges that connect the same pair of super-nodes, -// `policy.merge_edges(existing_id, fold_id)` returns the new heap priority -// for the surviving edge; the surviving edge's cached `weight` and heap -// entry are updated to that value. On the no-fold rekey branch the priority -// is recomputed via `policy.rekeyed_priority(...)` from the current heap -// value — policies whose priority depends on node-level state (e.g. the -// harmonic size factor) recompute; policies that don't keep the priority. -// -// `policy.merge_nodes(stable, removed)` is invoked once, before the per-fold -// loop, so policies that update node-level state (sizes, features, mutex -// storage) operate on the *pre-fold* roots — matching the order -// `existing_id` itself was created. template -inline std::size_t agglo_merge_dynamic_nodes( - multicut::detail::DynamicGraph &dynamic_graph, - util::UnionFind &sets, - ClusterPolicyBase::EdgeHeap &heap, - std::size_t u, - std::size_t v, - Policy &policy -) { - u = static_cast(sets.find(u)); - v = static_cast(sets.find(v)); - if (u == v) { - return u; +class ContractionObserver { +public: + ContractionObserver( + ClusterPolicyBase::Topology &topology, + util::UnionFind &sets, + ClusterPolicyBase::EdgeHeap &heap, + Policy &policy + ) + : topology_(topology), + sets_(sets), + heap_(heap), + policy_(policy) { } - auto stable = u; - auto removed = v; - if (dynamic_graph.adjacency[stable].size() < dynamic_graph.adjacency[removed].size()) { - std::swap(stable, removed); + void begin_contraction( + const std::uint64_t kept_node, + const std::uint64_t removed_node, + const std::uint64_t removed_edge + ) { + (void)removed_edge; + sets_.merge_to(kept_node, removed_node); + policy_.merge_nodes( + static_cast(kept_node), + static_cast(removed_node) + ); } - sets.merge_to(stable, removed); - policy.merge_nodes(stable, removed); - // Stamp stable's neighbors so each removed-neighbor lookup is O(1). - for (const auto &entry : dynamic_graph.adjacency[stable]) { - dynamic_graph.scratch_edge_id[entry.neighbor] = entry.edge_id; + void edge_deactivated(const std::uint64_t edge) { + heap_.erase(static_cast(edge)); } - // Erase the contracted edge from heap + stable adjacency. - const auto contracted_edge_id = dynamic_graph.scratch_edge_id[removed]; - heap.erase(contracted_edge_id); - dynamic_graph.scratch_edge_id[removed] = multicut::detail::no_edge; - multicut::detail::internal::erase_by_neighbor( - dynamic_graph.adjacency[stable], removed - ); - - // Snapshot removed's neighbors before mutating its adjacency. - const auto removed_neighbors = dynamic_graph.adjacency[removed]; - - for (const auto &entry : removed_neighbors) { - const auto neighbor = entry.neighbor; - const auto removed_edge_id = entry.edge_id; - if (neighbor == stable) { - continue; + void edge_rekeyed(const graph::detail::EdgeRekey &rekey) { + const auto edge = static_cast(rekey.edge); + if (!heap_.contains(edge)) { + return; } + const auto [u, v] = topology_.uv(rekey.edge); + const auto current_priority = heap_.priority_of(edge); + const auto new_priority = policy_.rekeyed_priority( + edge, + static_cast(u), + static_cast(v), + current_priority + ); + if (new_priority != current_priority) { + heap_.change(edge, new_priority); + } + } - const auto existing_id = dynamic_graph.scratch_edge_id[neighbor]; - if (existing_id == multicut::detail::no_edge) { - // Rekey: removed-side edge inherits its endpoint rename. The - // policy decides whether the priority changes (default: keep). - dynamic_graph.adjacency[stable].push_back({neighbor, removed_edge_id}); - dynamic_graph.scratch_edge_id[neighbor] = removed_edge_id; - multicut::detail::internal::rename_neighbor( - dynamic_graph.adjacency[neighbor], removed, stable - ); - auto &edge = dynamic_graph.edges[removed_edge_id]; - if (edge.u == removed) { - edge.u = stable; - } else { - edge.v = stable; - } - const auto current_priority = edge.weight; - const auto new_priority = policy.rekeyed_priority( - removed_edge_id, stable, neighbor, current_priority - ); - if (new_priority != current_priority) { - edge.weight = new_priority; - // The edge may have been previously popped via kRejectEdge - // (GASP cannot-link). In that case it must stay out of the heap - // (re-adding it would let it be re-popped and merged), so only - // update the priority when the edge is still present. - if (heap.contains(removed_edge_id)) { - heap.change(removed_edge_id, new_priority); - } - } - } else { - // Fold: both stable and removed had an edge to `neighbor`. Let the - // policy merge the per-edge state into `existing_id` and tell us - // the new heap priority; then drop the removed-side edge. - const auto new_priority = policy.merge_edges( - existing_id, removed_edge_id, stable, neighbor - ); - dynamic_graph.edges[existing_id].weight = new_priority; - heap.erase(removed_edge_id); - multicut::detail::internal::erase_by_neighbor( - dynamic_graph.adjacency[neighbor], removed - ); - if (heap.contains(existing_id)) { - heap.change(existing_id, new_priority); - } + void edges_folded( + const std::uint64_t kept_edge_id, + const std::uint64_t removed_edge_id + ) { + const auto kept_edge = static_cast(kept_edge_id); + const auto [u, v] = topology_.uv(kept_edge_id); + const auto new_priority = policy_.merge_edges( + kept_edge, + static_cast(removed_edge_id), + static_cast(u), + static_cast(v) + ); + if (heap_.contains(kept_edge)) { + heap_.change(kept_edge, new_priority); } } - // Clear scratch via the updated stable adjacency. - for (const auto &entry : dynamic_graph.adjacency[stable]) { - dynamic_graph.scratch_edge_id[entry.neighbor] = multicut::detail::no_edge; + void end_contraction(const std::uint64_t kept_node) { + policy_.contract_edge_done( + static_cast(kept_node), + topology_, + heap_ + ); } - dynamic_graph.adjacency[removed].clear(); - dynamic_graph.alive[removed] = false; - --dynamic_graph.alive_count; - policy.contract_edge_done(stable, dynamic_graph, heap); - return stable; +private: + ClusterPolicyBase::Topology &topology_; + util::UnionFind &sets_; + ClusterPolicyBase::EdgeHeap &heap_; + Policy &policy_; +}; + +template +inline std::uint64_t contract_edge( + ClusterPolicyBase::Topology &topology, + util::UnionFind &sets, + ClusterPolicyBase::EdgeHeap &heap, + const std::size_t edge, + Policy &policy, + Profiler &profile +) { + const auto edge_id = static_cast(edge); + const auto [kept, removed] = + topology.preferred_contraction_nodes(edge_id); + ContractionObserver observer(topology, sets, heap, policy); + std::uint64_t kept_node; + { + BIOIMAGE_PROFILE_SCOPE(profile, "contract"); + kept_node = + topology.contract_edge_observed_known_nodes( + edge_id, + kept, + removed, + observer + ); + } + return kept_node; } } // namespace bioimage_cpp::graph::agglomeration::detail diff --git a/include/bioimage_cpp/graph/agglomeration/edge_weighted.hxx b/include/bioimage_cpp/graph/agglomeration/edge_weighted.hxx index 9d7df5f..e5994f9 100644 --- a/include/bioimage_cpp/graph/agglomeration/edge_weighted.hxx +++ b/include/bioimage_cpp/graph/agglomeration/edge_weighted.hxx @@ -1,7 +1,6 @@ #pragma once #include "bioimage_cpp/graph/agglomeration/cluster_policy_base.hxx" -#include "bioimage_cpp/graph/multicut/detail.hxx" #include "bioimage_cpp/graph/undirected_graph.hxx" #include @@ -46,7 +45,6 @@ public: void initialize( const UndirectedGraph &graph, - DynamicGraph &dynamic_graph, EdgeHeap &heap ) override { const auto n_edges = static_cast(graph.number_of_edges()); @@ -72,31 +70,24 @@ public: const auto u = static_cast(uv.first); const auto v = static_cast(uv.second); const auto edge_index = static_cast(edge_id); - auto &edge = dynamic_graph.edges[edge_index]; - edge.u = u; - edge.v = v; const auto priority = priority_of(edge_index, u, v); - edge.weight = priority; - edge.is_constraint = 0; - dynamic_graph.adjacency[u].push_back({v, edge_index}); - dynamic_graph.adjacency[v].push_back({u, edge_index}); entries.push_back({edge_index, priority}); } heap.build_heap(std::move(entries)); } - bool is_done(const DynamicGraph &dynamic_graph) const override { - return dynamic_graph.alive_count <= num_clusters_stop_; + bool is_done(const Topology &topology) const override { + return topology.number_of_nodes() <= num_clusters_stop_; } Action next_action( std::size_t edge_id, double priority, - const DynamicGraph &dynamic_graph + const Topology &topology ) override { (void)edge_id; (void)priority; - (void)dynamic_graph; + (void)topology; return Action::kMerge; } @@ -134,22 +125,16 @@ public: void contract_edge_done( std::size_t stable, - DynamicGraph &dynamic_graph, + const Topology &topology, EdgeHeap &heap ) override { - // node_size_[stable] just changed; every edge incident to `stable` — - // including those whose endpoint was not renamed in the per-fold loop - // — needs its priority recomputed. - for (const auto &entry : dynamic_graph.adjacency[stable]) { - const auto edge_id = entry.edge_id; - const auto neighbor = entry.neighbor; + for (const auto &entry : topology.node_adjacency(stable)) { + const auto edge_id = static_cast(entry.edge); + const auto neighbor = static_cast(entry.node); const auto new_priority = priority_of(edge_id, stable, neighbor); - auto &edge = dynamic_graph.edges[edge_id]; - if (edge.weight != new_priority) { - edge.weight = new_priority; - if (heap.contains(edge_id)) { - heap.change(edge_id, new_priority); - } + if (heap.contains(edge_id) + && heap.priority_of(edge_id) != new_priority) { + heap.change(edge_id, new_priority); } } } diff --git a/include/bioimage_cpp/graph/agglomeration/gasp.hxx b/include/bioimage_cpp/graph/agglomeration/gasp.hxx index 7274b1f..0864af1 100644 --- a/include/bioimage_cpp/graph/agglomeration/gasp.hxx +++ b/include/bioimage_cpp/graph/agglomeration/gasp.hxx @@ -2,7 +2,6 @@ #include "bioimage_cpp/detail/mutex_storage.hxx" #include "bioimage_cpp/graph/agglomeration/cluster_policy_base.hxx" -#include "bioimage_cpp/graph/multicut/detail.hxx" #include "bioimage_cpp/graph/undirected_graph.hxx" #include @@ -80,7 +79,6 @@ public: void initialize( const UndirectedGraph &graph, - DynamicGraph &dynamic_graph, EdgeHeap &heap ) override { const auto n_edges = static_cast(graph.number_of_edges()); @@ -100,36 +98,26 @@ public: std::vector entries; entries.reserve(n_edges); for (std::uint64_t edge_id = 0; edge_id < graph.number_of_edges(); ++edge_id) { - const auto uv = graph.uv(edge_id); - const auto u = static_cast(uv.first); - const auto v = static_cast(uv.second); const auto edge_index = static_cast(edge_id); const double priority = priority_of(edge_weight_[edge_index]); - auto &edge = dynamic_graph.edges[edge_index]; - edge.u = u; - edge.v = v; - edge.weight = priority; - edge.is_constraint = 0; - dynamic_graph.adjacency[u].push_back({v, edge_index}); - dynamic_graph.adjacency[v].push_back({u, edge_index}); entries.push_back({edge_index, priority}); } heap.build_heap(std::move(entries)); } - bool is_done(const DynamicGraph &dynamic_graph) const override { - return dynamic_graph.alive_count <= num_clusters_stop_; + bool is_done(const Topology &topology) const override { + return topology.number_of_nodes() <= num_clusters_stop_; } Action next_action( std::size_t edge_id, double priority, - const DynamicGraph &dynamic_graph + const Topology &topology ) override { (void)priority; - const auto &edge = dynamic_graph.edges[edge_id]; - const auto u = static_cast(edge.u); - const auto v = static_cast(edge.v); + const auto [u, v] = topology.uv( + static_cast(edge_id) + ); if (check_mutex(u, v, cannot_link_)) { return Action::kRejectEdge; } diff --git a/include/bioimage_cpp/graph/agglomeration/mala.hxx b/include/bioimage_cpp/graph/agglomeration/mala.hxx index 644bbeb..62a26ec 100644 --- a/include/bioimage_cpp/graph/agglomeration/mala.hxx +++ b/include/bioimage_cpp/graph/agglomeration/mala.hxx @@ -1,7 +1,6 @@ #pragma once #include "bioimage_cpp/graph/agglomeration/cluster_policy_base.hxx" -#include "bioimage_cpp/graph/multicut/detail.hxx" #include "bioimage_cpp/graph/undirected_graph.hxx" #include @@ -50,8 +49,8 @@ public: num_clusters_stop_(num_clusters_stop), num_edges_stop_(num_edges_stop), threshold_(threshold) { - if (num_bins_ == 0) { - throw std::invalid_argument("num_bins must be >= 1"); + if (num_bins_ < 2) { + throw std::invalid_argument("num_bins must be >= 2"); } if (!(bin_max_ > bin_min_)) { throw std::invalid_argument( @@ -63,7 +62,6 @@ public: void initialize( const UndirectedGraph &graph, - DynamicGraph &dynamic_graph, EdgeHeap &heap ) override { const auto n_edges = static_cast(graph.number_of_edges()); @@ -75,35 +73,26 @@ public: ); } histograms_.assign(n_edges, std::vector(num_bins_, 0.0)); - active_edges_ = n_edges; std::vector entries; entries.reserve(n_edges); for (std::uint64_t edge_id = 0; edge_id < graph.number_of_edges(); ++edge_id) { - const auto uv = graph.uv(edge_id); - const auto u = static_cast(uv.first); - const auto v = static_cast(uv.second); const auto edge_index = static_cast(edge_id); const double indicator = initial_indicators_[edge_index]; insert_into(histograms_[edge_index], indicator, 1.0); const double priority = indicator; - auto &edge = dynamic_graph.edges[edge_index]; - edge.u = u; - edge.v = v; - edge.weight = priority; - edge.is_constraint = 0; - dynamic_graph.adjacency[u].push_back({v, edge_index}); - dynamic_graph.adjacency[v].push_back({u, edge_index}); entries.push_back({edge_index, priority}); } heap.build_heap(std::move(entries)); } - bool is_done(const DynamicGraph &dynamic_graph) const override { - if (num_clusters_stop_ > 0 && dynamic_graph.alive_count <= num_clusters_stop_) { + bool is_done(const Topology &topology) const override { + if (num_clusters_stop_ > 0 + && topology.number_of_nodes() <= num_clusters_stop_) { return true; } - if (num_edges_stop_ > 0 && active_edges_ <= num_edges_stop_) { + if (num_edges_stop_ > 0 + && topology.number_of_edges() <= num_edges_stop_) { return true; } return false; @@ -112,10 +101,10 @@ public: Action next_action( std::size_t edge_id, double priority, - const DynamicGraph &dynamic_graph + const Topology &topology ) override { (void)edge_id; - (void)dynamic_graph; + (void)topology; if (priority >= threshold_) { return Action::kStop; } @@ -140,7 +129,6 @@ public: for (std::size_t bin = 0; bin < num_bins_; ++bin) { target[bin] += source[bin]; } - --active_edges_; return median_of(target); } @@ -235,7 +223,6 @@ private: std::size_t num_edges_stop_; double threshold_; std::vector> histograms_; - std::size_t active_edges_ = 0; }; } // namespace bioimage_cpp::graph::agglomeration diff --git a/include/bioimage_cpp/graph/agglomeration/node_and_edge_weighted.hxx b/include/bioimage_cpp/graph/agglomeration/node_and_edge_weighted.hxx index 8629c80..f00ddc5 100644 --- a/include/bioimage_cpp/graph/agglomeration/node_and_edge_weighted.hxx +++ b/include/bioimage_cpp/graph/agglomeration/node_and_edge_weighted.hxx @@ -1,7 +1,6 @@ #pragma once #include "bioimage_cpp/graph/agglomeration/cluster_policy_base.hxx" -#include "bioimage_cpp/graph/multicut/detail.hxx" #include "bioimage_cpp/graph/undirected_graph.hxx" #include @@ -72,7 +71,6 @@ public: void initialize( const UndirectedGraph &graph, - DynamicGraph &dynamic_graph, EdgeHeap &heap ) override { const auto n_edges = static_cast(graph.number_of_edges()); @@ -98,31 +96,24 @@ public: const auto u = static_cast(uv.first); const auto v = static_cast(uv.second); const auto edge_index = static_cast(edge_id); - auto &edge = dynamic_graph.edges[edge_index]; - edge.u = u; - edge.v = v; const auto priority = priority_of(edge_index, u, v); - edge.weight = priority; - edge.is_constraint = 0; - dynamic_graph.adjacency[u].push_back({v, edge_index}); - dynamic_graph.adjacency[v].push_back({u, edge_index}); entries.push_back({edge_index, priority}); } heap.build_heap(std::move(entries)); } - bool is_done(const DynamicGraph &dynamic_graph) const override { - return dynamic_graph.alive_count <= num_clusters_stop_; + bool is_done(const Topology &topology) const override { + return topology.number_of_nodes() <= num_clusters_stop_; } Action next_action( std::size_t edge_id, double priority, - const DynamicGraph &dynamic_graph + const Topology &topology ) override { (void)edge_id; (void)priority; - (void)dynamic_graph; + (void)topology; return Action::kMerge; } @@ -171,19 +162,16 @@ public: void contract_edge_done( std::size_t stable, - DynamicGraph &dynamic_graph, + const Topology &topology, EdgeHeap &heap ) override { - for (const auto &entry : dynamic_graph.adjacency[stable]) { - const auto edge_id = entry.edge_id; - const auto neighbor = entry.neighbor; + for (const auto &entry : topology.node_adjacency(stable)) { + const auto edge_id = static_cast(entry.edge); + const auto neighbor = static_cast(entry.node); const auto new_priority = priority_of(edge_id, stable, neighbor); - auto &edge = dynamic_graph.edges[edge_id]; - if (edge.weight != new_priority) { - edge.weight = new_priority; - if (heap.contains(edge_id)) { - heap.change(edge_id, new_priority); - } + if (heap.contains(edge_id) + && heap.priority_of(edge_id) != new_priority) { + heap.change(edge_id, new_priority); } } } diff --git a/include/bioimage_cpp/graph/contraction_graph.hxx b/include/bioimage_cpp/graph/contraction_graph.hxx new file mode 100644 index 0000000..d740b8d --- /dev/null +++ b/include/bioimage_cpp/graph/contraction_graph.hxx @@ -0,0 +1,666 @@ +#pragma once + +#include "bioimage_cpp/graph/contraction_options.hxx" +#include "bioimage_cpp/graph/detail/contraction_topology.hxx" +#include "bioimage_cpp/graph/undirected_graph.hxx" +#include "bioimage_cpp/util/union_find.hxx" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace bioimage_cpp::graph { + +enum class Reduction { + Sum, + Mean, + Minimum, + Maximum, +}; + +template +struct MaterializedValues { + std::string name; + std::vector values; + std::vector shape; +}; + +using MaterializedValueMap = std::variant< + MaterializedValues, + MaterializedValues +>; + +struct MaterializedContraction { + UndirectedGraph graph; + std::vector node_values; + std::vector edge_values; + std::vector node_mapping; + std::vector edge_mapping; +}; + +template +class ReducedValueMap { +public: + ReducedValueMap( + std::string name, + const std::span values, + std::vector component_shape, + const std::size_t item_count, + const Reduction reduction + ) + : name_(std::move(name)), + component_shape_(std::move(component_shape)), + component_size_(shape_size(component_shape_)), + values_(values.begin(), values.end()), + counts_(reduction == Reduction::Mean ? item_count : 0, 1), + mean_sums_(reduction == Reduction::Mean ? values.size() : 0), + reduction_(reduction) { + if (values_.size() != checked_product(item_count, component_size_)) { + throw std::invalid_argument( + "values size does not match the item count and component shape" + ); + } + if (reduction_ == Reduction::Mean) { + std::transform( + values_.begin(), + values_.end(), + mean_sums_.begin(), + [](const T value) { + return static_cast(value); + } + ); + } + } + + [[nodiscard]] const std::string &name() const { + return name_; + } + + [[nodiscard]] const std::vector &component_shape() const { + return component_shape_; + } + + [[nodiscard]] std::size_t component_size() const { + return component_size_; + } + + [[nodiscard]] Reduction reduction() const { + return reduction_; + } + + void reduce(const std::uint64_t kept, const std::uint64_t removed) { + const auto kept_index = static_cast(kept); + const auto removed_index = static_cast(removed); + if (kept_index == removed_index) { + return; + } + const auto kept_offset = checked_product(kept_index, component_size_); + const auto removed_offset = checked_product(removed_index, component_size_); + + if (reduction_ == Reduction::Mean) { + const auto kept_count = counts_[kept_index]; + const auto removed_count = counts_[removed_index]; + const auto total = kept_count + removed_count; + const auto total_weight = static_cast(total); + for (std::size_t component = 0; component < component_size_; ++component) { + auto &sum = mean_sums_[kept_offset + component]; + sum += mean_sums_[removed_offset + component]; + values_[kept_offset + component] = + static_cast(sum / total_weight); + } + counts_[kept_index] = total; + return; + } + + for (std::size_t component = 0; component < component_size_; ++component) { + auto &a = values_[kept_offset + component]; + const auto b = values_[removed_offset + component]; + if (std::isnan(a) || std::isnan(b)) { + a = std::numeric_limits::quiet_NaN(); + } else if (reduction_ == Reduction::Sum) { + a += b; + } else if (reduction_ == Reduction::Minimum) { + a = std::min(a, b); + } else { + a = std::max(a, b); + } + } + } + + [[nodiscard]] MaterializedValues gather( + const std::span item_ids + ) const { + std::vector shape; + shape.reserve(component_shape_.size() + 1); + shape.push_back(item_ids.size()); + shape.insert(shape.end(), component_shape_.begin(), component_shape_.end()); + + std::vector output; + output.reserve(checked_product(item_ids.size(), component_size_)); + for (const auto item : item_ids) { + const auto offset = + checked_product(static_cast(item), component_size_); + output.insert( + output.end(), + values_.begin() + static_cast(offset), + values_.begin() + static_cast(offset + component_size_) + ); + } + return {name_, std::move(output), std::move(shape)}; + } + + [[nodiscard]] MaterializedValues value(const std::uint64_t item) const { + std::vector shape = component_shape_; + const auto offset = + checked_product(static_cast(item), component_size_); + std::vector output( + values_.begin() + static_cast(offset), + values_.begin() + static_cast(offset + component_size_) + ); + return {name_, std::move(output), std::move(shape)}; + } + +private: + static std::size_t checked_product( + const std::size_t first, + const std::size_t second + ) { + if (second != 0 && + first > std::numeric_limits::max() / second) { + throw std::overflow_error("value-map shape overflows size_t"); + } + return first * second; + } + + static std::size_t shape_size(const std::vector &shape) { + std::size_t size = 1; + for (const auto extent : shape) { + size = checked_product(size, extent); + } + return size; + } + + std::string name_; + std::vector component_shape_; + std::size_t component_size_; + std::vector values_; + std::vector counts_; + std::vector mean_sums_; + Reduction reduction_; +}; + +using ReducedValueMapVariant = std::variant< + ReducedValueMap, + ReducedValueMap +>; + +// Stateful graph contraction with named floating-point value maps. The input +// graph is simple. Parallel edges created by mutations can merge immediately or +// remain separate until materialization. +class ContractionGraph { +public: + using NodeId = std::uint64_t; + using EdgeId = std::uint64_t; + using Edge = UndirectedGraph::Edge; + + explicit ContractionGraph( + const UndirectedGraph &graph, + const ParallelEdgePolicy parallel_edge_policy = ParallelEdgePolicy::Merge + ) + : topology_(graph, parallel_edge_policy), + node_sets_(static_cast(graph.number_of_nodes())), + edge_parent_(static_cast(graph.number_of_edges())), + edge_state_( + static_cast(graph.number_of_edges()), + EdgeState::Active + ), + parallel_edge_policy_(parallel_edge_policy) { + initialize_edge_parents(); + } + + [[nodiscard]] std::size_t number_of_nodes() const { + return topology_.number_of_nodes(); + } + + [[nodiscard]] std::size_t number_of_edges() const { + return topology_.number_of_edges(); + } + + [[nodiscard]] std::vector nodes() const { + return topology_.active_nodes(); + } + + [[nodiscard]] std::vector edges() const { + return topology_.active_edges(); + } + + [[nodiscard]] Edge uv(const EdgeId edge) const { + return topology_.uv(edge); + } + + [[nodiscard]] std::int64_t find_edge(const NodeId u, const NodeId v) const { + return topology_.find_edge(u, v); + } + + [[nodiscard]] std::vector find_edges( + const NodeId u, + const NodeId v + ) const { + return topology_.find_edges(u, v); + } + + [[nodiscard]] ParallelEdgePolicy parallel_edge_policy() const { + return parallel_edge_policy_; + } + + [[nodiscard]] const std::vector &node_adjacency( + const NodeId node + ) const { + return topology_.node_adjacency(node); + } + + [[nodiscard]] std::size_t degree(const NodeId node) const { + return topology_.degree(node); + } + + [[nodiscard]] bool can_suppress_node(const NodeId node) const { + return topology_.can_suppress_node(node); + } + + [[nodiscard]] bool node_active(const NodeId node) const { + return topology_.node_active(node); + } + + [[nodiscard]] bool edge_active(const EdgeId edge) const { + return topology_.edge_active(edge); + } + + NodeId representative(const NodeId original_node) { + if (original_node >= node_sets_.size()) { + throw std::out_of_range("original_node is out of range"); + } + return node_sets_.find(original_node); + } + + template + void add_node_values( + std::string name, + const std::span values, + std::vector component_shape, + const Reduction reduction + ) { + require_registration_allowed(name, node_values_, "node"); + node_values_.emplace_back( + ReducedValueMap( + std::move(name), + values, + std::move(component_shape), + topology_.node_capacity(), + reduction + ) + ); + } + + template + void add_edge_values( + std::string name, + const std::span values, + std::vector component_shape, + const Reduction reduction + ) { + require_registration_allowed(name, edge_values_, "edge"); + edge_values_.emplace_back( + ReducedValueMap( + std::move(name), + values, + std::move(component_shape), + topology_.edge_capacity(), + reduction + ) + ); + } + + NodeId contract_edge( + const EdgeId edge, + const std::optional keep_node = std::nullopt + ) { + mutation_started_ = true; + const auto kept = topology_.contract_edge(edge, keep_node, change_); + apply_node_reduction(change_.kept_node, change_.removed_node); + node_sets_.merge_to(change_.kept_node, change_.removed_node); + apply_removed_edge_states(EdgeState::Internal); + apply_edge_folds(); + return kept; + } + + void erase_edge(const EdgeId edge) { + mutation_started_ = true; + topology_.erase_edge(edge, change_); + set_edge_state(edge, EdgeState::Deleted); + } + + EdgeId suppress_node(const NodeId node) { + mutation_started_ = true; + const auto replacement = topology_.suppress_node(node, change_); + apply_edge_folds(); + return replacement; + } + + [[nodiscard]] const ReducedValueMapVariant &node_value_map( + const std::string &name + ) const { + return find_value_map(node_values_, name, "node"); + } + + [[nodiscard]] const ReducedValueMapVariant &edge_value_map( + const std::string &name + ) const { + return find_value_map(edge_values_, name, "edge"); + } + + [[nodiscard]] MaterializedValueMap node_value( + const std::string &name, + const NodeId node + ) const { + if (!topology_.node_active(node)) { + throw std::invalid_argument("node is inactive"); + } + return std::visit([node](const auto &map) { + return MaterializedValueMap(map.value(node)); + }, node_value_map(name)); + } + + [[nodiscard]] MaterializedValueMap edge_value( + const std::string &name, + const EdgeId edge + ) const { + if (!topology_.edge_active(edge)) { + throw std::invalid_argument("edge is inactive"); + } + return std::visit([edge](const auto &map) { + return MaterializedValueMap(map.value(edge)); + }, edge_value_map(name)); + } + + [[nodiscard]] MaterializedValueMap active_node_values( + const std::string &name + ) const { + const auto ids = topology_.active_nodes(); + return std::visit([&ids](const auto &map) { + return MaterializedValueMap(map.gather(ids)); + }, node_value_map(name)); + } + + [[nodiscard]] MaterializedValueMap active_edge_values( + const std::string &name + ) const { + const auto ids = topology_.active_edges(); + return std::visit([&ids](const auto &map) { + return MaterializedValueMap(map.gather(ids)); + }, edge_value_map(name)); + } + + // Build an independent simple-graph snapshot. Parallel active edges fold + // only in the snapshot. Multiple input items can map to one output id. + MaterializedContraction materialize() { + const auto active_nodes = topology_.active_nodes(); + std::vector dense_node(topology_.node_capacity(), -1); + for (std::size_t dense = 0; dense < active_nodes.size(); ++dense) { + dense_node[static_cast(active_nodes[dense])] = + static_cast(dense); + } + + struct MaterializedEdge { + Edge uv; + EdgeId stable_edge; + }; + std::vector materialized_edges; + materialized_edges.reserve(topology_.number_of_edges()); + for (const auto edge : topology_.active_edges()) { + const auto uv = topology_.uv(edge); + const auto dense_u = + static_cast(dense_node[static_cast(uv.first)]); + const auto dense_v = + static_cast(dense_node[static_cast(uv.second)]); + if (dense_u == dense_v) { + throw std::runtime_error( + "cannot materialize a graph with active self edges" + ); + } + materialized_edges.push_back( + {{std::min(dense_u, dense_v), std::max(dense_u, dense_v)}, edge} + ); + } + std::sort( + materialized_edges.begin(), + materialized_edges.end(), + [](const auto &a, const auto &b) { + return a.uv != b.uv + ? a.uv < b.uv + : a.stable_edge < b.stable_edge; + } + ); + + std::vector output_edges; + std::vector active_edges; + std::vector> snapshot_folds; + output_edges.reserve(materialized_edges.size()); + active_edges.reserve(materialized_edges.size()); + snapshot_folds.reserve(materialized_edges.size()); + std::vector dense_edge(topology_.edge_capacity(), -1); + for (const auto &edge : materialized_edges) { + if (output_edges.empty() || output_edges.back() != edge.uv) { + output_edges.push_back(edge.uv); + active_edges.push_back(edge.stable_edge); + } else { + snapshot_folds.emplace_back(active_edges.back(), edge.stable_edge); + } + dense_edge[static_cast(edge.stable_edge)] = + static_cast(output_edges.size() - 1); + } + + auto graph = UndirectedGraph::from_sorted_unique_edges( + static_cast(active_nodes.size()), + std::move(output_edges), + true + ); + + std::vector node_mapping(topology_.node_capacity(), -1); + for (NodeId original = 0; original < topology_.node_capacity(); ++original) { + const auto root = node_sets_.find(original); + if (topology_.node_active(root)) { + node_mapping[static_cast(original)] = + dense_node[static_cast(root)]; + } + } + + std::vector edge_mapping(topology_.edge_capacity(), -1); + for (EdgeId original = 0; original < topology_.edge_capacity(); ++original) { + const auto root = edge_root(original); + if (edge_state_[static_cast(root)] == EdgeState::Active && + topology_.edge_active(root)) { + edge_mapping[static_cast(original)] = + dense_edge[static_cast(root)]; + } + } + + std::vector node_values; + node_values.reserve(node_values_.size()); + for (const auto &map : node_values_) { + node_values.push_back(std::visit( + [&active_nodes](const auto &typed) { + return MaterializedValueMap(typed.gather(active_nodes)); + }, + map + )); + } + + std::vector edge_values; + edge_values.reserve(edge_values_.size()); + for (const auto &map : edge_values_) { + edge_values.push_back(std::visit( + [&active_edges, &snapshot_folds](const auto &typed) { + auto snapshot = typed; + for (const auto &[kept, removed] : snapshot_folds) { + snapshot.reduce(kept, removed); + } + return MaterializedValueMap(snapshot.gather(active_edges)); + }, + map + )); + } + + return { + std::move(graph), + std::move(node_values), + std::move(edge_values), + std::move(node_mapping), + std::move(edge_mapping), + }; + } + +private: + enum class EdgeState : unsigned char { + Active, + Internal, + Deleted, + }; + + template + static decltype(auto) visit_value( + const ReducedValueMapVariant &map, + Function &&function + ) { + return std::visit(std::forward(function), map); + } + + static const std::string &map_name(const ReducedValueMapVariant &map) { + return visit_value(map, [](const auto &typed) -> const std::string & { + return typed.name(); + }); + } + + static const ReducedValueMapVariant &find_value_map( + const std::vector &maps, + const std::string &name, + const char *kind + ) { + for (const auto &map : maps) { + if (map_name(map) == name) { + return map; + } + } + throw std::invalid_argument( + std::string("unknown ") + kind + " value map: " + name + ); + } + + void require_registration_allowed( + const std::string &name, + const std::vector &maps, + const char *kind + ) const { + if (mutation_started_) { + throw std::invalid_argument( + "value maps must be registered before the first graph mutation" + ); + } + if (name.empty()) { + throw std::invalid_argument("value map name must not be empty"); + } + for (const auto &map : maps) { + if (map_name(map) == name) { + throw std::invalid_argument( + std::string(kind) + " value map already exists: " + name + ); + } + } + } + + void initialize_edge_parents() { + for (EdgeId edge = 0; edge < edge_parent_.size(); ++edge) { + edge_parent_[static_cast(edge)] = edge; + } + } + + EdgeId edge_root(EdgeId edge) { + auto current = static_cast(edge); + while (edge_parent_[current] != current) { + edge_parent_[current] = + edge_parent_[static_cast(edge_parent_[current])]; + current = static_cast(edge_parent_[current]); + } + return static_cast(current); + } + + void set_edge_state(const EdgeId edge, const EdgeState state) { + const auto root = edge_root(edge); + edge_state_[static_cast(root)] = state; + } + + void apply_removed_edge_states(const EdgeState state) { + for (const auto edge : change_.deactivated_edges) { + const auto folded = std::any_of( + change_.folded_edges.begin(), + change_.folded_edges.end(), + [edge](const auto &fold) { + return fold.removed_edge == edge; + } + ); + if (!folded) { + set_edge_state(edge, state); + } + } + } + + void apply_node_reduction(const NodeId kept, const NodeId removed) { + for (auto &map : node_values_) { + std::visit( + [kept, removed](auto &typed) { + typed.reduce(kept, removed); + }, + map + ); + } + } + + void apply_edge_folds() { + for (const auto &fold : change_.folded_edges) { + for (auto &map : edge_values_) { + std::visit( + [&fold](auto &typed) { + typed.reduce(fold.kept_edge, fold.removed_edge); + }, + map + ); + } + const auto kept_root = edge_root(fold.kept_edge); + const auto removed_root = edge_root(fold.removed_edge); + if (kept_root != removed_root) { + edge_parent_[static_cast(removed_root)] = kept_root; + } + } + } + + detail::ContractionTopology topology_; + util::UnionFind node_sets_; + std::vector edge_parent_; + std::vector edge_state_; + std::vector node_values_; + std::vector edge_values_; + detail::ContractionChange change_; + ParallelEdgePolicy parallel_edge_policy_ = ParallelEdgePolicy::Merge; + bool mutation_started_ = false; +}; + +} // namespace bioimage_cpp::graph diff --git a/include/bioimage_cpp/graph/contraction_options.hxx b/include/bioimage_cpp/graph/contraction_options.hxx new file mode 100644 index 0000000..516a394 --- /dev/null +++ b/include/bioimage_cpp/graph/contraction_options.hxx @@ -0,0 +1,10 @@ +#pragma once + +namespace bioimage_cpp::graph { + +enum class ParallelEdgePolicy { + Merge, + Keep, +}; + +} // namespace bioimage_cpp::graph diff --git a/include/bioimage_cpp/graph/detail/contraction_topology.hxx b/include/bioimage_cpp/graph/detail/contraction_topology.hxx new file mode 100644 index 0000000..56c6bd3 --- /dev/null +++ b/include/bioimage_cpp/graph/detail/contraction_topology.hxx @@ -0,0 +1,822 @@ +#pragma once + +#include "bioimage_cpp/graph/contraction_options.hxx" +#include "bioimage_cpp/graph/undirected_graph.hxx" + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace bioimage_cpp::graph::detail { + +inline constexpr std::uint64_t invalid_contraction_id = + std::numeric_limits::max(); + +struct ContractionAdjacency { + std::uint64_t node; + std::uint64_t edge; +}; + +struct EmptyContractionEdgePayload {}; + +template +struct BasicContractionEdge { + std::uint64_t u = 0; + std::uint64_t v = 0; + [[no_unique_address]] EdgePayload payload; +}; + +using ContractionEdge = + BasicContractionEdge; + +struct EdgeRekey { + std::uint64_t edge; + std::uint64_t old_u; + std::uint64_t old_v; + std::uint64_t new_u; + std::uint64_t new_v; +}; + +struct EdgeFold { + std::uint64_t kept_edge; + std::uint64_t removed_edge; +}; + +// Callers reuse this record across mutations to avoid one allocation per edge +// contraction. Solver adapters can use the same record to update heaps and +// objective-specific state after the topology changes. +struct ContractionChange { + std::uint64_t kept_node = invalid_contraction_id; + std::uint64_t removed_node = invalid_contraction_id; + std::uint64_t removed_edge = invalid_contraction_id; + std::uint64_t replacement_edge = invalid_contraction_id; + std::vector deactivated_edges; + std::vector rekeyed_edges; + std::vector folded_edges; + + void clear() { + kept_node = invalid_contraction_id; + removed_node = invalid_contraction_id; + removed_edge = invalid_contraction_id; + replacement_edge = invalid_contraction_id; + deactivated_edges.clear(); + rekeyed_edges.clear(); + folded_edges.clear(); + } +}; + +class ContractionChangeObserver { +public: + explicit ContractionChangeObserver(ContractionChange &change) + : change_(change) { + change_.clear(); + } + + void begin_contraction( + const std::uint64_t kept_node, + const std::uint64_t removed_node, + const std::uint64_t removed_edge + ) { + change_.kept_node = kept_node; + change_.removed_node = removed_node; + change_.removed_edge = removed_edge; + } + + void edge_deactivated(const std::uint64_t edge) { + change_.deactivated_edges.push_back(edge); + } + + void edge_rekeyed(const EdgeRekey &rekey) { + change_.rekeyed_edges.push_back(rekey); + } + + void edges_folded( + const std::uint64_t kept_edge, + const std::uint64_t removed_edge + ) { + change_.folded_edges.push_back({kept_edge, removed_edge}); + } + + void end_contraction(const std::uint64_t kept_node) { + (void)kept_node; + } + +private: + ContractionChange &change_; +}; + +// Mutable topology with stable sparse ids. Parallel-edge merging is optional. +// An edge payload keeps solver state next to its topology when this improves +// locality. The topology does not interpret the payload. +template +class BasicContractionTopology { +public: + using NodeId = std::uint64_t; + using EdgeId = std::uint64_t; + using Edge = UndirectedGraph::Edge; + + BasicContractionTopology() = default; + + explicit BasicContractionTopology( + const UndirectedGraph &graph, + const ParallelEdgePolicy parallel_policy = ParallelEdgePolicy::Merge + ) { + reset(graph, parallel_policy); + } + + BasicContractionTopology( + const NodeId number_of_nodes, + const std::vector &edges, + const ParallelEdgePolicy parallel_policy + ) + : adjacency_(static_cast(number_of_nodes)), + nodes_active_(static_cast(number_of_nodes), true), + active_node_count_(static_cast(number_of_nodes)), + parallel_policy_(parallel_policy), + scratch_edge_( + static_cast(number_of_nodes), + invalid_contraction_id + ) { + edges_.reserve(edges.size()); + for (const auto &edge : edges) { + add_initial_edge(edge.first, edge.second); + } + active_edge_count_ = edges_.size(); + } + + // Reset from a simple graph while retaining allocated storage. This path + // reads only the edge list, so it does not trigger a lazy CSR rebuild. + void reset( + const UndirectedGraph &graph, + const ParallelEdgePolicy parallel_policy = ParallelEdgePolicy::Merge + ) { + const auto number_of_nodes = graph.number_of_nodes(); + const auto &edges = graph.uv_ids(); + const auto node_count = static_cast(number_of_nodes); + + for (auto &adjacency : adjacency_) { + adjacency.clear(); + } + adjacency_.resize(node_count); + edges_.resize(edges.size()); + edges_active_.assign(edges.size(), true); + nodes_active_.assign(node_count, true); + active_node_count_ = node_count; + active_edge_count_ = edges.size(); + parallel_policy_ = parallel_policy; + + scratch_edge_.assign(node_count, 0); + for (const auto &[u, v] : edges) { + if (u >= number_of_nodes || v >= number_of_nodes) { + throw std::out_of_range("edge endpoint is out of range"); + } + if (u == v) { + throw std::invalid_argument("self edges are not supported"); + } + ++scratch_edge_[static_cast(u)]; + ++scratch_edge_[static_cast(v)]; + } + for (NodeId node = 0; node < number_of_nodes; ++node) { + adjacency_[static_cast(node)].reserve( + static_cast( + scratch_edge_[static_cast(node)] + ) + ); + } + std::fill( + scratch_edge_.begin(), + scratch_edge_.end(), + invalid_contraction_id + ); + + for (std::size_t edge = 0; edge < edges.size(); ++edge) { + const auto [u, v] = edges[edge]; + edges_[edge] = {u, v}; + add_adjacency(u, v, static_cast(edge)); + } + } + + [[nodiscard]] NodeId node_id_upper_bound() const { + return adjacency_.empty() ? 0 : static_cast(adjacency_.size() - 1); + } + + [[nodiscard]] EdgeId edge_id_upper_bound() const { + return edges_.empty() ? 0 : static_cast(edges_.size() - 1); + } + + [[nodiscard]] std::size_t node_capacity() const { + return adjacency_.size(); + } + + [[nodiscard]] std::size_t edge_capacity() const { + return edges_.size(); + } + + [[nodiscard]] std::size_t number_of_nodes() const { + return active_node_count_; + } + + [[nodiscard]] std::size_t number_of_edges() const { + return active_edge_count_; + } + + [[nodiscard]] bool node_active(const NodeId node) const { + validate_node_id(node); + return nodes_active_[static_cast(node)]; + } + + [[nodiscard]] bool edge_active(const EdgeId edge) const { + validate_edge_id(edge); + return edges_active_[static_cast(edge)]; + } + + [[nodiscard]] Edge uv(const EdgeId edge) const { + validate_active_edge(edge); + const auto &entry = edges_[static_cast(edge)]; + return {entry.u, entry.v}; + } + + [[nodiscard]] EdgePayload &edge_payload(const EdgeId edge) { + return edges_[static_cast(edge)].payload; + } + + [[nodiscard]] const EdgePayload &edge_payload(const EdgeId edge) const { + return edges_[static_cast(edge)].payload; + } + + [[nodiscard]] std::size_t degree(const NodeId node) const { + validate_active_node(node); + return adjacency_[static_cast(node)].size(); + } + + // Select the larger adjacency as the survivor. The edge must be active. + [[nodiscard]] std::pair preferred_contraction_nodes( + const EdgeId edge + ) const { + const auto &entry = edges_[static_cast(edge)]; + const auto degree_u = + adjacency_[static_cast(entry.u)].size(); + const auto degree_v = + adjacency_[static_cast(entry.v)].size(); + if (degree_u < degree_v) { + return {entry.v, entry.u}; + } + return {entry.u, entry.v}; + } + + [[nodiscard]] bool can_suppress_node(const NodeId node) const { + validate_node_id(node); + if (!nodes_active_[static_cast(node)]) { + return false; + } + const auto &incident = adjacency_[static_cast(node)]; + return incident.size() == 2 && + incident[0].edge != incident[1].edge && + incident[0].node != incident[1].node; + } + + [[nodiscard]] const std::vector &node_adjacency( + const NodeId node + ) const { + validate_active_node(node); + return adjacency_[static_cast(node)]; + } + + [[nodiscard]] std::vector active_nodes() const { + std::vector result; + result.reserve(active_node_count_); + for (NodeId node = 0; node < adjacency_.size(); ++node) { + if (nodes_active_[static_cast(node)]) { + result.push_back(node); + } + } + return result; + } + + [[nodiscard]] std::vector active_edges() const { + std::vector result; + result.reserve(active_edge_count_); + for (EdgeId edge = 0; edge < edges_.size(); ++edge) { + if (edges_active_[static_cast(edge)]) { + result.push_back(edge); + } + } + return result; + } + + [[nodiscard]] std::int64_t find_edge(const NodeId u, const NodeId v) const { + validate_active_node(u); + validate_active_node(v); + auto result = invalid_contraction_id; + for (const auto &adjacency : adjacency_[static_cast(u)]) { + if (adjacency.node == v) { + result = std::min(result, adjacency.edge); + } + } + return result == invalid_contraction_id + ? -1 + : static_cast(result); + } + + [[nodiscard]] std::vector find_edges( + const NodeId u, + const NodeId v + ) const { + validate_active_node(u); + validate_active_node(v); + std::vector result; + for (const auto &adjacency : adjacency_[static_cast(u)]) { + if (adjacency.node == v) { + result.push_back(adjacency.edge); + } + } + std::sort(result.begin(), result.end()); + return result; + } + + NodeId contract_edge( + const EdgeId edge, + const std::optional keep_node, + ContractionChange &change + ) { + ContractionChangeObserver observer(change); + return contract_edge_observed(edge, keep_node, observer); + } + + // Contract an edge and report mutation events as they occur. Solvers use + // this path to keep objective state in cache and avoid event-buffer writes. + template < + bool TrackEdgeActivity = true, + bool AssumeMergePolicy = false, + class Observer + > + NodeId contract_edge_observed( + const EdgeId edge, + const std::optional keep_node, + Observer &observer + ) { + validate_active_edge(edge); + const auto contracted = edges_[static_cast(edge)]; + if (contracted.u == contracted.v) { + throw std::invalid_argument("cannot contract a self edge"); + } + + auto kept = contracted.u; + auto removed = contracted.v; + if (keep_node.has_value()) { + if (*keep_node != contracted.u && *keep_node != contracted.v) { + throw std::invalid_argument( + "keep_node must be an endpoint of the contracted edge" + ); + } + kept = *keep_node; + removed = kept == contracted.u ? contracted.v : contracted.u; + } else { + const auto degree_u = adjacency_[static_cast(contracted.u)].size(); + const auto degree_v = adjacency_[static_cast(contracted.v)].size(); + if (degree_v > degree_u || + (degree_v == degree_u && contracted.v < contracted.u)) { + kept = contracted.v; + removed = contracted.u; + } + } + + return contract_edge_observed_known_nodes< + TrackEdgeActivity, + AssumeMergePolicy + >(edge, kept, removed, observer); + } + + // The caller has selected two current endpoints of `edge`. + // `TrackEdgeActivity = false` leaves edge activity and edge counts stale. + // `AssumeMergePolicy = true` requires a topology with merge policy. + template < + bool TrackEdgeActivity = true, + bool AssumeMergePolicy = false, + class Observer + > + NodeId contract_edge_observed_known_nodes( + const EdgeId edge, + const NodeId kept, + const NodeId removed, + Observer &observer + ) { + observer.begin_contraction(kept, removed, edge); + + if constexpr (AssumeMergePolicy) { + stamp_neighbors(kept); + } else if (parallel_policy_ == ParallelEdgePolicy::Merge) { + stamp_neighbors(kept); + } + + if constexpr (AssumeMergePolicy) { + erase_adjacency_neighbor( + adjacency_[static_cast(kept)], + removed + ); + } else if (parallel_policy_ == ParallelEdgePolicy::Merge) { + erase_adjacency_neighbor( + adjacency_[static_cast(kept)], + removed + ); + } else { + erase_adjacency_edge( + adjacency_[static_cast(kept)], + edge + ); + } + if constexpr (TrackEdgeActivity) { + edges_active_[static_cast(edge)] = false; + --active_edge_count_; + } + observer.edge_deactivated(edge); + if constexpr (AssumeMergePolicy) { + scratch_edge_[static_cast(removed)] = + invalid_contraction_id; + } else if (parallel_policy_ == ParallelEdgePolicy::Merge) { + scratch_edge_[static_cast(removed)] = + invalid_contraction_id; + } + + const auto &removed_adjacency = + adjacency_[static_cast(removed)]; + for (const auto &adjacency : removed_adjacency) { + const auto current_edge = adjacency.edge; + const auto neighbor = adjacency.node; + if (neighbor == kept) { + if constexpr (AssumeMergePolicy) { + continue; + } else if (current_edge != edge) { + if (parallel_policy_ == ParallelEdgePolicy::Merge) { + erase_adjacency_neighbor( + adjacency_[static_cast(kept)], + removed + ); + } else { + erase_adjacency_edge( + adjacency_[static_cast(kept)], + current_edge + ); + } + if constexpr (TrackEdgeActivity) { + edges_active_[ + static_cast(current_edge) + ] = false; + --active_edge_count_; + } + observer.edge_deactivated(current_edge); + } + continue; + } + + EdgeId existing; + if constexpr (AssumeMergePolicy) { + existing = scratch_edge_[static_cast(neighbor)]; + } else { + existing = + parallel_policy_ == ParallelEdgePolicy::Merge + ? scratch_edge_[static_cast(neighbor)] + : invalid_contraction_id; + } + if (existing != invalid_contraction_id) { + erase_adjacency_neighbor( + adjacency_[static_cast(neighbor)], + removed + ); + if constexpr (TrackEdgeActivity) { + edges_active_[static_cast(current_edge)] = + false; + --active_edge_count_; + } + observer.edge_deactivated(current_edge); + observer.edges_folded(existing, current_edge); + continue; + } + + const auto old_edge = + edges_[static_cast(current_edge)]; + adjacency_[static_cast(kept)].push_back( + {neighbor, current_edge} + ); + if constexpr (AssumeMergePolicy) { + scratch_edge_[static_cast(neighbor)] = current_edge; + } else if (parallel_policy_ == ParallelEdgePolicy::Merge) { + scratch_edge_[static_cast(neighbor)] = current_edge; + } + if constexpr (AssumeMergePolicy) { + rename_adjacency_neighbor_unchecked( + adjacency_[static_cast(neighbor)], + removed, + kept + ); + } else if (parallel_policy_ == ParallelEdgePolicy::Merge) { + rename_adjacency_neighbor( + adjacency_[static_cast(neighbor)], + removed, + kept + ); + } else { + rename_adjacency_edge( + adjacency_[static_cast(neighbor)], + current_edge, + kept + ); + } + auto &new_edge = + edges_[static_cast(current_edge)]; + if (new_edge.u == removed) { + new_edge.u = kept; + } else if constexpr (AssumeMergePolicy) { + new_edge.v = kept; + } else { + if (new_edge.v == removed) { + new_edge.v = kept; + } else { + throw std::runtime_error( + "edge is not incident to the rekeyed node" + ); + } + } + observer.edge_rekeyed( + { + current_edge, + old_edge.u, + old_edge.v, + new_edge.u, + new_edge.v, + } + ); + } + + if constexpr (AssumeMergePolicy) { + clear_stamps(kept); + } else if (parallel_policy_ == ParallelEdgePolicy::Merge) { + clear_stamps(kept); + } + adjacency_[static_cast(removed)].clear(); + nodes_active_[static_cast(removed)] = false; + --active_node_count_; + observer.end_contraction(kept); + return kept; + } + + void erase_edge(const EdgeId edge, ContractionChange &change) { + change.clear(); + validate_active_edge(edge); + change.removed_edge = edge; + deactivate_edge(edge); + change.deactivated_edges.push_back(edge); + } + + EdgeId suppress_node(const NodeId node, ContractionChange &change) { + change.clear(); + validate_active_node(node); + const auto incident = adjacency_[static_cast(node)]; + if (incident.size() != 2) { + throw std::invalid_argument( + "suppress_node requires degree 2, got degree=" + + std::to_string(incident.size()) + ); + } + if (incident[0].edge == incident[1].edge) { + throw std::invalid_argument("cannot suppress a node incident to a self edge"); + } + + const auto edge_a = incident[0].edge; + const auto edge_b = incident[1].edge; + const auto neighbor_a = incident[0].node; + const auto neighbor_b = incident[1].node; + if (neighbor_a == neighbor_b) { + throw std::invalid_argument( + "suppress_node would create a self edge" + ); + } + + change.removed_node = node; + deactivate_edge(edge_a); + deactivate_edge(edge_b); + change.deactivated_edges.push_back(edge_a); + change.deactivated_edges.push_back(edge_b); + + EdgeId replacement = invalid_contraction_id; + if (parallel_policy_ == ParallelEdgePolicy::Merge && + neighbor_a != neighbor_b) { + const auto found = find_edge(neighbor_a, neighbor_b); + if (found >= 0) { + replacement = static_cast(found); + change.folded_edges.push_back({replacement, edge_a}); + change.folded_edges.push_back({replacement, edge_b}); + } + } + + if (replacement == invalid_contraction_id) { + replacement = std::min(edge_a, edge_b); + const auto folded = replacement == edge_a ? edge_b : edge_a; + auto &replacement_entry = edges_[static_cast(replacement)]; + const auto old_u = replacement_entry.u; + const auto old_v = replacement_entry.v; + replacement_entry.u = neighbor_a; + replacement_entry.v = neighbor_b; + edges_active_[static_cast(replacement)] = true; + ++active_edge_count_; + add_adjacency(neighbor_a, neighbor_b, replacement); + if (old_u != node && old_v != node) { + throw std::runtime_error( + "suppressed edge is not incident to the suppressed node" + ); + } + change.rekeyed_edges.push_back( + { + replacement, + old_u, + old_v, + neighbor_a, + neighbor_b, + } + ); + change.folded_edges.push_back({replacement, folded}); + } + + adjacency_[static_cast(node)].clear(); + nodes_active_[static_cast(node)] = false; + --active_node_count_; + change.replacement_edge = replacement; + return replacement; + } + +private: + void validate_node_id(const NodeId node) const { + if (node >= adjacency_.size()) { + throw std::out_of_range( + "node id " + std::to_string(node) + " is out of range" + ); + } + } + + void validate_edge_id(const EdgeId edge) const { + if (edge >= edges_.size()) { + throw std::out_of_range( + "edge id " + std::to_string(edge) + " is out of range" + ); + } + } + + void validate_active_node(const NodeId node) const { + validate_node_id(node); + if (!nodes_active_[static_cast(node)]) { + throw std::invalid_argument( + "node " + std::to_string(node) + " is inactive" + ); + } + } + + void validate_active_edge(const EdgeId edge) const { + validate_edge_id(edge); + if (!edges_active_[static_cast(edge)]) { + throw std::invalid_argument( + "edge " + std::to_string(edge) + " is inactive" + ); + } + } + + void add_initial_edge(const NodeId u, const NodeId v) { + if (u >= adjacency_.size() || v >= adjacency_.size()) { + throw std::out_of_range("edge endpoint is out of range"); + } + if (u == v) { + throw std::invalid_argument("self edges are not supported"); + } + if (parallel_policy_ == ParallelEdgePolicy::Merge) { + for (const auto &adjacency : adjacency_[static_cast(u)]) { + if (adjacency.node == v) { + throw std::invalid_argument("parallel edges are not supported"); + } + } + } + const auto edge = static_cast(edges_.size()); + edges_.push_back({u, v}); + edges_active_.push_back(true); + add_adjacency(u, v, edge); + } + + void add_adjacency(const NodeId u, const NodeId v, const EdgeId edge) { + adjacency_[static_cast(u)].push_back({v, edge}); + adjacency_[static_cast(v)].push_back({u, edge}); + } + + static void erase_adjacency_edge( + std::vector &adjacency, + const EdgeId edge + ) { + for (std::size_t index = 0; index < adjacency.size(); ++index) { + if (adjacency[index].edge == edge) { + adjacency[index] = adjacency.back(); + adjacency.pop_back(); + return; + } + } + } + + static void erase_adjacency_neighbor( + std::vector &adjacency, + const NodeId node + ) { + for (std::size_t index = 0; index < adjacency.size(); ++index) { + if (adjacency[index].node == node) { + adjacency[index] = adjacency.back(); + adjacency.pop_back(); + return; + } + } + } + + static void rename_adjacency_neighbor( + std::vector &adjacency, + const NodeId old_node, + const NodeId new_node + ) { + for (auto &entry : adjacency) { + if (entry.node == old_node) { + entry.node = new_node; + return; + } + } + throw std::runtime_error("edge adjacency is inconsistent"); + } + + static void rename_adjacency_neighbor_unchecked( + std::vector &adjacency, + const NodeId old_node, + const NodeId new_node + ) { + for (auto &entry : adjacency) { + if (entry.node == old_node) { + entry.node = new_node; + return; + } + } + } + + static void rename_adjacency_edge( + std::vector &adjacency, + const EdgeId edge, + const NodeId new_node + ) { + for (auto &entry : adjacency) { + if (entry.edge == edge) { + entry.node = new_node; + return; + } + } + throw std::runtime_error("edge adjacency is inconsistent"); + } + + void deactivate_edge(const EdgeId edge) { + const auto edge_index = static_cast(edge); + auto &entry = edges_[edge_index]; + if (!edges_active_[edge_index]) { + return; + } + erase_adjacency_edge(adjacency_[static_cast(entry.u)], edge); + erase_adjacency_edge(adjacency_[static_cast(entry.v)], edge); + edges_active_[edge_index] = false; + --active_edge_count_; + } + + void stamp_neighbors(const NodeId node) { + for (const auto &adjacency : adjacency_[static_cast(node)]) { + if (adjacency.node != node) { + scratch_edge_[static_cast(adjacency.node)] = + adjacency.edge; + } + } + } + + void clear_stamps(const NodeId node) { + for (const auto &adjacency : adjacency_[static_cast(node)]) { + if (adjacency.node != node) { + scratch_edge_[static_cast(adjacency.node)] = + invalid_contraction_id; + } + } + } + + std::vector> adjacency_; + std::vector> edges_; + std::vector edges_active_; + std::vector nodes_active_; + std::size_t active_node_count_ = 0; + std::size_t active_edge_count_ = 0; + ParallelEdgePolicy parallel_policy_ = ParallelEdgePolicy::Merge; + std::vector scratch_edge_; +}; + +using ContractionTopology = BasicContractionTopology<>; + +} // namespace bioimage_cpp::graph::detail diff --git a/include/bioimage_cpp/graph/lifted_multicut/detail.hxx b/include/bioimage_cpp/graph/lifted_multicut/detail.hxx index d269aac..09330f4 100644 --- a/include/bioimage_cpp/graph/lifted_multicut/detail.hxx +++ b/include/bioimage_cpp/graph/lifted_multicut/detail.hxx @@ -1,260 +1,153 @@ #pragma once #include "bioimage_cpp/detail/indexed_heap.hxx" -#include "bioimage_cpp/util/union_find.hxx" +#include "bioimage_cpp/detail/profile.hxx" #include "bioimage_cpp/graph/connected_components.hxx" +#include "bioimage_cpp/graph/detail/contraction_topology.hxx" #include "bioimage_cpp/graph/undirected_graph.hxx" +#include "bioimage_cpp/util/union_find.hxx" -#include #include #include -#include #include #include #include namespace bioimage_cpp::graph::lifted_multicut::detail { -inline constexpr std::size_t no_edge = std::numeric_limits::max(); - -// Per-super-node adjacency entry. -struct NeighborEntry { - std::size_t neighbor; - std::size_t edge_id; -}; +using EdgeHeap = bioimage_cpp::detail::DenseIndexedHeap; -// Per-edge data stored in a flat vector indexed by stable edge_id. Same shape -// as multicut::detail::DynamicEdge but with `is_lifted` replacing the multicut -// constraint flag, since lifted edges have different heap semantics: -// * lifted edges are excluded from the heap until they become non-lifted; -// * lifted propagates iff *both* inputs are lifted (vs. constraint, which -// propagates if *either* input is constrained). -struct DynamicEdge { - std::size_t u = 0; - std::size_t v = 0; +struct ObjectiveEdge { double weight = 0.0; unsigned char is_lifted = 0; }; -using EdgeHeap = bioimage_cpp::detail::DenseIndexedHeap; - -struct DynamicGraph { - DynamicGraph() = default; - - explicit DynamicGraph(const UndirectedGraph &lifted_graph) { - reset(lifted_graph); - } +struct ContractionState { + graph::detail::BasicContractionTopology topology; + bioimage_cpp::util::UnionFind union_find{0}; + EdgeHeap heap; void reset(const UndirectedGraph &lifted_graph) { - const auto n_nodes = static_cast(lifted_graph.number_of_nodes()); - const auto n_edges = static_cast(lifted_graph.number_of_edges()); - - for (auto &adj : adjacency) { - adj.clear(); - } - adjacency.resize(n_nodes); - for (std::uint64_t node = 0; node < lifted_graph.number_of_nodes(); ++node) { - const auto degree = lifted_graph.node_adjacency(node).size(); - adjacency[static_cast(node)].reserve(degree); - } - - alive.assign(n_nodes, true); - alive_count = n_nodes; - scratch_edge_id.assign(n_nodes, no_edge); - edges.resize(n_edges); + topology.reset(lifted_graph, ParallelEdgePolicy::Merge); + union_find.reset( + static_cast(lifted_graph.number_of_nodes()) + ); + const auto number_of_edges = + static_cast(lifted_graph.number_of_edges()); + heap.reset_capacity(number_of_edges); } - - std::vector> adjacency; - std::vector edges; - std::vector alive; - std::size_t alive_count; - std::vector scratch_edge_id; }; -// Initialize the dynamic graph from a lifted graph + per-edge weights, where -// the first `n_base_edges` entries of the lifted graph are the base edges. -// Optional Gaussian noise is added to the weights, mirroring the multicut -// greedy additive flow. -// -// Lifted edges enter the dynamic graph but are *not* pushed onto the heap (the -// driver guards them via `is_lifted`). Non-lifted (base) edges are heap-pushed -// at their (possibly noisy) weight. -inline void initialize_dynamic_graph( +inline void initialize_contraction_state( const UndirectedGraph &lifted_graph, const std::vector &weights, - const std::uint64_t n_base_edges, - DynamicGraph &dynamic_graph, - EdgeHeap &heap, + const std::uint64_t number_of_base_edges, + ContractionState &state, const bool add_noise = false, const int seed = 42, const double sigma = 1.0 ) { - const auto n_edges = static_cast(lifted_graph.number_of_edges()); - heap.reset_capacity(n_edges); - std::vector heap_entries; - heap_entries.reserve(static_cast(n_base_edges)); + heap_entries.reserve(static_cast(number_of_base_edges)); std::mt19937 generator(seed); std::normal_distribution noise(0.0, sigma); - for (std::uint64_t edge = 0; edge < lifted_graph.number_of_edges(); ++edge) { - const auto uv = lifted_graph.uv(edge); - const auto u = static_cast(uv.first); - const auto v = static_cast(uv.second); + for (std::uint64_t edge = 0; + edge < lifted_graph.number_of_edges(); + ++edge) { double weight = weights[static_cast(edge)]; if (add_noise) { weight += noise(generator); } const auto edge_id = static_cast(edge); - auto &e = dynamic_graph.edges[edge_id]; - e.u = u; - e.v = v; - e.weight = weight; - e.is_lifted = (edge < n_base_edges) ? 0 : 1; - dynamic_graph.adjacency[u].push_back({v, edge_id}); - dynamic_graph.adjacency[v].push_back({u, edge_id}); - if (e.is_lifted == 0) { + auto &edge_payload = state.topology.edge_payload(edge); + edge_payload = { + weight, + static_cast(edge < number_of_base_edges ? 0 : 1) + }; + if (edge_payload.is_lifted == 0) { heap_entries.push_back({edge_id, weight}); } } - - heap.build_heap(std::move(heap_entries)); + state.heap.build_heap(std::move(heap_entries)); } -namespace internal { - -inline bool erase_by_neighbor(std::vector &list, const std::size_t target) { - for (std::size_t i = 0; i < list.size(); ++i) { - if (list[i].neighbor == target) { - list[i] = list.back(); - list.pop_back(); - return true; - } +class ContractionObserver { +public: + explicit ContractionObserver(ContractionState &state) + : state_(state) { } - return false; -} -inline void rename_neighbor( - std::vector &list, - const std::size_t from_node, - const std::size_t to_node -) { - for (auto &entry : list) { - if (entry.neighbor == from_node) { - entry.neighbor = to_node; - return; - } + void begin_contraction( + const std::uint64_t kept_node, + const std::uint64_t removed_node, + const std::uint64_t removed_edge + ) { + (void)removed_edge; + state_.union_find.merge_to(kept_node, removed_node); } -} -} // namespace internal - -// Contract the edge between u and v in the dynamic graph. Folds the -// smaller-degree super-node into the larger one. Heap is kept in sync with the -// following lifted-aware rules for parallel-edge merges: -// * both inputs lifted → result lifted (stays out of heap). -// * both inputs base → result base, priority = summed weight. -// * one of each kind → result base, edge re-enters or is updated in the -// heap with priority = summed weight. -// -// Precondition: the edge being contracted is non-lifted (the driver only -// pops base edges off the heap). -inline std::size_t merge_dynamic_nodes( - DynamicGraph &dynamic_graph, - bioimage_cpp::util::UnionFind &sets, - EdgeHeap &heap, - std::size_t u, - std::size_t v -) { - u = static_cast(sets.find(u)); - v = static_cast(sets.find(v)); - if (u == v) { - return u; - } - - auto stable = u; - auto removed = v; - if (dynamic_graph.adjacency[stable].size() < dynamic_graph.adjacency[removed].size()) { - std::swap(stable, removed); + void edge_deactivated(const std::uint64_t edge) { + state_.heap.erase(static_cast(edge)); } - sets.merge_to(stable, removed); - for (const auto &entry : dynamic_graph.adjacency[stable]) { - dynamic_graph.scratch_edge_id[entry.neighbor] = entry.edge_id; + void edge_rekeyed(const graph::detail::EdgeRekey &rekey) { + (void)rekey; } - const auto contracted_edge_id = dynamic_graph.scratch_edge_id[removed]; - heap.erase(contracted_edge_id); - dynamic_graph.scratch_edge_id[removed] = no_edge; - internal::erase_by_neighbor(dynamic_graph.adjacency[stable], removed); - - const auto removed_neighbors = dynamic_graph.adjacency[removed]; - - for (const auto &entry : removed_neighbors) { - const auto neighbor = entry.neighbor; - const auto removed_edge_id = entry.edge_id; - if (neighbor == stable) { - continue; - } - - const auto existing_id = dynamic_graph.scratch_edge_id[neighbor]; - if (existing_id == no_edge) { - // No stable-side edge to `neighbor`: re-key the removed-side edge - // by replacing `removed` with `stable`. Lifted flag and weight are - // preserved, so the heap state stays consistent (lifted edges stay - // off the heap; base edges keep their existing heap entry). - dynamic_graph.adjacency[stable].push_back({neighbor, removed_edge_id}); - dynamic_graph.scratch_edge_id[neighbor] = removed_edge_id; - internal::rename_neighbor(dynamic_graph.adjacency[neighbor], removed, stable); - auto &e = dynamic_graph.edges[removed_edge_id]; - if (e.u == removed) { - e.u = stable; - } else { - e.v = stable; - } + void edges_folded( + const std::uint64_t kept_edge_id, + const std::uint64_t removed_edge_id + ) { + const auto kept_edge = static_cast(kept_edge_id); + auto &kept = state_.topology.edge_payload(kept_edge_id); + const auto &removed = + state_.topology.edge_payload(removed_edge_id); + kept.weight += removed.weight; + kept.is_lifted = + (kept.is_lifted != 0 && removed.is_lifted != 0) + ? 1 + : 0; + if (kept.is_lifted != 0) { + state_.heap.erase(kept_edge); } else { - // Stable already has an edge to `neighbor`. Sum the weights and - // resolve the lifted flag. - auto &keep = dynamic_graph.edges[existing_id]; - const auto &fold = dynamic_graph.edges[removed_edge_id]; - const bool keep_was_lifted = keep.is_lifted != 0; - const bool fold_was_lifted = fold.is_lifted != 0; - keep.weight += fold.weight; - const bool result_lifted = keep_was_lifted && fold_was_lifted; - keep.is_lifted = result_lifted ? 1 : 0; - - internal::erase_by_neighbor(dynamic_graph.adjacency[neighbor], removed); - - if (result_lifted) { - // Both inputs were lifted: the result is still lifted, so the - // heap should not contain either side. (Both were already off.) - } else if (keep_was_lifted && !fold_was_lifted) { - // The fold side was the base edge and lives on the heap; we - // are dropping that id. The keep side (formerly lifted) needs - // to be inserted into the heap with the summed weight. - heap.erase(removed_edge_id); - heap.push(existing_id, keep.weight); - } else if (!keep_was_lifted && fold_was_lifted) { - // The keep side is on the heap; the fold side was lifted and - // is not. Update the keep entry's priority to the summed weight. - heap.change(existing_id, keep.weight); - } else { - // Both base edges: drop the removed-side heap entry and - // refresh the kept entry's priority. - heap.erase(removed_edge_id); - heap.change(existing_id, keep.weight); - } + state_.heap.push_or_change( + kept_edge, + kept.weight + ); } } - for (const auto &entry : dynamic_graph.adjacency[stable]) { - dynamic_graph.scratch_edge_id[entry.neighbor] = no_edge; + void end_contraction(const std::uint64_t kept_node) { + (void)kept_node; } - dynamic_graph.adjacency[removed].clear(); - dynamic_graph.alive[removed] = false; - --dynamic_graph.alive_count; - return stable; +private: + ContractionState &state_; +}; + +template +inline std::uint64_t contract_edge( + ContractionState &state, + const std::size_t edge, + Profiler &profile +) { + const auto edge_id = static_cast(edge); + const auto [kept, removed] = + state.topology.preferred_contraction_nodes(edge_id); + ContractionObserver observer(state); + std::uint64_t kept_node; + { + BIOIMAGE_PROFILE_SCOPE(profile, "contract"); + kept_node = + state.topology.contract_edge_observed_known_nodes( + edge_id, + kept, + removed, + observer + ); + } + return kept_node; } inline std::vector labels_from_sets( @@ -270,7 +163,9 @@ inline std::size_t stop_node_count( ) { return node_num_stop >= 1.0 ? static_cast(node_num_stop) - : static_cast(double(graph.number_of_nodes()) * node_num_stop + 0.5); + : static_cast( + double(graph.number_of_nodes()) * node_num_stop + 0.5 + ); } } // namespace bioimage_cpp::graph::lifted_multicut::detail diff --git a/include/bioimage_cpp/graph/lifted_multicut/fusion_move.hxx b/include/bioimage_cpp/graph/lifted_multicut/fusion_move.hxx index 5f91316..1497d11 100644 --- a/include/bioimage_cpp/graph/lifted_multicut/fusion_move.hxx +++ b/include/bioimage_cpp/graph/lifted_multicut/fusion_move.hxx @@ -91,12 +91,8 @@ public: return objective.labels(); } - // Proposal generators read base_graph.node_adjacency() concurrently in the - // stage-1 parallel region (the greedy-additive generator does, via - // DynamicGraph::reset). The lazy CSR rebuild is not thread-safe, and the - // warm-start below freezes the *lifted* graph, not the base graph, so freeze - // the base graph on this thread before fan-out. See UndirectedGraph - // thread-safety. (The lifted graph is only read by edge iteration here.) + // Proposal generators can read base adjacency in the stage-1 + // parallel region. Freeze the lazy CSR state before the fan-out. base_graph.freeze(); const auto effective_threads = ::bioimage_cpp::detail::normalize_thread_count( diff --git a/include/bioimage_cpp/graph/lifted_multicut/greedy_additive.hxx b/include/bioimage_cpp/graph/lifted_multicut/greedy_additive.hxx index 5e47917..1572adf 100644 --- a/include/bioimage_cpp/graph/lifted_multicut/greedy_additive.hxx +++ b/include/bioimage_cpp/graph/lifted_multicut/greedy_additive.hxx @@ -12,16 +12,10 @@ namespace bioimage_cpp::graph::lifted_multicut { // Reusable scratch state for `lifted_greedy_additive`. struct GreedyAdditiveWorkspace { - detail::DynamicGraph dynamic_graph; - bioimage_cpp::util::UnionFind union_find{0}; - detail::EdgeHeap heap; + detail::ContractionState state; void reset(const UndirectedGraph &lifted_graph) { - dynamic_graph.reset(lifted_graph); - union_find.reset(static_cast(lifted_graph.number_of_nodes())); - // The heap is sized by detail::initialize_dynamic_graph, which every - // caller runs immediately after reset(); resizing it here too would - // wipe the locator vector twice. + state.reset(lifted_graph); } }; @@ -49,35 +43,39 @@ inline std::vector greedy_additive( BIOIMAGE_PROFILE_SCOPE(profile, "workspace_reset"); workspace.reset(lifted_graph); } - auto &dynamic_graph = workspace.dynamic_graph; - auto &sets = workspace.union_find; - auto &heap = workspace.heap; + auto &state = workspace.state; { BIOIMAGE_PROFILE_SCOPE(profile, "initialize"); - detail::initialize_dynamic_graph( - lifted_graph, weights, n_base_edges, dynamic_graph, heap, add_noise, seed, sigma + detail::initialize_contraction_state( + lifted_graph, + weights, + n_base_edges, + state, + add_noise, + seed, + sigma ); } - { - BIOIMAGE_PROFILE_SCOPE(profile, "contraction_loop"); - while (!heap.empty() && dynamic_graph.alive_count > 1) { - const auto top = heap.top(); - if (top.priority <= weight_stop) { - break; - } - if (node_num_stop > 0.0 - && dynamic_graph.alive_count <= detail::stop_node_count(lifted_graph, node_num_stop)) { - break; - } - const auto &edge = dynamic_graph.edges[top.key]; - detail::merge_dynamic_nodes(dynamic_graph, sets, heap, edge.u, edge.v); + while (!state.heap.empty() && state.topology.number_of_nodes() > 1) { + const auto top = state.heap.top(); + if (top.priority <= weight_stop) { + break; + } + if (node_num_stop > 0.0 + && state.topology.number_of_nodes() + <= detail::stop_node_count( + lifted_graph, + node_num_stop + )) { + break; } + detail::contract_edge(state, top.key, profile); } std::vector labels; { BIOIMAGE_PROFILE_SCOPE(profile, "labels_from_sets"); - labels = detail::labels_from_sets(sets, lifted_graph); + labels = detail::labels_from_sets(state.union_find, lifted_graph); } BIOIMAGE_PROFILE_REPORT(profile); return labels; diff --git a/include/bioimage_cpp/graph/lifted_multicut/kernighan_lin.hxx b/include/bioimage_cpp/graph/lifted_multicut/kernighan_lin.hxx index 0a1b685..0492d63 100644 --- a/include/bioimage_cpp/graph/lifted_multicut/kernighan_lin.hxx +++ b/include/bioimage_cpp/graph/lifted_multicut/kernighan_lin.hxx @@ -3,6 +3,7 @@ #include "bioimage_cpp/detail/edge_hash.hxx" #include "bioimage_cpp/detail/indexed_heap.hxx" #include "bioimage_cpp/detail/profile.hxx" +#include "bioimage_cpp/detail/relabel.hxx" #include "bioimage_cpp/util/union_find.hxx" #include "bioimage_cpp/graph/connected_components.hxx" #include "bioimage_cpp/graph/lifted_multicut/objective.hxx" @@ -10,7 +11,6 @@ #include #include #include -#include #include #include #include @@ -86,28 +86,25 @@ struct ChainBuffers { stash_gain(n_nodes, 0.0) {} }; -// Per-edge entry of the per-node filtered adjacency built once during -// `chain_gain_init` and consumed by `chain_loop`. Lifted-graph adjacency is -// walked exactly once per chain (during init) and the few entries that -// survive the in-pair filter are appended here, with their weight and base- -// vs-lifted classification cached. The chain loop then iterates only these -// surviving entries — for typical small pairs in a large graph that's a -// >10× reduction in inner-loop iterations versus re-walking -// ``lifted_graph.node_adjacency(v)`` from scratch on every move. +// Cached neighbor data for one entry that survives the active-pair filter. struct FilteredAdj { std::uint64_t node; double weight; bool is_base; }; +struct ChainMove { + std::uint64_t node; + std::uint64_t new_label; +}; + struct ChainScratch { std::vector queue_nodes; + std::vector chain; bioimage_cpp::detail::DenseIndexedHeap heap; - // Per-node CSR-style range into ``filtered_entries``. ``filtered_count[v]`` - // is set whenever v is part of the current chain's pair; readers must - // therefore index only into nodes known to be in-pair (i.e. nodes popped - // from the heap, all of which were pushed during init). + // Per-node ranges into `filtered_entries`. Only active pair nodes have + // valid ranges. std::vector filtered_offset; std::vector filtered_count; std::vector filtered_entries; @@ -150,9 +147,11 @@ inline double run_chain( } auto &queue_nodes = scratch.queue_nodes; + auto &chain = scratch.chain; auto &heap = scratch.heap; auto &filtered_entries = scratch.filtered_entries; queue_nodes.clear(); + chain.clear(); heap.clear(); filtered_entries.clear(); @@ -235,11 +234,6 @@ inline double run_chain( } // end chain_gain_init scope const double gain_from_merging = is_split ? 0.0 : 0.5 * gain_from_merging_double; - struct Move { - std::uint64_t node; - std::uint64_t new_label; - }; - std::vector chain; chain.reserve(queue_nodes.size()); double cumulative = 0.0; @@ -265,10 +259,8 @@ inline double run_chain( best_prefix = chain.size(); } - // Walk the pre-built in-pair adjacency; each entry is guaranteed - // ``bufs.in_pair[u_key] == 1`` so we only need to filter on - // ``bufs.moved``. ``was_in_heap`` is cached so the three subsequent - // heap operations don't each re-query the locator. + // Walk the pre-built in-pair adjacency. Each entry is in the current + // pair, so the loop only needs to check whether the node moved. const auto offset = scratch.filtered_offset[v_key]; const auto count = scratch.filtered_count[v_key]; for (std::uint32_t i = 0; i < count; ++i) { @@ -311,8 +303,9 @@ inline double run_chain( for (const auto v : queue_nodes) { const auto v_key = static_cast(v); bufs.in_pair[v_key] = 0; - bufs.moved[v_key] = 0; - bufs.cross_count[v_key] = 0; + } + for (const auto &move : chain) { + bufs.moved[static_cast(move.node)] = 0; } } // end chain_cleanup scope @@ -340,70 +333,6 @@ inline double run_chain( return 0.0; } -// Mark which clusters in ``labels`` differ in node-set from any cluster in -// ``previous_labels``. A new cluster c is "unchanged" iff every node in c -// shares the same previous label c_old AND ``|c| == |c_old|`` — i.e. the -// partition was neither split nor merged during the last outer iter. -// -// Used to gate pair-chains and cluster-splits: a pair (A, B) whose inputs -// are identical to the last iteration's must produce the same chain result -// (which was "no improvement" — otherwise the partitions would have changed). -// Skipping such pairs is the only major algorithmic optimization in nifty's -// outer KL driver that we previously lacked. -inline std::vector compute_cluster_changed( - const std::vector &labels, - const std::vector &previous_labels, - const std::uint64_t number_of_clusters -) { - std::vector changed(static_cast(number_of_clusters), 0); - if (number_of_clusters == 0) { - return changed; - } - if (previous_labels.size() != labels.size()) { - // First iter or shape change — treat everything as changed. - std::fill(changed.begin(), changed.end(), 1); - return changed; - } - - const auto max_prev = *std::max_element(previous_labels.begin(), previous_labels.end()); - std::vector prev_size(static_cast(max_prev) + 1, 0); - for (const auto p : previous_labels) { - ++prev_size[static_cast(p)]; - } - - constexpr auto SENTINEL = std::numeric_limits::max(); - std::vector map_new_to_old( - static_cast(number_of_clusters), SENTINEL - ); - std::vector new_size(static_cast(number_of_clusters), 0); - - for (std::size_t v = 0; v < labels.size(); ++v) { - const auto c_new = static_cast(labels[v]); - const auto c_old = previous_labels[v]; - ++new_size[c_new]; - if (changed[c_new]) { - continue; - } - if (map_new_to_old[c_new] == SENTINEL) { - map_new_to_old[c_new] = c_old; - } else if (map_new_to_old[c_new] != c_old) { - changed[c_new] = 1; - } - } - - for (std::size_t c = 0; c < changed.size(); ++c) { - if (changed[c]) { - continue; - } - const auto c_old = map_new_to_old[c]; - if (c_old == SENTINEL - || prev_size[static_cast(c_old)] != new_size[c]) { - changed[c] = 1; - } - } - return changed; -} - // Re-split labels so that every cluster is connected in the base graph. // Returns the new labeling (dense relabeled). inline std::vector enforce_base_connectivity( @@ -474,6 +403,9 @@ inline std::vector kernighan_lin( { BIOIMAGE_PROFILE_SCOPE(profile, "compute_pairs"); pairs = detail_kl::compute_base_cluster_pairs(base_graph, labels); + } + { + BIOIMAGE_PROFILE_SCOPE(profile, "build_cluster_index"); number_of_clusters = labels.empty() ? std::uint64_t{0} : (*std::max_element(labels.begin(), labels.end()) + 1); @@ -489,12 +421,38 @@ inline std::vector kernighan_lin( changed.assign(static_cast(number_of_clusters), 1); } - { - BIOIMAGE_PROFILE_SCOPE(profile, "pair_chains"); - for (const auto &pair : pairs) { - if (!changed[static_cast(pair.a)] - && !changed[static_cast(pair.b)]) { - continue; + for (const auto &pair : pairs) { + if (!changed[static_cast(pair.a)] + && !changed[static_cast(pair.b)]) { + continue; + } + const auto delta = detail_kl::run_chain( + base_graph, + lifted_graph, + lifted_weights, + n_base_edges, + labels, + cluster_to_nodes, + bufs, + scratch, + pair.a, + pair.b, + epsilon, + profile + ); + if (delta > epsilon) { + improved = true; + } + } + + std::uint64_t next_label = number_of_clusters; + for (std::uint64_t cluster = 0; cluster < number_of_clusters; ++cluster) { + if (!changed[static_cast(cluster)]) { + continue; + } + while (true) { + if (next_label >= cluster_to_nodes.size()) { + cluster_to_nodes.resize(static_cast(next_label) + 1); } const auto delta = detail_kl::run_chain( base_graph, @@ -505,48 +463,16 @@ inline std::vector kernighan_lin( cluster_to_nodes, bufs, scratch, - pair.a, - pair.b, + cluster, + next_label, epsilon, profile ); - if (delta > epsilon) { - improved = true; - } - } - } - - { - BIOIMAGE_PROFILE_SCOPE(profile, "cluster_splits"); - std::uint64_t next_label = number_of_clusters; - for (std::uint64_t cluster = 0; cluster < number_of_clusters; ++cluster) { - if (!changed[static_cast(cluster)]) { - continue; - } - while (true) { - if (next_label >= cluster_to_nodes.size()) { - cluster_to_nodes.resize(static_cast(next_label) + 1); - } - const auto delta = detail_kl::run_chain( - base_graph, - lifted_graph, - lifted_weights, - n_base_edges, - labels, - cluster_to_nodes, - bufs, - scratch, - cluster, - next_label, - epsilon, - profile - ); - if (delta <= epsilon) { - break; - } - improved = true; - ++next_label; + if (delta <= epsilon) { + break; } + improved = true; + ++next_label; } } @@ -564,7 +490,7 @@ inline std::vector kernighan_lin( const auto new_n_clusters = labels.empty() ? std::uint64_t{0} : (*std::max_element(labels.begin(), labels.end()) + 1); - changed = detail_kl::compute_cluster_changed( + changed = bioimage_cpp::detail::changed_clusters( labels, prev_iter_labels, new_n_clusters ); prev_iter_labels = labels; diff --git a/include/bioimage_cpp/graph/multicut.hxx b/include/bioimage_cpp/graph/multicut.hxx index 1e75948..7b4ff61 100644 --- a/include/bioimage_cpp/graph/multicut.hxx +++ b/include/bioimage_cpp/graph/multicut.hxx @@ -1,5 +1,7 @@ #pragma once +#include "bioimage_cpp/graph/multicut/chained.hxx" +#include "bioimage_cpp/graph/multicut/decomposer.hxx" #include "bioimage_cpp/graph/multicut/greedy_additive.hxx" #include "bioimage_cpp/graph/multicut/greedy_fixation.hxx" #include "bioimage_cpp/graph/multicut/kernighan_lin.hxx" diff --git a/include/bioimage_cpp/graph/multicut/chained.hxx b/include/bioimage_cpp/graph/multicut/chained.hxx new file mode 100644 index 0000000..296a241 --- /dev/null +++ b/include/bioimage_cpp/graph/multicut/chained.hxx @@ -0,0 +1,63 @@ +#pragma once + +#include "bioimage_cpp/graph/multicut/objective.hxx" + +#include +#include +#include +#include +#include + +namespace bioimage_cpp::graph::multicut { + +class ChainedSolver final : public CloneableSolverBase { +public: + explicit ChainedSolver( + const std::vector &solvers + ) { + if (solvers.empty()) { + throw std::invalid_argument("solvers must not be empty"); + } + solvers_.reserve(solvers.size()); + for (const auto *solver : solvers) { + if (solver == nullptr) { + throw std::invalid_argument("solvers must not contain null"); + } + solvers_.push_back(clone_solver(*solver)); + } + } + + ChainedSolver(const ChainedSolver &) = delete; + ChainedSolver &operator=(const ChainedSolver &) = delete; + ChainedSolver(ChainedSolver &&) noexcept = default; + ChainedSolver &operator=(ChainedSolver &&) noexcept = default; + + std::vector optimize(Objective &objective) const override { + for (const auto &solver : solvers_) { + solver->optimize(objective); + } + return objective.labels(); + } + + std::unique_ptr clone() const override { + std::vector> cloned; + cloned.reserve(solvers_.size()); + for (const auto &solver : solvers_) { + cloned.push_back(clone_solver(*solver)); + } + return std::unique_ptr( + new ChainedSolver(std::move(cloned)) + ); + } + +private: + explicit ChainedSolver( + std::vector> solvers + ) + : solvers_(std::move(solvers)) { + } + + std::vector> solvers_; +}; + +} // namespace bioimage_cpp::graph::multicut diff --git a/include/bioimage_cpp/graph/multicut/decomposer.hxx b/include/bioimage_cpp/graph/multicut/decomposer.hxx new file mode 100644 index 0000000..20e2e0e --- /dev/null +++ b/include/bioimage_cpp/graph/multicut/decomposer.hxx @@ -0,0 +1,298 @@ +#pragma once + +#include "bioimage_cpp/detail/profile.hxx" +#include "bioimage_cpp/detail/threading.hxx" +#include "bioimage_cpp/graph/connected_components.hxx" +#include "bioimage_cpp/graph/multicut/greedy_additive.hxx" +#include "bioimage_cpp/graph/multicut/objective.hxx" +#include "bioimage_cpp/graph/undirected_graph.hxx" + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace bioimage_cpp::graph::multicut { + +class DecomposerSolver final : public SolverBase { +public: + DecomposerSolver( + const CloneableSolverBase &sub_solver, + const CloneableSolverBase *fallthrough_solver = nullptr, + const std::size_t number_of_threads = 0 + ) + : sub_solver_(clone_solver(sub_solver)), + fallthrough_solver_( + fallthrough_solver == nullptr + ? nullptr + : clone_solver(*fallthrough_solver) + ), + number_of_threads_(number_of_threads) { + } + + std::vector optimize(Objective &objective) const override { + BIOIMAGE_PROFILE_INIT(profile); + + if ( + fallthrough_solver_ == nullptr + && dynamic_cast( + sub_solver_.get() + ) != nullptr + ) { + auto labels = sub_solver_->optimize(objective); + BIOIMAGE_PROFILE_REPORT(profile); + return labels; + } + + const auto &graph = objective.graph(); + const auto &costs = objective.costs(); + const auto number_of_nodes = + static_cast(graph.number_of_nodes()); + const auto number_of_edges = + static_cast(graph.number_of_edges()); + + std::vector component_of_node; + { + BIOIMAGE_PROFILE_SCOPE(profile, "component_find"); + std::vector positive_edge(number_of_edges); + for (std::size_t edge = 0; edge < number_of_edges; ++edge) { + positive_edge[edge] = costs[edge] > 0.0 ? 1 : 0; + } + component_of_node = connected_components( + graph, + positive_edge.empty() ? nullptr : positive_edge.data() + ); + } + + const auto number_of_components = component_of_node.empty() + ? std::size_t{0} + : static_cast( + *std::max_element(component_of_node.begin(), component_of_node.end()) + ) + 1; + if (number_of_components <= 1) { + const auto &solver = fallthrough_solver_ == nullptr + ? sub_solver_ + : fallthrough_solver_; + auto labels = solver->optimize(objective); + BIOIMAGE_PROFILE_REPORT(profile); + return labels; + } + + std::vector node_offsets(number_of_components + 1, 0); + std::vector edge_offsets(number_of_components + 1, 0); + std::vector grouped_nodes(number_of_nodes); + std::vector global_to_local(number_of_nodes); + std::vector grouped_edge_ids; + + { + BIOIMAGE_PROFILE_SCOPE(profile, "component_group"); + for (const auto component : component_of_node) { + ++node_offsets[static_cast(component) + 1]; + } + for (std::size_t component = 0; + component < number_of_components; + ++component) { + node_offsets[component + 1] += node_offsets[component]; + } + + auto node_cursor = node_offsets; + for (std::size_t node = 0; node < number_of_nodes; ++node) { + const auto component = + static_cast(component_of_node[node]); + const auto position = node_cursor[component]++; + grouped_nodes[position] = static_cast(node); + global_to_local[node] = + static_cast(position - node_offsets[component]); + } + + for (std::size_t edge = 0; edge < number_of_edges; ++edge) { + const auto uv = graph.uv(static_cast(edge)); + const auto u_component = + component_of_node[static_cast(uv.first)]; + const auto v_component = + component_of_node[static_cast(uv.second)]; + if (u_component == v_component) { + ++edge_offsets[static_cast(u_component) + 1]; + } + } + for (std::size_t component = 0; + component < number_of_components; + ++component) { + edge_offsets[component + 1] += edge_offsets[component]; + } + + grouped_edge_ids.resize(edge_offsets.back()); + auto edge_cursor = edge_offsets; + for (std::size_t edge = 0; edge < number_of_edges; ++edge) { + const auto uv = graph.uv(static_cast(edge)); + const auto u_component = + component_of_node[static_cast(uv.first)]; + const auto v_component = + component_of_node[static_cast(uv.second)]; + if (u_component != v_component) { + continue; + } + grouped_edge_ids[ + edge_cursor[static_cast(u_component)]++ + ] = static_cast(edge); + } + } + + struct Task { + std::size_t component; + std::size_t node_begin; + std::size_t node_end; + std::size_t edge_begin; + std::size_t edge_end; + std::unique_ptr solver; + }; + + std::vector tasks; + tasks.reserve(number_of_components); + for (std::size_t component = 0; + component < number_of_components; + ++component) { + const auto node_begin = node_offsets[component]; + const auto node_end = node_offsets[component + 1]; + if (node_end - node_begin <= 1) { + continue; + } + tasks.push_back( + Task{ + component, + node_begin, + node_end, + edge_offsets[component], + edge_offsets[component + 1], + clone_solver(*sub_solver_) + } + ); + } + std::stable_sort( + tasks.begin(), + tasks.end(), + [](const Task &left, const Task &right) { + const auto left_edges = left.edge_end - left.edge_begin; + const auto right_edges = right.edge_end - right.edge_begin; + if (left_edges != right_edges) { + return left_edges > right_edges; + } + return left.component < right.component; + } + ); + + std::vector> component_labels( + number_of_components + ); + const auto solve_task = [&](Task &task) { + std::vector local_edges; + std::vector local_costs; + const auto task_edge_count = task.edge_end - task.edge_begin; + local_edges.reserve(task_edge_count); + local_costs.reserve(task_edge_count); + + for (std::size_t position = task.edge_begin; + position < task.edge_end; + ++position) { + const auto edge = grouped_edge_ids[position]; + const auto uv = graph.uv(edge); + auto local_u = + global_to_local[static_cast(uv.first)]; + auto local_v = + global_to_local[static_cast(uv.second)]; + if (local_v < local_u) { + std::swap(local_u, local_v); + } + local_edges.emplace_back(local_u, local_v); + local_costs.push_back(costs[static_cast(edge)]); + } + + auto subgraph = UndirectedGraph::from_unique_edges( + static_cast(task.node_end - task.node_begin), + std::move(local_edges), + false + ); + Objective sub_objective(subgraph, std::move(local_costs)); + component_labels[task.component] = dense_relabel( + task.solver->optimize(sub_objective) + ); + }; + + { + BIOIMAGE_PROFILE_SCOPE(profile, "parallel_solve"); + const auto effective_threads = + ::bioimage_cpp::detail::normalize_thread_count( + number_of_threads_, tasks.size() + ); + std::atomic next_task{0}; + ::bioimage_cpp::detail::parallel_for_chunks( + effective_threads, + effective_threads, + [&]( + const std::size_t, + const std::size_t, + const std::size_t + ) { + while (true) { + const auto task_index = next_task.fetch_add( + 1, std::memory_order_relaxed + ); + if (task_index >= tasks.size()) { + break; + } + solve_task(tasks[task_index]); + } + } + ); + } + + std::vector global_labels(number_of_nodes); + { + BIOIMAGE_PROFILE_SCOPE(profile, "label_assembly"); + std::uint64_t label_offset = 0; + for (std::size_t component = 0; + component < number_of_components; + ++component) { + const auto begin = node_offsets[component]; + const auto end = node_offsets[component + 1]; + if (end - begin == 1) { + global_labels[ + static_cast(grouped_nodes[begin]) + ] = label_offset++; + continue; + } + + const auto &labels = component_labels[component]; + if (labels.size() != end - begin || labels.empty()) { + throw std::runtime_error( + "component solver returned an invalid label count" + ); + } + for (std::size_t local_node = 0; + local_node < labels.size(); + ++local_node) { + const auto global_node = grouped_nodes[begin + local_node]; + global_labels[static_cast(global_node)] = + labels[local_node] + label_offset; + } + label_offset += + *std::max_element(labels.begin(), labels.end()) + 1; + } + } + + objective.set_labels(std::move(global_labels)); + BIOIMAGE_PROFILE_REPORT(profile); + return objective.labels(); + } + +private: + std::unique_ptr sub_solver_; + std::unique_ptr fallthrough_solver_; + std::size_t number_of_threads_; +}; + +} // namespace bioimage_cpp::graph::multicut diff --git a/include/bioimage_cpp/graph/multicut/detail.hxx b/include/bioimage_cpp/graph/multicut/detail.hxx index 5ed3bb9..8ed1fd9 100644 --- a/include/bioimage_cpp/graph/multicut/detail.hxx +++ b/include/bioimage_cpp/graph/multicut/detail.hxx @@ -1,289 +1,180 @@ #pragma once #include "bioimage_cpp/detail/indexed_heap.hxx" -#include "bioimage_cpp/util/union_find.hxx" +#include "bioimage_cpp/detail/profile.hxx" #include "bioimage_cpp/graph/connected_components.hxx" -#include "bioimage_cpp/graph/multicut/objective.hxx" +#include "bioimage_cpp/graph/detail/contraction_topology.hxx" #include "bioimage_cpp/graph/undirected_graph.hxx" +#include "bioimage_cpp/util/union_find.hxx" #include #include #include -#include #include #include #include namespace bioimage_cpp::graph::multicut::detail { -inline constexpr std::size_t no_edge = std::numeric_limits::max(); - -// Per-super-node adjacency entry. -struct NeighborEntry { - std::size_t neighbor; - std::size_t edge_id; -}; +using EdgeHeap = bioimage_cpp::detail::DenseIndexedHeap; -// Per-edge data stored in a flat vector indexed by stable edge_id. Edge ids -// never grow past the original number of input edges — when two edges fold -// into one during a merge, one id survives and the other becomes orphaned -// (no adjacency entry or heap entry references it again). -struct DynamicEdge { - std::size_t u = 0; - std::size_t v = 0; +struct ObjectiveEdge { double weight = 0.0; unsigned char is_constraint = 0; }; -// Heap keyed by dense stable edge_id. The vector-backed DenseLocator updates -// in O(1) per sift step, which was the missing ingredient for greedy_additive -// to beat the std::priority_queue baseline. -using EdgeHeap = bioimage_cpp::detail::DenseIndexedHeap; - -struct DynamicGraph { - DynamicGraph() = default; - - explicit DynamicGraph(const UndirectedGraph &graph) { - reset(graph); - } +struct ContractionState { + graph::detail::BasicContractionTopology topology; + bioimage_cpp::util::UnionFind union_find{0}; + EdgeHeap heap; - // Reuse the buffers of an existing DynamicGraph for a new input graph. - // Inner adjacency vectors are `clear()`-ed (keeping capacity) and the - // outer container is resized; degree-based reserves prevent any per-edge - // adjacency growth during initialise. void reset(const UndirectedGraph &graph) { - const auto n_nodes = static_cast(graph.number_of_nodes()); - const auto n_edges = static_cast(graph.number_of_edges()); - - for (auto &adj : adjacency) { - adj.clear(); - } - adjacency.resize(n_nodes); - for (std::uint64_t node = 0; node < graph.number_of_nodes(); ++node) { - const auto degree = graph.node_adjacency(node).size(); - adjacency[static_cast(node)].reserve(degree); - } - - alive.assign(n_nodes, true); - alive_count = n_nodes; - scratch_edge_id.assign(n_nodes, no_edge); - edges.resize(n_edges); - } - - // O(degree(u)). Returns no_edge when (u, v) is not an edge. - [[nodiscard]] std::size_t find_edge(const std::size_t u, const std::size_t v) const { - if (u >= adjacency.size() || v >= adjacency.size()) { - return no_edge; - } - for (const auto &entry : adjacency[u]) { - if (entry.neighbor == v) { - return entry.edge_id; - } - } - return no_edge; - } - - [[nodiscard]] bool edge_exists(const std::size_t u, const std::size_t v) const { - return find_edge(u, v) != no_edge; - } - - [[nodiscard]] bool has_constraint(const std::size_t u, const std::size_t v) const { - const auto id = find_edge(u, v); - return id != no_edge && edges[id].is_constraint != 0; + topology.reset(graph, ParallelEdgePolicy::Merge); + union_find.reset(static_cast(graph.number_of_nodes())); + const auto number_of_edges = + static_cast(graph.number_of_edges()); + heap.reset_capacity(number_of_edges); } - - std::vector> adjacency; - std::vector edges; - std::vector alive; - std::size_t alive_count; - // Sized to #super-nodes, all entries == no_edge between merges. Used by - // merge_dynamic_nodes to find the existing stable-side edge for each of - // removed's neighbors in O(1) without hashing. - std::vector scratch_edge_id; }; -inline double priority_for(const double weight, const bool absolute_priority) { +inline double priority_for( + const double weight, + const bool absolute_priority +) { return absolute_priority ? std::abs(weight) : weight; } -inline void initialize_dynamic_graph( +inline void initialize_contraction_state( const UndirectedGraph &graph, const std::vector &costs, - DynamicGraph &dynamic_graph, - EdgeHeap &heap, + ContractionState &state, const bool absolute_priority, const bool add_noise = false, const int seed = 42, const double sigma = 1.0 ) { - const auto n_edges = static_cast(graph.number_of_edges()); - heap.reset_capacity(n_edges); - std::vector heap_entries; - heap_entries.reserve(n_edges); + heap_entries.reserve(static_cast(graph.number_of_edges())); std::mt19937 generator(seed); std::normal_distribution noise(0.0, sigma); for (std::uint64_t edge = 0; edge < graph.number_of_edges(); ++edge) { - const auto uv = graph.uv(edge); - const auto u = static_cast(uv.first); - const auto v = static_cast(uv.second); double weight = costs[static_cast(edge)]; if (add_noise) { weight += noise(generator); } const auto edge_id = static_cast(edge); - auto &e = dynamic_graph.edges[edge_id]; - e.u = u; - e.v = v; - e.weight = weight; - e.is_constraint = 0; - dynamic_graph.adjacency[u].push_back({v, edge_id}); - dynamic_graph.adjacency[v].push_back({u, edge_id}); - heap_entries.push_back({edge_id, priority_for(weight, absolute_priority)}); + state.topology.edge_payload(edge) = {weight, 0}; + heap_entries.push_back( + {edge_id, priority_for(weight, absolute_priority)} + ); } - - // Floyd's heapify is O(n_edges) — meaningfully faster than n_edges - // successive `push` calls when n_edges is in the hundreds of thousands - // (the common case for graphs we run the multicut on). - heap.build_heap(std::move(heap_entries)); -} - -inline std::vector labels_from_sets( - bioimage_cpp::util::UnionFind &sets, - const UndirectedGraph &graph -) { - return dense_labels_from_union_find(sets, graph.number_of_nodes()); + state.heap.build_heap(std::move(heap_entries)); } -inline std::size_t stop_node_count(const UndirectedGraph &graph, const double node_num_stop) { - return node_num_stop >= 1.0 - ? static_cast(node_num_stop) - : static_cast(double(graph.number_of_nodes()) * node_num_stop + 0.5); -} - -namespace internal { - -// Erase the first entry with neighbor == target from `list` (swap-with-back). -// Returns whether anything was erased. -inline bool erase_by_neighbor(std::vector &list, const std::size_t target) { - for (std::size_t i = 0; i < list.size(); ++i) { - if (list[i].neighbor == target) { - list[i] = list.back(); - list.pop_back(); - return true; - } +class ContractionObserver { +public: + ContractionObserver( + ContractionState &state, + const bool absolute_priority + ) + : state_(state), + absolute_priority_(absolute_priority) { } - return false; -} - -// Rename the first entry whose neighbor == from_node to point to to_node. -inline void rename_neighbor( - std::vector &list, - const std::size_t from_node, - const std::size_t to_node -) { - for (auto &entry : list) { - if (entry.neighbor == from_node) { - entry.neighbor = to_node; - return; - } - } -} -} // namespace internal - -// Contract the edge between u and v in the dynamic graph. The smaller-degree -// super-node is folded into the larger-degree one to keep the per-super-node -// adjacency growth amortized. The heap is kept in sync without staleness: each -// edge id appears at most once. -inline std::size_t merge_dynamic_nodes( - DynamicGraph &dynamic_graph, - bioimage_cpp::util::UnionFind &sets, - EdgeHeap &heap, - std::size_t u, - std::size_t v, - const bool absolute_priority -) { - u = static_cast(sets.find(u)); - v = static_cast(sets.find(v)); - if (u == v) { - return u; + void begin_contraction( + const std::uint64_t kept_node, + const std::uint64_t removed_node, + const std::uint64_t removed_edge + ) { + (void)removed_edge; + state_.union_find.merge_to(kept_node, removed_node); } - auto stable = u; - auto removed = v; - if (dynamic_graph.adjacency[stable].size() < dynamic_graph.adjacency[removed].size()) { - std::swap(stable, removed); + void edge_deactivated(const std::uint64_t edge) { + state_.heap.erase(static_cast(edge)); } - sets.merge_to(stable, removed); - // Stamp stable's neighbors so each removed-neighbor lookup is O(1). - for (const auto &entry : dynamic_graph.adjacency[stable]) { - dynamic_graph.scratch_edge_id[entry.neighbor] = entry.edge_id; + void edge_rekeyed(const graph::detail::EdgeRekey &rekey) { + (void)rekey; } - // The contracted edge (stable, removed) must be in stable's adjacency. - const auto contracted_edge_id = dynamic_graph.scratch_edge_id[removed]; - heap.erase(contracted_edge_id); - dynamic_graph.scratch_edge_id[removed] = no_edge; - internal::erase_by_neighbor(dynamic_graph.adjacency[stable], removed); - - // Snapshot removed's neighbors before mutating its adjacency. - const auto removed_neighbors = dynamic_graph.adjacency[removed]; - - for (const auto &entry : removed_neighbors) { - const auto neighbor = entry.neighbor; - const auto removed_edge_id = entry.edge_id; - if (neighbor == stable) { - continue; + void edges_folded( + const std::uint64_t kept_edge_id, + const std::uint64_t removed_edge_id + ) { + const auto kept_edge = static_cast(kept_edge_id); + auto &kept = state_.topology.edge_payload(kept_edge_id); + const auto &removed = + state_.topology.edge_payload(removed_edge_id); + kept.weight += removed.weight; + kept.is_constraint = + (kept.is_constraint != 0 || removed.is_constraint != 0) + ? 1 + : 0; + if (kept.is_constraint != 0) { + state_.heap.erase(kept_edge); + } else { + state_.heap.change( + kept_edge, + priority_for( + kept.weight, + absolute_priority_ + ) + ); } + } - const auto existing_id = dynamic_graph.scratch_edge_id[neighbor]; - if (existing_id == no_edge) { - // No existing stable-side edge to `neighbor`. Re-key the - // removed-side edge by replacing `removed` with `stable` on both - // sides. The edge_id, weight and constraint flag are preserved, - // so the heap entry remains valid without any priority change. - dynamic_graph.adjacency[stable].push_back({neighbor, removed_edge_id}); - dynamic_graph.scratch_edge_id[neighbor] = removed_edge_id; - internal::rename_neighbor(dynamic_graph.adjacency[neighbor], removed, stable); - auto &e = dynamic_graph.edges[removed_edge_id]; - if (e.u == removed) { - e.u = stable; - } else { - e.v = stable; - } - } else { - // Stable already has an edge to `neighbor`; fold removed's weight - // (and constraint flag) into it. Then drop the removed-side edge. - auto &keep = dynamic_graph.edges[existing_id]; - const auto &fold = dynamic_graph.edges[removed_edge_id]; - keep.weight += fold.weight; - const bool propagated_constraint = - keep.is_constraint != 0 || fold.is_constraint != 0; - keep.is_constraint = propagated_constraint ? 1 : 0; + void end_contraction(const std::uint64_t kept_node) { + (void)kept_node; + } - heap.erase(removed_edge_id); - internal::erase_by_neighbor(dynamic_graph.adjacency[neighbor], removed); +private: + ContractionState &state_; + bool absolute_priority_; +}; - if (propagated_constraint) { - heap.erase(existing_id); - } else { - heap.change(existing_id, priority_for(keep.weight, absolute_priority)); - } - } +template +inline std::uint64_t contract_edge( + ContractionState &state, + const std::size_t edge, + const bool absolute_priority, + Profiler &profile +) { + const auto edge_id = static_cast(edge); + const auto [kept, removed] = + state.topology.preferred_contraction_nodes(edge_id); + ContractionObserver observer(state, absolute_priority); + std::uint64_t kept_node; + { + BIOIMAGE_PROFILE_SCOPE(profile, "contract"); + kept_node = + state.topology.contract_edge_observed_known_nodes( + edge_id, + kept, + removed, + observer + ); } + return kept_node; +} - // Clear scratch via the updated stable adjacency (includes appended entries). - for (const auto &entry : dynamic_graph.adjacency[stable]) { - dynamic_graph.scratch_edge_id[entry.neighbor] = no_edge; - } +inline std::vector labels_from_sets( + bioimage_cpp::util::UnionFind &sets, + const UndirectedGraph &graph +) { + return dense_labels_from_union_find(sets, graph.number_of_nodes()); +} - dynamic_graph.adjacency[removed].clear(); - dynamic_graph.alive[removed] = false; - --dynamic_graph.alive_count; - return stable; +inline std::size_t stop_node_count( + const UndirectedGraph &graph, + const double node_num_stop +) { + return node_num_stop >= 1.0 + ? static_cast(node_num_stop) + : static_cast( + double(graph.number_of_nodes()) * node_num_stop + 0.5 + ); } } // namespace bioimage_cpp::graph::multicut::detail diff --git a/include/bioimage_cpp/graph/multicut/fusion_move.hxx b/include/bioimage_cpp/graph/multicut/fusion_move.hxx index 4377e08..8aebd63 100644 --- a/include/bioimage_cpp/graph/multicut/fusion_move.hxx +++ b/include/bioimage_cpp/graph/multicut/fusion_move.hxx @@ -77,11 +77,8 @@ public: return objective.labels(); } - // Proposal generators may read graph.node_adjacency() concurrently in the - // stage-1 parallel region (the greedy-additive generator does, via - // DynamicGraph::reset). The lazy CSR rebuild is not thread-safe, and the - // warm-start below only freezes the graph for a singleton initial labeling, - // so freeze on this thread before fan-out. See UndirectedGraph thread-safety. + // Proposal generators can read adjacency in the stage-1 parallel + // region. Freeze the lazy CSR state before the fan-out. graph.freeze(); // One workspace per worker thread; reused across the warm-start, every diff --git a/include/bioimage_cpp/graph/multicut/greedy_additive.hxx b/include/bioimage_cpp/graph/multicut/greedy_additive.hxx index 259a1c6..f461a5e 100644 --- a/include/bioimage_cpp/graph/multicut/greedy_additive.hxx +++ b/include/bioimage_cpp/graph/multicut/greedy_additive.hxx @@ -1,29 +1,22 @@ #pragma once +#include "bioimage_cpp/detail/profile.hxx" #include "bioimage_cpp/graph/multicut/detail.hxx" #include "bioimage_cpp/graph/multicut/objective.hxx" #include #include +#include #include namespace bioimage_cpp::graph::multicut { -// Reusable scratch state for `greedy_additive`. Construct once and call -// `greedy_additive(..., workspace)` repeatedly to avoid per-call allocation -// of the DynamicGraph, UnionFind, and EdgeHeap. Capacities only grow; the -// internal vectors are reset (not freed) between calls. +// Reusable scratch state for `greedy_additive`. struct GreedyAdditiveWorkspace { - detail::DynamicGraph dynamic_graph; - bioimage_cpp::util::UnionFind union_find{0}; - detail::EdgeHeap heap; + detail::ContractionState state; void reset(const UndirectedGraph &graph) { - dynamic_graph.reset(graph); - union_find.reset(static_cast(graph.number_of_nodes())); - // The heap is sized by detail::initialize_dynamic_graph, which every - // caller runs immediately after reset(); resizing it here too would - // wipe the locator vector twice. + state.reset(graph); } }; @@ -37,26 +30,39 @@ inline std::vector greedy_additive( const double sigma, GreedyAdditiveWorkspace &workspace ) { + BIOIMAGE_PROFILE_INIT(profile); validate_costs(graph, costs); - workspace.reset(graph); - auto &dynamic_graph = workspace.dynamic_graph; - auto &sets = workspace.union_find; - auto &heap = workspace.heap; - detail::initialize_dynamic_graph(graph, costs, dynamic_graph, heap, false, add_noise, seed, sigma); + { + BIOIMAGE_PROFILE_SCOPE(profile, "workspace_reset"); + workspace.reset(graph); + } + auto &state = workspace.state; + { + BIOIMAGE_PROFILE_SCOPE(profile, "initialize"); + detail::initialize_contraction_state( + graph, costs, state, false, add_noise, seed, sigma + ); + } - while (!heap.empty() && dynamic_graph.alive_count > 1) { - const auto top = heap.top(); + while (!state.heap.empty() && state.topology.number_of_nodes() > 1) { + const auto top = state.heap.top(); if (top.priority <= weight_stop) { break; } if (node_num_stop > 0.0 - && dynamic_graph.alive_count <= detail::stop_node_count(graph, node_num_stop)) { + && state.topology.number_of_nodes() + <= detail::stop_node_count(graph, node_num_stop)) { break; } - const auto &edge = dynamic_graph.edges[top.key]; - detail::merge_dynamic_nodes(dynamic_graph, sets, heap, edge.u, edge.v, false); + detail::contract_edge(state, top.key, false, profile); } - return detail::labels_from_sets(sets, graph); + std::vector labels; + { + BIOIMAGE_PROFILE_SCOPE(profile, "labels_from_sets"); + labels = detail::labels_from_sets(state.union_find, graph); + } + BIOIMAGE_PROFILE_REPORT(profile); + return labels; } inline std::vector greedy_additive( @@ -74,7 +80,7 @@ inline std::vector greedy_additive( ); } -class GreedyAdditiveSolver final : public SolverBase { +class GreedyAdditiveSolver final : public CloneableSolverBase { public: GreedyAdditiveSolver( const double weight_stop = 0.0, @@ -104,6 +110,10 @@ public: return labels; } + std::unique_ptr clone() const override { + return std::make_unique(*this); + } + private: double weight_stop_; double node_num_stop_; diff --git a/include/bioimage_cpp/graph/multicut/greedy_fixation.hxx b/include/bioimage_cpp/graph/multicut/greedy_fixation.hxx index 948d41a..6938322 100644 --- a/include/bioimage_cpp/graph/multicut/greedy_fixation.hxx +++ b/include/bioimage_cpp/graph/multicut/greedy_fixation.hxx @@ -1,10 +1,12 @@ #pragma once +#include "bioimage_cpp/detail/profile.hxx" #include "bioimage_cpp/graph/multicut/detail.hxx" #include "bioimage_cpp/graph/multicut/objective.hxx" #include #include +#include #include namespace bioimage_cpp::graph::multicut { @@ -15,37 +17,49 @@ inline std::vector greedy_fixation( const double weight_stop, const double node_num_stop ) { + BIOIMAGE_PROFILE_INIT(profile); validate_costs(graph, costs); - detail::DynamicGraph dynamic_graph(graph); - bioimage_cpp::util::UnionFind sets(static_cast(graph.number_of_nodes())); - detail::EdgeHeap heap; - detail::initialize_dynamic_graph(graph, costs, dynamic_graph, heap, true); + detail::ContractionState state; + { + BIOIMAGE_PROFILE_SCOPE(profile, "workspace_reset"); + state.reset(graph); + } + { + BIOIMAGE_PROFILE_SCOPE(profile, "initialize"); + detail::initialize_contraction_state(graph, costs, state, true); + } - while (!heap.empty() && dynamic_graph.alive_count > 1) { - const auto top = heap.top(); + while (!state.heap.empty() && state.topology.number_of_nodes() > 1) { + const auto top = state.heap.top(); // Priority is |weight|, so this also handles weight == 0 (stop). if (top.priority <= weight_stop) { break; } if (node_num_stop > 0.0 - && dynamic_graph.alive_count <= detail::stop_node_count(graph, node_num_stop)) { + && state.topology.number_of_nodes() + <= detail::stop_node_count(graph, node_num_stop)) { break; } const auto edge_id = top.key; - const auto &edge = dynamic_graph.edges[edge_id]; - if (edge.weight > 0.0) { - detail::merge_dynamic_nodes(dynamic_graph, sets, heap, edge.u, edge.v, true); + auto &objective_edge = state.topology.edge_payload(edge_id); + if (objective_edge.weight > 0.0) { + detail::contract_edge(state, edge_id, true, profile); } else { - // weight < 0: forbid merging through this edge. merge_dynamic_nodes - // propagates the flag onto any merged successor edges. - heap.pop(); - dynamic_graph.edges[edge_id].is_constraint = 1; + // A negative edge installs a persistent constraint. + state.heap.pop(); + objective_edge.is_constraint = 1; } } - return detail::labels_from_sets(sets, graph); + std::vector labels; + { + BIOIMAGE_PROFILE_SCOPE(profile, "labels_from_sets"); + labels = detail::labels_from_sets(state.union_find, graph); + } + BIOIMAGE_PROFILE_REPORT(profile); + return labels; } -class GreedyFixationSolver final : public SolverBase { +class GreedyFixationSolver final : public CloneableSolverBase { public: GreedyFixationSolver(const double weight_stop = 0.0, const double node_num_stop = -1.0) : weight_stop_(weight_stop), @@ -58,6 +72,10 @@ public: return labels; } + std::unique_ptr clone() const override { + return std::make_unique(*this); + } + private: double weight_stop_; double node_num_stop_; diff --git a/include/bioimage_cpp/graph/multicut/kernighan_lin.hxx b/include/bioimage_cpp/graph/multicut/kernighan_lin.hxx index 4b8cc9f..7cc8e53 100644 --- a/include/bioimage_cpp/graph/multicut/kernighan_lin.hxx +++ b/include/bioimage_cpp/graph/multicut/kernighan_lin.hxx @@ -2,12 +2,16 @@ #include "bioimage_cpp/detail/edge_hash.hxx" #include "bioimage_cpp/detail/indexed_heap.hxx" -#include "bioimage_cpp/util/union_find.hxx" +#include "bioimage_cpp/detail/profile.hxx" +#include "bioimage_cpp/detail/relabel.hxx" +#include "bioimage_cpp/graph/multicut/greedy_additive.hxx" #include "bioimage_cpp/graph/multicut/objective.hxx" +#include "bioimage_cpp/util/union_find.hxx" #include #include #include +#include #include #include #include @@ -69,9 +73,7 @@ inline std::vector> build_cluster_to_nodes( // Per-node scratch reused across chains. // // - `in_pair` : 1 while the node sits in (A ∪ B) for the current chain. -// - `moved` : 1 once the node has been popped (tentatively moved this -// chain). Pairs with `in_pair` to distinguish "moved" from -// "non-bordered, never pushed". +// - `moved` : 1 once the node has been popped. // - `cross_count` : number of cross-side bipartition neighbors. Matches // nifty's `referenced_by`: a node is only allowed in the // heap once this is positive, which restricts the chain to @@ -97,11 +99,22 @@ struct ChainBuffers { stash_gain(n_nodes, 0.0) {} }; +struct FilteredAdjacency { + std::uint64_t node; + double weight; +}; + struct ChainScratch { std::vector queue_nodes; bioimage_cpp::detail::DenseIndexedHeap heap; - - explicit ChainScratch(const std::size_t n_nodes) : heap(n_nodes) {} + std::vector filtered_offset; + std::vector filtered_count; + std::vector filtered_entries; + + explicit ChainScratch(const std::size_t n_nodes) + : heap(n_nodes), + filtered_offset(n_nodes, 0), + filtered_count(n_nodes, 0) {} }; // Run a Kernighan-Lin move-chain on the bipartition (cluster_a, cluster_b). @@ -115,6 +128,7 @@ struct ChainScratch { // Also handles single-cluster splits: pass `cluster_b` as a fresh label // (no live members) and the chain will try to peel off a subset of `cluster_a` // into the new label. +template inline double run_chain( const UndirectedGraph &graph, const std::vector &costs, @@ -124,7 +138,8 @@ inline double run_chain( ChainScratch &scratch, const std::uint64_t cluster_a, const std::uint64_t cluster_b, - const double epsilon + const double epsilon, + [[maybe_unused]] ProfilerT &profile ) { if (cluster_a == cluster_b) { return 0.0; @@ -132,27 +147,32 @@ inline double run_chain( auto &queue_nodes = scratch.queue_nodes; auto &heap = scratch.heap; + auto &filtered_entries = scratch.filtered_entries; queue_nodes.clear(); heap.clear(); - - const auto &stale_a = cluster_to_nodes[static_cast(cluster_a)]; - const auto &stale_b = cluster_to_nodes[static_cast(cluster_b)]; - queue_nodes.reserve(stale_a.size() + stale_b.size()); + filtered_entries.clear(); std::size_t live_a = 0; std::size_t live_b = 0; - for (const auto v : stale_a) { - if (labels[static_cast(v)] == cluster_a) { - queue_nodes.push_back(v); - bufs.in_pair[static_cast(v)] = 1; - ++live_a; + { + BIOIMAGE_PROFILE_SCOPE(profile, "chain_init"); + const auto &stale_a = cluster_to_nodes[static_cast(cluster_a)]; + const auto &stale_b = cluster_to_nodes[static_cast(cluster_b)]; + queue_nodes.reserve(stale_a.size() + stale_b.size()); + + for (const auto v : stale_a) { + if (labels[static_cast(v)] == cluster_a) { + queue_nodes.push_back(v); + bufs.in_pair[static_cast(v)] = 1; + ++live_a; + } } - } - for (const auto v : stale_b) { - if (labels[static_cast(v)] == cluster_b) { - queue_nodes.push_back(v); - bufs.in_pair[static_cast(v)] = 1; - ++live_b; + for (const auto v : stale_b) { + if (labels[static_cast(v)] == cluster_b) { + queue_nodes.push_back(v); + bufs.in_pair[static_cast(v)] = 1; + ++live_b; + } } } // Skip if no non-trivial move exists: the cluster_b == fresh-label split @@ -169,34 +189,46 @@ inline double run_chain( // A node is eligible because there is no border yet — the first move has // to peel off the weakest-attached interior node. const bool is_split = (live_b == 0); - for (const auto v : queue_nodes) { - double w_to_a = 0.0; - double w_to_b = 0.0; - std::uint32_t cross = 0; - const auto v_label = labels[static_cast(v)]; - for (const auto adj : graph.node_adjacency(v)) { - const auto u_key = static_cast(adj.node); - if (!bufs.in_pair[u_key]) { - continue; - } - const auto c = costs[static_cast(adj.edge)]; - const auto u_label = labels[u_key]; - if (u_label == cluster_a) { - w_to_a += c; - } else { - w_to_b += c; + { + BIOIMAGE_PROFILE_SCOPE(profile, "chain_gain_init"); + for (const auto v : queue_nodes) { + double w_to_a = 0.0; + double w_to_b = 0.0; + std::uint32_t cross = 0; + const auto v_label = labels[static_cast(v)]; + const auto v_key = static_cast(v); + const auto filter_start = filtered_entries.size(); + for (const auto adj : graph.node_adjacency(v)) { + const auto u_key = static_cast(adj.node); + if (!bufs.in_pair[u_key]) { + continue; + } + const auto c = costs[static_cast(adj.edge)]; + const auto u_label = labels[u_key]; + filtered_entries.push_back({adj.node, c}); + if (u_label == cluster_a) { + w_to_a += c; + } else { + w_to_b += c; + } + if (u_label != v_label) { + ++cross; + } } - if (u_label != v_label) { - ++cross; + scratch.filtered_offset[v_key] = + static_cast(filter_start); + scratch.filtered_count[v_key] = + static_cast( + filtered_entries.size() - filter_start + ); + const double gain_v = + (v_label == cluster_a) ? (w_to_b - w_to_a) : (w_to_a - w_to_b); + bufs.stash_gain[v_key] = gain_v; + bufs.cross_count[v_key] = cross; + if (is_split || cross > 0) { + heap.push(v_key, gain_v); } } - const double gain_v = (v_label == cluster_a) ? (w_to_b - w_to_a) : (w_to_a - w_to_b); - const auto v_key = static_cast(v); - bufs.stash_gain[v_key] = gain_v; - bufs.cross_count[v_key] = cross; - if (is_split || cross > 0) { - heap.push(v_key, gain_v); - } } struct Move { @@ -210,61 +242,75 @@ inline double run_chain( double best_cumulative = 0.0; std::size_t best_prefix = 0; - while (!heap.empty()) { - const auto top = heap.pop(); - const auto v = static_cast(top.key); - const auto gain_v = top.priority; - const auto v_key = static_cast(v); - const auto old_label = labels[v_key]; - const auto new_label = (old_label == cluster_a) ? cluster_b : cluster_a; - - bufs.moved[v_key] = 1; - cumulative += gain_v; - chain.push_back({v, new_label}); - - if (cumulative > best_cumulative + epsilon) { - best_cumulative = cumulative; - best_prefix = chain.size(); - } - - for (const auto adj : graph.node_adjacency(v)) { - const auto u_key = static_cast(adj.node); - if (!bufs.in_pair[u_key] || bufs.moved[u_key]) { - continue; + { + BIOIMAGE_PROFILE_SCOPE(profile, "chain_loop"); + while (!heap.empty()) { + const auto top = heap.pop(); + const auto v = static_cast(top.key); + const auto gain_v = top.priority; + const auto v_key = static_cast(v); + const auto old_label = labels[v_key]; + const auto new_label = (old_label == cluster_a) ? cluster_b : cluster_a; + + bufs.moved[v_key] = 1; + cumulative += gain_v; + chain.push_back({v, new_label}); + + if (cumulative > best_cumulative + epsilon) { + best_cumulative = cumulative; + best_prefix = chain.size(); } - const auto c = costs[static_cast(adj.edge)]; - const auto u_label = labels[u_key]; - const double delta = (u_label == old_label) ? 2.0 * c : -2.0 * c; - bufs.stash_gain[u_key] += delta; - if (heap.contains(u_key)) { - heap.change(u_key, bufs.stash_gain[u_key]); - } - // Border maintenance. For pair-chains, only nodes that are - // currently bordered may be popped. A node becomes bordered when - // it gains its first cross-side neighbor (cross_count 0 -> 1) and - // un-borders when it loses its last (cross_count -> 0). - if (u_label == old_label) { - ++bufs.cross_count[u_key]; - if (!is_split && !heap.contains(u_key)) { - heap.push(u_key, bufs.stash_gain[u_key]); + + const auto offset = scratch.filtered_offset[v_key]; + const auto count = scratch.filtered_count[v_key]; + for (std::uint32_t index = 0; index < count; ++index) { + const auto &adjacency = filtered_entries[offset + index]; + const auto u_key = + static_cast(adjacency.node); + if (bufs.moved[u_key]) { + continue; } - } else { - if (bufs.cross_count[u_key] > 0) { - --bufs.cross_count[u_key]; + const auto u_label = labels[u_key]; + const double delta = (u_label == old_label) + ? 2.0 * adjacency.weight + : -2.0 * adjacency.weight; + bufs.stash_gain[u_key] += delta; + // Each unmoved split node stays in the heap. An unmoved pair + // node is in the heap exactly when it has a cross-side edge. + const bool was_in_heap = + is_split || bufs.cross_count[u_key] > 0; + if (was_in_heap) { + heap.change(u_key, bufs.stash_gain[u_key]); } - if (!is_split && bufs.cross_count[u_key] == 0 - && heap.contains(u_key)) { - heap.erase(u_key); + // Border maintenance. For pair-chains, only nodes that are + // currently bordered may be popped. + if (u_label == old_label) { + ++bufs.cross_count[u_key]; + if (!is_split && !was_in_heap) { + heap.push(u_key, bufs.stash_gain[u_key]); + } + } else { + if (bufs.cross_count[u_key] > 0) { + --bufs.cross_count[u_key]; + } + if (!is_split && bufs.cross_count[u_key] == 0 + && was_in_heap) { + heap.erase(u_key); + } } } } } - for (const auto v : queue_nodes) { - const auto v_key = static_cast(v); - bufs.in_pair[v_key] = 0; - bufs.moved[v_key] = 0; - bufs.cross_count[v_key] = 0; + { + BIOIMAGE_PROFILE_SCOPE(profile, "chain_cleanup"); + for (const auto v : queue_nodes) { + const auto v_key = static_cast(v); + bufs.in_pair[v_key] = 0; + } + for (const auto &move : chain) { + bufs.moved[static_cast(move.node)] = 0; + } } if (best_cumulative > epsilon) { @@ -374,29 +420,68 @@ inline std::vector kernighan_lin( const std::uint64_t number_of_outer_iterations, const double epsilon ) { + BIOIMAGE_PROFILE_INIT(profile); validate_costs(graph, costs); validate_labels(graph, labels); - labels = dense_relabel(labels); + { + BIOIMAGE_PROFILE_SCOPE(profile, "initial_relabel"); + labels = dense_relabel(labels); + } const auto n_nodes = static_cast(graph.number_of_nodes()); detail_kl::ChainBuffers bufs(n_nodes); detail_kl::ChainScratch scratch(n_nodes); + std::vector changed; + auto previous_labels = labels; + for (std::uint64_t iteration = 0; iteration < number_of_outer_iterations; ++iteration) { bool improved = false; - - const auto pairs_for_chain = detail_kl::compute_cluster_pairs(graph, costs, labels); - const auto number_of_clusters = labels.empty() - ? std::uint64_t{0} - : (*std::max_element(labels.begin(), labels.end()) + 1); - auto cluster_to_nodes = detail_kl::build_cluster_to_nodes(labels, number_of_clusters); - + // Disable the gate in the last configured iteration. This preserves + // the fixed output when an unchanged cluster becomes splittable. + const bool use_changed_gate = + iteration + 1 < number_of_outer_iterations; + + std::vector pairs_for_chain; + { + BIOIMAGE_PROFILE_SCOPE(profile, "compute_pairs_chain"); + pairs_for_chain = detail_kl::compute_cluster_pairs(graph, costs, labels); + } + std::uint64_t number_of_clusters = 0; + std::vector> cluster_to_nodes; + { + BIOIMAGE_PROFILE_SCOPE(profile, "build_cluster_index"); + number_of_clusters = labels.empty() + ? std::uint64_t{0} + : (*std::max_element(labels.begin(), labels.end()) + 1); + cluster_to_nodes = + detail_kl::build_cluster_to_nodes(labels, number_of_clusters); + } + if (iteration == 0) { + changed.assign(static_cast(number_of_clusters), 1); + } for (const auto &pair : pairs_for_chain) { + if (use_changed_gate + && !changed[static_cast(pair.a)] + && !changed[static_cast(pair.b)]) { + continue; + } const auto delta = detail_kl::run_chain( - graph, costs, labels, cluster_to_nodes, bufs, scratch, pair.a, pair.b, epsilon + graph, + costs, + labels, + cluster_to_nodes, + bufs, + scratch, + pair.a, + pair.b, + epsilon, + profile ); if (delta > epsilon) { improved = true; + changed[static_cast(pair.a)] = 1; + changed[static_cast(pair.b)] = 1; } } @@ -409,12 +494,25 @@ inline std::vector kernighan_lin( // nodes. std::uint64_t next_label = number_of_clusters; for (std::uint64_t cluster = 0; cluster < number_of_clusters; ++cluster) { + if (use_changed_gate + && !changed[static_cast(cluster)]) { + continue; + } while (true) { if (next_label >= cluster_to_nodes.size()) { cluster_to_nodes.resize(static_cast(next_label) + 1); } const auto delta = detail_kl::run_chain( - graph, costs, labels, cluster_to_nodes, bufs, scratch, cluster, next_label, epsilon + graph, + costs, + labels, + cluster_to_nodes, + bufs, + scratch, + cluster, + next_label, + epsilon, + profile ); if (delta <= epsilon) { break; @@ -424,41 +522,81 @@ inline std::vector kernighan_lin( } } - const auto pairs_for_join = detail_kl::compute_cluster_pairs(graph, costs, labels); + std::vector pairs_for_join; + { + BIOIMAGE_PROFILE_SCOPE(profile, "compute_pairs_join"); + pairs_for_join = detail_kl::compute_cluster_pairs(graph, costs, labels); + } const auto current_number_of_clusters = labels.empty() ? std::uint64_t{0} : (*std::max_element(labels.begin(), labels.end()) + 1); - if (detail_kl::apply_joins(labels, pairs_for_join, current_number_of_clusters, epsilon)) { - improved = true; + { + BIOIMAGE_PROFILE_SCOPE(profile, "joins"); + if (detail_kl::apply_joins( + labels, pairs_for_join, current_number_of_clusters, epsilon + )) { + improved = true; + } } - if (detail_kl::single_node_polish(graph, costs, labels, epsilon)) { - improved = true; + { + BIOIMAGE_PROFILE_SCOPE(profile, "polish"); + if (detail_kl::single_node_polish(graph, costs, labels, epsilon)) { + improved = true; + } } - labels = dense_relabel(labels); + { + BIOIMAGE_PROFILE_SCOPE(profile, "relabel"); + labels = dense_relabel(labels); + } + { + BIOIMAGE_PROFILE_SCOPE(profile, "compute_changed"); + const auto new_number_of_clusters = labels.empty() + ? std::uint64_t{0} + : (*std::max_element(labels.begin(), labels.end()) + 1); + changed = bioimage_cpp::detail::changed_clusters( + labels, previous_labels, new_number_of_clusters + ); + previous_labels = labels; + } if (!improved) { break; } } + BIOIMAGE_PROFILE_REPORT(profile); return labels; } -class KernighanLinSolver final : public SolverBase { +class KernighanLinSolver final : public CloneableSolverBase { public: KernighanLinSolver( const std::uint64_t number_of_outer_iterations = 100, - const double epsilon = 1.0e-6 + const double epsilon = 1.0e-6, + const bool warm_start_greedy = false ) : number_of_outer_iterations_(number_of_outer_iterations), - epsilon_(epsilon) { + epsilon_(epsilon), + warm_start_greedy_(warm_start_greedy) { } std::vector optimize(Objective &objective) const override { + auto initial_labels = objective.labels(); + if (warm_start_greedy_ && is_singleton_labeling(initial_labels)) { + initial_labels = greedy_additive( + objective.graph(), + objective.costs(), + 0.0, + -1.0, + false, + 42, + 1.0 + ); + } auto labels = kernighan_lin( objective.graph(), objective.costs(), - objective.labels(), + std::move(initial_labels), number_of_outer_iterations_, epsilon_ ); @@ -466,9 +604,23 @@ public: return labels; } + std::unique_ptr clone() const override { + return std::make_unique(*this); + } + private: + static bool is_singleton_labeling(const std::vector &labels) { + for (std::size_t index = 0; index < labels.size(); ++index) { + if (labels[index] != static_cast(index)) { + return false; + } + } + return true; + } + std::uint64_t number_of_outer_iterations_; double epsilon_; + bool warm_start_greedy_; }; } // namespace bioimage_cpp::graph::multicut diff --git a/include/bioimage_cpp/graph/multicut/objective.hxx b/include/bioimage_cpp/graph/multicut/objective.hxx index 6a67492..18b4456 100644 --- a/include/bioimage_cpp/graph/multicut/objective.hxx +++ b/include/bioimage_cpp/graph/multicut/objective.hxx @@ -5,6 +5,7 @@ #include #include +#include #include #include #include @@ -111,4 +112,20 @@ public: virtual std::vector optimize(Objective &objective) const = 0; }; +class CloneableSolverBase : public SolverBase { +public: + // Return an independent solver for one worker. + virtual std::unique_ptr clone() const = 0; +}; + +inline std::unique_ptr clone_solver( + const CloneableSolverBase &solver +) { + auto cloned = solver.clone(); + if (cloned == nullptr) { + throw std::runtime_error("solver clone must not be null"); + } + return cloned; +} + } // namespace bioimage_cpp::graph::multicut diff --git a/include/bioimage_cpp/graph/undirected_graph.hxx b/include/bioimage_cpp/graph/undirected_graph.hxx index 715cccd..fa446ad 100644 --- a/include/bioimage_cpp/graph/undirected_graph.hxx +++ b/include/bioimage_cpp/graph/undirected_graph.hxx @@ -35,7 +35,7 @@ struct Adjacency { // CSR is rebuilt lazily: incremental `insert_edge*` only appends to `edges_` // (and `edge_lookup_`) and marks the adjacency dirty; the first subsequent // `node_adjacency` read triggers a single bulk rebuild. Bulk construction -// paths (`from_sorted_unique_edges`, subclass `build_edges` overrides) call +// paths (`from_unique_edges`, subclass `build_edges` overrides) call // `rebuild_adjacency_from_edges()` explicitly, which keeps reads cheap and // thread-safe. // @@ -45,13 +45,10 @@ struct Adjacency { // overwriting `adjacency_offsets_` — which corrupts the CSR (garbage neighbor // ids, out-of-bounds reads) and intermittently segfaults. The rule: // -// Any algorithm that reads `node_adjacency` (directly, or via -// `breadth_first_search`, `extract_subgraph_from_nodes`, or a sub-solver -// such as `multicut::greedy_additive`'s `DynamicGraph::reset`) from -// `parallel_for_chunks` or other threads MUST `freeze()` the graph on the -// calling thread *before* the fan-out. +// Any algorithm that reads `node_adjacency` directly or indirectly from +// worker threads MUST call `freeze()` before the fan-out. // -// Once frozen (or built via `from_sorted_unique_edges`, which rebuilds the CSR +// Once frozen (or built via `from_unique_edges`, which rebuilds the CSR // eagerly), the graph has no mutable read path and is safe to share by // `const&` across reader threads. Graphs built incrementally via `insert_edge*` // (including the `from_edges` binding and `region_adjacency_graph`) start dirty. @@ -166,6 +163,16 @@ public: ); } + [[nodiscard]] std::vector node_degrees() const { + ensure_adjacency_built(); + std::vector result(static_cast(number_of_nodes_)); + for (NodeId node = 0; node < number_of_nodes_; ++node) { + const auto index = static_cast(node); + result[index] = adjacency_offsets_[index + 1] - adjacency_offsets_[index]; + } + return result; + } + virtual EdgeId insert_edge(const NodeId u, const NodeId v) { validate_node(u); validate_node(v); @@ -245,20 +252,15 @@ public: return copy; } - // Fast construction from a pre-sorted, deduplicated edge list. + // Fast construction from a deduplicated edge list. // - // Precondition: `edges` is sorted ascending by `(u, v)` with `u < v` in - // every entry, and contains no duplicates. No node id may equal or exceed - // `number_of_nodes`. The call takes ownership of `edges` and uses it as - // the graph's edge storage, bypassing the per-edge hash dedup that - // `insert_edge` performs — useful when bulk-building a contracted graph - // whose unique edges are already known. + // Precondition: every edge has `u < v`, no edge is duplicated, and all + // node ids are less than `number_of_nodes`. The function preserves input + // edge order and bypasses the hash-based deduplication in `insert_edge`. // - // When `populate_lookup` is false, the edge-lookup hash map is left empty - // and `find_edge`/`insert_edge` are not available on the returned graph. - // Used by the fusion-move contraction primitive, whose sub-solver only - // walks edges and adjacency lists. - static UndirectedGraph from_sorted_unique_edges( + // When `populate_lookup` is false, `find_edge` cannot resolve these edges. + // A later `insert_edge` call cannot deduplicate against them. + static UndirectedGraph from_unique_edges( const NodeId number_of_nodes, std::vector edges, const bool populate_lookup = true @@ -281,6 +283,17 @@ public: return graph; } + // Fast construction when the unique edge list is also sorted by `(u, v)`. + static UndirectedGraph from_sorted_unique_edges( + const NodeId number_of_nodes, + std::vector edges, + const bool populate_lookup = true + ) { + return from_unique_edges( + number_of_nodes, std::move(edges), populate_lookup + ); + } + [[nodiscard]] std::pair, std::vector> extract_subgraph_from_nodes(const std::vector &nodes) const { std::unordered_set node_set; diff --git a/include/bioimage_cpp/label_multiset/downsample.hxx b/include/bioimage_cpp/label_multiset/downsample.hxx index ce814c0..1bafa1e 100644 --- a/include/bioimage_cpp/label_multiset/downsample.hxx +++ b/include/bioimage_cpp/label_multiset/downsample.hxx @@ -4,6 +4,7 @@ #include "bioimage_cpp/blocking.hxx" #include "bioimage_cpp/label_multiset/multiset.hxx" #include "bioimage_cpp/label_multiset/read_subset.hxx" +#include "bioimage_cpp/label_multiset/validation.hxx" #include #include @@ -41,6 +42,12 @@ inline void downsample_multiset( std::vector &new_ids, std::vector &new_counts ) { + validate_downsample_input( + blocking, offsets, entry_sizes, entry_offsets, ids, counts + ); + validate_downsample_output( + blocking, new_argmax, new_offsets, new_entry_offsets + ); using Key = HashKey; std::unordered_map, HashKeyHash> candidate_dict; @@ -56,9 +63,10 @@ inline void downsample_multiset( std::vector this_ids; std::vector this_counts; const auto block = blocking.get_block(static_cast(block_id)); - read_subset_block(block, strides, - offsets, entry_sizes, entry_offsets, ids, counts, - this_ids, this_counts, /*argsort=*/true); + detail::read_subset_block_unchecked( + block, strides, offsets, entry_sizes, entry_offsets, ids, counts, + this_ids, this_counts, /*argsort=*/true + ); IdT max_label{}; CountT max_count{}; diff --git a/include/bioimage_cpp/label_multiset/merger.hxx b/include/bioimage_cpp/label_multiset/merger.hxx index b14d3e2..d7754ef 100644 --- a/include/bioimage_cpp/label_multiset/merger.hxx +++ b/include/bioimage_cpp/label_multiset/merger.hxx @@ -2,11 +2,13 @@ #include "bioimage_cpp/array_view.hxx" #include "bioimage_cpp/label_multiset/multiset.hxx" +#include "bioimage_cpp/label_multiset/validation.hxx" #include #include #include #include +#include #include #include @@ -24,11 +26,12 @@ public: const ConstArrayView &entry_sizes, const ConstArrayView &ids, const ConstArrayView &counts - ) - : offsets_(offsets.data, offsets.data + offsets.shape[0]), - entry_sizes_(entry_sizes.data, entry_sizes.data + entry_sizes.shape[0]), - ids_(ids.data, ids.data + ids.shape[0]), - counts_(counts.data, counts.data + counts.shape[0]) { + ) { + validate_merger_entries(offsets, entry_sizes, ids, counts); + copy_view(offsets, offsets_); + copy_view(entry_sizes, entry_sizes_); + copy_view(ids, ids_); + copy_view(counts, counts_); init_hashed(); } @@ -40,7 +43,7 @@ public: // Ingest a batch of *unique* entries described by (unique_offsets, // entry_sizes, ids, counts), then rewrite `offsets` so that each element // (which was indexed by "entry id within the input batch") becomes the - // absolute byte offset into the deduplicated ids_/counts_ arrays. + // absolute element offset into the deduplicated ids_/counts_ arrays. void update( const ConstArrayView &unique_offsets, const ConstArrayView &batch_entry_sizes, @@ -49,12 +52,15 @@ public: ArrayView &offsets ) { const std::size_t n_entries = static_cast(unique_offsets.shape[0]); - // Maps batch entry id → absolute byte offset in ids_/counts_. - std::unordered_map new_offset_dict; + validate_merger_entries( + unique_offsets, batch_entry_sizes, batch_ids, batch_counts + ); + validate_update_entry_indices(offsets, n_entries); + std::vector new_offsets(n_entries); for (std::size_t entry = 0; entry < n_entries; ++entry) { - const OffsetT off = unique_offsets.data[entry]; - const OffsetT size = batch_entry_sizes.data[entry]; + const auto off = static_cast(unique_offsets.data[entry]); + const auto size = static_cast(batch_entry_sizes.data[entry]); const IdT *ids_begin = batch_ids.data + off; const IdT *ids_end = ids_begin + size; @@ -84,7 +90,7 @@ public: } if (match) { new_entry = false; - new_offset_dict[static_cast(entry)] = static_cast(c_offset); + new_offsets[entry] = static_cast(c_offset); break; } } @@ -95,7 +101,7 @@ public: const std::size_t this_offset = ids_.size(); offsets_.emplace_back(static_cast(this_offset)); entry_sizes_.emplace_back(static_cast(this_size)); - new_offset_dict[static_cast(entry)] = static_cast(this_offset); + new_offsets[entry] = static_cast(this_offset); ids_.insert(ids_.end(), ids_begin, ids_end); counts_.insert(counts_.end(), counts_begin, counts_end); const std::size_t this_id = offsets_.size() - 1; @@ -110,11 +116,23 @@ public: const std::size_t n_off = static_cast(offsets.shape[0]); for (std::size_t i = 0; i < n_off; ++i) { - offsets.data[i] = new_offset_dict[offsets.data[i]]; + offsets.data[i] = new_offsets[static_cast(offsets.data[i])]; } } private: + template + static void copy_view( + const ConstArrayView &view, + std::vector &output + ) { + const auto size = static_cast(view.shape[0]); + output.resize(size); + if (size != 0) { + std::copy_n(view.data, size, output.data()); + } + } + void init_hashed() { const std::size_t n_entries = offsets_.size(); for (std::size_t entry = 0; entry < n_entries; ++entry) { diff --git a/include/bioimage_cpp/label_multiset/read_subset.hxx b/include/bioimage_cpp/label_multiset/read_subset.hxx index 55928af..7c216a4 100644 --- a/include/bioimage_cpp/label_multiset/read_subset.hxx +++ b/include/bioimage_cpp/label_multiset/read_subset.hxx @@ -3,9 +3,11 @@ #include "bioimage_cpp/array_view.hxx" #include "bioimage_cpp/blocking.hxx" #include "bioimage_cpp/label_multiset/multiset.hxx" +#include "bioimage_cpp/label_multiset/validation.hxx" #include #include +#include #include #include @@ -21,13 +23,15 @@ inline void read_subset( std::vector &counts_out, const bool argsort = true ) { + validate_read_subset(offsets, sizes, ids, counts); std::unordered_map count_dict; const std::size_t n_offsets = static_cast(offsets.shape[0]); for (std::size_t off_id = 0; off_id < n_offsets; ++off_id) { const std::size_t offset = static_cast(offsets.data[off_id]); const std::size_t size = static_cast(sizes.data[off_id]); - for (std::size_t pos = offset; pos < offset + size; ++pos) { + for (std::size_t relative = 0; relative < size; ++relative) { + const auto pos = offset + relative; const IdT id = ids.data[pos]; const CountT count = counts.data[pos]; auto it = count_dict.find(id); @@ -52,11 +56,10 @@ inline void read_subset( } } -// Block-aware variant: collect (offset, size) pairs for every spatial position -// in the block, then call the flat overload. C-order strides over the *full* -// spatial domain. +namespace detail { + template -inline void read_subset_block( +inline void read_subset_block_unchecked( const Block &block, const std::vector &strides, const ConstArrayView &offsets, @@ -66,8 +69,13 @@ inline void read_subset_block( const ConstArrayView &counts, std::vector &ids_out, std::vector &counts_out, - const bool argsort = true + const bool argsort ) { + if (strides.size() != block.begin().size()) { + throw std::invalid_argument( + "strides length must match block dimensionality" + ); + } std::unordered_map count_dict; const auto &begin = block.begin(); const auto &end = block.end(); @@ -80,10 +88,16 @@ inline void read_subset_block( for (std::size_t d = 0; d < ndim; ++d) { index += static_cast(coord[d]) * strides[d]; } + if (index >= static_cast(offsets.shape[0])) { + throw std::invalid_argument( + "block coordinates exceed the flat multiset spatial extent" + ); + } const std::size_t off = static_cast(offsets.data[index]); const std::size_t entry_idx = static_cast(entry_offsets.data[index]); const std::size_t size = static_cast(entry_sizes.data[entry_idx]); - for (std::size_t pos = off; pos < off + size; ++pos) { + for (std::size_t relative = 0; relative < size; ++relative) { + const auto pos = off + relative; const IdT id = ids.data[pos]; const CountT count = counts.data[pos]; auto it = count_dict.find(id); @@ -121,6 +135,37 @@ inline void read_subset_block( } } +} // namespace detail + +// Block-aware variant. C-order strides cover the full spatial domain. +template +inline void read_subset_block( + const Block &block, + const std::vector &strides, + const ConstArrayView &offsets, + const ConstArrayView &entry_sizes, + const ConstArrayView &entry_offsets, + const ConstArrayView &ids, + const ConstArrayView &counts, + std::vector &ids_out, + std::vector &counts_out, + const bool argsort = true +) { + validate_flat_multiset(offsets, entry_sizes, entry_offsets, ids, counts); + detail::read_subset_block_unchecked( + block, + strides, + offsets, + entry_sizes, + entry_offsets, + ids, + counts, + ids_out, + counts_out, + argsort + ); +} + inline std::vector c_order_strides_for_shape(const CoordinateVector &shape) { const std::size_t ndim = shape.size(); std::vector strides(ndim, 1); diff --git a/include/bioimage_cpp/label_multiset/validation.hxx b/include/bioimage_cpp/label_multiset/validation.hxx new file mode 100644 index 0000000..4396a98 --- /dev/null +++ b/include/bioimage_cpp/label_multiset/validation.hxx @@ -0,0 +1,324 @@ +#pragma once + +#include "bioimage_cpp/array_view.hxx" +#include "bioimage_cpp/blocking.hxx" + +#include +#include +#include +#include +#include +#include +#include + +namespace bioimage_cpp::label_multiset { + +namespace validation_detail { + +template +inline std::size_t view_length( + const ConstArrayView &view, + const char *name +) { + if (view.ndim() != 1) { + throw std::invalid_argument( + std::string(name) + " must have ndim 1, got ndim=" + + std::to_string(view.ndim()) + ); + } + if (view.shape[0] < 0) { + throw std::invalid_argument(std::string(name) + " has a negative length"); + } + return static_cast(view.shape[0]); +} + +template +inline std::size_t view_length( + const ArrayView &view, + const char *name +) { + if (view.ndim() != 1) { + throw std::invalid_argument( + std::string(name) + " must have ndim 1, got ndim=" + + std::to_string(view.ndim()) + ); + } + if (view.shape[0] < 0) { + throw std::invalid_argument(std::string(name) + " has a negative length"); + } + return static_cast(view.shape[0]); +} + +template +inline std::size_t as_size( + const Integer value, + const char *name, + const std::size_t index +) { + static_assert(std::is_integral_v); + if constexpr (std::is_signed_v) { + if (value < 0) { + throw std::invalid_argument( + std::string(name) + "[" + std::to_string(index) + + "] must be non-negative" + ); + } + } + if (!std::in_range(value)) { + throw std::invalid_argument( + std::string(name) + "[" + std::to_string(index) + + "] does not fit size_t" + ); + } + return static_cast(value); +} + +inline std::size_t full_grid_size(const Blocking &blocking) { + std::size_t result = 1; + const auto &shape = blocking.roi_end(); + for (std::size_t axis = 0; axis < shape.size(); ++axis) { + const auto extent = as_size(shape[axis], "blocking.roi_end", axis); + if (extent != 0 && result > std::numeric_limits::max() / extent) { + throw std::invalid_argument( + "product(blocking.roi_end) does not fit size_t" + ); + } + result *= extent; + } + return result; +} + +} // namespace validation_detail + +template +inline std::size_t validate_flat_storage( + const ConstArrayView &ids, + const ConstArrayView &counts +) { + const auto n_ids = validation_detail::view_length(ids, "ids"); + const auto n_counts = validation_detail::view_length(counts, "counts"); + if (n_ids != n_counts) { + throw std::invalid_argument( + "ids and counts must have the same length, got ids length=" + + std::to_string(n_ids) + ", counts length=" + std::to_string(n_counts) + ); + } + return n_ids; +} + +template +inline void validate_flat_ranges( + const ConstArrayView &offsets, + const ConstArrayView &sizes, + const std::size_t storage_length, + const bool require_nonempty +) { + const auto n_offsets = validation_detail::view_length(offsets, "offsets"); + const auto n_sizes = validation_detail::view_length(sizes, "sizes"); + if (n_offsets != n_sizes) { + throw std::invalid_argument( + "offsets and sizes must have the same length, got offsets length=" + + std::to_string(n_offsets) + ", sizes length=" + std::to_string(n_sizes) + ); + } + for (std::size_t index = 0; index < n_offsets; ++index) { + const auto offset = + validation_detail::as_size(offsets.data[index], "offsets", index); + const auto size = + validation_detail::as_size(sizes.data[index], "sizes", index); + if (require_nonempty && size == 0) { + throw std::invalid_argument( + "sizes[" + std::to_string(index) + "] must be greater than zero" + ); + } + if (offset > storage_length) { + throw std::invalid_argument( + "offsets[" + std::to_string(index) + + "] exceeds flat storage length " + std::to_string(storage_length) + ); + } + if (size > storage_length - offset) { + throw std::invalid_argument( + "range at index " + std::to_string(index) + + " exceeds flat storage length " + std::to_string(storage_length) + ); + } + } +} + +template +inline void validate_read_subset( + const ConstArrayView &offsets, + const ConstArrayView &sizes, + const ConstArrayView &ids, + const ConstArrayView &counts +) { + const auto storage_length = validate_flat_storage(ids, counts); + validate_flat_ranges(offsets, sizes, storage_length, false); +} + +template +inline void validate_flat_multiset( + const ConstArrayView &offsets, + const ConstArrayView &entry_sizes, + const ConstArrayView &entry_offsets, + const ConstArrayView &ids, + const ConstArrayView &counts +) { + const auto n_offsets = validation_detail::view_length(offsets, "offsets"); + const auto n_entry_offsets = + validation_detail::view_length(entry_offsets, "entry_offsets"); + const auto n_entries = + validation_detail::view_length(entry_sizes, "entry_sizes"); + const auto storage_length = validate_flat_storage(ids, counts); + + if (n_offsets != n_entry_offsets) { + throw std::invalid_argument( + "offsets and entry_offsets must have the same length, got offsets length=" + + std::to_string(n_offsets) + ", entry_offsets length=" + + std::to_string(n_entry_offsets) + ); + } + if (n_offsets == 0) { + if (n_entries != 0 || storage_length != 0) { + throw std::invalid_argument( + "an empty multiset must have empty entry_sizes, ids, and counts" + ); + } + return; + } + if (n_entries == 0) { + throw std::invalid_argument( + "entry_sizes must not be empty when spatial entries are present" + ); + } + + std::vector referenced(n_entries, false); + for (std::size_t entry = 0; entry < n_entries; ++entry) { + const auto size = validation_detail::as_size( + entry_sizes.data[entry], "entry_sizes", entry + ); + if (size == 0) { + throw std::invalid_argument( + "entry_sizes[" + std::to_string(entry) + + "] must be greater than zero" + ); + } + } + for (std::size_t spatial = 0; spatial < n_offsets; ++spatial) { + const auto entry = validation_detail::as_size( + entry_offsets.data[spatial], "entry_offsets", spatial + ); + if (entry >= n_entries) { + throw std::invalid_argument( + "entry_offsets[" + std::to_string(spatial) + + "] must be less than entry_sizes length " + + std::to_string(n_entries) + ); + } + referenced[entry] = true; + const auto offset = + validation_detail::as_size(offsets.data[spatial], "offsets", spatial); + const auto size = validation_detail::as_size( + entry_sizes.data[entry], "entry_sizes", entry + ); + if (offset > storage_length) { + throw std::invalid_argument( + "offsets[" + std::to_string(spatial) + + "] exceeds flat storage length " + std::to_string(storage_length) + ); + } + if (size > storage_length - offset) { + throw std::invalid_argument( + "entry range at spatial index " + std::to_string(spatial) + + " exceeds flat storage length " + std::to_string(storage_length) + ); + } + } + for (std::size_t entry = 0; entry < n_entries; ++entry) { + if (!referenced[entry]) { + throw std::invalid_argument( + "entry_sizes[" + std::to_string(entry) + + "] is not referenced by entry_offsets" + ); + } + } +} + +template +inline void validate_downsample_input( + const Blocking &blocking, + const ConstArrayView &offsets, + const ConstArrayView &entry_sizes, + const ConstArrayView &entry_offsets, + const ConstArrayView &ids, + const ConstArrayView &counts +) { + validate_flat_multiset(offsets, entry_sizes, entry_offsets, ids, counts); + const auto n_spatial = validation_detail::view_length(offsets, "offsets"); + const auto expected = validation_detail::full_grid_size(blocking); + if (n_spatial != expected) { + throw std::invalid_argument( + "offsets length must equal product(blocking.roi_end), got offsets length=" + + std::to_string(n_spatial) + ", expected length=" + + std::to_string(expected) + ); + } +} + +template +inline void validate_downsample_output( + const Blocking &blocking, + const ArrayView &new_argmax, + const ArrayView &new_offsets, + const ArrayView &new_entry_offsets +) { + const auto expected = validation_detail::as_size( + blocking.number_of_blocks(), "blocking.number_of_blocks", 0 + ); + const auto n_argmax = + validation_detail::view_length(new_argmax, "new_argmax"); + const auto n_offsets = + validation_detail::view_length(new_offsets, "new_offsets"); + const auto n_entry_offsets = + validation_detail::view_length(new_entry_offsets, "new_entry_offsets"); + if (n_argmax != expected || n_offsets != expected || n_entry_offsets != expected) { + throw std::invalid_argument( + "downsample output lengths must equal blocking.number_of_blocks" + ); + } +} + +template +inline void validate_merger_entries( + const ConstArrayView &unique_offsets, + const ConstArrayView &entry_sizes, + const ConstArrayView &ids, + const ConstArrayView &counts +) { + const auto storage_length = validate_flat_storage(ids, counts); + validate_flat_ranges(unique_offsets, entry_sizes, storage_length, true); +} + +template +inline void validate_update_entry_indices( + const ArrayView &entry_indices, + const std::size_t number_of_entries +) { + const auto n_indices = + validation_detail::view_length(entry_indices, "offsets"); + for (std::size_t index = 0; index < n_indices; ++index) { + const auto entry = validation_detail::as_size( + entry_indices.data[index], "offsets", index + ); + if (entry >= number_of_entries) { + throw std::invalid_argument( + "offsets[" + std::to_string(index) + + "] must be less than the number of batch entries " + + std::to_string(number_of_entries) + ); + } + } +} + +} // namespace bioimage_cpp::label_multiset diff --git a/src/bindings/filters.cxx b/src/bindings/filters.cxx index 34b9761..0eefbf5 100644 --- a/src/bindings/filters.cxx +++ b/src/bindings/filters.cxx @@ -7,6 +7,7 @@ #include #include +#include #include #include @@ -468,9 +469,75 @@ Image structure_tensor_eigenvalues_3d( return out; } +void ev3_symmetric_float32(ConstImage components, Image out) { + const char *fn = "_filters_ev3_symmetric_float32"; + require_ndim(components, 2, fn); + if (components.shape(0) != 6) { + throw std::invalid_argument( + std::string(fn) + ": components must have shape (6, n), got first " + "dimension=" + std::to_string(components.shape(0)) + ); + } + + if (out.ndim() != 2 || out.shape(0) != components.shape(1) || + out.shape(1) != 3) { + throw std::invalid_argument( + std::string(fn) + ": out must have shape (n, 3), got ndim=" + + std::to_string(out.ndim()) + + (out.ndim() == 2 + ? ", shape=(" + std::to_string(out.shape(0)) + ", " + + std::to_string(out.shape(1)) + ")" + : "") + ); + } + + const std::size_t n = components.shape(1); + if (n > static_cast(std::numeric_limits::max())) { + throw std::invalid_argument( + std::string(fn) + ": n is too large" + ); + } + const float *components_ptr = components.data(); + for (std::size_t i = 0; i < 6 * n; ++i) { + if (!std::isfinite(components_ptr[i])) { + throw std::invalid_argument( + std::string(fn) + ": components must contain only finite values" + ); + } + } + + float *out_ptr = out.data(); + { + nb::gil_scoped_release release; + filters::ev3_symmetric_descending_interleaved( + components_ptr, + components_ptr + n, + components_ptr + 2 * n, + components_ptr + 3 * n, + components_ptr + 4 * n, + components_ptr + 5 * n, + out_ptr, + static_cast(n) + ); + } +} + } // namespace void bind_filters(nb::module_ &m) { + m.def( + "_filters_convolution_backend", &filters::convolution_backend, + "Return the selected internal filter convolution backend." + ); + m.def( + "_filters_eigenvalue_backend", &filters::eigenvalue_backend, + "Return the selected internal filter eigenvalue backend." + ); + m.def( + "_filters_ev3_symmetric_float32", &ev3_symmetric_float32, + nb::arg("components"), nb::arg("out"), + "Compute sorted eigenvalues for symmetric 3x3 float32 matrices." + ); m.def( "_gaussian_smoothing_2d_float32", &gaussian_smoothing_2d, nb::arg("image"), nb::arg("sigma_y"), nb::arg("sigma_x"), diff --git a/src/bindings/graph.cxx b/src/bindings/graph.cxx index 76f2764..2fabd5c 100644 --- a/src/bindings/graph.cxx +++ b/src/bindings/graph.cxx @@ -546,6 +546,10 @@ UInt64Array graph_node_adjacency(const Graph &graph, const std::uint64_t node) { return result; } +UInt64Array graph_node_degrees(const Graph &graph) { + return detail::copy_vector_to_array(graph.node_degrees()); +} + UInt64Array graph_serialize(const Graph &graph) { auto result = make_uint64_array({static_cast(graph.serialization_size())}); auto *data = result.data(); @@ -793,6 +797,38 @@ UInt64Array multicut_kernighan_lin( return vector_to_uint64_array(label_vector); } +UInt64Array multicut_decomposer( + const Graph &graph, + ConstDoubleArray costs, + ConstUInt64Array initial_labels, + const graph::multicut::CloneableSolverBase *sub_solver, + const graph::multicut::CloneableSolverBase *fallthrough_solver, + const std::size_t number_of_threads +) { + if (sub_solver == nullptr) { + throw std::invalid_argument("sub_solver must not be null"); + } + auto cost_vector = + double_array_to_vector(costs, "edge_costs", graph.number_of_edges()); + auto label_vector = + uint64_array_to_vector( + initial_labels, "initial_labels", graph.number_of_nodes() + ); + graph::multicut::DecomposerSolver solver( + *sub_solver, fallthrough_solver, number_of_threads + ); + + std::vector result; + { + nb::gil_scoped_release release; + graph::multicut::Objective objective( + graph, std::move(cost_vector), std::move(label_vector) + ); + result = solver.optimize(objective); + } + return vector_to_uint64_array(result); +} + std::pair graph_breadth_first_search( const Graph &graph, const std::uint64_t source, @@ -1895,6 +1931,11 @@ void bind_graph(nb::module_ &m) { .def("insert_edges", &graph_insert_edges, nb::arg("uvs")) .def("find_edges", &graph_find_edges, nb::arg("uvs")) .def("node_adjacency", &graph_node_adjacency, nb::arg("node")) + .def( + "node_degrees", + &graph_node_degrees, + "Return the degree of each node as a uint64 array." + ) .def_prop_ro("serialization_size", &Graph::serialization_size) .def("serialize", &graph_serialize) .def( @@ -2160,10 +2201,17 @@ void bind_graph(nb::module_ &m) { nb::arg("epsilon") ); - // Multicut sub-solver hierarchy used by fusion moves. The classes are - // opaque to Python; constructors carry per-solver settings. + // Native solver hierarchy used by fusion moves and decomposition. The + // classes are opaque to Python; constructors carry solver settings. nb::class_(m, "_MulticutSolverBase"); - nb::class_( + nb::class_< + graph::multicut::CloneableSolverBase, + graph::multicut::SolverBase + >(m, "_CloneableMulticutSolverBase"); + nb::class_< + graph::multicut::GreedyAdditiveSolver, + graph::multicut::CloneableSolverBase + >( m, "_GreedyAdditiveMulticutSubSolver" ) .def( @@ -2174,7 +2222,10 @@ void bind_graph(nb::module_ &m) { nb::arg("seed") = 42, nb::arg("sigma") = 1.0 ); - nb::class_( + nb::class_< + graph::multicut::GreedyFixationSolver, + graph::multicut::CloneableSolverBase + >( m, "_GreedyFixationMulticutSubSolver" ) .def( @@ -2182,14 +2233,41 @@ void bind_graph(nb::module_ &m) { nb::arg("weight_stop") = 0.0, nb::arg("node_num_stop") = -1.0 ); - nb::class_( + nb::class_< + graph::multicut::KernighanLinSolver, + graph::multicut::CloneableSolverBase + >( m, "_KernighanLinMulticutSubSolver" ) .def( - nb::init(), + nb::init(), nb::arg("number_of_outer_iterations") = 100, - nb::arg("epsilon") = 1.0e-6 + nb::arg("epsilon") = 1.0e-6, + nb::arg("warm_start_greedy") = false ); + nb::class_< + graph::multicut::ChainedSolver, + graph::multicut::CloneableSolverBase + >(m, "_ChainedMulticutSubSolver") + .def( + nb::init< + const std::vector< + const graph::multicut::CloneableSolverBase * + > & + >(), + nb::arg("solvers") + ); + + m.def( + "_multicut_decomposer", + &multicut_decomposer, + nb::arg("graph"), + nb::arg("edge_costs"), + nb::arg("initial_labels"), + nb::arg("sub_solver"), + nb::arg("fallthrough_solver") = nullptr, + nb::arg("number_of_threads") = 0 + ); // Proposal generators used by fusion moves. nb::class_(m, "_ProposalGeneratorBase"); diff --git a/src/bindings/graph.hxx b/src/bindings/graph.hxx index 8badc5e..dd4c48f 100644 --- a/src/bindings/graph.hxx +++ b/src/bindings/graph.hxx @@ -5,5 +5,6 @@ namespace bioimage_cpp::bindings { void bind_graph(nanobind::module_ &m); +void bind_graph_contraction(nanobind::module_ &m); } // namespace bioimage_cpp::bindings diff --git a/src/bindings/graph_contraction.cxx b/src/bindings/graph_contraction.cxx new file mode 100644 index 0000000..16480e6 --- /dev/null +++ b/src/bindings/graph_contraction.cxx @@ -0,0 +1,337 @@ +#include "graph.hxx" +#include "ndarray.hxx" + +#include "bioimage_cpp/graph/contraction_graph.hxx" + +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace nb = nanobind; + +namespace bioimage_cpp::bindings { +namespace { + +using Graph = graph::UndirectedGraph; +using ContractionGraph = graph::ContractionGraph; +using UInt64Array = nb::ndarray; + +template +using ConstFloatingArray = nb::ndarray; + +graph::ParallelEdgePolicy parallel_edge_policy_from_code(const int code) { + switch (code) { + case 0: + return graph::ParallelEdgePolicy::Merge; + case 1: + return graph::ParallelEdgePolicy::Keep; + default: + throw std::invalid_argument( + "unknown parallel edge policy code: " + std::to_string(code) + ); + } +} + +graph::Reduction reduction_from_code(const int code) { + switch (code) { + case 0: + return graph::Reduction::Sum; + case 1: + return graph::Reduction::Mean; + case 2: + return graph::Reduction::Minimum; + case 3: + return graph::Reduction::Maximum; + default: + throw std::invalid_argument("unknown reduction code: " + std::to_string(code)); + } +} + +template +void add_values( + ContractionGraph &contraction, + const std::string &name, + const ConstFloatingArray values, + const int reduction_code, + const bool node_values +) { + const auto expected = node_values + ? contraction.nodes().size() + : contraction.edges().size(); + if (values.ndim() == 0) { + throw std::invalid_argument("values must have at least one dimension"); + } + if (values.shape(0) != expected) { + throw std::invalid_argument( + "values.shape[0] must match the original number of " + + std::string(node_values ? "nodes" : "edges") + + ", got " + std::to_string(values.shape(0)) + + " for " + std::to_string(expected) + ); + } + + std::vector component_shape; + component_shape.reserve(values.ndim() - 1); + std::size_t size = 1; + for (std::size_t axis = 0; axis < values.ndim(); ++axis) { + size *= values.shape(axis); + if (axis > 0) { + component_shape.push_back(values.shape(axis)); + } + } + const auto input = std::span(values.data(), size); + if (node_values) { + contraction.add_node_values( + name, + input, + std::move(component_shape), + reduction_from_code(reduction_code) + ); + } else { + contraction.add_edge_values( + name, + input, + std::move(component_shape), + reduction_from_code(reduction_code) + ); + } +} + +UInt64Array ids_to_array(const std::vector &ids) { + return detail::copy_vector_to_array(ids); +} + +UInt64Array adjacency_to_array( + const std::vector &adjacency +) { + auto output = detail::make_array_for_overwrite( + {adjacency.size(), std::size_t{2}} + ); + for (std::size_t index = 0; index < adjacency.size(); ++index) { + output.data()[2 * index] = adjacency[index].node; + output.data()[2 * index + 1] = adjacency[index].edge; + } + return output; +} + +template +nb::object values_to_array(const graph::MaterializedValues &values) { + return nb::cast(detail::copy_vector_to_array(values.values, values.shape)); +} + +nb::object values_to_array(const graph::MaterializedValueMap &values) { + return std::visit( + [](const auto &typed) { + return values_to_array(typed); + }, + values + ); +} + +nb::tuple active_node_values( + const ContractionGraph &contraction, + const std::string &name +) { + return nb::make_tuple( + ids_to_array(contraction.nodes()), + values_to_array(contraction.active_node_values(name)) + ); +} + +nb::tuple active_edge_values( + const ContractionGraph &contraction, + const std::string &name +) { + return nb::make_tuple( + ids_to_array(contraction.edges()), + values_to_array(contraction.active_edge_values(name)) + ); +} + +UInt64Array graph_uv_ids(const Graph &graph) { + auto output = detail::make_array_for_overwrite( + {static_cast(graph.number_of_edges()), std::size_t{2}} + ); + for (std::uint64_t edge = 0; edge < graph.number_of_edges(); ++edge) { + const auto uv = graph.uv(edge); + output.data()[2 * edge] = uv.first; + output.data()[2 * edge + 1] = uv.second; + } + return output; +} + +nb::dict materialized_values_to_dict( + const std::vector &values +) { + nb::dict output; + for (const auto &value : values) { + std::visit( + [&output](const auto &typed) { + output[nb::str(typed.name.c_str())] = values_to_array(typed); + }, + value + ); + } + return output; +} + +nb::tuple materialize(ContractionGraph &contraction) { + auto result = contraction.materialize(); + const auto number_of_nodes = result.graph.number_of_nodes(); + auto uvs = graph_uv_ids(result.graph); + auto node_values = materialized_values_to_dict(result.node_values); + auto edge_values = materialized_values_to_dict(result.edge_values); + auto node_mapping = detail::copy_vector_to_array(result.node_mapping); + auto edge_mapping = detail::copy_vector_to_array(result.edge_mapping); + return nb::make_tuple( + number_of_nodes, + std::move(uvs), + std::move(node_values), + std::move(edge_values), + std::move(node_mapping), + std::move(edge_mapping) + ); +} + +} // namespace + +void bind_graph_contraction(nb::module_ &m) { + nb::class_(m, "_ContractionGraph") + .def( + "__init__", + [](ContractionGraph *self, const Graph &graph, const int parallel_edges) { + new (self) ContractionGraph( + graph, + parallel_edge_policy_from_code(parallel_edges) + ); + }, + nb::arg("graph"), + nb::arg("parallel_edges") = 0 + ) + .def_prop_ro("number_of_nodes", &ContractionGraph::number_of_nodes) + .def_prop_ro("number_of_edges", &ContractionGraph::number_of_edges) + .def("nodes", &ContractionGraph::nodes) + .def("edges", &ContractionGraph::edges) + .def("uv", &ContractionGraph::uv, nb::arg("edge")) + .def("find_edge", &ContractionGraph::find_edge, nb::arg("u"), nb::arg("v")) + .def( + "find_edges", + [](const ContractionGraph &self, const std::uint64_t u, + const std::uint64_t v) { + return ids_to_array(self.find_edges(u, v)); + }, + nb::arg("u"), + nb::arg("v") + ) + .def( + "node_adjacency", + [](const ContractionGraph &self, const std::uint64_t node) { + return adjacency_to_array(self.node_adjacency(node)); + }, + nb::arg("node") + ) + .def("degree", &ContractionGraph::degree, nb::arg("node")) + .def( + "can_suppress_node", + &ContractionGraph::can_suppress_node, + nb::arg("node") + ) + .def("is_node_active", &ContractionGraph::node_active, nb::arg("node")) + .def("is_edge_active", &ContractionGraph::edge_active, nb::arg("edge")) + .def( + "representative", + &ContractionGraph::representative, + nb::arg("original_node") + ) + .def( + "_add_node_values_float32", + [](ContractionGraph &self, const std::string &name, + const ConstFloatingArray values, const int reduction) { + add_values(self, name, values, reduction, true); + }, + nb::arg("name"), + nb::arg("values"), + nb::arg("reduction") + ) + .def( + "_add_node_values_float64", + [](ContractionGraph &self, const std::string &name, + const ConstFloatingArray values, const int reduction) { + add_values(self, name, values, reduction, true); + }, + nb::arg("name"), + nb::arg("values"), + nb::arg("reduction") + ) + .def( + "_add_edge_values_float32", + [](ContractionGraph &self, const std::string &name, + const ConstFloatingArray values, const int reduction) { + add_values(self, name, values, reduction, false); + }, + nb::arg("name"), + nb::arg("values"), + nb::arg("reduction") + ) + .def( + "_add_edge_values_float64", + [](ContractionGraph &self, const std::string &name, + const ConstFloatingArray values, const int reduction) { + add_values(self, name, values, reduction, false); + }, + nb::arg("name"), + nb::arg("values"), + nb::arg("reduction") + ) + .def( + "contract_edge", + &ContractionGraph::contract_edge, + nb::arg("edge"), + nb::arg("keep_node") = std::nullopt + ) + .def("erase_edge", &ContractionGraph::erase_edge, nb::arg("edge")) + .def("suppress_node", &ContractionGraph::suppress_node, nb::arg("node")) + .def( + "node_value", + [](const ContractionGraph &self, const std::string &name, + const std::uint64_t node) { + return values_to_array(self.node_value(name, node)); + }, + nb::arg("name"), + nb::arg("node") + ) + .def( + "edge_value", + [](const ContractionGraph &self, const std::string &name, + const std::uint64_t edge) { + return values_to_array(self.edge_value(name, edge)); + }, + nb::arg("name"), + nb::arg("edge") + ) + .def( + "active_node_values", + &active_node_values, + nb::arg("name") + ) + .def( + "active_edge_values", + &active_edge_values, + nb::arg("name") + ) + .def("materialize", &materialize); +} + +} // namespace bioimage_cpp::bindings diff --git a/src/bindings/label_multiset.cxx b/src/bindings/label_multiset.cxx index 142a500..8b0884e 100644 --- a/src/bindings/label_multiset.cxx +++ b/src/bindings/label_multiset.cxx @@ -7,6 +7,7 @@ #include "bioimage_cpp/label_multiset/from_labels.hxx" #include "bioimage_cpp/label_multiset/merger.hxx" #include "bioimage_cpp/label_multiset/read_subset.hxx" +#include "bioimage_cpp/label_multiset/validation.hxx" #include #include @@ -77,12 +78,7 @@ bind_read_subset_flat(OffsetArray offsets, OffsetArray sizes, auto sizes_v = view_1d(sizes, "sizes"); auto ids_v = view_1d(ids, "ids"); auto counts_v = view_1d(counts, "counts"); - if (offsets_v.shape[0] != sizes_v.shape[0]) { - throw std::invalid_argument("offsets and sizes must have the same length"); - } - if (ids_v.shape[0] != counts_v.shape[0]) { - throw std::invalid_argument("ids and counts must have the same length"); - } + label_multiset::validate_read_subset(offsets_v, sizes_v, ids_v, counts_v); std::vector ids_out; std::vector counts_out; @@ -115,14 +111,9 @@ bind_downsample_multiset(const Blocking &blocking, auto entry_offsets_v = view_1d(entry_offsets, "entry_offsets"); auto ids_v = view_1d(ids, "ids"); auto counts_v = view_1d(counts, "counts"); - if (ids_v.shape[0] != counts_v.shape[0]) { - throw std::invalid_argument("ids and counts must have the same length"); - } - if (offsets_v.shape[0] != entry_offsets_v.shape[0]) { - throw std::invalid_argument( - "offsets and entry_offsets must have the same length" - ); - } + label_multiset::validate_downsample_input( + blocking, offsets_v, entry_sizes_v, entry_offsets_v, ids_v, counts_v + ); const std::size_t n_blocks = static_cast(blocking.number_of_blocks()); @@ -255,6 +246,12 @@ class PyMultisetMerger { offsets.data(), {static_cast(offsets.shape(0))}, {}}; + label_multiset::validate_merger_entries( + uo_v, es_v, ids_v, counts_v + ); + label_multiset::validate_update_entry_indices( + off_view, static_cast(uo_v.shape[0]) + ); { nb::gil_scoped_release release; merger_.update(uo_v, es_v, ids_v, counts_v, off_view); diff --git a/src/bindings/module.cxx b/src/bindings/module.cxx index 1c8b56e..8a95fbe 100644 --- a/src/bindings/module.cxx +++ b/src/bindings/module.cxx @@ -25,6 +25,7 @@ NB_MODULE(_core, m) { bioimage_cpp::bindings::bind_filters(m); bioimage_cpp::bindings::bind_flow(m); bioimage_cpp::bindings::bind_graph(m); + bioimage_cpp::bindings::bind_graph_contraction(m); bioimage_cpp::bindings::bind_ground_truth(m); bioimage_cpp::bindings::bind_label_multiset(m); bioimage_cpp::bindings::bind_mesh(m); diff --git a/src/bioimage_cpp/_validation.py b/src/bioimage_cpp/_validation.py index 4489b17..e5e7456 100644 --- a/src/bioimage_cpp/_validation.py +++ b/src/bioimage_cpp/_validation.py @@ -27,8 +27,6 @@ def strict_integer_array(values, name: str, *, dtype: np.dtype, shape: tuple[int | None, ...] | None = None, non_negative: bool = False) -> np.ndarray: array = np.asarray(values) - if not np.issubdtype(array.dtype, np.integer) or np.issubdtype(array.dtype, np.bool_): - raise TypeError(f"{name} must contain integers, got dtype={array.dtype}") if ndim is not None and array.ndim != ndim: raise ValueError(f"{name} must have ndim={ndim}, got ndim={array.ndim}") if shape is not None: @@ -37,11 +35,36 @@ def strict_integer_array(values, name: str, *, dtype: np.dtype, for actual, expected in zip(array.shape, shape) ): raise ValueError(f"{name} must have shape {shape}, got shape={array.shape}") + target = np.dtype(dtype) + if array.dtype == np.dtype(object): + converted = np.empty(array.shape, dtype=object) + for index, value in enumerate(array.flat): + if isinstance(value, (bool, np.bool_)): + raise TypeError(f"{name} must contain integers, got bool") + try: + converted.flat[index] = operator.index(value) + except TypeError as error: + raise TypeError( + f"{name} must contain integers, got value={value!r}" + ) from error + if converted.size: + info = np.iinfo(target) + minimum = min(converted.flat) + maximum = max(converted.flat) + if non_negative and minimum < 0: + raise ValueError(f"{name} must contain non-negative integers") + if minimum < info.min or maximum > info.max: + raise ValueError( + f"{name} values must fit dtype {target}, got range " + f"[{minimum}, {maximum}]" + ) + return np.ascontiguousarray(converted, dtype=target) + if not np.issubdtype(array.dtype, np.integer) or np.issubdtype(array.dtype, np.bool_): + raise TypeError(f"{name} must contain integers, got dtype={array.dtype}") if non_negative and np.issubdtype(array.dtype, np.signedinteger): if array.size and np.any(array < 0): raise ValueError(f"{name} must contain non-negative integers") - target = np.dtype(dtype) # Fast path: a lossless (safe) cast is representable for every value by # definition, so skip the O(n) min()/max() range scan. This keeps hot # callers such as `objective.energy(labels)` (uint64->uint64, diff --git a/src/bioimage_cpp/graph/__init__.py b/src/bioimage_cpp/graph/__init__.py index e6fc841..6380e96 100644 --- a/src/bioimage_cpp/graph/__init__.py +++ b/src/bioimage_cpp/graph/__init__.py @@ -2,8 +2,9 @@ Top-level surface: -- Graph structures: :class:`UndirectedGraph`, :class:`GridGraph2D`, - :class:`GridGraph3D`, :class:`RegionAdjacencyGraph`. +- Graph structures: :class:`UndirectedGraph`, :class:`ContractionGraph`, + :class:`GridGraph2D`, :class:`GridGraph3D`, + :class:`RegionAdjacencyGraph`. - Constructors: :func:`undirected_graph`, :func:`grid_graph`, :func:`region_adjacency_graph`. - Algorithms: :func:`connected_components`, :func:`breadth_first_search`, @@ -40,6 +41,8 @@ from __future__ import annotations +from dataclasses import dataclass + import numpy as np from .. import _core @@ -54,6 +57,8 @@ _as_uv_array, _normalize_labels, _normalize_number_of_threads, + _require_finite_weights, + _resolve_weight_dtype, ) @@ -120,6 +125,261 @@ def deserialize(cls, serialization): return cls.from_edges(number_of_nodes, uvs) +@dataclass(frozen=True) +class ContractionResult: + """Materialized graph and values from a :class:`ContractionGraph`. + + Rows in each node or edge value array match the corresponding ids in + ``graph``. ``node_mapping`` and ``edge_mapping`` map ids from the input + graph to these dense ids. Multiple input ids can map to one output id. An + entry is ``-1`` when no materialized item represents the input item. + """ + + graph: UndirectedGraph + node_values: dict[str, np.ndarray] + edge_values: dict[str, np.ndarray] + node_mapping: np.ndarray + edge_mapping: np.ndarray + + +_CONTRACTION_REDUCTIONS = { + "sum": 0, + "mean": 1, + "min": 2, + "max": 3, +} + +_CONTRACTION_PARALLEL_EDGES = { + "merge": 0, + "keep": 1, +} + + +def _contraction_values(values, name: str, expected_items: int) -> np.ndarray: + array = _resolve_weight_dtype(values, name) + if array.ndim == 0: + raise ValueError(f"{name} must have at least one dimension") + if array.shape[0] != expected_items: + raise ValueError( + f"{name}.shape[0] must be {expected_items}, got {array.shape[0]}" + ) + return np.ascontiguousarray(array) + + +def _contraction_reduction(reduction: str) -> int: + try: + return _CONTRACTION_REDUCTIONS[reduction] + except KeyError as error: + supported = ", ".join(_CONTRACTION_REDUCTIONS) + raise ValueError( + f"reduction must be one of ({supported}), got {reduction!r}" + ) from error + + +def _contraction_parallel_edges(parallel_edges: str) -> int: + try: + return _CONTRACTION_PARALLEL_EDGES[parallel_edges] + except KeyError as error: + supported = ", ".join(_CONTRACTION_PARALLEL_EDGES) + raise ValueError( + f"parallel_edges must be one of ({supported}), got {parallel_edges!r}" + ) from error + + +class ContractionGraph: + """Mutable graph for edge contractions and value reduction. + + The graph copies the input topology. Node and edge ids stay stable while + they are active. Mutations can leave gaps in the id range. The input graph + must be simple. + + Register value maps before the first mutation. A map must have shape + ``(number_of_items, ...)`` and a floating dtype. ``float32`` and + ``float64`` are preserved. The reduction is component-wise: + + - ``"sum"`` adds represented values. + - ``"mean"`` computes the mean over represented input items. + - ``"min"`` and ``"max"`` select the extrema. + + Node maps reduce when an edge contraction merges its endpoints. Edge maps + reduce when two surviving edges are folded into one. The contracted edge + has no surviving target, so its value is discarded. + + ``parallel_edges="merge"`` folds parallel edges during each mutation. + ``parallel_edges="keep"`` keeps them separate in the mutable graph. + :meth:`materialize` always creates a simple graph and folds remaining + parallel edges only in the returned snapshot. + + This class is not thread-safe. + """ + + def __init__( + self, + graph: UndirectedGraph, + *, + parallel_edges: str = "merge", + ): + parallel_edges_code = _contraction_parallel_edges(parallel_edges) + self._core = _core._ContractionGraph(graph, parallel_edges_code) + self._original_number_of_nodes = int(graph.number_of_nodes) + self._original_number_of_edges = int(graph.number_of_edges) + self._parallel_edges = parallel_edges + self._mutation_started = False + + @property + def parallel_edges(self) -> str: + """Policy for parallel edges created by mutations.""" + return self._parallel_edges + + @property + def number_of_nodes(self) -> int: + """Number of active nodes.""" + return int(self._core.number_of_nodes) + + @property + def number_of_edges(self) -> int: + """Number of active edges.""" + return int(self._core.number_of_edges) + + def nodes(self) -> np.ndarray: + """Return active stable node ids in ascending order.""" + return np.asarray(self._core.nodes(), dtype=np.uint64) + + def edges(self) -> np.ndarray: + """Return active stable edge ids in ascending order.""" + return np.asarray(self._core.edges(), dtype=np.uint64) + + def uv(self, edge: int) -> tuple[int, int]: + u, v = self._core.uv(int(edge)) + return int(u), int(v) + + def find_edge(self, u: int, v: int) -> int: + """Return the smallest matching active edge id, or ``-1``.""" + return int(self._core.find_edge(int(u), int(v))) + + def find_edges(self, u: int, v: int) -> np.ndarray: + """Return all matching active edge ids in ascending order.""" + return np.asarray(self._core.find_edges(int(u), int(v)), dtype=np.uint64) + + def node_adjacency(self, node: int) -> np.ndarray: + """Return ``(neighbor, edge)`` rows for an active node.""" + return self._core.node_adjacency(int(node)) + + def degree(self, node: int) -> int: + return int(self._core.degree(int(node))) + + def can_suppress_node(self, node: int) -> bool: + """Return whether :meth:`suppress_node` accepts this node.""" + return bool(self._core.can_suppress_node(int(node))) + + def is_node_active(self, node: int) -> bool: + return bool(self._core.is_node_active(int(node))) + + def is_edge_active(self, edge: int) -> bool: + return bool(self._core.is_edge_active(int(edge))) + + def representative(self, original_node: int) -> int: + """Return the current representative of an input node.""" + return int(self._core.representative(int(original_node))) + + def add_node_values(self, name: str, values, *, reduction: str) -> None: + """Register a node value map before the first mutation.""" + if self._mutation_started: + raise ValueError( + "value maps must be registered before the first graph mutation" + ) + array = _contraction_values( + values, + name, + self._original_number_of_nodes, + ) + code = _contraction_reduction(reduction) + getattr(self._core, f"_add_node_values_{array.dtype.name}")( + str(name), array, code + ) + + def add_edge_values(self, name: str, values, *, reduction: str) -> None: + """Register an edge value map before the first mutation.""" + if self._mutation_started: + raise ValueError( + "value maps must be registered before the first graph mutation" + ) + array = _contraction_values( + values, + name, + self._original_number_of_edges, + ) + code = _contraction_reduction(reduction) + getattr(self._core, f"_add_edge_values_{array.dtype.name}")( + str(name), array, code + ) + + def contract_edge(self, edge: int, *, keep_node: int | None = None) -> int: + """Contract an active edge and return the retained node id.""" + keep = None if keep_node is None else int(keep_node) + result = int(self._core.contract_edge(int(edge), keep)) + self._mutation_started = True + return result + + def erase_edge(self, edge: int) -> None: + """Erase an active edge without merging its endpoints.""" + self._core.erase_edge(int(edge)) + self._mutation_started = True + + def suppress_node(self, node: int) -> int: + """Suppress a degree-2 node and return the replacement edge id. + + The operation removes the node and its two incident edges. It connects + the two neighbors and reduces both edge value rows into the replacement + edge. The suppressed node value has no target and is discarded. + """ + result = int(self._core.suppress_node(int(node))) + self._mutation_started = True + return result + + def node_value(self, name: str, node: int) -> np.ndarray: + """Return a copy of one active node value.""" + return self._core.node_value(str(name), int(node)) + + def edge_value(self, name: str, edge: int) -> np.ndarray: + """Return a copy of one active edge value.""" + return self._core.edge_value(str(name), int(edge)) + + def active_node_values(self, name: str) -> tuple[np.ndarray, np.ndarray]: + """Return active node ids and their value rows.""" + return self._core.active_node_values(str(name)) + + def active_edge_values(self, name: str) -> tuple[np.ndarray, np.ndarray]: + """Return active edge ids and their value rows.""" + return self._core.active_edge_values(str(name)) + + def materialize(self) -> ContractionResult: + """Create a dense graph and aligned copies of all registered values. + + The output graph is simple. The snapshot folds active parallel edges + with each edge map's registered reduction. + + Materialization does not change this contraction graph. The result + remains valid after later mutations. + """ + ( + number_of_nodes, + uvs, + node_values, + edge_values, + node_mapping, + edge_mapping, + ) = self._core.materialize() + graph = UndirectedGraph.from_unique_edges(int(number_of_nodes), uvs) + return ContractionResult( + graph=graph, + node_values=dict(node_values), + edge_values=dict(edge_values), + node_mapping=node_mapping, + edge_mapping=edge_mapping, + ) + + def _normalize_projection_offsets(offsets, ndim: int) -> list[list[int]]: return [list(offset) for offset in strict_offsets(offsets, ndim)] @@ -515,7 +775,8 @@ def edge_weighted_watershed( edge_weights: 1D array of length ``graph.number_of_edges``. Supported dtypes are ``float32`` and ``float64``. Other floating dtypes are cast to - ``float32`` (matches nifty); other dtypes raise ``TypeError``. + ``float32`` (matches nifty); other dtypes raise ``TypeError``. All + values must be finite. seeds: 1D array of length ``graph.number_of_nodes``. Supported dtypes are ``uint32``, ``uint64``, ``int32``, ``int64``. ``0`` marks unlabeled @@ -555,6 +816,7 @@ def edge_weighted_watershed( weight_array = _as_1d_array( weight_array, weight_array.dtype, "edge_weights", int(graph.number_of_edges) ) + _require_finite_weights(weight_array, "edge_weights") seed_array = _as_1d_array( seed_array, seed_array.dtype, "seeds", int(graph.number_of_nodes) ) @@ -678,6 +940,8 @@ def project_node_labels_to_pixels( __all__ = [ + "ContractionGraph", + "ContractionResult", "GridGraph2D", "GridGraph3D", "RagCoordinates", diff --git a/src/bioimage_cpp/graph/_shared.py b/src/bioimage_cpp/graph/_shared.py index 2ece8e7..8ba08bd 100644 --- a/src/bioimage_cpp/graph/_shared.py +++ b/src/bioimage_cpp/graph/_shared.py @@ -92,7 +92,7 @@ def _as_edge_costs(edge_costs, graph) -> np.ndarray: raise ValueError("edge_costs must be a 1D array") if array.shape[0] != graph.number_of_edges: raise ValueError("edge_costs length must match graph number_of_edges") - return np.ascontiguousarray(array) + return _require_finite_weights(np.ascontiguousarray(array), "edge_costs") def _as_node_labels(labels, graph) -> np.ndarray: @@ -117,26 +117,10 @@ def _as_1d_array(values, dtype, name: str, expected_size: int) -> np.ndarray: return np.ascontiguousarray(array) -def _dense_labels(labels) -> np.ndarray: - labels = strict_integer_array( - labels, "labels", dtype=np.uint64, non_negative=True - ) - _, dense = np.unique(labels, return_inverse=True) - return np.ascontiguousarray(dense.astype(np.uint64, copy=False)) - - -def _subproblem_from_edges(number_of_nodes: int, nodes, uvs, edge_costs): - # Local import to avoid a circular dependency with the multicut submodule - # at module-load time (this helper is only called from the decomposer). - from . import UndirectedGraph - - local_ids = np.full(int(number_of_nodes), -1, dtype=np.int64) - local_ids[nodes] = np.arange(nodes.size, dtype=np.int64) - local_uvs = local_ids[np.asarray(uvs, dtype=np.uint64)] - sub_graph = UndirectedGraph(int(nodes.size), int(len(edge_costs))) - if local_uvs.size: - sub_graph.insert_edges(np.ascontiguousarray(local_uvs.astype(np.uint64, copy=False))) - return sub_graph, np.ascontiguousarray(np.asarray(edge_costs, dtype=np.float64)) +def _require_finite_weights(array: np.ndarray, name: str) -> np.ndarray: + if array.size and not np.all(np.isfinite(array)): + raise ValueError(f"{name} must contain only finite values") + return array def _normalize_labels(labels: np.ndarray) -> np.ndarray: diff --git a/src/bioimage_cpp/graph/agglomeration.py b/src/bioimage_cpp/graph/agglomeration.py index 75d1055..586f88a 100644 --- a/src/bioimage_cpp/graph/agglomeration.py +++ b/src/bioimage_cpp/graph/agglomeration.py @@ -7,6 +7,7 @@ All policies operate on an :class:`bioimage_cpp.graph.UndirectedGraph` or a subclass (``RegionAdjacencyGraph``, ``GridGraph2D``/``GridGraph3D``). +All floating inputs that affect heap priorities must contain finite values. """ from __future__ import annotations @@ -16,8 +17,10 @@ import numpy as np from .. import _core +from .._validation import strict_index from ._shared import ( _as_1d_array, + _require_finite_weights, _resolve_weight_dtype, ) @@ -51,16 +54,27 @@ def optimize(self, graph, *args, **kwargs) -> np.ndarray: """Run the agglomeration on ``graph`` and return dense node labels.""" +def _finite_float(value, name: str) -> float: + result = float(value) + if not np.isfinite(result): + raise ValueError(f"{name} must be finite") + return result + + def _ensure_edge_array(values, name, n_edges, dtype): if values is None: return np.ones(int(n_edges), dtype=dtype) - return _as_1d_array(values, dtype, name, int(n_edges)) + return _require_finite_weights( + _as_1d_array(values, dtype, name, int(n_edges)), name + ) def _ensure_node_array(values, name, n_nodes, dtype): if values is None: return np.ones(int(n_nodes), dtype=dtype) - return _as_1d_array(values, dtype, name, int(n_nodes)) + return _require_finite_weights( + _as_1d_array(values, dtype, name, int(n_nodes)), name + ) class EdgeWeightedClusterPolicy(ClusterPolicy): @@ -84,7 +98,7 @@ class EdgeWeightedClusterPolicy(ClusterPolicy): def __init__(self, *, num_clusters_stop: int = 1, size_regularizer: float = 1.0): self.num_clusters_stop = int(num_clusters_stop) - self.size_regularizer = float(size_regularizer) + self.size_regularizer = _finite_float(size_regularizer, "size_regularizer") def optimize( self, @@ -99,6 +113,7 @@ def optimize( indicator_array = _as_1d_array( indicator_array, dtype, "edge_indicators", int(graph.number_of_edges) ) + _require_finite_weights(indicator_array, "edge_indicators") edge_size_array = _ensure_edge_array( edge_sizes, "edge_sizes", graph.number_of_edges, dtype ) @@ -144,8 +159,8 @@ def __init__( beta: float = 0.5, ): self.num_clusters_stop = int(num_clusters_stop) - self.size_regularizer = float(size_regularizer) - self.beta = float(beta) + self.size_regularizer = _finite_float(size_regularizer, "size_regularizer") + self.beta = _finite_float(beta, "beta") def optimize( self, @@ -165,6 +180,7 @@ def optimize( indicator_array = _as_1d_array( indicator_array, dtype, "edge_indicators", int(graph.number_of_edges) ) + _require_finite_weights(indicator_array, "edge_indicators") edge_size_array = _ensure_edge_array( edge_sizes, "edge_sizes", graph.number_of_edges, dtype ) @@ -177,6 +193,7 @@ def optimize( "node_features must have shape (number_of_nodes, n_channels), got " f"shape={feature_array.shape}, number_of_nodes={int(graph.number_of_nodes)}" ) + _require_finite_weights(feature_array, "node_features") run = _NODE_AND_EDGE_WEIGHTED_BY_DTYPE[dtype] return run( graph, @@ -201,7 +218,8 @@ class MalaClusterPolicy(ClusterPolicy): Parameters ---------- num_bins: - Number of histogram bins covering ``[bin_min, bin_max]``. + Number of histogram bins covering ``[bin_min, bin_max]``. Must be at + least ``2``. bin_min, bin_max: Range covered by the histogram. Values outside the range fall into the boundary bins. @@ -226,12 +244,12 @@ def __init__( num_edges_stop: int = 0, threshold: float = 0.5, ): - self.num_bins = int(num_bins) - self.bin_min = float(bin_min) - self.bin_max = float(bin_max) + self.num_bins = strict_index(num_bins, "num_bins", minimum=2) + self.bin_min = _finite_float(bin_min, "bin_min") + self.bin_max = _finite_float(bin_max, "bin_max") self.num_clusters_stop = int(num_clusters_stop) self.num_edges_stop = int(num_edges_stop) - self.threshold = float(threshold) + self.threshold = _finite_float(threshold, "threshold") def optimize(self, graph, edge_indicators) -> np.ndarray: indicator_array = _resolve_weight_dtype(edge_indicators, "edge_indicators") @@ -239,6 +257,7 @@ def optimize(self, graph, edge_indicators) -> np.ndarray: indicator_array = _as_1d_array( indicator_array, dtype, "edge_indicators", int(graph.number_of_edges) ) + _require_finite_weights(indicator_array, "edge_indicators") run = _MALA_BY_DTYPE[dtype] return run( graph, @@ -308,6 +327,7 @@ def optimize( weight_array = _as_1d_array( weight_array, dtype, "edge_weights", int(graph.number_of_edges) ) + _require_finite_weights(weight_array, "edge_weights") edge_size_array = _ensure_edge_array( edge_sizes, "edge_sizes", graph.number_of_edges, dtype ) diff --git a/src/bioimage_cpp/graph/lifted_multicut/__init__.py b/src/bioimage_cpp/graph/lifted_multicut/__init__.py index c5a4dcb..2b03a25 100644 --- a/src/bioimage_cpp/graph/lifted_multicut/__init__.py +++ b/src/bioimage_cpp/graph/lifted_multicut/__init__.py @@ -37,6 +37,7 @@ _as_node_labels, _as_uv_array, _normalize_number_of_threads, + _require_finite_weights, ) @@ -151,13 +152,13 @@ def lifted_edges_from_node_labels( class LiftedMulticutObjective: """Lifted multicut objective. - Stores a base graph + base edge costs together with an internal *lifted + Stores a base graph and finite base costs together with an internal *lifted graph* that is a superset of the base graph (base edges occupy ids ``0 .. base.number_of_edges - 1``; lifted edges follow). The energy of a node labeling is the sum of base + lifted edge weights across cut edges. - The lifted edges can be supplied either as explicit ``(uvs, costs)`` - arrays, via a ``bfs_distance=k`` constructor argument that inserts a + All explicit costs must be finite. Lifted edges can be supplied as + ``(uvs, costs)`` arrays, via a ``bfs_distance=k`` argument that inserts a zero-weight lifted edge for every pair of nodes within ``k`` hops of each other in the base graph, or by calling :meth:`set_cost` after construction. """ @@ -237,11 +238,14 @@ def __init__( f"lifted_uvs.shape[0]={uv_array.shape[0]}, " f"lifted_costs.shape[0]={cost_array.shape[0]}" ) + cost_array = _require_finite_weights( + np.ascontiguousarray(cost_array), "lifted_costs" + ) _add_lifted_edges( lifted_graph, weights_list, uv_array, - np.ascontiguousarray(cost_array), + cost_array, overwrite_existing=overwrite_existing, ) @@ -250,6 +254,7 @@ def __init__( self._n_base_edges = int(base_graph.number_of_edges) self._weights = np.ascontiguousarray(np.concatenate(weights_list)) \ if len(weights_list) > 1 else weights_list[0] + _require_finite_weights(self._weights, "weights") if initial_labels is None: self._labels = np.arange(base_graph.number_of_nodes, dtype=np.uint64) else: @@ -304,25 +309,34 @@ def set_cost( previously inserted lifted edge), the weight is accumulated unless ``overwrite=True``. """ - pre = int(self._lifted_graph.number_of_edges) + weight_value = float(weight) + if not np.isfinite(weight_value): + raise ValueError("weight must be finite") + edge = int(self._lifted_graph.find_edge(int(u), int(v))) + if edge >= 0: + new_value = ( + weight_value if overwrite else float(self._weights[edge]) + weight_value + ) + if not np.isfinite(new_value): + raise ValueError("accumulated weight must be finite") + self._weights[edge] = new_value + return edge, False + edge = int(self._lifted_graph.insert_edge(int(u), int(v))) - if int(self._lifted_graph.number_of_edges) > pre: + if edge == int(self._weights.size): self._weights = np.concatenate( - [self._weights, np.asarray([float(weight)], dtype=np.float64)] + [self._weights, np.asarray([weight_value], dtype=np.float64)] ) return edge, True - if overwrite: - self._weights[edge] = float(weight) - else: - self._weights[edge] = self._weights[edge] + float(weight) - return edge, False + raise RuntimeError("lifted graph insertion did not append a new edge") def energy(self, labels=None) -> float: label_array = ( self._labels if labels is None else _as_node_labels(labels, self._base_graph) ) + weights = _require_finite_weights(self._weights, "weights") return float( - _core._lifted_multicut_energy(self._lifted_graph, self._weights, label_array) + _core._lifted_multicut_energy(self._lifted_graph, weights, label_array) ) @@ -448,9 +462,10 @@ def __init__( self.sigma = float(sigma) def optimize(self, objective: LiftedMulticutObjective) -> np.ndarray: + weights = _require_finite_weights(objective.weights, "weights") labels = _core._lifted_multicut_greedy_additive( objective.lifted_graph, - objective.weights, + weights, objective.number_of_base_edges, self.weight_stop, self.node_num_stop, @@ -491,6 +506,7 @@ def __init__( self.epsilon = float(epsilon) def optimize(self, objective: LiftedMulticutObjective) -> np.ndarray: + weights = _require_finite_weights(objective.weights, "weights") initial_labels = objective.labels if np.array_equal( initial_labels, @@ -498,7 +514,7 @@ def optimize(self, objective: LiftedMulticutObjective) -> np.ndarray: ): initial_labels = _core._lifted_multicut_greedy_additive( objective.lifted_graph, - objective.weights, + weights, objective.number_of_base_edges, 0.0, -1.0, @@ -509,7 +525,7 @@ def optimize(self, objective: LiftedMulticutObjective) -> np.ndarray: labels = _core._lifted_multicut_kernighan_lin( objective.graph, objective.lifted_graph, - objective.weights, + weights, objective.number_of_base_edges, initial_labels, self.number_of_outer_iterations, @@ -595,9 +611,10 @@ def __init__( def optimize(self, objective: LiftedMulticutObjective) -> np.ndarray: n_base = objective.number_of_base_edges + weights = _require_finite_weights(objective.weights, "weights") # The base costs back the proposal generators (the lifted weights # cannot drive base-graph contraction or watershed segmentation). - base_costs = np.ascontiguousarray(objective.weights[:n_base]) + base_costs = np.ascontiguousarray(weights[:n_base]) cpp_pgens = [ self.proposal_generator._build_for_thread( objective.graph, base_costs, slot @@ -610,7 +627,7 @@ def optimize(self, objective: LiftedMulticutObjective) -> np.ndarray: labels = _core._lifted_multicut_fusion_move( objective.graph, objective.lifted_graph, - objective.weights, + weights, n_base, objective.labels, cpp_pgens, diff --git a/src/bioimage_cpp/graph/multicut/__init__.py b/src/bioimage_cpp/graph/multicut/__init__.py index 8c73fc4..3d61b60 100644 --- a/src/bioimage_cpp/graph/multicut/__init__.py +++ b/src/bioimage_cpp/graph/multicut/__init__.py @@ -36,14 +36,13 @@ _as_edge_costs, _as_node_labels, _copy_graph, - _dense_labels, _normalize_number_of_threads, - _subproblem_from_edges, + _require_finite_weights, ) class MulticutObjective: - """Multicut objective for an undirected graph and edge costs.""" + """Multicut objective for an undirected graph and finite edge costs.""" def __init__( self, @@ -82,7 +81,8 @@ def reset_labels(self) -> None: def energy(self, labels=None) -> float: label_array = self._labels if labels is None else _as_node_labels(labels, self._graph) - return float(_core._multicut_energy(self._graph, self._edge_costs, label_array)) + costs = _require_finite_weights(self._edge_costs, "edge_costs") + return float(_core._multicut_energy(self._graph, costs, label_array)) class MulticutSolver(ABC): @@ -116,9 +116,10 @@ def __init__( self.sigma = float(sigma) def optimize(self, objective: MulticutObjective) -> np.ndarray: + costs = _require_finite_weights(objective.edge_costs, "edge_costs") labels = _core._multicut_greedy_additive( objective.graph, - objective.edge_costs, + costs, self.weight_stop, self.node_num_stop, self.add_noise, @@ -137,6 +138,9 @@ def _build_cpp_sub_solver(self): sigma=self.sigma, ) + def _build_cpp_decomposer_solver(self): + return self._build_cpp_sub_solver() + class GreedyFixationMulticut(MulticutSolver): """Greedy fixation multicut solver. @@ -151,9 +155,10 @@ def __init__(self, *, weight_stop: float = 0.0, node_num_stop: float = -1.0): self.node_num_stop = float(node_num_stop) def optimize(self, objective: MulticutObjective) -> np.ndarray: + costs = _require_finite_weights(objective.edge_costs, "edge_costs") labels = _core._multicut_greedy_fixation( objective.graph, - objective.edge_costs, + costs, self.weight_stop, self.node_num_stop, ) @@ -166,6 +171,9 @@ def _build_cpp_sub_solver(self): node_num_stop=self.node_num_stop, ) + def _build_cpp_decomposer_solver(self): + return self._build_cpp_sub_solver() + class KernighanLinMulticut(MulticutSolver): """Kernighan-Lin multicut solver. @@ -190,6 +198,7 @@ def __init__( self.epsilon = float(epsilon) def optimize(self, objective: MulticutObjective) -> np.ndarray: + costs = _require_finite_weights(objective.edge_costs, "edge_costs") initial_labels = objective.labels if np.array_equal( initial_labels, @@ -197,7 +206,7 @@ def optimize(self, objective: MulticutObjective) -> np.ndarray: ): initial_labels = _core._multicut_greedy_additive( objective.graph, - objective.edge_costs, + costs, 0.0, -1.0, False, @@ -206,7 +215,7 @@ def optimize(self, objective: MulticutObjective) -> np.ndarray: ) labels = _core._multicut_kernighan_lin( objective.graph, - objective.edge_costs, + costs, initial_labels, self.number_of_outer_iterations, self.epsilon, @@ -218,6 +227,14 @@ def _build_cpp_sub_solver(self): return _core._KernighanLinMulticutSubSolver( number_of_outer_iterations=self.number_of_outer_iterations, epsilon=self.epsilon, + warm_start_greedy=False, + ) + + def _build_cpp_decomposer_solver(self): + return _core._KernighanLinMulticutSubSolver( + number_of_outer_iterations=self.number_of_outer_iterations, + epsilon=self.epsilon, + warm_start_greedy=True, ) @@ -375,12 +392,13 @@ def __init__( raise ValueError("stop_if_no_improvement must be >= 1") def optimize(self, objective: MulticutObjective) -> np.ndarray: + costs = _require_finite_weights(objective.edge_costs, "edge_costs") # Build one C++ proposal generator per parallel slot, each with a # distinct seed offset, so parallel streams are independent and # reproducible. cpp_pgens = [ self.proposal_generator._build_for_thread( - objective.graph, objective.edge_costs, slot + objective.graph, costs, slot ) for slot in range(self.number_of_parallel_proposals) ] @@ -389,7 +407,7 @@ def optimize(self, objective: MulticutObjective) -> np.ndarray: ) labels = _core._multicut_fusion_move( objective.graph, - objective.edge_costs, + costs, objective.labels, cpp_pgens, cpp_sub_solver, @@ -403,6 +421,8 @@ def optimize(self, objective: MulticutObjective) -> np.ndarray: class ChainedMulticutSolvers(MulticutSolver): + """Run a sequence of multicut solvers on one objective.""" + def __init__(self, solvers): self.solvers = list(solvers) if len(self.solvers) == 0: @@ -416,6 +436,33 @@ def optimize(self, objective: MulticutObjective) -> np.ndarray: labels = solver.optimize(objective) return labels + def _build_cpp_decomposer_solver(self): + return _core._ChainedMulticutSubSolver( + [ + solver._build_cpp_decomposer_solver() + for solver in self.solvers + ] + ) + + +def _require_decomposer_solver(solver: MulticutSolver, name: str) -> None: + supported = ( + GreedyAdditiveMulticut, + GreedyFixationMulticut, + KernighanLinMulticut, + ) + if isinstance(solver, supported): + return + if isinstance(solver, ChainedMulticutSolvers): + for index, child in enumerate(solver.solvers): + _require_decomposer_solver(child, f"{name}.solvers[{index}]") + return + raise TypeError( + f"{name} must be GreedyAdditiveMulticut, " + "GreedyFixationMulticut, KernighanLinMulticut, or a " + "ChainedMulticutSolvers containing only these solvers" + ) + class MulticutDecomposer(MulticutSolver): """Decomposition-based multicut solver. @@ -426,6 +473,13 @@ class MulticutDecomposer(MulticutSolver): Splits the multicut problem into connected components (based on positive edge costs) and solves each component independently with ``sub_solver``. + + ``number_of_threads=0`` uses available hardware concurrency. Native + workers use independent C++ solver instances. Results are assembled in + component order, independent of worker scheduling. + + Component and fallthrough solvers must be greedy additive, greedy + fixation, Kernighan-Lin, or a chain composed only from these solvers. """ def __init__( @@ -437,54 +491,38 @@ def __init__( ): if not isinstance(sub_solver, MulticutSolver): raise TypeError("sub_solver must inherit from MulticutSolver") - if fallthrough_solver is not None and not isinstance(fallthrough_solver, MulticutSolver): + if fallthrough_solver is not None and not isinstance( + fallthrough_solver, MulticutSolver + ): raise TypeError("fallthrough_solver must inherit from MulticutSolver") + _require_decomposer_solver(sub_solver, "sub_solver") + if fallthrough_solver is not None: + _require_decomposer_solver( + fallthrough_solver, "fallthrough_solver" + ) self.sub_solver = sub_solver self.fallthrough_solver = fallthrough_solver self.number_of_threads = _normalize_number_of_threads(number_of_threads) def optimize(self, objective: MulticutObjective) -> np.ndarray: - # Local import to avoid a circular dependency at module-load time: - # ``connected_components`` lives in the top-level ``bioimage_cpp.graph`` - # namespace, which itself imports this submodule. - from .. import connected_components - - if self.fallthrough_solver is None and isinstance(self.sub_solver, GreedyAdditiveMulticut): - return self.sub_solver.optimize(objective) - - component_labels = connected_components( + edge_costs = _require_finite_weights( + objective.edge_costs, "edge_costs" + ) + cpp_sub_solver = self.sub_solver._build_cpp_decomposer_solver() + cpp_fallthrough_solver = ( + None + if self.fallthrough_solver is None + else self.fallthrough_solver._build_cpp_decomposer_solver() + ) + labels = _core._multicut_decomposer( objective.graph, - edge_mask=objective.edge_costs > 0.0, + edge_costs, + objective.labels, + cpp_sub_solver, + cpp_fallthrough_solver, + self.number_of_threads, ) - number_of_components = int(component_labels.max()) + 1 if component_labels.size else 0 - if number_of_components <= 1: - solver = self.fallthrough_solver or self.sub_solver - return solver.optimize(objective) - - global_labels = np.empty(objective.graph.number_of_nodes, dtype=np.uint64) - label_offset = 0 - all_uvs = objective.graph.uv_ids() - for component in range(number_of_components): - nodes = np.flatnonzero(component_labels == component).astype(np.uint64) - if nodes.size == 1: - global_labels[int(nodes[0])] = label_offset - label_offset += 1 - continue - - edge_ids = objective.graph.edges_from_node_list(nodes) - sub_graph, sub_costs = _subproblem_from_edges( - objective.graph.number_of_nodes, - nodes, - all_uvs[edge_ids], - objective.edge_costs[edge_ids], - ) - sub_objective = MulticutObjective(sub_graph, sub_costs) - sub_labels = self.sub_solver.optimize(sub_objective) - sub_labels = _dense_labels(sub_labels) - global_labels[nodes] = sub_labels + label_offset - label_offset += int(sub_labels.max()) + 1 - - objective.labels = _dense_labels(global_labels) + objective.labels = labels return objective.labels diff --git a/src/bioimage_cpp/graph/mutex_watershed.py b/src/bioimage_cpp/graph/mutex_watershed.py index 61138a5..d8a9fcc 100644 --- a/src/bioimage_cpp/graph/mutex_watershed.py +++ b/src/bioimage_cpp/graph/mutex_watershed.py @@ -16,6 +16,7 @@ from ._shared import ( _as_1d_array, _as_uv_array, + _require_finite_weights, _resolve_weight_dtype, ) @@ -67,13 +68,13 @@ def mutex_watershed_clustering( edge_costs: 1D array of length ``graph.number_of_edges``. Supported dtypes are ``float32`` and ``float64``; other floating dtypes are cast to - ``float32``. Higher values are more attractive. + ``float32``. Values must be finite. Higher values are more attractive. mutex_uvs: ``(n_mutex, 2)`` uint64 array of (u, v) pairs for the mutex edges. mutex_costs: 1D array of length ``n_mutex``. Same dtype rules as ``edge_costs``; - if the two dtypes differ both are promoted to ``float64``. Higher - values are stronger repulsions. + if the two dtypes differ both are promoted to ``float64``. Values must + be finite. Higher values are stronger repulsions. Returns ------- @@ -103,6 +104,8 @@ def mutex_watershed_clustering( "mutex_costs", int(mutex_uv_array.shape[0]), ) + _require_finite_weights(edge_cost_array, "edge_costs") + _require_finite_weights(mutex_cost_array, "mutex_costs") run = _MUTEX_WATERSHED_CLUSTERING_BY_DTYPE[edge_cost_array.dtype] return run(graph, edge_cost_array, mutex_uv_array, mutex_cost_array) @@ -148,7 +151,8 @@ def semantic_mutex_watershed_clustering( semantic_costs: 1D array of length ``n_semantic``. Same dtype rules as ``edge_costs``; if the floating dtypes of the three weight arrays - do not all agree, all three are promoted to ``float64``. + do not all agree, all three are promoted to ``float64``. Values must + be finite. Returns ------- @@ -193,6 +197,9 @@ def semantic_mutex_watershed_clustering( "semantic_costs", int(semantic_uv_array.shape[0]), ) + _require_finite_weights(edge_cost_array, "edge_costs") + _require_finite_weights(mutex_cost_array, "mutex_costs") + _require_finite_weights(semantic_cost_array, "semantic_costs") run = _SEMANTIC_MUTEX_WATERSHED_CLUSTERING_BY_DTYPE[edge_cost_array.dtype] return run( diff --git a/src/bioimage_cpp/label_multiset/__init__.py b/src/bioimage_cpp/label_multiset/__init__.py index dc5b96a..72bb397 100644 --- a/src/bioimage_cpp/label_multiset/__init__.py +++ b/src/bioimage_cpp/label_multiset/__init__.py @@ -11,7 +11,7 @@ Storage layout (mirrors nifty): -- ``offsets`` length ``n_spatial``: spatial position to byte offset into ``ids`` / ``counts`` +- ``offsets`` length ``n_spatial``: spatial position to element offset into ``ids`` / ``counts`` - ``entry_offsets`` length ``n_spatial``: spatial position to unique-entry index - ``entry_sizes`` length ``n_unique``: number of ``(id, count)`` pairs per entry - ``ids`` length ``total_elems``: concatenated label ids (sorted within each entry) @@ -28,15 +28,106 @@ from .. import _core from .._core import Blocking +from .._validation import strict_integer_array _ID_DTYPE = np.dtype(np.uint64) _COUNT_DTYPE = np.dtype(np.uint32) _OFFSET_DTYPE = np.dtype(np.uint64) +def _as_integer_metadata(values, name: str, dtype: np.dtype) -> np.ndarray: + untyped = np.asarray(values) + if untyped.size == 0 and untyped.ndim == 1: + return np.ascontiguousarray(untyped, dtype=dtype) + return strict_integer_array( + values, + name, + dtype=dtype, + ndim=1, + non_negative=True, + ) + + +def _validate_multiset_arrays( + argmax, + offsets, + entry_offsets, + entry_sizes, + ids, + counts, +) -> tuple[np.ndarray, ...]: + arrays = ( + _as_integer_metadata(argmax, "argmax", _ID_DTYPE), + _as_integer_metadata(offsets, "offsets", _OFFSET_DTYPE), + _as_integer_metadata(entry_offsets, "entry_offsets", _OFFSET_DTYPE), + _as_integer_metadata(entry_sizes, "entry_sizes", _OFFSET_DTYPE), + _as_integer_metadata(ids, "ids", _ID_DTYPE), + _as_integer_metadata(counts, "counts", _COUNT_DTYPE), + ) + argmax_array, offset_array, entry_offset_array, size_array, id_array, count_array = arrays + + if id_array.size != count_array.size: + raise ValueError( + "ids and counts must have the same length, got " + f"{id_array.size} and {count_array.size}" + ) + n_spatial = offset_array.size + if argmax_array.size != n_spatial or entry_offset_array.size != n_spatial: + raise ValueError( + "argmax, offsets, and entry_offsets must have the same length, got " + f"{argmax_array.size}, {n_spatial}, and {entry_offset_array.size}" + ) + if n_spatial == 0: + if size_array.size != 0 or id_array.size != 0: + raise ValueError( + "an empty LabelMultiset must have empty entry_sizes, ids, and counts" + ) + return arrays + if size_array.size == 0: + raise ValueError( + "entry_sizes must not be empty when spatial entries are present" + ) + if np.any(size_array == 0): + raise ValueError("entry_sizes must contain only positive values") + if np.any(entry_offset_array >= size_array.size): + raise ValueError( + "entry_offsets values must be less than the length of entry_sizes" + ) + referenced = np.bincount( + entry_offset_array.astype(np.intp, copy=False), + minlength=size_array.size, + ) + if np.any(referenced == 0): + raise ValueError("every entry_sizes element must be referenced by entry_offsets") + if np.any(offset_array > id_array.size): + raise ValueError("offsets values must not exceed the flat storage length") + selected_sizes = size_array[entry_offset_array.astype(np.intp, copy=False)] + remaining = id_array.size - offset_array + if np.any(selected_sizes > remaining): + raise ValueError("an entry range exceeds the flat storage length") + return arrays + + +def _validated_multiset(multiset: "LabelMultiset") -> tuple[np.ndarray, ...]: + if not isinstance(multiset, LabelMultiset): + raise TypeError("multiset must be a LabelMultiset") + return _validate_multiset_arrays( + multiset.argmax, + multiset.offsets, + multiset.entry_offsets, + multiset.entry_sizes, + multiset.ids, + multiset.counts, + ) + + @dataclass class LabelMultiset: - """A deduplicated label-histogram representation over a spatial grid.""" + """A validated, deduplicated label histogram over a spatial grid. + + Construction converts integer inputs to the storage dtypes. Native + operations validate the arrays again because the dataclass is mutable. + """ argmax: np.ndarray # shape (n_spatial,), dtype uint64 offsets: np.ndarray # shape (n_spatial,), dtype uint64 @@ -45,6 +136,23 @@ class LabelMultiset: ids: np.ndarray # shape (total_elems,), dtype uint64 counts: np.ndarray # shape (total_elems,), dtype uint32 + def __post_init__(self) -> None: + ( + self.argmax, + self.offsets, + self.entry_offsets, + self.entry_sizes, + self.ids, + self.counts, + ) = _validate_multiset_arrays( + self.argmax, + self.offsets, + self.entry_offsets, + self.entry_sizes, + self.ids, + self.counts, + ) + @property def n_spatial(self) -> int: return int(self.offsets.shape[0]) @@ -117,14 +225,22 @@ def downsample_multiset( ``blocking`` must be defined over the same spatial extent as the input multiset (i.e. ``prod(blocking.roi_end) == multiset.n_spatial``). """ + ( + _, + offsets, + entry_offsets, + entry_sizes, + ids, + counts, + ) = _validated_multiset(multiset) argmax, new_offsets, new_entry_offsets, new_entry_sizes, new_ids, new_counts = ( _core._downsample_multiset( blocking, - multiset.offsets, - multiset.entry_sizes, - multiset.entry_offsets, - multiset.ids, - multiset.counts, + offsets, + entry_sizes, + entry_offsets, + ids, + counts, restrict_set, ) ) @@ -149,19 +265,23 @@ def read_subset( Returns the summed ``(ids, counts)``, sorted by id if ``argsort``. """ - offsets = np.ascontiguousarray(offsets, dtype=_OFFSET_DTYPE) - sizes = np.ascontiguousarray(sizes, dtype=_OFFSET_DTYPE) - ids = np.ascontiguousarray(ids, dtype=_ID_DTYPE) - counts = np.ascontiguousarray(counts, dtype=_COUNT_DTYPE) + offsets = _as_integer_metadata(offsets, "offsets", _OFFSET_DTYPE) + sizes = _as_integer_metadata(sizes, "sizes", _OFFSET_DTYPE) + ids = _as_integer_metadata(ids, "ids", _ID_DTYPE) + counts = _as_integer_metadata(counts, "counts", _COUNT_DTYPE) return _core._read_subset(offsets, sizes, ids, counts, argsort) -def _unique_offsets_of(multiset: "LabelMultiset") -> np.ndarray: - """For each unique entry of ``multiset``, return the byte offset into ids/counts.""" - n = multiset.n_entries +def _unique_offsets_of( + offsets: np.ndarray, + entry_offsets: np.ndarray, + n_entries: int, +) -> np.ndarray: + """Return one element offset for each unique multiset entry.""" + n = int(n_entries) out = np.empty(n, dtype=_OFFSET_DTYPE) for e in range(n): - out[e] = multiset.offsets[np.where(multiset.entry_offsets == e)[0][0]] + out[e] = offsets[np.flatnonzero(entry_offsets == e)[0]] return out @@ -175,7 +295,7 @@ class MultisetMerger: Call :meth:`update` with subsequent batches; each call extends the internal storage with any genuinely new entries and rewrites the passed-in ``offsets`` array so each spatial position points at its - final deduplicated byte offset. + final deduplicated element offset. """ def __init__( @@ -185,8 +305,12 @@ def __init__( ids: np.ndarray, counts: np.ndarray, ) -> None: - unique_offsets = np.ascontiguousarray(unique_offsets, dtype=_OFFSET_DTYPE) - entry_sizes = np.ascontiguousarray(entry_sizes, dtype=_OFFSET_DTYPE) + unique_offsets = _as_integer_metadata( + unique_offsets, "unique_offsets", _OFFSET_DTYPE + ) + entry_sizes = _as_integer_metadata( + entry_sizes, "entry_sizes", _OFFSET_DTYPE + ) if unique_offsets.shape != entry_sizes.shape: raise ValueError( "unique_offsets and entry_sizes must have the same length " @@ -196,18 +320,21 @@ def __init__( self._impl = _core._MultisetMerger( unique_offsets, entry_sizes, - np.ascontiguousarray(ids, dtype=_ID_DTYPE), - np.ascontiguousarray(counts, dtype=_COUNT_DTYPE), + _as_integer_metadata(ids, "ids", _ID_DTYPE), + _as_integer_metadata(counts, "counts", _COUNT_DTYPE), ) @classmethod def from_multiset(cls, multiset: "LabelMultiset") -> "MultisetMerger": """Build a merger seeded with the unique entries of ``multiset``.""" + _, offsets, entry_offsets, entry_sizes, ids, counts = _validated_multiset( + multiset + ) return cls( - _unique_offsets_of(multiset), - multiset.entry_sizes, - multiset.ids, - multiset.counts, + _unique_offsets_of(offsets, entry_offsets, entry_sizes.size), + entry_sizes, + ids, + counts, ) def update( @@ -222,15 +349,21 @@ def update( ``offsets`` is mutated in-place and also returned. """ - if offsets.dtype != _OFFSET_DTYPE or not offsets.flags["C_CONTIGUOUS"]: + if ( + not isinstance(offsets, np.ndarray) + or offsets.dtype != _OFFSET_DTYPE + or offsets.ndim != 1 + or not offsets.flags["C_CONTIGUOUS"] + or not offsets.flags["WRITEABLE"] + ): raise TypeError( - "offsets must be a contiguous uint64 array (it is modified in place)" + "offsets must be a writable contiguous 1D uint64 array" ) return self._impl.update( - np.ascontiguousarray(unique_offsets, dtype=_OFFSET_DTYPE), - np.ascontiguousarray(entry_sizes, dtype=_OFFSET_DTYPE), - np.ascontiguousarray(ids, dtype=_ID_DTYPE), - np.ascontiguousarray(counts, dtype=_COUNT_DTYPE), + _as_integer_metadata(unique_offsets, "unique_offsets", _OFFSET_DTYPE), + _as_integer_metadata(entry_sizes, "entry_sizes", _OFFSET_DTYPE), + _as_integer_metadata(ids, "ids", _ID_DTYPE), + _as_integer_metadata(counts, "counts", _COUNT_DTYPE), offsets, ) diff --git a/src/bioimage_cpp/skeleton/postprocessing.py b/src/bioimage_cpp/skeleton/postprocessing.py index d70f802..fe35ee7 100644 --- a/src/bioimage_cpp/skeleton/postprocessing.py +++ b/src/bioimage_cpp/skeleton/postprocessing.py @@ -1,18 +1,14 @@ """Post-processing for skeleton graphs. -Resolve junction nodes so each filament becomes its own connected component. -Degree-3 and degree-4 nodes are split or pruned based on the angles between their -incident edges: the straightest pair is kept as the through-going filament and -the remaining arm(s) are either separated, or for short dead-ends (spurs), pruned. +Cut degree-3 branches and split degree-4 crossings based on the angles between +their incident edges. Short dead-end branches can also be pruned by length. """ from __future__ import annotations -from collections import defaultdict - import numpy as np -from ..graph import connected_components +from ..graph import ContractionGraph, UndirectedGraph, connected_components from ..utils import UnionFind from ._graph import skeleton_to_graph @@ -94,14 +90,17 @@ def clean_filament_graph( min_join_angle: float = 175.0, save_intermediates: list | None = None, ) -> tuple[np.ndarray, np.ndarray, np.ndarray | None]: - """Split skeleton junctions so each filament is its own component. + """Clean a skeleton graph by cutting branches and splitting crossings. Postprocessing steps, in order: 1. If ``tick_length > 0``, prune dead-end branches shorter than it via :func:`remove_ticks`. - 2. Split each degree-3 junction, separating the odd arm when it diverges - from the through pair by at least ``min_branch_angle``. + 2. At each degree-3 junction, identify the straightest pair of arms. If the + remaining arm diverges by at least ``min_branch_angle``, remove its edge + adjacent to the junction. This cuts the arm from the through-going + filament and removes its first segment. A one-edge arm is removed + completely. 3. Split each degree-4 crossing, separating its two through pairs when they are collinear to within ``min_through_angle``. 4. If ``join_dist > 0``, join collinear endpoints across gaps up to @@ -120,8 +119,9 @@ def clean_filament_graph( min_through_angle: Minimum through-pair angle (degrees) for a degree-4 crossing to split. min_branch_angle: - Minimum angle (degrees) between a degree-3 node's odd arm and its - through pair for the odd arm to be separated. + Minimum angle in degrees between a degree-3 node's odd arm and its + through pair. When the angle meets this threshold, the edge adjacent + to the junction on the odd arm is removed. tick_length: If > 0, prune dead-end branches shorter than this (physical) distance. join_dist: @@ -155,7 +155,7 @@ def _snapshot(name): edges = edges.copy() _snapshot("ticks") graph = skeleton_to_graph(vertices, edges) - degrees = np.bincount(edges.reshape(-1), minlength=len(vertices)) + degrees = graph.node_degrees() splits, prune_edges = [], set() for v in np.where(degrees == 3)[0]: @@ -211,6 +211,30 @@ def _adjacency(num_nodes, edges): return indptr, dst, eid, degrees +def _nodes_in_noncycle_components(indptr, dst, degrees): + eligible = np.zeros(len(degrees), dtype=bool) + visited = np.zeros(len(degrees), dtype=bool) + for start in range(len(degrees)): + if visited[start]: + continue + component = [] + stack = [start] + visited[start] = True + has_critical_node = False + while stack: + node = stack.pop() + component.append(node) + has_critical_node |= degrees[node] != 2 + for index in range(indptr[node], indptr[node + 1]): + neighbor = int(dst[index]) + if not visited[neighbor]: + visited[neighbor] = True + stack.append(neighbor) + if has_critical_node: + eligible[component] = True + return eligible + + def remove_ticks(vertices, edges, tick_length, radii=None): """Prune short dead-end branches ("ticks") from a skeleton graph. @@ -227,7 +251,8 @@ def remove_ticks(vertices, edges, tick_length, radii=None): vertices: Float array with shape ``(V, D)`` of skeleton vertex coordinates. edges: - Integer array with shape ``(E, 2)`` indexing ``vertices``. + Integer array with shape ``(E, 2)`` indexing ``vertices``. Self-edges + and duplicate undirected edges are not supported. tick_length: Maximum branch length (physical) that may be pruned. radii: @@ -240,75 +265,70 @@ def remove_ticks(vertices, edges, tick_length, radii=None): ``None`` when no input radii were given. """ vertices = np.asarray(vertices, dtype=np.float64) - edges = np.asarray(edges, dtype=np.int64) + if vertices.ndim != 2: + raise ValueError(f"vertices must be a 2D array, got ndim={vertices.ndim}") + raw_edges = np.asarray(edges) + if not np.issubdtype(raw_edges.dtype, np.integer): + raise TypeError(f"edges must have an integer dtype, got dtype={raw_edges.dtype}") + if raw_edges.ndim != 2 or raw_edges.shape[1] != 2: + raise ValueError(f"edges must have shape (E, 2), got shape={raw_edges.shape}") + if np.issubdtype(raw_edges.dtype, np.signedinteger) and np.any(raw_edges < 0): + raise ValueError("edges must not contain negative node ids") + edges_u64 = np.ascontiguousarray(raw_edges, dtype=np.uint64) num_nodes = len(vertices) + if edges_u64.size and int(edges_u64.max()) >= num_nodes: + raise IndexError( + f"edge endpoint must be smaller than len(vertices)={num_nodes}" + ) + if radii is not None and len(radii) != num_nodes: + raise ValueError( + f"radii length must be {num_nodes}, got length={len(radii)}" + ) + edges = edges_u64.astype(np.int64, copy=False) if len(edges) == 0: return vertices, edges.copy(), radii - indptr, dst, eid, degrees = _adjacency(num_nodes, edges) - - # Distance supergraph: sid -> [end_a, end_b, length, edge_ids]. - supers = {} - incident = defaultdict(set) - edge_used = np.zeros(len(edges), dtype=bool) - sid = 0 - for node in map(int, np.where(degrees != 2)[0]): - for k in range(indptr[node], indptr[node + 1]): - if edge_used[eid[k]]: - continue - prev, cur = node, int(dst[k]) - path = [int(eid[k])] - length = float(np.linalg.norm(vertices[cur] - vertices[node])) - while degrees[cur] == 2: - s, e = indptr[cur], indptr[cur + 1] - nbrs, eids = dst[s:e], eid[s:e] - pick = 0 if int(nbrs[0]) != prev else 1 - path.append(int(eids[pick])) - nxt = int(nbrs[pick]) - length += float(np.linalg.norm(vertices[nxt] - vertices[cur])) - prev, cur = cur, nxt - for pe in path: - edge_used[pe] = True - supers[sid] = [node, cur, length, path] - incident[node].add(sid) - incident[cur].add(sid) - sid += 1 - - dropped = set() + graph = UndirectedGraph.from_edges(num_nodes, edges_u64) + if graph.number_of_edges != len(edges): + raise ValueError("edges must not contain duplicate undirected edges") + + indptr, dst, _, degrees = _adjacency(num_nodes, edges) + lengths = np.linalg.norm( + vertices[edges[:, 1]] - vertices[edges[:, 0]], + axis=1, + ) + work = ContractionGraph(graph, parallel_edges="keep") + work.add_edge_values("length", lengths, reduction="sum") + + eligible = _nodes_in_noncycle_components(indptr, dst, degrees) + for node in np.where((degrees == 2) & eligible)[0]: + node = int(node) + if work.can_suppress_node(node): + work.suppress_node(node) + + threshold = float(tick_length) while True: - best, best_len = None, tick_length - for s, (a, b, length, _) in supers.items(): - terminal_a, terminal_b = len(incident[a]) == 1, len(incident[b]) == 1 - if (terminal_a ^ terminal_b) and length < best_len: - best, best_len = s, length + edge_ids, active_lengths = work.active_edge_values("length") + best = None + for edge, length in zip(edge_ids, active_lengths, strict=True): + edge = int(edge) + length = float(length) + a, b = work.uv(edge) + terminal_a = work.degree(a) == 1 + terminal_b = work.degree(b) == 1 + if terminal_a ^ terminal_b and length < threshold: + candidate = (length, edge, a, b) + if best is None or candidate[:2] < best[:2]: + best = candidate if best is None: break - a, b, length, path = supers.pop(best) - incident[a].discard(best) - incident[b].discard(best) - dropped.update(path) + _, edge, a, b = best + work.erase_edge(edge) for node in (a, b): - if len(incident[node]) == 2: - s1, s2 = incident[node] - a1, b1, l1, p1 = supers.pop(s1) - a2, b2, l2, p2 = supers.pop(s2) - far1 = b1 if a1 == node else a1 - far2 = b2 if a2 == node else a2 - for x in (far1, far2, node): - incident[x].discard(s1) - incident[x].discard(s2) - supers[sid] = [far1, far2, l1 + l2, p1 + p2] - incident[far1].add(sid) - incident[far2].add(sid) - sid += 1 - - if dropped: - keep = np.ones(len(edges), dtype=bool) - keep[list(dropped)] = False - edges = edges[keep] - else: - edges = edges.copy() + if work.can_suppress_node(node): + work.suppress_node(node) + edges = edges[work.materialize().edge_mapping >= 0] vertices, edges, radii = _compact(vertices, edges, radii) return vertices, edges, radii @@ -360,7 +380,12 @@ def join_close_components(vertices, edges, dist, *, min_join_angle=175.0, vertices, edges, radii: ``vertices`` and ``radii`` unchanged; ``edges`` has the join edges added. """ - from scipy.spatial import cKDTree + try: + from scipy.spatial import cKDTree + except ImportError as error: + raise ImportError( + "join_close_components requires scipy; install scipy to use this function" + ) from error vertices = np.asarray(vertices, dtype=np.float64) edges = np.asarray(edges, dtype=np.int64) @@ -432,7 +457,8 @@ def draw_instances(vertices, edges, labels, shape, radius=1): Integer array with shape ``(E, 2)`` indexing ``vertices``. labels: Per-vertex integer labels, e.g. from - :func:`bioimage_cpp.graph.connected_components`. + :func:`bioimage_cpp.graph.connected_components`. Both endpoints of + each edge should have the same label. shape: Output volume shape ``(Z, Y, X)``. radius: diff --git a/src/cpp/filters/convolve_avx2.cxx b/src/cpp/filters/convolve_avx2.cxx new file mode 100644 index 0000000..d8118ee --- /dev/null +++ b/src/cpp/filters/convolve_avx2.cxx @@ -0,0 +1,486 @@ +#include "bioimage_cpp/filters/convolve.hxx" + +#if !defined(BIOIMAGE_FILTERS_AVX2_DISPATCH) +#error "convolve_avx2.cxx requires BIOIMAGE_FILTERS_AVX2_DISPATCH" +#endif + +#include +#include + +namespace bioimage_cpp::filters::avx2 { +namespace { + +template +void convolve_x_radius_avx2( + const float *__restrict in, + float *__restrict out, + const std::ptrdiff_t n_rows, + const std::ptrdiff_t n_cols, + const float *__restrict half_coefs +) { + if (n_cols <= 0 || n_rows <= 0) return; + + const std::ptrdiff_t prologue_end = std::min(R, n_cols); + const std::ptrdiff_t epilogue_start = + std::max(prologue_end, n_cols - R); + __m256 h[R + 1]; + for (int k = 0; k <= R; ++k) { + h[k] = _mm256_set1_ps(half_coefs[k]); + } + + for (std::ptrdiff_t row = 0; row < n_rows; ++row) { + const float *__restrict in_row = in + row * n_cols; + float *__restrict out_row = out + row * n_cols; + detail::convolve_x_border_range( + in_row, out_row, 0, prologue_end, n_cols, half_coefs + ); + + std::ptrdiff_t x = prologue_end; + for (; x + 8 <= epilogue_start; x += 8) { + __m256 acc; + if constexpr (Symmetric) { + acc = _mm256_mul_ps(_mm256_loadu_ps(in_row + x), h[0]); + } else { + acc = _mm256_setzero_ps(); + } + for (int k = 1; k <= R; ++k) { + const __m256 left = _mm256_loadu_ps(in_row + x - k); + const __m256 right = _mm256_loadu_ps(in_row + x + k); + const __m256 pair = Symmetric + ? _mm256_add_ps(left, right) + : _mm256_sub_ps(right, left); + acc = _mm256_fmadd_ps(h[k], pair, acc); + } + _mm256_storeu_ps(out_row + x, acc); + } + detail::convolve_x_main_range( + in_row, out_row, x, epilogue_start, half_coefs + ); + detail::convolve_x_border_range( + in_row, out_row, epilogue_start, n_cols, n_cols, half_coefs + ); + } +} + +template +void convolve_x( + const float *in, + float *out, + const std::ptrdiff_t n_rows, + const std::ptrdiff_t n_cols, + const int radius, + const float *half_coefs +) { +#define BIOIMAGE_FILTERS_AVX2_RADIUS_X(R) \ + case R: \ + convolve_x_radius_avx2( \ + in, out, n_rows, n_cols, half_coefs \ + ); \ + return; + + switch (radius) { + BIOIMAGE_FILTERS_AVX2_RADIUS_X(1) + BIOIMAGE_FILTERS_AVX2_RADIUS_X(2) + BIOIMAGE_FILTERS_AVX2_RADIUS_X(3) + BIOIMAGE_FILTERS_AVX2_RADIUS_X(4) + BIOIMAGE_FILTERS_AVX2_RADIUS_X(5) + BIOIMAGE_FILTERS_AVX2_RADIUS_X(6) + BIOIMAGE_FILTERS_AVX2_RADIUS_X(7) + BIOIMAGE_FILTERS_AVX2_RADIUS_X(8) + BIOIMAGE_FILTERS_AVX2_RADIUS_X(9) + BIOIMAGE_FILTERS_AVX2_RADIUS_X(10) + BIOIMAGE_FILTERS_AVX2_RADIUS_X(11) + BIOIMAGE_FILTERS_AVX2_RADIUS_X(12) + default: + return; + } +#undef BIOIMAGE_FILTERS_AVX2_RADIUS_X +} + +template +void convolve_strided_radius_avx2( + const float *__restrict in, + float *__restrict out, + const std::ptrdiff_t n_outer, + const std::ptrdiff_t n_axis, + const std::ptrdiff_t n_inner, + const float *__restrict half_coefs +) { + if (n_axis <= 0 || n_inner <= 0 || n_outer <= 0) return; + + const std::ptrdiff_t outer_stride = n_axis * n_inner; + const std::ptrdiff_t prologue_end = std::min(R, n_axis); + const std::ptrdiff_t epilogue_start = + std::max(prologue_end, n_axis - R); + __m256 h[R + 1]; + for (int k = 0; k <= R; ++k) { + h[k] = _mm256_set1_ps(half_coefs[k]); + } + + for (std::ptrdiff_t o = 0; o < n_outer; ++o) { + const float *__restrict in_o = in + o * outer_stride; + float *__restrict out_o = out + o * outer_stride; + + for (std::ptrdiff_t y = prologue_end; y < epilogue_start; ++y) { + const float *__restrict center = in_o + y * n_inner; + float *__restrict out_row = out_o + y * n_inner; + std::ptrdiff_t i = 0; + for (; i + 8 <= n_inner; i += 8) { + __m256 acc; + if constexpr (Symmetric) { + acc = _mm256_mul_ps(_mm256_loadu_ps(center + i), h[0]); + } else { + acc = _mm256_setzero_ps(); + } + for (int k = 1; k <= R; ++k) { + const __m256 up = _mm256_loadu_ps( + in_o + (y - k) * n_inner + i + ); + const __m256 dn = _mm256_loadu_ps( + in_o + (y + k) * n_inner + i + ); + const __m256 pair = Symmetric + ? _mm256_add_ps(up, dn) + : _mm256_sub_ps(dn, up); + acc = _mm256_fmadd_ps(h[k], pair, acc); + } + _mm256_storeu_ps(out_row + i, acc); + } + for (; i < n_inner; ++i) { + float acc = Symmetric ? half_coefs[0] * center[i] : 0.0f; + for (int k = 1; k <= R; ++k) { + const float up = in_o[(y - k) * n_inner + i]; + const float dn = in_o[(y + k) * n_inner + i]; + acc += half_coefs[k] * (Symmetric ? up + dn : dn - up); + } + out_row[i] = acc; + } + } + + for (std::ptrdiff_t y = 0; y < prologue_end; ++y) { + detail::convolve_strided_border_row( + in_o, out_o, y, n_axis, n_inner, half_coefs + ); + } + for (std::ptrdiff_t y = epilogue_start; y < n_axis; ++y) { + detail::convolve_strided_border_row( + in_o, out_o, y, n_axis, n_inner, half_coefs + ); + } + } +} + +template +void convolve_strided( + const float *in, + float *out, + const std::ptrdiff_t n_outer, + const std::ptrdiff_t n_axis, + const std::ptrdiff_t n_inner, + const int radius, + const float *half_coefs +) { +#define BIOIMAGE_FILTERS_AVX2_RADIUS_S(R) \ + case R: \ + convolve_strided_radius_avx2( \ + in, out, n_outer, n_axis, n_inner, half_coefs \ + ); \ + return; + + switch (radius) { + BIOIMAGE_FILTERS_AVX2_RADIUS_S(1) + BIOIMAGE_FILTERS_AVX2_RADIUS_S(2) + BIOIMAGE_FILTERS_AVX2_RADIUS_S(3) + BIOIMAGE_FILTERS_AVX2_RADIUS_S(4) + BIOIMAGE_FILTERS_AVX2_RADIUS_S(5) + BIOIMAGE_FILTERS_AVX2_RADIUS_S(6) + BIOIMAGE_FILTERS_AVX2_RADIUS_S(7) + BIOIMAGE_FILTERS_AVX2_RADIUS_S(8) + BIOIMAGE_FILTERS_AVX2_RADIUS_S(9) + BIOIMAGE_FILTERS_AVX2_RADIUS_S(10) + BIOIMAGE_FILTERS_AVX2_RADIUS_S(11) + BIOIMAGE_FILTERS_AVX2_RADIUS_S(12) + default: + return; + } +#undef BIOIMAGE_FILTERS_AVX2_RADIUS_S +} + +} // namespace + +void convolve_axis_x( + const float *in, + float *out, + const std::ptrdiff_t n_rows, + const std::ptrdiff_t n_cols, + const int radius, + const bool symmetric, + const float *half_coefs +) { + if (symmetric) { + convolve_x(in, out, n_rows, n_cols, radius, half_coefs); + } else { + convolve_x(in, out, n_rows, n_cols, radius, half_coefs); + } +} + +void convolve_axis_strided( + const float *in, + float *out, + const std::ptrdiff_t n_outer, + const std::ptrdiff_t n_axis, + const std::ptrdiff_t n_inner, + const int radius, + const bool symmetric, + const float *half_coefs +) { + if (symmetric) { + convolve_strided( + in, out, n_outer, n_axis, n_inner, radius, half_coefs + ); + } else { + convolve_strided( + in, out, n_outer, n_axis, n_inner, radius, half_coefs + ); + } +} + +template +void convolve_outer_products_radius_avx2( + const std::array &gradients, + const std::array> &out, + const std::ptrdiff_t n_outer, + const std::ptrdiff_t n_axis, + const std::ptrdiff_t n_inner, + const float *half_coefs +) { + static_assert(D == 2 || D == 3); + if (n_axis <= 0 || n_inner <= 0 || n_outer <= 0) return; + + constexpr std::size_t C = detail::kSymmetricComponents; + const std::ptrdiff_t outer_stride = n_axis * n_inner; + __m256 h[R + 1]; + for (int k = 0; k <= R; ++k) { + h[k] = _mm256_set1_ps(half_coefs[k]); + } + + for (std::ptrdiff_t o = 0; o < n_outer; ++o) { + std::array in_o{}; + std::array out_o{}; + for (std::size_t d = 0; d < D; ++d) { + in_o[d] = gradients[d] + o * outer_stride; + } + for (std::size_t c = 0; c < C; ++c) { + out_o[c] = out[c] + o * outer_stride; + } + + for (std::ptrdiff_t y = 0; y < n_axis; ++y) { + std::ptrdiff_t i = 0; + for (; i + 8 <= n_inner; i += 8) { + const std::ptrdiff_t center_index = y * n_inner + i; + const __m256 g0 = _mm256_loadu_ps(in_o[0] + center_index); + const __m256 g1 = _mm256_loadu_ps(in_o[1] + center_index); + __m256 acc0 = _mm256_mul_ps(h[0], _mm256_mul_ps(g0, g0)); + __m256 acc1 = _mm256_mul_ps(h[0], _mm256_mul_ps(g0, g1)); + __m256 acc2; + __m256 acc3; + __m256 acc4; + __m256 acc5; + if constexpr (D == 2) { + acc2 = _mm256_mul_ps(h[0], _mm256_mul_ps(g1, g1)); + } else { + const __m256 g2 = _mm256_loadu_ps(in_o[2] + center_index); + acc2 = _mm256_mul_ps(h[0], _mm256_mul_ps(g0, g2)); + acc3 = _mm256_mul_ps(h[0], _mm256_mul_ps(g1, g1)); + acc4 = _mm256_mul_ps(h[0], _mm256_mul_ps(g1, g2)); + acc5 = _mm256_mul_ps(h[0], _mm256_mul_ps(g2, g2)); + } + + for (int k = 1; k <= R; ++k) { + const std::ptrdiff_t y_up = detail::mirror_index(y - k, n_axis); + const std::ptrdiff_t y_dn = detail::mirror_index(y + k, n_axis); + const std::ptrdiff_t up_index = y_up * n_inner + i; + const std::ptrdiff_t dn_index = y_dn * n_inner + i; + const __m256 g0u = _mm256_loadu_ps(in_o[0] + up_index); + const __m256 g0d = _mm256_loadu_ps(in_o[0] + dn_index); + const __m256 g1u = _mm256_loadu_ps(in_o[1] + up_index); + const __m256 g1d = _mm256_loadu_ps(in_o[1] + dn_index); + acc0 = _mm256_fmadd_ps( + h[k], + _mm256_add_ps( + _mm256_mul_ps(g0u, g0u), _mm256_mul_ps(g0d, g0d) + ), + acc0 + ); + acc1 = _mm256_fmadd_ps( + h[k], + _mm256_add_ps( + _mm256_mul_ps(g0u, g1u), _mm256_mul_ps(g0d, g1d) + ), + acc1 + ); + if constexpr (D == 2) { + acc2 = _mm256_fmadd_ps( + h[k], + _mm256_add_ps( + _mm256_mul_ps(g1u, g1u), _mm256_mul_ps(g1d, g1d) + ), + acc2 + ); + } else { + const __m256 g2u = _mm256_loadu_ps(in_o[2] + up_index); + const __m256 g2d = _mm256_loadu_ps(in_o[2] + dn_index); + acc2 = _mm256_fmadd_ps( + h[k], + _mm256_add_ps( + _mm256_mul_ps(g0u, g2u), _mm256_mul_ps(g0d, g2d) + ), + acc2 + ); + acc3 = _mm256_fmadd_ps( + h[k], + _mm256_add_ps( + _mm256_mul_ps(g1u, g1u), _mm256_mul_ps(g1d, g1d) + ), + acc3 + ); + acc4 = _mm256_fmadd_ps( + h[k], + _mm256_add_ps( + _mm256_mul_ps(g1u, g2u), _mm256_mul_ps(g1d, g2d) + ), + acc4 + ); + acc5 = _mm256_fmadd_ps( + h[k], + _mm256_add_ps( + _mm256_mul_ps(g2u, g2u), _mm256_mul_ps(g2d, g2d) + ), + acc5 + ); + } + } + + _mm256_storeu_ps(out_o[0] + center_index, acc0); + _mm256_storeu_ps(out_o[1] + center_index, acc1); + _mm256_storeu_ps(out_o[2] + center_index, acc2); + if constexpr (D == 3) { + _mm256_storeu_ps(out_o[3] + center_index, acc3); + _mm256_storeu_ps(out_o[4] + center_index, acc4); + _mm256_storeu_ps(out_o[5] + center_index, acc5); + } + } + + for (; i < n_inner; ++i) { + const std::ptrdiff_t center_index = y * n_inner + i; + const float g0 = in_o[0][center_index]; + const float g1 = in_o[1][center_index]; + float acc[C]; + acc[0] = half_coefs[0] * g0 * g0; + acc[1] = half_coefs[0] * g0 * g1; + if constexpr (D == 2) { + acc[2] = half_coefs[0] * g1 * g1; + } else { + const float g2 = in_o[2][center_index]; + acc[2] = half_coefs[0] * g0 * g2; + acc[3] = half_coefs[0] * g1 * g1; + acc[4] = half_coefs[0] * g1 * g2; + acc[5] = half_coefs[0] * g2 * g2; + } + + for (int k = 1; k <= R; ++k) { + const std::ptrdiff_t y_up = detail::mirror_index(y - k, n_axis); + const std::ptrdiff_t y_dn = detail::mirror_index(y + k, n_axis); + const std::ptrdiff_t up_index = y_up * n_inner + i; + const std::ptrdiff_t dn_index = y_dn * n_inner + i; + const float g0u = in_o[0][up_index]; + const float g0d = in_o[0][dn_index]; + const float g1u = in_o[1][up_index]; + const float g1d = in_o[1][dn_index]; + const float hk = half_coefs[k]; + acc[0] += hk * (g0u * g0u + g0d * g0d); + acc[1] += hk * (g0u * g1u + g0d * g1d); + if constexpr (D == 2) { + acc[2] += hk * (g1u * g1u + g1d * g1d); + } else { + const float g2u = in_o[2][up_index]; + const float g2d = in_o[2][dn_index]; + acc[2] += hk * (g0u * g2u + g0d * g2d); + acc[3] += hk * (g1u * g1u + g1d * g1d); + acc[4] += hk * (g1u * g2u + g1d * g2d); + acc[5] += hk * (g2u * g2u + g2d * g2d); + } + } + for (std::size_t c = 0; c < C; ++c) { + out_o[c][center_index] = acc[c]; + } + } + } + } +} + +template +void convolve_outer_products( + const std::array &gradients, + const std::array> &out, + const std::ptrdiff_t n_outer, + const std::ptrdiff_t n_axis, + const std::ptrdiff_t n_inner, + const int radius, + const float *half_coefs +) { +#define BIOIMAGE_FILTERS_AVX2_RADIUS_OP(R) \ + case R: \ + convolve_outer_products_radius_avx2( \ + gradients, out, n_outer, n_axis, n_inner, half_coefs \ + ); \ + return; + + switch (radius) { + BIOIMAGE_FILTERS_AVX2_RADIUS_OP(1) + BIOIMAGE_FILTERS_AVX2_RADIUS_OP(2) + BIOIMAGE_FILTERS_AVX2_RADIUS_OP(3) + BIOIMAGE_FILTERS_AVX2_RADIUS_OP(4) + BIOIMAGE_FILTERS_AVX2_RADIUS_OP(5) + BIOIMAGE_FILTERS_AVX2_RADIUS_OP(6) + BIOIMAGE_FILTERS_AVX2_RADIUS_OP(7) + BIOIMAGE_FILTERS_AVX2_RADIUS_OP(8) + BIOIMAGE_FILTERS_AVX2_RADIUS_OP(9) + BIOIMAGE_FILTERS_AVX2_RADIUS_OP(10) + BIOIMAGE_FILTERS_AVX2_RADIUS_OP(11) + BIOIMAGE_FILTERS_AVX2_RADIUS_OP(12) + default: + return; + } +#undef BIOIMAGE_FILTERS_AVX2_RADIUS_OP +} + +void convolve_outer_products_2d( + const std::array &gradients, + const std::array &out, + const std::ptrdiff_t n_outer, + const std::ptrdiff_t n_axis, + const std::ptrdiff_t n_inner, + const int radius, + const float *half_coefs +) { + convolve_outer_products<2>( + gradients, out, n_outer, n_axis, n_inner, radius, half_coefs + ); +} + +void convolve_outer_products_3d( + const std::array &gradients, + const std::array &out, + const std::ptrdiff_t n_outer, + const std::ptrdiff_t n_axis, + const std::ptrdiff_t n_inner, + const int radius, + const float *half_coefs +) { + convolve_outer_products<3>( + gradients, out, n_outer, n_axis, n_inner, radius, half_coefs + ); +} + +} // namespace bioimage_cpp::filters::avx2 diff --git a/src/cpp/filters/dispatch.cxx b/src/cpp/filters/dispatch.cxx new file mode 100644 index 0000000..0d6aa6c --- /dev/null +++ b/src/cpp/filters/dispatch.cxx @@ -0,0 +1,209 @@ +#include "bioimage_cpp/filters/dispatch.hxx" + +#include +#include + +#if defined(_MSC_VER) +#include +#include +#endif + +namespace bioimage_cpp::filters::avx2 { + +void convolve_axis_x( + const float *in, + float *out, + std::ptrdiff_t n_rows, + std::ptrdiff_t n_cols, + int radius, + bool symmetric, + const float *half_coefs +); + +void convolve_axis_strided( + const float *in, + float *out, + std::ptrdiff_t n_outer, + std::ptrdiff_t n_axis, + std::ptrdiff_t n_inner, + int radius, + bool symmetric, + const float *half_coefs +); + +void convolve_outer_products_2d( + const std::array &gradients, + const std::array &out, + std::ptrdiff_t n_outer, + std::ptrdiff_t n_axis, + std::ptrdiff_t n_inner, + int radius, + const float *half_coefs +); + +void convolve_outer_products_3d( + const std::array &gradients, + const std::array &out, + std::ptrdiff_t n_outer, + std::ptrdiff_t n_axis, + std::ptrdiff_t n_inner, + int radius, + const float *half_coefs +); + +void ev3_symmetric_descending_interleaved( + const float *__restrict a00, + const float *__restrict a01, + const float *__restrict a02, + const float *__restrict a11, + const float *__restrict a12, + const float *__restrict a22, + float *__restrict out, + std::ptrdiff_t n +); + +} // namespace bioimage_cpp::filters::avx2 + +namespace bioimage_cpp::filters::dispatch { +namespace { + +bool force_scalar_requested() { + const char *value = std::getenv("BIOIMAGE_CPP_FILTERS_FORCE_SCALAR"); + return value != nullptr && value[0] != '\0' && std::strcmp(value, "0") != 0; +} + +bool runtime_avx2_fma_supported() noexcept { + static const bool supported = []() noexcept { +#if defined(_MSC_VER) + int registers[4]{}; + __cpuid(registers, 1); + constexpr int fma_bit = 1 << 12; + constexpr int osxsave_bit = 1 << 27; + constexpr int avx_bit = 1 << 28; + if ((registers[2] & (fma_bit | osxsave_bit | avx_bit)) != + (fma_bit | osxsave_bit | avx_bit)) { + return false; + } + if ((_xgetbv(0) & 0x6) != 0x6) { + return false; + } + __cpuidex(registers, 7, 0); + constexpr int avx2_bit = 1 << 5; + return (registers[1] & avx2_bit) != 0; +#elif defined(__GNUC__) || defined(__clang__) + __builtin_cpu_init(); + return __builtin_cpu_supports("avx2") && __builtin_cpu_supports("fma"); +#else + return false; +#endif + }(); + return supported; +} + +bool use_avx2() { + return !force_scalar_requested() && runtime_avx2_fma_supported(); +} + +} // namespace + +bool try_convolve_axis_x( + const float *in, + float *out, + const std::ptrdiff_t n_rows, + const std::ptrdiff_t n_cols, + const int radius, + const bool symmetric, + const float *half_coefs +) { + if (!use_avx2() || radius < 1 || radius > 12) { + return false; + } + avx2::convolve_axis_x( + in, out, n_rows, n_cols, radius, symmetric, half_coefs + ); + return true; +} + +bool try_convolve_axis_strided( + const float *in, + float *out, + const std::ptrdiff_t n_outer, + const std::ptrdiff_t n_axis, + const std::ptrdiff_t n_inner, + const int radius, + const bool symmetric, + const float *half_coefs +) { + if (!use_avx2() || radius < 1 || radius > 12) { + return false; + } + avx2::convolve_axis_strided( + in, out, n_outer, n_axis, n_inner, radius, symmetric, half_coefs + ); + return true; +} + +bool try_convolve_outer_products_2d( + const std::array &gradients, + const std::array &out, + const std::ptrdiff_t n_outer, + const std::ptrdiff_t n_axis, + const std::ptrdiff_t n_inner, + const int radius, + const float *half_coefs +) { + if (!use_avx2() || radius < 1 || radius > 12) { + return false; + } + avx2::convolve_outer_products_2d( + gradients, out, n_outer, n_axis, n_inner, radius, half_coefs + ); + return true; +} + +bool try_convolve_outer_products_3d( + const std::array &gradients, + const std::array &out, + const std::ptrdiff_t n_outer, + const std::ptrdiff_t n_axis, + const std::ptrdiff_t n_inner, + const int radius, + const float *half_coefs +) { + if (!use_avx2() || radius < 1 || radius > 12) { + return false; + } + avx2::convolve_outer_products_3d( + gradients, out, n_outer, n_axis, n_inner, radius, half_coefs + ); + return true; +} + +bool try_ev3_symmetric_descending_interleaved( + const float *a00, + const float *a01, + const float *a02, + const float *a11, + const float *a12, + const float *a22, + float *out, + const std::ptrdiff_t n +) { + if (!use_avx2() || n < 8) { + return false; + } + avx2::ev3_symmetric_descending_interleaved( + a00, a01, a02, a11, a12, a22, out, n + ); + return true; +} + +const char *convolution_backend() { + return use_avx2() ? "avx2" : "scalar"; +} + +const char *eigenvalue_backend() { + return use_avx2() ? "avx2" : "scalar"; +} + +} // namespace bioimage_cpp::filters::dispatch diff --git a/src/cpp/filters/eigenvalues_avx2.cxx b/src/cpp/filters/eigenvalues_avx2.cxx new file mode 100644 index 0000000..1f55565 --- /dev/null +++ b/src/cpp/filters/eigenvalues_avx2.cxx @@ -0,0 +1,281 @@ +#include "bioimage_cpp/filters/eigenvalues.hxx" + +#if !defined(BIOIMAGE_FILTERS_AVX2_DISPATCH) +#error "eigenvalues_avx2.cxx requires BIOIMAGE_FILTERS_AVX2_DISPATCH" +#endif + +#include +#include +#include + +namespace bioimage_cpp::filters::avx2 { +namespace { + +inline __m256 abs_ps(const __m256 value) { + const __m256 sign = _mm256_set1_ps(-0.0f); + return _mm256_andnot_ps(sign, value); +} + +inline __m256 approximate_acos(const __m256 value) { + constexpr float coefficients[] = { + 1.4866664409637451f, + -0.07770606875419617f, + 0.005770874209702015f, + -0.0005768424598500133f, + 0.00006643808592343703f, + -0.000008315640116052236f, + 0.0000010878551393034286f, + -0.00000014935945102934056f, + }; + + const __m256 one = _mm256_set1_ps(1.0f); + const __m256 t = abs_ps(value); + const __m256 z = _mm256_sub_ps(_mm256_add_ps(t, t), one); + const __m256 two_z = _mm256_add_ps(z, z); + + __m256 b1 = _mm256_setzero_ps(); + __m256 b2 = _mm256_setzero_ps(); + for (int k = 7; k >= 1; --k) { + const __m256 ck = _mm256_set1_ps(coefficients[k]); + const __m256 b0 = _mm256_fmadd_ps(two_z, b1, _mm256_sub_ps(ck, b2)); + b2 = b1; + b1 = b0; + } + const __m256 q = _mm256_fmadd_ps( + z, b1, _mm256_sub_ps(_mm256_set1_ps(coefficients[0]), b2) + ); + const __m256 base = _mm256_mul_ps( + _mm256_sqrt_ps(_mm256_max_ps(_mm256_setzero_ps(), _mm256_sub_ps(one, t))), + q + ); + const __m256 reflected = _mm256_sub_ps(_mm256_set1_ps(3.14159265358979323846f), base); + const __m256 negative = _mm256_cmp_ps(value, _mm256_setzero_ps(), _CMP_LT_OQ); + return _mm256_blendv_ps(base, reflected, negative); +} + +inline __m256 approximate_cos_phi(const __m256 phi) { + constexpr float coefficients[] = { + 1.0f, + -0.4999999701976776f, + 0.041666433215141296f, + -0.0013882536441087723f, + 0.000024095119442790747f, + }; + + const __m256 u = _mm256_mul_ps(phi, phi); + __m256 result = _mm256_set1_ps(coefficients[4]); + result = _mm256_fmadd_ps(result, u, _mm256_set1_ps(coefficients[3])); + result = _mm256_fmadd_ps(result, u, _mm256_set1_ps(coefficients[2])); + result = _mm256_fmadd_ps(result, u, _mm256_set1_ps(coefficients[1])); + return _mm256_fmadd_ps(result, u, _mm256_set1_ps(coefficients[0])); +} + +inline __m256 approximate_sin_phi(const __m256 phi) { + constexpr float coefficients[] = { + 1.0f, + -0.1666666567325592f, + 0.008333305828273296f, + -0.0001983467664103955f, + 0.0000026878346943703946f, + }; + + const __m256 u = _mm256_mul_ps(phi, phi); + __m256 sinc = _mm256_set1_ps(coefficients[4]); + sinc = _mm256_fmadd_ps(sinc, u, _mm256_set1_ps(coefficients[3])); + sinc = _mm256_fmadd_ps(sinc, u, _mm256_set1_ps(coefficients[2])); + sinc = _mm256_fmadd_ps(sinc, u, _mm256_set1_ps(coefficients[1])); + sinc = _mm256_fmadd_ps(sinc, u, _mm256_set1_ps(coefficients[0])); + return _mm256_mul_ps(phi, sinc); +} + +inline void store_four_triples( + __m128 e0, + __m128 e1, + __m128 e2, + float *out +) { + __m128 zero = _mm_setzero_ps(); + _MM_TRANSPOSE4_PS(e0, e1, e2, zero); + + const __m128 first_next = _mm_shuffle_ps(e1, e1, _MM_SHUFFLE(0, 0, 0, 0)); + const __m128 out0 = _mm_blend_ps(e0, first_next, 0b1000); + const __m128 out1 = _mm_shuffle_ps(e1, e2, _MM_SHUFFLE(1, 0, 2, 1)); + const __m128 out2_candidate = + _mm_shuffle_ps(e2, zero, _MM_SHUFFLE(2, 1, 0, 2)); + const __m128 fourth_first = + _mm_shuffle_ps(zero, zero, _MM_SHUFFLE(0, 0, 0, 0)); + const __m128 out2 = _mm_blend_ps(out2_candidate, fourth_first, 0b0010); + + _mm_storeu_ps(out, out0); + _mm_storeu_ps(out + 4, out1); + _mm_storeu_ps(out + 8, out2); +} + +inline void store_eight_triples( + const __m256 e0, + const __m256 e1, + const __m256 e2, + float *out +) { + store_four_triples( + _mm256_castps256_ps128(e0), + _mm256_castps256_ps128(e1), + _mm256_castps256_ps128(e2), + out + ); + store_four_triples( + _mm256_extractf128_ps(e0, 1), + _mm256_extractf128_ps(e1, 1), + _mm256_extractf128_ps(e2, 1), + out + 12 + ); +} + +} // namespace + +void ev3_symmetric_descending_interleaved( + const float *__restrict a00, + const float *__restrict a01, + const float *__restrict a02, + const float *__restrict a11, + const float *__restrict a12, + const float *__restrict a22, + float *__restrict out, + const std::ptrdiff_t n +) { + const __m256 zero = _mm256_setzero_ps(); + const __m256 one = _mm256_set1_ps(1.0f); + const __m256 one_third = _mm256_set1_ps(1.0f / 3.0f); + const __m256 one_sixth = _mm256_set1_ps(1.0f / 6.0f); + const __m256 two = _mm256_set1_ps(2.0f); + const __m256 minus_one = _mm256_set1_ps(-1.0f); + const __m256 minus_half = _mm256_set1_ps(-0.5f); + const __m256 minus_sqrt_three_over_two = + _mm256_set1_ps(-0.8660254037844386f); + + const std::ptrdiff_t vector_end = n - n % 8; + std::ptrdiff_t i = 0; + for (; i < vector_end; i += 8) { + __m256 v00 = _mm256_loadu_ps(a00 + i); + __m256 v01 = _mm256_loadu_ps(a01 + i); + __m256 v02 = _mm256_loadu_ps(a02 + i); + __m256 v11 = _mm256_loadu_ps(a11 + i); + __m256 v12 = _mm256_loadu_ps(a12 + i); + __m256 v22 = _mm256_loadu_ps(a22 + i); + + __m256 scale = abs_ps(v00); + scale = _mm256_max_ps(scale, abs_ps(v01)); + scale = _mm256_max_ps(scale, abs_ps(v02)); + scale = _mm256_max_ps(scale, abs_ps(v11)); + scale = _mm256_max_ps(scale, abs_ps(v12)); + scale = _mm256_max_ps(scale, abs_ps(v22)); + + const __m256 zero_scale = _mm256_cmp_ps(scale, zero, _CMP_EQ_OQ); + const __m256 positive_scale = _mm256_cmp_ps(scale, zero, _CMP_GT_OQ); + const __m256 subnormal_scale = _mm256_and_ps( + positive_scale, + _mm256_cmp_ps( + scale, + _mm256_set1_ps(std::numeric_limits::min()), + _CMP_LT_OQ + ) + ); + if (_mm256_movemask_ps(subnormal_scale) != 0) { + for (std::ptrdiff_t j = i; j < i + 8; ++j) { + detail::ev3_one_descending( + a00[j], a01[j], a02[j], a11[j], a12[j], a22[j], + out[3 * j], out[3 * j + 1], out[3 * j + 2] + ); + } + continue; + } + const __m256 safe_scale = _mm256_blendv_ps(scale, one, zero_scale); + const __m256 inv_scale = _mm256_div_ps(one, safe_scale); + v00 = _mm256_mul_ps(v00, inv_scale); + v01 = _mm256_mul_ps(v01, inv_scale); + v02 = _mm256_mul_ps(v02, inv_scale); + v11 = _mm256_mul_ps(v11, inv_scale); + v12 = _mm256_mul_ps(v12, inv_scale); + v22 = _mm256_mul_ps(v22, inv_scale); + + const __m256 mean = _mm256_mul_ps( + _mm256_add_ps(_mm256_add_ps(v00, v11), v22), one_third + ); + const __m256 b00 = _mm256_sub_ps(v00, mean); + const __m256 b11 = _mm256_sub_ps(v11, mean); + const __m256 b22 = _mm256_sub_ps(v22, mean); + + __m256 trace_b2 = _mm256_mul_ps(b00, b00); + trace_b2 = _mm256_fmadd_ps(b11, b11, trace_b2); + trace_b2 = _mm256_fmadd_ps(b22, b22, trace_b2); + __m256 off_diagonal = _mm256_mul_ps(v01, v01); + off_diagonal = _mm256_fmadd_ps(v02, v02, off_diagonal); + off_diagonal = _mm256_fmadd_ps(v12, v12, off_diagonal); + trace_b2 = _mm256_fmadd_ps(two, off_diagonal, trace_b2); + const __m256 p2 = _mm256_mul_ps(trace_b2, one_sixth); + + const __m256 valid_p2 = _mm256_cmp_ps(p2, zero, _CMP_GT_OQ); + const __m256 safe_p2 = _mm256_blendv_ps(one, p2, valid_p2); + const __m256 p = _mm256_sqrt_ps(safe_p2); + + const __m256 minor0 = _mm256_fnmadd_ps(v12, v12, _mm256_mul_ps(b11, b22)); + const __m256 minor1 = _mm256_fnmadd_ps(v12, v02, _mm256_mul_ps(v01, b22)); + const __m256 minor2 = _mm256_fnmadd_ps(b11, v02, _mm256_mul_ps(v01, v12)); + __m256 det_b = _mm256_mul_ps(b00, minor0); + det_b = _mm256_fnmadd_ps(v01, minor1, det_b); + det_b = _mm256_fmadd_ps(v02, minor2, det_b); + + const __m256 denominator = _mm256_mul_ps(two, _mm256_mul_ps(safe_p2, p)); + const __m256 valid_denominator = + _mm256_cmp_ps(denominator, zero, _CMP_GT_OQ); + const __m256 valid = _mm256_and_ps(valid_p2, valid_denominator); + const __m256 safe_denominator = + _mm256_blendv_ps(one, denominator, valid); + __m256 r = _mm256_div_ps(det_b, safe_denominator); + r = _mm256_min_ps(one, _mm256_max_ps(minus_one, r)); + + const __m256 phi = _mm256_mul_ps(approximate_acos(r), one_third); + const __m256 cos_phi = approximate_cos_phi(phi); + const __m256 sin_phi = approximate_sin_phi(phi); + const __m256 two_p = _mm256_add_ps(p, p); + + __m256 root_large = _mm256_fmadd_ps(two_p, cos_phi, mean); + const __m256 small_angle = _mm256_fmadd_ps( + minus_sqrt_three_over_two, sin_phi, + _mm256_mul_ps(minus_half, cos_phi) + ); + __m256 root_small = _mm256_fmadd_ps(two_p, small_angle, mean); + __m256 root_mid = _mm256_sub_ps( + _mm256_mul_ps(_mm256_set1_ps(3.0f), mean), + _mm256_add_ps(root_large, root_small) + ); + + const __m256 equal_root = _mm256_mul_ps(mean, scale); + root_large = _mm256_mul_ps(root_large, scale); + root_mid = _mm256_mul_ps(root_mid, scale); + root_small = _mm256_mul_ps(root_small, scale); + const __m256 invalid = _mm256_or_ps( + zero_scale, _mm256_cmp_ps(valid, zero, _CMP_EQ_OQ) + ); + root_large = _mm256_blendv_ps(root_large, equal_root, invalid); + root_mid = _mm256_blendv_ps(root_mid, equal_root, invalid); + root_small = _mm256_blendv_ps(root_small, equal_root, invalid); + + const __m256 high01 = _mm256_max_ps(root_large, root_mid); + const __m256 low01 = _mm256_min_ps(root_large, root_mid); + const __m256 high = _mm256_max_ps(high01, root_small); + const __m256 middle_candidate = _mm256_min_ps(high01, root_small); + const __m256 middle = _mm256_max_ps(low01, middle_candidate); + const __m256 low = _mm256_min_ps(low01, middle_candidate); + store_eight_triples(high, middle, low, out + 3 * i); + } + + for (; i < n; ++i) { + detail::ev3_one_descending( + a00[i], a01[i], a02[i], a11[i], a12[i], a22[i], + out[3 * i], out[3 * i + 1], out[3 * i + 2] + ); + } +} + +} // namespace bioimage_cpp::filters::avx2 diff --git a/tests/graph/agglomeration/test_mala.py b/tests/graph/agglomeration/test_mala.py index 09c0448..fc9acc2 100644 --- a/tests/graph/agglomeration/test_mala.py +++ b/tests/graph/agglomeration/test_mala.py @@ -51,6 +51,35 @@ def test_num_clusters_stop_respected(): assert len(np.unique(labels)) == 3 +def test_num_edges_stop_counts_contracted_edges_on_chain(): + graph = chain_graph(5) + indicators = np.array([0.1, 0.2, 0.3, 0.4], dtype=np.float64) + + labels = bic.graph.agglomeration.MalaClusterPolicy( + threshold=1.0, + num_clusters_stop=1, + num_edges_stop=3, + ).optimize(graph, indicators) + + assert len(np.unique(labels)) == 4 + + +def test_num_edges_stop_counts_contracted_and_folded_edges(): + graph = bic.graph.UndirectedGraph.from_edges( + 3, + [[0, 1], [0, 2], [1, 2]], + ) + indicators = np.array([0.1, 0.2, 0.3], dtype=np.float64) + + labels = bic.graph.agglomeration.MalaClusterPolicy( + threshold=1.0, + num_clusters_stop=1, + num_edges_stop=1, + ).optimize(graph, indicators) + + assert len(np.unique(labels)) == 2 + + def test_float32_and_float64_match(): graph = two_clusters_graph() indicators_f32 = np.array( @@ -74,12 +103,34 @@ def test_bad_bin_range_raises(): ).optimize(graph, np.array([0.1, 0.1], dtype=np.float64)) -def test_zero_bins_raises(): +@pytest.mark.parametrize("num_bins", [0, 1, -1]) +def test_invalid_bin_count_raises(num_bins): + with pytest.raises(ValueError, match="num_bins must be >= 2"): + bic.graph.agglomeration.MalaClusterPolicy(num_bins=num_bins) + + +@pytest.mark.parametrize("num_bins", [True, 1.5]) +def test_non_integral_bin_count_raises(num_bins): + with pytest.raises(TypeError, match="num_bins must be an integer"): + bic.graph.agglomeration.MalaClusterPolicy(num_bins=num_bins) + + +@pytest.mark.parametrize( + ("threshold", "expected_clusters"), + [ + (-0.1, 3), + (0.5, 2), + (1.1, 1), + ], +) +def test_two_bin_threshold_behavior(threshold, expected_clusters): graph = chain_graph(3) - with pytest.raises(Exception): - bic.graph.agglomeration.MalaClusterPolicy(num_bins=0).optimize( - graph, np.array([0.1, 0.1], dtype=np.float64) - ) + labels = bic.graph.agglomeration.MalaClusterPolicy( + num_bins=2, + threshold=threshold, + num_clusters_stop=1, + ).optimize(graph, np.array([0.2, 0.8], dtype=np.float64)) + assert np.unique(labels).size == expected_clusters def test_indicator_length_mismatch_raises(): diff --git a/tests/graph/lifted_multicut/test_fusion_move.py b/tests/graph/lifted_multicut/test_fusion_move.py index 3bda8eb..ff015db 100644 --- a/tests/graph/lifted_multicut/test_fusion_move.py +++ b/tests/graph/lifted_multicut/test_fusion_move.py @@ -313,13 +313,8 @@ def test_fusion_move_default_parallel_proposals_tracks_threads(): def test_greedy_proposals_parallel_is_deterministic_on_dirty_base_graph(): # Regression guard for the lazy-CSR-adjacency data race on the *base* graph. - # The greedy-additive proposal generator reads base_graph.node_adjacency() - # (via DynamicGraph::reset); with T>1 the parallel proposal slots used to - # race on the first rebuild of a not-yet-frozen base graph. Unlike the - # multicut driver, here the singleton warm-start only freezes the *lifted* - # graph, so the race is reachable from the default start. The solver now - # freezes the base graph before fan-out; the multi-threaded result must equal - # the single-threaded reference on every run. + # Proposal generators can read base adjacency from parallel slots. The + # solver must freeze a dirty base graph before the fan-out. # # Note: a regression here can surface as a process crash (it is a data race), # not just a value mismatch. diff --git a/tests/graph/lifted_multicut/test_greedy_additive.py b/tests/graph/lifted_multicut/test_greedy_additive.py index 4629b5e..b28b90a 100644 --- a/tests/graph/lifted_multicut/test_greedy_additive.py +++ b/tests/graph/lifted_multicut/test_greedy_additive.py @@ -86,3 +86,39 @@ def test_greedy_additive_lifted_attractive_merges_through_base_path(): ) labels = bic.graph.lifted_multicut.LiftedGreedyAdditiveMulticut().optimize(objective) same_partition(labels, [0, 0, 0]) + + +def test_folded_lifted_edges_do_not_become_merge_candidates(): + base = bic.graph.UndirectedGraph.from_edges(3, [[0, 1]]) + objective = bic.graph.lifted_multicut.LiftedMulticutObjective( + base, + np.array([10.0], dtype=np.float64), + lifted_uvs=np.array([[0, 2], [1, 2]], dtype=np.uint64), + lifted_costs=np.array([8.0, 9.0], dtype=np.float64), + ) + + labels = bic.graph.lifted_multicut.LiftedGreedyAdditiveMulticut().optimize( + objective + ) + + assert labels[0] == labels[1] + assert labels[0] != labels[2] + + +def test_base_and_lifted_fold_remains_a_base_merge_candidate(): + base = bic.graph.UndirectedGraph.from_edges( + 3, + [[0, 1], [0, 2]], + ) + objective = bic.graph.lifted_multicut.LiftedMulticutObjective( + base, + np.array([10.0, -5.0], dtype=np.float64), + lifted_uvs=np.array([[1, 2]], dtype=np.uint64), + lifted_costs=np.array([10.0], dtype=np.float64), + ) + + labels = bic.graph.lifted_multicut.LiftedGreedyAdditiveMulticut().optimize( + objective + ) + + assert len(np.unique(labels)) == 1 diff --git a/tests/graph/lifted_multicut/test_kernighan_lin.py b/tests/graph/lifted_multicut/test_kernighan_lin.py index 43a1c7e..85b6d6d 100644 --- a/tests/graph/lifted_multicut/test_kernighan_lin.py +++ b/tests/graph/lifted_multicut/test_kernighan_lin.py @@ -97,3 +97,32 @@ def test_kl_warm_starts_from_singleton(): number_of_outer_iterations=5 ).optimize(objective) same_partition(labels, [0, 0, 0]) + + +def test_kl_is_deterministic_with_lifted_only_cross_neighbors(): + base = bic.graph.UndirectedGraph.from_edges( + 4, [[0, 1], [1, 2], [2, 3]] + ) + base_costs = np.array([4.0, -1.0, 4.0], dtype=np.float64) + lifted_uvs = np.array([[0, 3]], dtype=np.uint64) + lifted_costs = np.array([-3.0], dtype=np.float64) + initial_labels = np.array([0, 0, 1, 1], dtype=np.uint64) + + results = [] + for _ in range(3): + objective = bic.graph.lifted_multicut.LiftedMulticutObjective( + base, + base_costs, + lifted_uvs=lifted_uvs, + lifted_costs=lifted_costs, + initial_labels=initial_labels, + ) + results.append( + bic.graph.lifted_multicut.LiftedKernighanLinMulticut( + number_of_outer_iterations=10 + ).optimize(objective) + ) + + same_partition(results[0], initial_labels) + for labels in results[1:]: + np.testing.assert_array_equal(labels, results[0]) diff --git a/tests/graph/multicut/test_chain_decomposer.py b/tests/graph/multicut/test_chain_decomposer.py index d05b025..fce4303 100644 --- a/tests/graph/multicut/test_chain_decomposer.py +++ b/tests/graph/multicut/test_chain_decomposer.py @@ -3,7 +3,114 @@ import bioimage_cpp as bic -from ._helpers import same_partition +from ._helpers import edge_cut_labels, same_partition + + +def _copy_supported_solver(solver): + multicut = bic.graph.multicut + if isinstance(solver, multicut.GreedyAdditiveMulticut): + return multicut.GreedyAdditiveMulticut( + weight_stop=solver.weight_stop, + node_num_stop=solver.node_num_stop, + add_noise=solver.add_noise, + seed=solver.seed, + sigma=solver.sigma, + ) + if isinstance(solver, multicut.GreedyFixationMulticut): + return multicut.GreedyFixationMulticut( + weight_stop=solver.weight_stop, + node_num_stop=solver.node_num_stop, + ) + if isinstance(solver, multicut.KernighanLinMulticut): + return multicut.KernighanLinMulticut( + number_of_outer_iterations=solver.number_of_outer_iterations, + number_of_inner_iterations=solver.number_of_inner_iterations, + epsilon=solver.epsilon, + ) + if isinstance(solver, multicut.ChainedMulticutSolvers): + return multicut.ChainedMulticutSolvers( + [_copy_supported_solver(child) for child in solver.solvers] + ) + raise TypeError(f"unsupported test solver: {type(solver).__name__}") + + +def _python_reference_decomposition( + graph, + edge_costs, + sub_solver, + fallthrough_solver, +): + objective = bic.graph.multicut.MulticutObjective(graph, edge_costs) + components = bic.graph.connected_components( + objective.graph, + edge_mask=objective.edge_costs > 0.0, + ) + number_of_components = int(components.max()) + 1 if components.size else 0 + if number_of_components <= 1: + solver = fallthrough_solver or sub_solver + return _copy_supported_solver(solver).optimize(objective) + + component_index = components.astype(np.intp, copy=False) + node_counts = np.bincount( + component_index, minlength=number_of_components + ) + node_offsets = np.concatenate( + [np.array([0], dtype=np.intp), np.cumsum(node_counts, dtype=np.intp)] + ) + grouped_nodes = np.argsort(component_index, kind="stable").astype( + np.uint64, copy=False + ) + global_to_local = np.empty(graph.number_of_nodes, dtype=np.uint64) + for component in range(number_of_components): + nodes = grouped_nodes[ + node_offsets[component] : node_offsets[component + 1] + ] + global_to_local[nodes] = np.arange(nodes.size, dtype=np.uint64) + + uvs = graph.uv_ids() + u_components = component_index[uvs[:, 0]] + v_components = component_index[uvs[:, 1]] + internal_edges = np.flatnonzero(u_components == v_components) + edge_order = np.argsort(u_components[internal_edges], kind="stable") + grouped_edges = internal_edges[edge_order] + edge_counts = np.bincount( + u_components[internal_edges], minlength=number_of_components + ) + edge_offsets = np.concatenate( + [np.array([0], dtype=np.intp), np.cumsum(edge_counts, dtype=np.intp)] + ) + + labels = np.empty(graph.number_of_nodes, dtype=np.uint64) + label_offset = 0 + for component in range(number_of_components): + nodes = grouped_nodes[ + node_offsets[component] : node_offsets[component + 1] + ] + if nodes.size == 1: + labels[int(nodes[0])] = label_offset + label_offset += 1 + continue + + edge_ids = grouped_edges[ + edge_offsets[component] : edge_offsets[component + 1] + ] + local_uvs = global_to_local[uvs[edge_ids]] + subgraph = bic.graph.UndirectedGraph( + int(nodes.size), int(edge_ids.size) + ) + subgraph.insert_edges(local_uvs) + sub_objective = bic.graph.multicut.MulticutObjective( + subgraph, objective.edge_costs[edge_ids] + ) + sub_labels = _copy_supported_solver(sub_solver).optimize( + sub_objective + ) + _, sub_labels = np.unique(sub_labels, return_inverse=True) + sub_labels = sub_labels.astype(np.uint64, copy=False) + labels[nodes] = sub_labels + label_offset + label_offset += int(sub_labels.max()) + 1 + + return labels def test_chained_multicut_solvers(chain_problem): @@ -12,7 +119,9 @@ def test_chained_multicut_solvers(chain_problem): solver = bic.graph.multicut.ChainedMulticutSolvers( [ bic.graph.multicut.GreedyAdditiveMulticut(), - bic.graph.multicut.KernighanLinMulticut(number_of_outer_iterations=5), + bic.graph.multicut.KernighanLinMulticut( + number_of_outer_iterations=5 + ), ] ) @@ -28,9 +137,15 @@ def test_chained_solver_rejects_empty_chain(): def test_multicut_decomposer_solves_positive_components(): - graph = bic.graph.UndirectedGraph.from_edges(4, [[0, 1], [1, 2], [2, 3]]) - objective = bic.graph.multicut.MulticutObjective(graph, [1.0, -5.0, 1.0]) - solver = bic.graph.multicut.MulticutDecomposer(bic.graph.multicut.GreedyAdditiveMulticut()) + graph = bic.graph.UndirectedGraph.from_edges( + 4, [[0, 1], [1, 2], [2, 3]] + ) + objective = bic.graph.multicut.MulticutObjective( + graph, [1.0, -5.0, 1.0] + ) + solver = bic.graph.multicut.MulticutDecomposer( + bic.graph.multicut.GreedyAdditiveMulticut() + ) labels = solver.optimize(objective) @@ -38,17 +153,14 @@ def test_multicut_decomposer_solves_positive_components(): assert objective.energy() == pytest.approx(-5.0) -def test_multicut_decomposer_uses_fallthrough_solver_for_single_component(): - class SingletonSolver(bic.graph.multicut.MulticutSolver): - def optimize(self, objective): - objective.labels = np.arange(objective.graph.number_of_nodes, dtype=np.uint64) - return objective.labels - +def test_multicut_decomposer_uses_fallthrough_for_single_component(): graph = bic.graph.UndirectedGraph.from_edges(2, [[0, 1]]) objective = bic.graph.multicut.MulticutObjective(graph, [1.0]) solver = bic.graph.multicut.MulticutDecomposer( bic.graph.multicut.GreedyAdditiveMulticut(), - fallthrough_solver=SingletonSolver(), + fallthrough_solver=bic.graph.multicut.GreedyAdditiveMulticut( + weight_stop=2.0 + ), ) labels = solver.optimize(objective) @@ -63,7 +175,9 @@ def test_decomposer_on_external_toy_problem(external_toy_problem): bic.graph.multicut.ChainedMulticutSolvers( [ bic.graph.multicut.GreedyAdditiveMulticut(), - bic.graph.multicut.KernighanLinMulticut(number_of_outer_iterations=10), + bic.graph.multicut.KernighanLinMulticut( + number_of_outer_iterations=10 + ), ] ) ) @@ -76,8 +190,233 @@ def test_decomposer_on_external_toy_problem(external_toy_problem): def test_decomposer_energy_bound_on_grid_problem(grid_problem): graph, costs = grid_problem objective = bic.graph.multicut.MulticutObjective(graph, costs) - solver = bic.graph.multicut.MulticutDecomposer(bic.graph.multicut.GreedyAdditiveMulticut()) + solver = bic.graph.multicut.MulticutDecomposer( + bic.graph.multicut.GreedyAdditiveMulticut() + ) labels = solver.optimize(objective) assert objective.energy(labels) <= -20.0 + + +def _two_component_objective(): + graph = bic.graph.UndirectedGraph.from_edges( + 4, + [[0, 1], [1, 2], [2, 3]], + ) + return bic.graph.multicut.MulticutObjective( + graph, [1.0, -5.0, 1.0] + ) + + +@pytest.mark.parametrize("number_of_threads", [0, 1, 2, 8]) +def test_decomposer_is_deterministic_across_thread_counts(number_of_threads): + objective = _two_component_objective() + labels = bic.graph.multicut.MulticutDecomposer( + bic.graph.multicut.GreedyFixationMulticut(), + fallthrough_solver=bic.graph.multicut.GreedyAdditiveMulticut(), + number_of_threads=number_of_threads, + ).optimize(objective) + + np.testing.assert_array_equal(labels, [0, 0, 1, 1]) + assert objective.energy() == pytest.approx(-5.0) + + +def test_decomposer_handles_singleton_components(): + graph = bic.graph.UndirectedGraph(3) + objective = bic.graph.multicut.MulticutObjective(graph, []) + labels = bic.graph.multicut.MulticutDecomposer( + bic.graph.multicut.KernighanLinMulticut( + number_of_outer_iterations=0 + ), + fallthrough_solver=bic.graph.multicut.GreedyAdditiveMulticut(), + number_of_threads=2, + ).optimize(objective) + + np.testing.assert_array_equal(labels, [0, 1, 2]) + + +def test_decomposer_handles_empty_graph(): + graph = bic.graph.UndirectedGraph() + objective = bic.graph.multicut.MulticutObjective(graph, []) + labels = bic.graph.multicut.MulticutDecomposer( + bic.graph.multicut.KernighanLinMulticut(), + fallthrough_solver=bic.graph.multicut.GreedyFixationMulticut(), + number_of_threads=0, + ).optimize(objective) + + np.testing.assert_array_equal(labels, np.array([], dtype=np.uint64)) + + +def test_decomposer_kernighan_lin_keeps_greedy_warm_start(): + objective = _two_component_objective() + labels = bic.graph.multicut.MulticutDecomposer( + bic.graph.multicut.KernighanLinMulticut( + number_of_outer_iterations=0 + ), + fallthrough_solver=bic.graph.multicut.GreedyAdditiveMulticut(), + number_of_threads=2, + ).optimize(objective) + + np.testing.assert_array_equal(labels, [0, 0, 1, 1]) + + +class _CustomSolver(bic.graph.multicut.MulticutSolver): + def optimize(self, objective): + return objective.labels + + +@pytest.mark.parametrize( + "unsupported_solver", + [ + pytest.param(_CustomSolver(), id="custom-python"), + pytest.param( + bic.graph.multicut.FusionMoveMulticut( + proposal_generator=( + bic.graph.multicut.WatershedProposalGenerator() + ) + ), + id="fusion-move", + ), + pytest.param( + bic.graph.multicut.MulticutDecomposer( + bic.graph.multicut.GreedyAdditiveMulticut() + ), + id="nested-decomposer", + ), + pytest.param( + bic.graph.multicut.ChainedMulticutSolvers( + [ + bic.graph.multicut.GreedyAdditiveMulticut(), + _CustomSolver(), + ] + ), + id="chain-with-custom-python", + ), + ], +) +def test_decomposer_rejects_unsupported_sub_solver(unsupported_solver): + with pytest.raises(TypeError, match="sub_solver"): + bic.graph.multicut.MulticutDecomposer(unsupported_solver) + + +def test_decomposer_rejects_unsupported_fallthrough_solver(): + with pytest.raises(TypeError, match="fallthrough_solver"): + bic.graph.multicut.MulticutDecomposer( + bic.graph.multicut.GreedyAdditiveMulticut(), + fallthrough_solver=_CustomSolver(), + ) + + +def test_decomposer_rejects_negative_thread_count(): + with pytest.raises(ValueError, match="number_of_threads"): + bic.graph.multicut.MulticutDecomposer( + bic.graph.multicut.GreedyAdditiveMulticut(), + number_of_threads=-1, + ) + + +def test_multicut_solvers_do_not_expose_clone_api(): + assert not hasattr(bic.graph.multicut.MulticutSolver, "clone") + + +@pytest.mark.parametrize( + "sub_solver", + [ + pytest.param( + bic.graph.multicut.GreedyAdditiveMulticut( + weight_stop=0.05, + add_noise=True, + seed=7, + sigma=0.2, + ), + id="greedy-additive", + ), + pytest.param( + bic.graph.multicut.GreedyFixationMulticut(weight_stop=0.05), + id="greedy-fixation", + ), + pytest.param( + bic.graph.multicut.KernighanLinMulticut( + number_of_outer_iterations=5, + epsilon=1.0e-5, + ), + id="kernighan-lin", + ), + pytest.param( + bic.graph.multicut.ChainedMulticutSolvers( + [ + bic.graph.multicut.GreedyAdditiveMulticut( + weight_stop=0.02, + add_noise=True, + seed=11, + sigma=0.1, + ), + bic.graph.multicut.KernighanLinMulticut( + number_of_outer_iterations=3, + epsilon=1.0e-5, + ), + ] + ), + id="chain", + ), + ], +) +def test_native_decomposer_matches_python_reference(sub_solver): + rng = np.random.default_rng(27) + all_edges = np.array( + [ + (u, v) + for u in range(12) + for v in range(u + 1, 12) + ], + dtype=np.uint64, + ) + fallthrough_solver = bic.graph.multicut.GreedyFixationMulticut( + weight_stop=2.0 + ) + + for _ in range(8): + edge_indices = rng.choice( + all_edges.shape[0], size=24, replace=False + ) + rng.shuffle(edge_indices) + edges = all_edges[edge_indices] + costs = rng.normal(size=edges.shape[0]) + costs[rng.random(edges.shape[0]) < 0.4] -= 2.0 + graph = bic.graph.UndirectedGraph.from_edges(12, edges) + + expected = _python_reference_decomposition( + graph, + costs, + sub_solver, + fallthrough_solver, + ) + objective = bic.graph.multicut.MulticutObjective(graph, costs) + actual = bic.graph.multicut.MulticutDecomposer( + sub_solver, + fallthrough_solver=fallthrough_solver, + number_of_threads=4, + ).optimize(objective) + + np.testing.assert_array_equal( + edge_cut_labels(graph, actual), + edge_cut_labels(graph, expected), + ) + assert objective.energy(actual) == pytest.approx( + objective.energy(expected) + ) + + +def test_decomposer_preserves_labels_after_binding_validation_error(): + objective = _two_component_objective() + original_labels = objective.labels.copy() + objective._edge_costs[0] = np.nan + with pytest.raises(ValueError, match="finite"): + bic.graph.multicut.MulticutDecomposer( + bic.graph.multicut.GreedyAdditiveMulticut(), + fallthrough_solver=bic.graph.multicut.GreedyAdditiveMulticut(), + number_of_threads=2, + ).optimize(objective) + + np.testing.assert_array_equal(objective.labels, original_labels) diff --git a/tests/graph/multicut/test_fusion_move.py b/tests/graph/multicut/test_fusion_move.py index 4c00416..423ce57 100644 --- a/tests/graph/multicut/test_fusion_move.py +++ b/tests/graph/multicut/test_fusion_move.py @@ -268,12 +268,8 @@ def test_runs_on_graph_without_negative_edges(chain_problem): def test_greedy_proposals_parallel_is_deterministic_on_dirty_graph(): - # Smoke-test the parallel greedy-additive-proposal path on a dirty graph - # with a non-singleton initial labeling (which skips the calling-thread - # warm-start that would otherwise freeze the graph). The lazy CSR rebuild - # is not thread-safe; the solver now freezes the graph before fan-out, so - # the multi-threaded result must equal the single-threaded reference on - # every run. + # Exercise parallel proposal generation on a dirty graph. The solver must + # freeze the lazy CSR state before the fan-out. # # Note: the multicut race is hard to trigger deterministically from Python # (the calling thread typically wins the rebuild before OS-spawned worker diff --git a/tests/graph/multicut/test_greedy_fixation.py b/tests/graph/multicut/test_greedy_fixation.py index 043cc56..8fbd483 100644 --- a/tests/graph/multicut/test_greedy_fixation.py +++ b/tests/graph/multicut/test_greedy_fixation.py @@ -1,3 +1,6 @@ +import numpy as np +import pytest + import bioimage_cpp as bic @@ -11,6 +14,21 @@ def test_greedy_fixation_respects_negative_constraints(frustrated_triangle): assert objective.energy(labels) <= -3.0 +def test_greedy_fixation_propagates_two_folded_constraints(): + graph = bic.graph.UndirectedGraph.from_edges( + 3, + [[0, 1], [0, 2], [1, 2]], + ) + costs = np.array([3.0, -5.0, -4.0], dtype=np.float64) + objective = bic.graph.multicut.MulticutObjective(graph, costs) + + labels = bic.graph.multicut.GreedyFixationMulticut().optimize(objective) + + assert labels[0] == labels[1] + assert labels[0] != labels[2] + assert objective.energy(labels) == pytest.approx(-9.0) + + def test_greedy_fixation_node_num_stop(chain_problem): graph, costs = chain_problem objective = bic.graph.multicut.MulticutObjective(graph, costs) diff --git a/tests/graph/multicut/test_kernighan_lin.py b/tests/graph/multicut/test_kernighan_lin.py index 82479fe..3adcb1a 100644 --- a/tests/graph/multicut/test_kernighan_lin.py +++ b/tests/graph/multicut/test_kernighan_lin.py @@ -52,3 +52,58 @@ def test_kernighan_lin_energy_bound_on_grid_problem(grid_problem): # Regression guard pinned to the move-chain implementation's converged energy. assert objective.energy(labels) <= -34.0 + + +def test_kernighan_lin_multi_iteration_regression(): + edges = np.array( + [ + [0, 2], + [0, 3], + [0, 6], + [0, 7], + [1, 3], + [1, 4], + [1, 5], + [2, 3], + [2, 4], + [2, 5], + [2, 6], + [3, 4], + [3, 6], + [3, 7], + [4, 5], + [4, 6], + [4, 7], + [5, 6], + [5, 7], + [6, 7], + ], + dtype=np.uint64, + ) + costs = np.array( + [ + 4, 3, -1, 3, 1, -5, 0, 0, -4, -5, + -3, 5, -4, 0, 3, 5, 5, -5, 5, -4, + ], + dtype=np.float64, + ) + graph = bic.graph.UndirectedGraph.from_edges(8, edges) + + expected = { + 1: (np.array([0, 0, 0, 1, 1, 1, 2, 1], dtype=np.uint64), -19.0), + 2: (np.array([0, 1, 1, 0, 0, 0, 2, 0], dtype=np.uint64), -21.0), + 10: (np.array([0, 1, 1, 0, 0, 0, 2, 0], dtype=np.uint64), -21.0), + } + for number_of_outer_iterations, ( + expected_labels, + expected_energy, + ) in expected.items(): + outputs = [] + for _ in range(3): + objective = bic.graph.multicut.MulticutObjective(graph, costs) + labels = bic.graph.multicut.KernighanLinMulticut( + number_of_outer_iterations=number_of_outer_iterations + ).optimize(objective) + outputs.append(labels) + assert objective.energy(labels) == pytest.approx(expected_energy) + assert all(np.array_equal(labels, expected_labels) for labels in outputs) diff --git a/tests/graph/test_contraction_graph.py b/tests/graph/test_contraction_graph.py new file mode 100644 index 0000000..e02eb5a --- /dev/null +++ b/tests/graph/test_contraction_graph.py @@ -0,0 +1,301 @@ +import numpy as np +import pytest + +import bioimage_cpp as bic + + +def _triangle(): + return bic.graph.UndirectedGraph.from_edges( + 3, + np.array([[0, 1], [1, 2], [0, 2]], dtype=np.uint64), + ) + + +def _parallel_paths(): + return bic.graph.UndirectedGraph.from_edges( + 4, + np.array([[0, 2], [1, 2], [0, 3], [1, 3]], dtype=np.uint64), + ) + + +def test_contract_edge_materializes_aligned_values_and_mappings(): + graph = _triangle() + work = bic.graph.ContractionGraph(graph) + work.add_node_values( + "position", + np.array([[0.0, 2.0], [2.0, 4.0], [10.0, 12.0]], dtype=np.float32), + reduction="mean", + ) + work.add_edge_values( + "weight", + np.array([100.0, 2.0, 3.0], dtype=np.float64), + reduction="sum", + ) + + assert work.contract_edge(0, keep_node=0) == 0 + result = work.materialize() + + assert isinstance(result.graph, bic.graph.UndirectedGraph) + np.testing.assert_array_equal(result.graph.uv_ids(), [[0, 1]]) + np.testing.assert_allclose( + result.node_values["position"], + [[1.0, 3.0], [10.0, 12.0]], + ) + assert result.node_values["position"].dtype == np.float32 + np.testing.assert_array_equal(result.edge_values["weight"], [5.0]) + assert result.edge_values["weight"].dtype == np.float64 + np.testing.assert_array_equal(result.node_mapping, [0, 0, 1]) + np.testing.assert_array_equal(result.edge_mapping, [-1, 0, 0]) + + +@pytest.mark.parametrize("dtype", [np.float32, np.float64]) +@pytest.mark.parametrize( + "reduction, expected", + [ + ("sum", 8.0), + ("mean", 4.0), + ("min", 3.0), + ("max", 5.0), + ], +) +def test_edge_reductions_preserve_dtype(dtype, reduction, expected): + work = bic.graph.ContractionGraph(_triangle()) + work.add_edge_values( + "value", + np.array([100.0, 3.0, 5.0], dtype=dtype), + reduction=reduction, + ) + + work.contract_edge(0, keep_node=0) + values = work.materialize().edge_values["value"] + + assert values.dtype == np.dtype(dtype) + np.testing.assert_allclose(values, [expected]) + + +@pytest.mark.parametrize( + "reduction, expected", + [ + ("sum", 6.0), + ("mean", 3.0), + ("min", 2.0), + ("max", 4.0), + ], +) +def test_node_reductions(reduction, expected): + graph = bic.graph.UndirectedGraph.from_edges(2, [[0, 1]]) + work = bic.graph.ContractionGraph(graph) + work.add_node_values( + "value", + np.array([2.0, 4.0], dtype=np.float32), + reduction=reduction, + ) + + work.contract_edge(0) + + np.testing.assert_allclose(work.materialize().node_values["value"], [expected]) + + +def test_mean_tracks_the_number_of_represented_edges(): + graph = bic.graph.UndirectedGraph.from_edges( + 4, + [[0, 2], [1, 2], [0, 1], [0, 3], [2, 3]], + ) + work = bic.graph.ContractionGraph(graph) + work.add_edge_values( + "value", + np.array([1.0, 3.0, 100.0, 10.0, 100.0], dtype=np.float64), + reduction="mean", + ) + + work.contract_edge(2, keep_node=0) + work.contract_edge(4, keep_node=2) + + np.testing.assert_allclose( + work.materialize().edge_values["value"], + [(1.0 + 3.0 + 10.0) / 3.0], + ) + + +def test_suppress_node_reduces_edges_and_drops_node_value(): + graph = bic.graph.UndirectedGraph.from_edges(3, [[0, 1], [1, 2]]) + work = bic.graph.ContractionGraph(graph) + work.add_node_values( + "node", + np.array([1.0, 2.0, 3.0], dtype=np.float64), + reduction="sum", + ) + work.add_edge_values( + "length", + np.array([2.0, 3.0], dtype=np.float32), + reduction="sum", + ) + + assert work.suppress_node(1) == 0 + result = work.materialize() + + np.testing.assert_array_equal(result.graph.uv_ids(), [[0, 1]]) + np.testing.assert_array_equal(result.node_values["node"], [1.0, 3.0]) + np.testing.assert_array_equal(result.edge_values["length"], [5.0]) + np.testing.assert_array_equal(result.node_mapping, [0, -1, 1]) + np.testing.assert_array_equal(result.edge_mapping, [0, 0]) + + +def test_materialized_snapshot_is_independent(): + graph = bic.graph.UndirectedGraph.from_edges(3, [[0, 1], [1, 2]]) + work = bic.graph.ContractionGraph(graph) + work.add_edge_values( + "length", + np.array([2.0, 3.0], dtype=np.float64), + reduction="sum", + ) + replacement = work.suppress_node(1) + first = work.materialize() + + work.erase_edge(replacement) + second = work.materialize() + + assert first.graph.number_of_edges == 1 + np.testing.assert_array_equal(first.edge_values["length"], [5.0]) + assert second.graph.number_of_edges == 0 + assert second.edge_values["length"].shape == (0,) + np.testing.assert_array_equal(second.edge_mapping, [-1, -1]) + + +def test_active_value_queries_align_with_stable_ids(): + work = bic.graph.ContractionGraph(_triangle()) + work.add_edge_values( + "value", + np.array([10.0, 20.0, 30.0], dtype=np.float32), + reduction="sum", + ) + work.contract_edge(0, keep_node=0) + + edge_ids, values = work.active_edge_values("value") + + np.testing.assert_array_equal(edge_ids, [2]) + np.testing.assert_array_equal(values, [50.0]) + assert work.edge_value("value", 2).shape == () + assert float(work.edge_value("value", 2)) == 50.0 + + +def test_input_graph_is_copied(): + graph = bic.graph.UndirectedGraph.from_edges(3, [[0, 1]]) + work = bic.graph.ContractionGraph(graph) + + graph.insert_edge(1, 2) + + assert work.number_of_edges == 1 + assert work.find_edge(1, 2) == -1 + + +@pytest.mark.parametrize("dtype", [np.float32, np.float64]) +@pytest.mark.parametrize( + "reduction, expected_active, expected_materialized", + [ + ("sum", [3.0, 7.0], 10.0), + ("mean", [1.5, 3.5], 2.5), + ("min", [1.0, 3.0], 1.0), + ("max", [2.0, 4.0], 4.0), + ], +) +def test_keep_parallel_edges_fold_only_in_materialized_snapshot( + dtype, + reduction, + expected_active, + expected_materialized, +): + work = bic.graph.ContractionGraph( + _parallel_paths(), + parallel_edges="keep", + ) + work.add_edge_values( + "value", + np.array([1.0, 2.0, 3.0, 4.0], dtype=dtype), + reduction=reduction, + ) + + work.suppress_node(2) + work.suppress_node(3) + + assert work.parallel_edges == "keep" + assert work.find_edge(0, 1) == 0 + np.testing.assert_array_equal(work.find_edges(0, 1), [0, 2]) + edge_ids, active_values = work.active_edge_values("value") + np.testing.assert_array_equal(edge_ids, [0, 2]) + np.testing.assert_allclose(active_values, expected_active) + + result = work.materialize() + + np.testing.assert_array_equal(result.graph.uv_ids(), [[0, 1]]) + np.testing.assert_allclose( + result.edge_values["value"], + [expected_materialized], + ) + assert result.edge_values["value"].dtype == np.dtype(dtype) + np.testing.assert_array_equal(result.node_mapping, [0, 1, -1, -1]) + np.testing.assert_array_equal(result.edge_mapping, [0, 0, 0, 0]) + + assert work.number_of_edges == 2 + np.testing.assert_array_equal(work.edges(), [0, 2]) + np.testing.assert_allclose( + work.active_edge_values("value")[1], + expected_active, + ) + + +def test_keep_parallel_edges_rejects_self_edge_from_suppression(): + work = bic.graph.ContractionGraph(_triangle(), parallel_edges="keep") + assert work.can_suppress_node(0) + work.suppress_node(0) + + assert not work.can_suppress_node(0) + assert not work.can_suppress_node(1) + with pytest.raises(ValueError, match="self edge"): + work.suppress_node(1) + + +def test_parallel_edge_policy_validation_and_public_surface(): + with pytest.raises(ValueError, match="parallel_edges"): + bic.graph.ContractionGraph(_triangle(), parallel_edges="invalid") + + assert not hasattr(bic.graph.ContractionGraph, "_from_edges") + assert not hasattr( + bic.graph.ContractionGraph(_triangle()), + "_deleted_original_edges", + ) + + +def test_nan_values_propagate(): + work = bic.graph.ContractionGraph(_triangle()) + work.add_edge_values( + "value", + np.array([0.0, np.nan, 1.0], dtype=np.float64), + reduction="min", + ) + work.contract_edge(0, keep_node=0) + + assert np.isnan(work.materialize().edge_values["value"][0]) + + +def test_invalid_value_registration_and_inactive_ids(): + work = bic.graph.ContractionGraph(_triangle()) + + with pytest.raises(TypeError, match="floating dtype"): + work.add_edge_values("value", np.ones(3, dtype=np.int64), reduction="sum") + with pytest.raises(ValueError, match=r"shape\[0\]"): + work.add_edge_values("value", np.ones(2, dtype=np.float32), reduction="sum") + with pytest.raises(ValueError, match="reduction"): + work.add_edge_values("value", np.ones(3, dtype=np.float32), reduction="median") + + work.add_edge_values("value", np.ones(3, dtype=np.float32), reduction="sum") + with pytest.raises(ValueError, match="already exists"): + work.add_edge_values("value", np.ones(3, dtype=np.float32), reduction="sum") + + work.contract_edge(0, keep_node=0) + with pytest.raises(ValueError, match="before the first graph mutation"): + work.add_node_values("late", np.ones(3, dtype=np.float32), reduction="sum") + with pytest.raises(ValueError, match="inactive"): + work.edge_value("value", 0) + with pytest.raises(ValueError, match="degree 2"): + work.suppress_node(2) diff --git a/tests/graph/test_non_finite_weights.py b/tests/graph/test_non_finite_weights.py new file mode 100644 index 0000000..0e0e333 --- /dev/null +++ b/tests/graph/test_non_finite_weights.py @@ -0,0 +1,215 @@ +from __future__ import annotations + +import numpy as np +import pytest + +import bioimage_cpp as bic + + +NON_FINITE = [np.nan, np.inf, -np.inf] + + +def _chain_graph(): + return bic.graph.UndirectedGraph.from_edges(3, [[0, 1], [1, 2]]) + + +@pytest.mark.parametrize("value", NON_FINITE) +def test_multicut_rejects_non_finite_costs(value): + graph = _chain_graph() + with pytest.raises(ValueError, match="edge_costs.*finite"): + bic.graph.multicut.MulticutObjective(graph, [1.0, value]) + + +@pytest.mark.parametrize( + "solver", + [ + bic.graph.multicut.GreedyAdditiveMulticut(), + bic.graph.multicut.GreedyFixationMulticut(), + bic.graph.multicut.KernighanLinMulticut(), + bic.graph.multicut.FusionMoveMulticut( + proposal_generator=bic.graph.multicut.WatershedProposalGenerator(), + number_of_iterations=1, + ), + ], +) +def test_multicut_solvers_revalidate_mutated_costs(solver): + objective = bic.graph.multicut.MulticutObjective(_chain_graph(), [1.0, -1.0]) + objective.edge_costs[1] = np.nan + with pytest.raises(ValueError, match="edge_costs.*finite"): + solver.optimize(objective) + with pytest.raises(ValueError, match="edge_costs.*finite"): + objective.energy() + + +@pytest.mark.parametrize("value", NON_FINITE) +def test_lifted_multicut_rejects_non_finite_costs(value): + graph = _chain_graph() + with pytest.raises(ValueError, match="edge_costs.*finite"): + bic.graph.lifted_multicut.LiftedMulticutObjective(graph, [1.0, value]) + with pytest.raises(ValueError, match="lifted_costs.*finite"): + bic.graph.lifted_multicut.LiftedMulticutObjective( + graph, + [1.0, 1.0], + lifted_uvs=[[0, 2]], + lifted_costs=[value], + ) + + +@pytest.mark.parametrize("value", NON_FINITE) +def test_lifted_set_cost_rejects_non_finite_without_mutation(value): + objective = bic.graph.lifted_multicut.LiftedMulticutObjective( + _chain_graph(), [1.0, 1.0] + ) + with pytest.raises(ValueError, match="weight.*finite"): + objective.set_cost(0, 2, value) + assert objective.number_of_lifted_edges == 0 + + +def test_lifted_set_cost_rejects_non_finite_accumulation_without_mutation(): + objective = bic.graph.lifted_multicut.LiftedMulticutObjective( + _chain_graph(), + [1.0, 1.0], + lifted_uvs=[[0, 2]], + lifted_costs=[1.0e308], + ) + with pytest.raises(ValueError, match="accumulated weight.*finite"): + objective.set_cost(0, 2, 1.0e308) + assert objective.weights[-1] == 1.0e308 + + +def test_lifted_solver_revalidates_mutated_weights(): + objective = bic.graph.lifted_multicut.LiftedMulticutObjective( + _chain_graph(), + [1.0, 1.0], + lifted_uvs=[[0, 2]], + lifted_costs=[-1.0], + ) + objective.weights[-1] = np.nan + solver = bic.graph.lifted_multicut.FusionMoveLiftedMulticut( + proposal_generator=bic.graph.multicut.WatershedProposalGenerator(), + number_of_iterations=1, + ) + with pytest.raises(ValueError, match="weights.*finite"): + solver.optimize(objective) + with pytest.raises(ValueError, match="weights.*finite"): + objective.energy() + + +@pytest.mark.parametrize("value", NON_FINITE) +def test_edge_weighted_watershed_rejects_non_finite_weights(value): + with pytest.raises(ValueError, match="edge_weights.*finite"): + bic.graph.edge_weighted_watershed( + _chain_graph(), + np.array([0.1, value], dtype=np.float64), + np.array([1, 0, 2], dtype=np.uint64), + ) + + +@pytest.mark.parametrize("argument", ["edge_costs", "mutex_costs"]) +def test_mutex_watershed_rejects_non_finite_weights(argument): + kwargs = { + "graph": _chain_graph(), + "edge_costs": np.array([1.0, 1.0], dtype=np.float64), + "mutex_uvs": np.array([[0, 2]], dtype=np.uint64), + "mutex_costs": np.array([1.0], dtype=np.float64), + } + kwargs[argument][-1] = np.nan + with pytest.raises(ValueError, match=rf"{argument}.*finite"): + bic.graph.mutex_watershed.mutex_watershed_clustering(**kwargs) + + +@pytest.mark.parametrize( + "argument", + ["edge_costs", "mutex_costs", "semantic_costs"], +) +def test_semantic_mutex_watershed_rejects_non_finite_weights(argument): + kwargs = { + "graph": _chain_graph(), + "edge_costs": np.array([1.0, 1.0], dtype=np.float64), + "mutex_uvs": np.array([[0, 2]], dtype=np.uint64), + "mutex_costs": np.array([1.0], dtype=np.float64), + "semantic_node_classes": np.array([[0, 1]], dtype=np.uint64), + "semantic_costs": np.array([1.0], dtype=np.float64), + } + kwargs[argument][-1] = np.nan + with pytest.raises(ValueError, match=rf"{argument}.*finite"): + bic.graph.mutex_watershed.semantic_mutex_watershed_clustering(**kwargs) + + +@pytest.mark.parametrize( + ("make_policy", "arguments", "name"), + [ + ( + bic.graph.agglomeration.EdgeWeightedClusterPolicy, + {"edge_indicators": [0.1, np.nan]}, + "edge_indicators", + ), + ( + bic.graph.agglomeration.GaspClusterPolicy, + {"edge_weights": [1.0, np.nan]}, + "edge_weights", + ), + ( + bic.graph.agglomeration.MalaClusterPolicy, + {"edge_indicators": [0.1, np.nan]}, + "edge_indicators", + ), + ], +) +def test_agglomeration_rejects_non_finite_primary_weights( + make_policy, arguments, name +): + with pytest.raises(ValueError, match=rf"{name}.*finite"): + make_policy().optimize(_chain_graph(), **arguments) + + +@pytest.mark.parametrize( + ("argument", "value"), + [ + ("edge_sizes", [1.0, np.nan]), + ("node_sizes", [1.0, 1.0, np.nan]), + ], +) +def test_agglomeration_rejects_non_finite_auxiliary_weights(argument, value): + kwargs = { + "edge_indicators": [0.1, 0.2], + argument: value, + } + with pytest.raises(ValueError, match=rf"{argument}.*finite"): + bic.graph.agglomeration.EdgeWeightedClusterPolicy().optimize( + _chain_graph(), **kwargs + ) + + +def test_node_and_edge_agglomeration_rejects_non_finite_features(): + features = np.array([[0.0], [np.nan], [1.0]], dtype=np.float64) + with pytest.raises(ValueError, match="node_features.*finite"): + bic.graph.agglomeration.NodeAndEdgeWeightedClusterPolicy().optimize( + _chain_graph(), [0.1, 0.2], features + ) + + +@pytest.mark.parametrize( + ("policy", "name"), + [ + ( + lambda: bic.graph.agglomeration.EdgeWeightedClusterPolicy( + size_regularizer=np.nan + ), + "size_regularizer", + ), + ( + lambda: bic.graph.agglomeration.NodeAndEdgeWeightedClusterPolicy( + beta=np.inf + ), + "beta", + ), + ( + lambda: bic.graph.agglomeration.MalaClusterPolicy(threshold=-np.inf), + "threshold", + ), + ], +) +def test_agglomeration_rejects_non_finite_policy_parameters(policy, name): + with pytest.raises(ValueError, match=rf"{name}.*finite"): + policy() diff --git a/tests/graph/test_undirected_graph.py b/tests/graph/test_undirected_graph.py index 51286ee..512e89d 100644 --- a/tests/graph/test_undirected_graph.py +++ b/tests/graph/test_undirected_graph.py @@ -55,6 +55,19 @@ def test_undirected_graph_node_adjacency(): ) +def test_undirected_graph_node_degrees_updates_after_insertion(): + graph = bic.graph.UndirectedGraph.from_edges(5, [[0, 1], [1, 2], [1, 3]]) + + degrees = graph.node_degrees() + + assert degrees.dtype == np.uint64 + np.testing.assert_array_equal(degrees, [1, 3, 1, 1, 0]) + + graph.insert_edge(3, 4) + + np.testing.assert_array_equal(graph.node_degrees(), [1, 3, 1, 2, 1]) + + def test_undirected_graph_serialize_and_deserialize(): graph = bic.graph.UndirectedGraph.from_edges(3, [[0, 2], [0, 1]]) diff --git a/tests/label_multiset/test_merger.py b/tests/label_multiset/test_merger.py index cc964b2..2a986e9 100644 --- a/tests/label_multiset/test_merger.py +++ b/tests/label_multiset/test_merger.py @@ -57,7 +57,7 @@ def test_merger_deduplicates_identical_entry(): def test_merger_update_with_identical_entries_is_idempotent(): # Building a merger from a multiset and feeding the same multiset back as # an update should leave ids/counts unchanged and rewrite spatial offsets - # to point at the original byte offsets. + # to point at the original element offsets. rng = np.random.default_rng(3) labels = rng.integers(0, 4, size=(4, 4), dtype=np.uint64) ms = multiset_from_labels(labels, (2, 2)) diff --git a/tests/label_multiset/test_validation.py b/tests/label_multiset/test_validation.py new file mode 100644 index 0000000..7643c78 --- /dev/null +++ b/tests/label_multiset/test_validation.py @@ -0,0 +1,180 @@ +from __future__ import annotations + +import subprocess +import sys + +import numpy as np +import pytest + +from bioimage_cpp._core import Blocking +from bioimage_cpp.label_multiset import ( + LabelMultiset, + MultisetMerger, + downsample_multiset, + read_subset, +) + + +def _valid_multiset() -> LabelMultiset: + return LabelMultiset( + argmax=[5, 7], + offsets=[0, 1], + entry_offsets=[0, 1], + entry_sizes=[1, 1], + ids=[5, 7], + counts=[2, 3], + ) + + +@pytest.mark.parametrize( + ("offsets", "sizes", "match"), + [ + ([2], [1], "offsets"), + ([1], [2], "range"), + ([np.iinfo(np.uint64).max], [2], "offsets"), + ], +) +def test_read_subset_rejects_invalid_ranges(offsets, sizes, match): + with pytest.raises(ValueError, match=match): + read_subset(offsets, sizes, [1], [1]) + + +def test_read_subset_rejects_mismatched_lengths(): + with pytest.raises(ValueError, match="offsets and sizes"): + read_subset([0, 1], [1], [1, 2], [1, 1]) + with pytest.raises(ValueError, match="ids and counts"): + read_subset([0], [1], [1, 2], [1]) + + +@pytest.mark.parametrize( + ("argument", "values", "match"), + [ + ("offsets", [-1], "non-negative"), + ("offsets", [2**64], "fit dtype uint64"), + ("counts", [2**32], "fit dtype uint32"), + ], +) +def test_read_subset_checks_unsigned_conversion(argument, values, match): + kwargs = { + "offsets": [0], + "sizes": [1], + "ids": [1], + "counts": [1], + } + kwargs[argument] = values + with pytest.raises(ValueError, match=match): + read_subset(**kwargs) + + +def test_read_subset_accepts_last_storage_element(): + ids, counts = read_subset([1], [1], [4, 9], [2, 7]) + np.testing.assert_array_equal(ids, [9]) + np.testing.assert_array_equal(counts, [7]) + + +@pytest.mark.parametrize( + "replacement", + [ + {"counts": [1]}, + {"argmax": [5]}, + {"entry_offsets": [0, 2]}, + {"entry_sizes": [3, 1]}, + {"entry_sizes": [1, 0]}, + ], +) +def test_label_multiset_rejects_invalid_structure(replacement): + values = { + "argmax": [5, 7], + "offsets": [0, 1], + "entry_offsets": [0, 1], + "entry_sizes": [1, 1], + "ids": [5, 7], + "counts": [2, 3], + } + values.update(replacement) + with pytest.raises(ValueError): + LabelMultiset(**values) + + +def test_label_multiset_accepts_empty_top_level(): + multiset = LabelMultiset( + argmax=[], + offsets=[], + entry_offsets=[], + entry_sizes=[], + ids=[], + counts=[], + ) + assert multiset.n_spatial == 0 + assert multiset.n_entries == 0 + assert multiset.ids.dtype == np.uint64 + assert multiset.counts.dtype == np.uint32 + + +def test_downsample_revalidates_mutated_multiset(): + multiset = _valid_multiset() + multiset.offsets[1] = 99 + blocking = Blocking([0], [2], [2]) + with pytest.raises(ValueError, match="offsets"): + downsample_multiset(multiset, blocking) + + +def test_downsample_rejects_blocking_extent_mismatch(): + multiset = _valid_multiset() + blocking = Blocking([0], [3], [2]) + with pytest.raises(ValueError, match=r"product\(blocking.roi_end\)"): + downsample_multiset(multiset, blocking) + + +def test_merger_rejects_update_length_and_entry_index(): + merger = MultisetMerger([0], [1], [5], [2]) + output_offsets = np.array([0], dtype=np.uint64) + with pytest.raises(ValueError, match="offsets and sizes"): + merger.update([0, 1], [1], [5, 7], [2, 3], output_offsets) + with pytest.raises(ValueError, match="batch entries"): + merger.update([0], [1], [7], [3], np.array([1], dtype=np.uint64)) + + +def test_merger_from_multiset_revalidates_mutation(): + multiset = _valid_multiset() + multiset.entry_offsets[1] = 7 + with pytest.raises(ValueError, match="entry_offsets"): + MultisetMerger.from_multiset(multiset) + + +@pytest.mark.parametrize( + "code", + [ + """ +import numpy as np +from bioimage_cpp.label_multiset import MultisetMerger +try: + MultisetMerger( + np.array([0], dtype=np.uint64), + np.array([0], dtype=np.uint64), + np.array([], dtype=np.uint64), + np.array([], dtype=np.uint32), + ) +except ValueError: + raise SystemExit(0) +raise SystemExit(3) +""", + """ +import numpy as np +from bioimage_cpp.label_multiset import read_subset +try: + read_subset( + np.array([99], dtype=np.uint64), + np.array([1], dtype=np.uint64), + np.array([1], dtype=np.uint64), + np.array([1], dtype=np.uint32), + ) +except ValueError: + raise SystemExit(0) +raise SystemExit(3) +""", + ], +) +def test_former_native_crash_cases_raise_in_subprocess(code): + result = subprocess.run([sys.executable, "-c", code], check=False) + assert result.returncode == 0 diff --git a/tests/mesh/test_smoothing.py b/tests/mesh/test_smoothing.py index 00e080b..7db5b60 100644 --- a/tests/mesh/test_smoothing.py +++ b/tests/mesh/test_smoothing.py @@ -1,6 +1,3 @@ -import os -import sys - import numpy as np import pytest @@ -215,38 +212,3 @@ def test_rejects_mismatched_verts_normals_dtype(): faces = np.array([[0, 1, 2]], dtype=np.int64) with pytest.raises(TypeError, match="same dtype"): bic.mesh.smooth_mesh(verts, normals, faces, iterations=1) - - -def test_parity_with_python_reference(): - """Compare against the nifty-based Python reference. Skipped if nifty missing. - - Only one iteration is compared. The reference has an aliasing quirk - (``current_verts = new_verts`` makes them refer to the same buffer) that - turns iterations 1+ into in-place Gauss-Seidel smoothing, whereas this - implementation does textbook Jacobi smoothing (independent read/write - buffers). The two agree exactly at ``iterations=1``. - """ - pytest.importorskip("nifty") - - repo_root = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) - dev_path = os.path.join(repo_root, "development", "mesh") - sys.path.insert(0, dev_path) - try: - from _mesh_smoothing_reference import smooth_mesh as smooth_mesh_reference - finally: - sys.path.remove(dev_path) - - rng = np.random.default_rng(2026) - scipy_spatial = pytest.importorskip("scipy.spatial") - n_points = 80 - raw = rng.standard_normal((n_points, 3)) - points = raw / np.linalg.norm(raw, axis=1, keepdims=True) - hull = scipy_spatial.ConvexHull(points) - verts = points.astype(np.float64) - faces = hull.simplices.astype(np.int64) - normals = verts.copy() - - out_v, out_n = bic.mesh.smooth_mesh(verts, normals, faces, iterations=1) - ref_v, ref_n = smooth_mesh_reference(verts, normals, faces, 1) - np.testing.assert_allclose(out_v, ref_v, rtol=1e-10, atol=1e-12) - np.testing.assert_allclose(out_n, ref_n, rtol=1e-10, atol=1e-12) diff --git a/tests/skeleton/test_postprocessing.py b/tests/skeleton/test_postprocessing.py index f245220..7f2f970 100644 --- a/tests/skeleton/test_postprocessing.py +++ b/tests/skeleton/test_postprocessing.py @@ -1,3 +1,5 @@ +import builtins + import numpy as np import pytest @@ -93,6 +95,125 @@ def test_remove_ticks_empty_edges(): assert len(out_edges) == 0 +def test_remove_ticks_rejects_duplicate_edges(): + vertices = np.array([[0.0, 0.0], [1.0, 0.0]]) + edges = np.array([[0, 1], [1, 0]], dtype=np.int64) + + with pytest.raises(ValueError, match="duplicate undirected edges"): + bic.skeleton.remove_ticks(vertices, edges, tick_length=1.0) + + +def test_remove_ticks_prunes_spurs_at_two_branch_points(): + vertices = np.array( + [ + [0.0, 0.0], + [1.0, 0.0], + [2.0, 0.0], + [3.0, 0.0], + [4.0, 0.0], + [1.0, 1.0], + [3.0, 1.0], + ] + ) + edges = np.array( + [[0, 1], [1, 2], [2, 3], [3, 4], [1, 5], [3, 6]], + dtype=np.int64, + ) + + out_vertices, out_edges, _ = bic.skeleton.remove_ticks( + vertices, + edges, + tick_length=1.5, + ) + + assert len(out_vertices) == 5 + assert len(out_edges) == 4 + + +def test_remove_ticks_keeps_pure_cycle(): + vertices = np.array( + [[0.0, 0.0], [0.0, 1.0], [1.0, 1.0], [1.0, 0.0]] + ) + edges = np.array([[0, 1], [1, 2], [2, 3], [3, 0]], dtype=np.int64) + + out_vertices, out_edges, _ = bic.skeleton.remove_ticks( + vertices, + edges, + tick_length=100.0, + ) + + np.testing.assert_array_equal(out_vertices, vertices) + np.testing.assert_array_equal(out_edges, edges) + + +def test_remove_ticks_preserves_parallel_paths_while_pruning_spur(): + vertices = np.array( + [ + [0.0, 0.0], + [4.0, 0.0], + [2.0, 1.0], + [2.0, 0.0], + [2.0, -1.0], + [0.0, 0.5], + ] + ) + edges = np.array( + [[0, 2], [2, 1], [0, 3], [3, 1], [0, 4], [4, 1], [0, 5]], + dtype=np.int64, + ) + + out_vertices, out_edges, _ = bic.skeleton.remove_ticks( + vertices, + edges, + tick_length=1.0, + ) + + assert len(out_vertices) == 5 + assert len(out_edges) == 6 + + +def test_remove_ticks_preserves_two_parallel_paths_while_pruning_spur(): + vertices = np.array( + [ + [0.0, 0.0], + [4.0, 0.0], + [2.0, 1.0], + [2.0, -1.0], + [0.0, 0.5], + ] + ) + edges = np.array( + [[0, 2], [2, 1], [0, 3], [3, 1], [0, 4]], + dtype=np.int64, + ) + + out_vertices, out_edges, _ = bic.skeleton.remove_ticks( + vertices, + edges, + tick_length=1.0, + ) + + assert len(out_vertices) == 4 + assert len(out_edges) == 4 + + +def test_remove_ticks_preserves_radii_dtype(): + vertices = np.array([[0.0, 0.0], [1.0, 0.0], [1.0, 0.5], [2.0, 0.0]]) + edges = np.array([[0, 1], [1, 2], [1, 3]], dtype=np.int64) + radii = np.arange(4, dtype=np.float32) + + out_vertices, out_edges, out_radii = bic.skeleton.remove_ticks( + vertices, + edges, + tick_length=1.0, + radii=radii, + ) + + assert len(out_vertices) == 3 + assert len(out_edges) == 2 + assert out_radii.dtype == np.float32 + + def test_join_close_components_links_collinear_endpoints(): # Two collinear fragments along x with a gap of 2 between node 1 and node 2. vertices = np.array([[0.0, 0.0, 0.0], [0.0, 0.0, 1.0], [0.0, 0.0, 3.0], [0.0, 0.0, 4.0]]) @@ -132,6 +253,24 @@ def test_join_close_components_skips_same_component(): assert len(out_edges) == 2 +def test_join_close_components_reports_missing_scipy(monkeypatch): + original_import = builtins.__import__ + + def import_without_scipy(name, *args, **kwargs): + if name == "scipy.spatial": + raise ModuleNotFoundError("No module named 'scipy'") + return original_import(name, *args, **kwargs) + + monkeypatch.setattr(builtins, "__import__", import_without_scipy) + + with pytest.raises(ImportError, match="requires scipy"): + bic.skeleton.join_close_components( + np.array([[0.0, 0.0], [1.0, 0.0]]), + np.array([[0, 1]], dtype=np.int64), + dist=1.0, + ) + + def test_clean_graph_splits_crossing(): vertices = np.array( [[0.0, 0.0, 0.0], [0.0, 0.0, 1.0], [0.0, 0.0, -1.0], [0.0, 1.0, 0.0], [0.0, -1.0, 0.0]], @@ -149,6 +288,39 @@ def test_clean_graph_splits_crossing(): assert len(out_radii) == len(out_vertices) +def test_clean_graph_cuts_degree3_junction_edge(): + vertices = np.array( + [[0.0, 0.0], [1.0, 0.0], [-1.0, 0.0], [0.0, 1.0], [0.0, 2.0]] + ) + edges = np.array([[0, 1], [0, 2], [0, 3], [3, 4]], dtype=np.int64) + + out_vertices, out_edges, _ = bic.skeleton.clean_filament_graph( + vertices, + edges, + direction_span=1, + ) + + np.testing.assert_array_equal(out_vertices, vertices) + np.testing.assert_array_equal(out_edges, [[0, 1], [0, 2], [3, 4]]) + assert _component_count(out_vertices, out_edges) == 2 + + +def test_clean_graph_removes_one_edge_degree3_arm(): + vertices = np.array( + [[0.0, 0.0], [1.0, 0.0], [-1.0, 0.0], [0.0, 1.0]] + ) + edges = np.array([[0, 1], [0, 2], [0, 3]], dtype=np.int64) + + out_vertices, out_edges, _ = bic.skeleton.clean_filament_graph( + vertices, + edges, + direction_span=1, + ) + + np.testing.assert_array_equal(out_vertices, vertices[:3]) + np.testing.assert_array_equal(out_edges, [[0, 1], [0, 2]]) + + def test_draw_instances_labels_edge(): vertices = np.array([[0.0, 0.0, 0.0], [0.0, 0.0, 3.0]]) edges = np.array([[0, 1]], dtype=np.int64) diff --git a/tests/test_filters.py b/tests/test_filters.py index bce5a6d..fe08a56 100644 --- a/tests/test_filters.py +++ b/tests/test_filters.py @@ -6,10 +6,13 @@ trigonometric closed-form rounding. """ +from concurrent.futures import ThreadPoolExecutor + import numpy as np import pytest from scipy import ndimage +from bioimage_cpp import _core import bioimage_cpp.filters as bf @@ -197,16 +200,271 @@ def test_structure_tensor_eigenvalues_2d_matches_reference(): np.testing.assert_allclose(got, ref, atol=2e-3) -def test_structure_tensor_eigenvalues_3d_shape_and_order(): +def _structure_tensor_eigenvalues_reference_3d(vol, inner, outer): + gz = ndimage.gaussian_filter(vol, inner, order=[1, 0, 0], mode="mirror") + gy = ndimage.gaussian_filter(vol, inner, order=[0, 1, 0], mode="mirror") + gx = ndimage.gaussian_filter(vol, inner, order=[0, 0, 1], mode="mirror") + szz = ndimage.gaussian_filter(gz * gz, outer, mode="mirror") + szy = ndimage.gaussian_filter(gz * gy, outer, mode="mirror") + szx = ndimage.gaussian_filter(gz * gx, outer, mode="mirror") + syy = ndimage.gaussian_filter(gy * gy, outer, mode="mirror") + syx = ndimage.gaussian_filter(gy * gx, outer, mode="mirror") + sxx = ndimage.gaussian_filter(gx * gx, outer, mode="mirror") + mat = np.stack([ + np.stack([szz, szy, szx], axis=-1), + np.stack([szy, syy, syx], axis=-1), + np.stack([szx, syx, sxx], axis=-1), + ], axis=-2) + return np.linalg.eigvalsh(mat)[..., ::-1].astype(np.float32) + + +def _symmetric_components(matrices): + return np.ascontiguousarray( + np.stack( + [ + matrices[:, 0, 0], + matrices[:, 0, 1], + matrices[:, 0, 2], + matrices[:, 1, 1], + matrices[:, 1, 2], + matrices[:, 2, 2], + ] + ), + dtype=np.float32, + ) + + +def _direct_ev3(components): + out = np.empty((components.shape[1], 3), dtype=np.float32) + _core._filters_ev3_symmetric_float32(components, out) + return out + + +@pytest.mark.parametrize("n", [0, 1, 7, 8, 9, 15, 16, 17]) +def test_direct_ev3_handles_vector_boundaries(n): + rng = np.random.default_rng(42 + n) + matrices = rng.normal(size=(n, 3, 3)).astype(np.float32) + matrices += matrices.transpose(0, 2, 1) + got = _direct_ev3(_symmetric_components(matrices)) + ref = np.linalg.eigvalsh(matrices)[..., ::-1].astype(np.float32) + np.testing.assert_allclose(got, ref, rtol=2e-5, atol=2e-6) + + +def test_direct_ev3_matches_numpy_across_scales(): + rng = np.random.default_rng(91) + n = 1003 + components = rng.normal(size=(6, n)).astype(np.float32) + scales = np.power(10.0, rng.uniform(-15.0, 15.0, size=n)).astype(np.float32) + components *= scales + + matrices = np.empty((n, 3, 3), dtype=np.float32) + matrices[:, 0, 0] = components[0] + matrices[:, 0, 1] = matrices[:, 1, 0] = components[1] + matrices[:, 0, 2] = matrices[:, 2, 0] = components[2] + matrices[:, 1, 1] = components[3] + matrices[:, 1, 2] = matrices[:, 2, 1] = components[4] + matrices[:, 2, 2] = components[5] + + got = _direct_ev3(components) + ref = np.linalg.eigvalsh(matrices)[..., ::-1].astype(np.float32) + scale = np.maximum( + np.max(np.abs(ref), axis=1), np.finfo(np.float32).tiny + ) + scaled_error = np.max(np.abs(got - ref), axis=1) / scale + assert np.max(scaled_error) < 2e-5 + assert np.all(got[:, :-1] >= got[:, 1:]) + + +def test_direct_ev3_handles_repeated_and_near_repeated_roots(): + rng = np.random.default_rng(23) + spectra = [] + smallest_subnormal = float( + np.nextafter(np.float32(0.0), np.float32(1.0)) + ) + for scale in ( + smallest_subnormal, + np.finfo(np.float32).tiny, + 1e-20, + 1e-8, + 1.0, + 1e8, + 1e20, + ): + spectra.extend( + [ + [0.0, 0.0, 0.0], + [scale, scale, scale], + [3.0 * scale, 3.0 * scale, -2.0 * scale], + [3.0 * scale, -2.0 * scale, -2.0 * scale], + [scale, scale * (1.0 + 2e-6), -0.5 * scale], + [scale, scale * (1.0 - 2e-6), -0.5 * scale], + ] + ) + + matrices = [] + for eigenvalues in spectra: + q, _ = np.linalg.qr(rng.normal(size=(3, 3))) + matrices.append( + (q @ np.diag(eigenvalues) @ q.T).astype(np.float32) + ) + matrices = np.stack(matrices) + + got = _direct_ev3(_symmetric_components(matrices)) + ref = np.linalg.eigvalsh(matrices)[..., ::-1].astype(np.float32) + scale = np.maximum( + np.max(np.abs(ref), axis=1), np.finfo(np.float32).tiny + ) + scaled_error = np.max(np.abs(got - ref), axis=1) / scale + assert np.isfinite(got).all() + assert np.max(scaled_error) < 3e-4 + assert np.all(got[:, :-1] >= got[:, 1:]) + + +def test_direct_ev3_forced_scalar_matches_automatic(monkeypatch): + monkeypatch.delenv("BIOIMAGE_CPP_FILTERS_FORCE_SCALAR", raising=False) + if _core._filters_eigenvalue_backend() != "avx2": + pytest.skip("AVX2 eigenvalue backend is not available") + + rng = np.random.default_rng(7) + components = rng.normal(size=(6, 1003)).astype(np.float32) + automatic = _direct_ev3(components) + + monkeypatch.setenv("BIOIMAGE_CPP_FILTERS_FORCE_SCALAR", "1") + assert _core._filters_eigenvalue_backend() == "scalar" + scalar = _direct_ev3(components) + np.testing.assert_allclose(automatic, scalar, rtol=2e-5, atol=3e-5) + + +@pytest.mark.parametrize("value", [np.nan, np.inf]) +def test_direct_ev3_rejects_non_finite_components(value): + components = np.zeros((6, 8), dtype=np.float32) + components[2, 3] = value + with pytest.raises(ValueError, match="finite"): + _direct_ev3(components) + + +def test_direct_ev3_validates_shapes_and_writability(): + with pytest.raises(ValueError, match=r"shape \(6, n\)"): + _direct_ev3(np.zeros((5, 8), dtype=np.float32)) + + components = np.zeros((6, 8), dtype=np.float32) + with pytest.raises(ValueError, match=r"shape \(n, 3\)"): + _core._filters_ev3_symmetric_float32( + components, np.empty((8, 2), dtype=np.float32) + ) + + out = np.empty((8, 3), dtype=np.float32) + out.flags.writeable = False + with pytest.raises(TypeError): + _core._filters_ev3_symmetric_float32(components, out) + + +def test_structure_tensor_eigenvalues_3d_matches_reference(): vol = _random_image((8, 16, 16)) got = bf.structure_tensor_eigenvalues(vol, 1.0, 2.0) + ref = _structure_tensor_eigenvalues_reference_3d(vol, 1.0, 2.0) assert got.shape == vol.shape + (3,) + np.testing.assert_allclose(got, ref, atol=5e-3) assert np.all(got[..., 0] >= got[..., 1]) assert np.all(got[..., 1] >= got[..., 2]) - # All eigenvalues of a positive-semidefinite tensor are >= 0. assert np.all(got >= -1e-6) +@pytest.mark.parametrize("shape", [(1, 5, 7), (2, 3, 4)]) +def test_eigenvalue_filters_support_short_axes(shape): + vol = _random_image(shape) + got_hessian = bf.hessian_of_gaussian_eigenvalues(vol, 1.0) + ref_hessian = _hessian_eigenvalues_reference_3d(vol, 1.0) + np.testing.assert_allclose(got_hessian, ref_hessian, atol=5e-3) + + got_structure = bf.structure_tensor_eigenvalues(vol, 1.0, 1.5) + ref_structure = _structure_tensor_eigenvalues_reference_3d(vol, 1.0, 1.5) + np.testing.assert_allclose(got_structure, ref_structure, atol=5e-3) + + +def test_runtime_radius_fallback_matches_scipy(): + img = _random_image((40, 48)) + got = bf.gaussian_smoothing(img, 5.0, window_size=3.0) + ref = ndimage.gaussian_filter(img, 5.0, mode="mirror", truncate=3.0) + np.testing.assert_allclose(got, ref, atol=1e-3) + + +def test_forced_scalar_matches_automatic_backend(monkeypatch): + monkeypatch.delenv("BIOIMAGE_CPP_FILTERS_FORCE_SCALAR", raising=False) + if _core._filters_convolution_backend() != "avx2": + pytest.skip("AVX2 filter backend is not available") + assert _core._filters_eigenvalue_backend() == "avx2" + + vol = _random_image((7, 11, 17)) + functions = [ + lambda: bf.gaussian_smoothing(vol, 1.5), + lambda: bf.gaussian_derivative(vol, 1.5, [1, 0, 0]), + lambda: bf.gaussian_gradient_magnitude(vol, 1.5), + lambda: bf.laplacian_of_gaussian(vol, 1.5), + lambda: bf.hessian_of_gaussian_eigenvalues(vol, 1.5), + lambda: bf.structure_tensor_eigenvalues(vol, 1.0, 2.0), + ] + automatic = [function() for function in functions] + + monkeypatch.setenv("BIOIMAGE_CPP_FILTERS_FORCE_SCALAR", "1") + assert _core._filters_convolution_backend() == "scalar" + assert _core._filters_eigenvalue_backend() == "scalar" + scalar = [function() for function in functions] + + for got, expected in zip(automatic, scalar, strict=True): + np.testing.assert_allclose(got, expected, atol=1e-5) + + +@pytest.mark.parametrize("radius", range(1, 13)) +def test_specialized_radii_match_scalar_backend(monkeypatch, radius): + monkeypatch.delenv("BIOIMAGE_CPP_FILTERS_FORCE_SCALAR", raising=False) + if _core._filters_convolution_backend() != "avx2": + pytest.skip("AVX2 filter backend is not available") + + image = _random_image((25, 31)) + automatic_smoothing = bf.gaussian_smoothing( + image, float(radius), window_size=1.0 + ) + automatic_derivative = bf.gaussian_derivative( + image, float(radius), [1, 0], window_size=1.0 + ) + + monkeypatch.setenv("BIOIMAGE_CPP_FILTERS_FORCE_SCALAR", "1") + scalar_smoothing = bf.gaussian_smoothing( + image, float(radius), window_size=1.0 + ) + scalar_derivative = bf.gaussian_derivative( + image, float(radius), [1, 0], window_size=1.0 + ) + np.testing.assert_allclose(automatic_smoothing, scalar_smoothing, atol=1e-5) + np.testing.assert_allclose(automatic_derivative, scalar_derivative, atol=1e-5) + + +def test_concurrent_filter_calls_are_deterministic(): + vol = _random_image((8, 12, 16)) + expected = bf.structure_tensor_eigenvalues(vol, 1.0, 2.0) + with ThreadPoolExecutor(max_workers=4) as executor: + results = list( + executor.map( + lambda _: bf.structure_tensor_eigenvalues(vol, 1.0, 2.0), + range(8), + ) + ) + for result in results: + np.testing.assert_array_equal(result, expected) + + +@pytest.mark.parametrize("shape", [(0, 4), (2, 0, 3)]) +def test_filters_support_empty_inputs(shape): + image = np.empty(shape, dtype=np.float32) + assert bf.gaussian_smoothing(image, 1.0).shape == shape + assert bf.hessian_of_gaussian_eigenvalues(image, 1.0).shape == shape + (len(shape),) + assert ( + bf.structure_tensor_eigenvalues(image, 1.0, 2.0).shape + == shape + (len(shape),) + ) + + # --------------------------------------------------------------------------- # dtype handling # ---------------------------------------------------------------------------