diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f62e76d8..7428f9e2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -100,12 +100,12 @@ jobs: PY # ---------------------------------------------------------------------- # - # (c) CPU-only tests: the CPU subset of the suite. GPU, network, and - # undeclared-dep tests are ignored (see BASELINE). The two known - # np.unicode_ test-only failures are deselected until restructure-tests - # (PR D) fixes the NumPy-2 alias. - # TODO(PR D): once pytest markers land, replace the --ignore/--deselect - # list with `-m "not gpu and not slow and not network"`. + # (c) CPU-only tests: the CPU subset of the suite, selected by pytest + # markers (`-m "not gpu and not slow and not network"`). GPU tests are + # additionally skipped at collection when cupy/ccc_cuda_ext are absent + # (tests/gpu/conftest.py). The research-only modules import undeclared + # heavy deps (requests/minepy/IPython) and are excluded from the wheel, + # so they are still `--ignore`d here (owned by improve-docs / PR E). # ---------------------------------------------------------------------- # test-cpu: name: CPU tests @@ -141,11 +141,9 @@ jobs: PATH: /opt/conda/bin:/usr/local/cuda/bin:/usr/bin:/bin run: | python -m pytest tests/ \ - --ignore=tests/gpu \ --ignore=tests/test_giant.py \ --ignore=tests/test_methods.py \ --ignore=tests/test_plots.py \ --ignore=tests/test_corr.py \ - --deselect "tests/test_coef_pval.py::test_cm_numerical_and_categorical_features_perfect_relationship_pvalue" \ - --deselect "tests/test_coef_pval.py::test_cm_numerical_and_categorical_features_weakly_relationship_pvalue" \ + -m "not gpu and not slow and not network" \ -q -ra -p no:cacheprovider -o addopts="" diff --git a/CMakeLists.txt b/CMakeLists.txt index 1f073ed6..8d3acc08 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,6 +1,11 @@ cmake_minimum_required(VERSION 3.15...3.26) project(${SKBUILD_PROJECT_NAME} LANGUAGES CUDA CXX) +# When ON, build the CUDA C++ gtests under tests/cuda_ext/ and register them with +# ctest. OFF by default so the wheel build never fetches googletest or compiles +# the native tests. Enable with: cmake -S . -B build -DCCC_BUILD_TESTS=ON +option(CCC_BUILD_TESTS "Build the CUDA C++ gtests and register them with ctest" OFF) + # Export compile_commands.json so clang-tidy (and editors/LSP) can analyze the # CUDA/C++ sources. Run clang-tidy host-only, e.g.: # clang-tidy -p build --extra-arg=--cuda-host-only libs/ccc_cuda_ext/*.cu @@ -42,14 +47,10 @@ find_package(Python REQUIRED COMPONENTS Interpreter Development.Module) find_package(pybind11 CONFIG REQUIRED) set(PYBIND11_NEWPYTHON ON) -# Download and configure Google Test +# Google Test is fetched only when building the native tests (CCC_BUILD_TESTS); +# the default wheel build must not touch the network for gtest. See the guarded +# test block near the end of this file. include(FetchContent) -FetchContent_Declare( - googletest - GIT_REPOSITORY https://github.com/google/googletest.git - GIT_TAG v1.15.2 # Adjust version as needed -) -FetchContent_MakeAvailable(googletest) # Download and configure spdlog FetchContent_Declare( spdlog @@ -62,54 +63,6 @@ FetchContent_MakeAvailable(spdlog) # include_directories("${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES}") -# Setup Gtest -enable_testing() -# Function to automatically add tests from a directory -function(add_tests_from_directory TEST_DIR) - # Find all test files in the directory - file(GLOB_RECURSE TEST_FILES - "${TEST_DIR}/*_test.cu" # Files ending with _test.cu - "${TEST_DIR}/test_*.cu" # Files starting with test_ - ) - - # Loop through each test file - foreach(TEST_FILE ${TEST_FILES}) - # Get the filename without extension - get_filename_component(TEST_NAME ${TEST_FILE} NAME_WE) - - # Create an executable for this test - add_executable(${TEST_NAME} ${TEST_FILE} ${headers} ${sources}) - - # target_include_directories(${TEST_NAME} PRIVATE - # ${PROJECT_INCLUDE_DIR} # Add this line - # ${Python_INCLUDE_DIRS} - # ) - - target_link_libraries(${TEST_NAME} PRIVATE - GTest::gtest_main - GTest::gtest - pybind11::headers - pybind11::embed - Python::Python - # Add your other project libraries here - # project_lib - ) - - # Add the test to CTest - add_test(NAME ${TEST_NAME} COMMAND ${TEST_NAME}) - - # Set test properties (optional) - # Set test properties (optional) - set_tests_properties(${TEST_NAME} PROPERTIES - TIMEOUT 10 # Timeout in seconds - WORKING_DIRECTORY "${CMAKE_RUNTIME_OUTPUT_DIRECTORY}" - ) - endforeach() -endfunction() - -# Specify your test directory and call the function -# add_tests_from_directory(${CMAKE_CURRENT_SOURCE_DIR}/tests) - # Optional: Set output directories set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/bin) set(CMAKE_LIBRARY_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/lib) @@ -127,3 +80,68 @@ target_link_libraries(${CUDA_EXT_MODULE_NAME} PRIVATE ) install(TARGETS ${CUDA_EXT_MODULE_NAME} LIBRARY DESTINATION .) + +# --------------------------------------------------------------------------- # +# CUDA C++ gtests (opt-in). Guarded by CCC_BUILD_TESTS so the wheel build never +# fetches googletest or compiles the native tests. +# cmake -S . -B build -DCCC_BUILD_TESTS=ON && cmake --build build && ctest --test-dir build +# --------------------------------------------------------------------------- # +if(CCC_BUILD_TESTS) + message(STATUS "CCC_BUILD_TESTS=ON: configuring CUDA C++ gtests") + + # Embedding a Python interpreter (test_ari_random.cu) needs Development.Embed, + # which the default wheel build (Development.Module only) does not require. + find_package(Python REQUIRED COMPONENTS Interpreter Development.Embed Development.Module) + + FetchContent_Declare( + googletest + GIT_REPOSITORY https://github.com/google/googletest.git + GIT_TAG v1.15.2 + ) + FetchContent_MakeAvailable(googletest) + include(GoogleTest) + enable_testing() + + # The gtests exercise the ARI kernels in metrics.cu. Compile that translation + # unit once into a static library and link it into each test executable so we + # do not recompile the (slow) .cu file per test. coef.cu / binder.cu are not + # needed by the current tests and are intentionally excluded. + add_library(ccc_cuda_ext_testlib STATIC ${CUDA_EXT_DIR}/metrics.cu) + target_link_libraries(ccc_cuda_ext_testlib PUBLIC + spdlog::spdlog + pybind11::headers + Python::Python + ) + set_target_properties(ccc_cuda_ext_testlib PROPERTIES + POSITION_INDEPENDENT_CODE ON + ) + + # Discover the gtest source files. test_ari_py.cu is a legacy manual driver + # with its own main() (not a gtest); it would clash with gtest_main, so skip it. + file(GLOB CCC_TEST_FILES "${CMAKE_CURRENT_SOURCE_DIR}/tests/cuda_ext/test_*.cu") + foreach(TEST_FILE ${CCC_TEST_FILES}) + get_filename_component(TEST_NAME ${TEST_FILE} NAME_WE) + if(TEST_NAME STREQUAL "test_ari_py") + continue() + endif() + + add_executable(${TEST_NAME} ${TEST_FILE}) + target_link_libraries(${TEST_NAME} PRIVATE + ccc_cuda_ext_testlib + GTest::gtest_main + GTest::gtest + pybind11::headers + pybind11::embed + Python::Python + ) + + # gtest_discover_tests registers each TEST_P case with ctest. The default + # 10s ctest timeout was too tight for the Python-reference tests; raise it. + gtest_discover_tests(${TEST_NAME} + PROPERTIES + TIMEOUT 300 + WORKING_DIRECTORY "${CMAKE_RUNTIME_OUTPUT_DIRECTORY}" + DISCOVERY_TIMEOUT 120 + ) + endforeach() +endif() diff --git a/libs/ccc/bench/__init__.py b/libs/ccc/bench/__init__.py new file mode 100644 index 00000000..ad01525e --- /dev/null +++ b/libs/ccc/bench/__init__.py @@ -0,0 +1,8 @@ +"""Standalone benchmark CLI for CCC (``ccc-gpu-bench`` / ``python -m ccc.bench``). + +Reproducible, structured GPU-vs-CPU performance measurement decoupled from the +test suite: end-to-end coefficient benchmarks, ARI-kernel micro-benchmarks and +CPU parallelism scaling, emitted as JSON Lines / CSV with full environment +metadata. Runs degrade gracefully on machines without a GPU (CPU-side modes work; +GPU modes fail fast with a clear message). See ``ccc-gpu-bench --help``. +""" diff --git a/libs/ccc/bench/__main__.py b/libs/ccc/bench/__main__.py new file mode 100644 index 00000000..a45b456a --- /dev/null +++ b/libs/ccc/bench/__main__.py @@ -0,0 +1,8 @@ +"""Enable ``python -m ccc.bench``.""" + +import sys + +from .cli import main + +if __name__ == "__main__": + sys.exit(main()) diff --git a/libs/ccc/bench/cli.py b/libs/ccc/bench/cli.py new file mode 100644 index 00000000..66f070e2 --- /dev/null +++ b/libs/ccc/bench/cli.py @@ -0,0 +1,237 @@ +"""``ccc-gpu-bench`` command-line interface (also ``python -m ccc.bench``). + +Reproducible, structured performance measurement decoupled from the pytest +suite. Modes: + + coef end-to-end GPU-vs-CPU coefficient benchmark (grid over + features x samples x n_jobs), with ``--pvalue-n-perms`` and + ``--return-parts`` variants. + ari kernel-level ARI GPU-vs-CPU micro-benchmark. + scaling CPU parallelism (n_jobs) scaling. + +Output is JSON Lines (default) or CSV, written incrementally, one record per +case with full config + environment metadata + seed. Only stdlib + already +present deps are used. +""" + +import argparse +import sys + +from . import runners +from .env import capture_environment, gpu_available +from .output import open_writer +from .presets import get_preset + + +def _add_common(sub: argparse.ArgumentParser) -> None: + sub.add_argument( + "--preset", + choices=["smoke", "paper"], + default=None, + help="named grid; explicit grid args override individual axes", + ) + sub.add_argument( + "-o", + "--output", + default=None, + help="output path (default: stdout). '-' also means stdout", + ) + sub.add_argument( + "--format", + choices=["jsonl", "csv"], + default="jsonl", + help="output format (default: jsonl)", + ) + sub.add_argument("--seed", type=int, default=42, help="random seed (default: 42)") + sub.add_argument( + "--repeats", type=int, default=3, help="timed repeats per case (default: 3)" + ) + sub.add_argument( + "--warmup", type=int, default=1, help="untimed warmup calls (default: 1)" + ) + sub.add_argument( + "--profile", + action="store_true", + help="CPU category profile (partitioning/ARI/...) + nsys hint for GPU", + ) + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser( + prog="ccc-gpu-bench", + description="Structured GPU/CPU benchmarks for the CCC coefficient.", + ) + subparsers = parser.add_subparsers(dest="mode", required=True) + + # coef ------------------------------------------------------------------ # + coef = subparsers.add_parser("coef", help="end-to-end GPU vs CPU coefficient") + _add_common(coef) + coef.add_argument("--features", type=int, nargs="+", help="feature counts") + coef.add_argument("--samples", type=int, nargs="+", help="sample (object) counts") + coef.add_argument("--n-jobs", type=int, nargs="+", help="CPU worker counts") + coef.add_argument("--pvalue-n-perms", type=int, default=None) + coef.add_argument("--return-parts", action="store_true") + coef.add_argument("--gpu-only", action="store_true") + coef.add_argument("--cpu-only", action="store_true") + + # ari ------------------------------------------------------------------- # + ari = subparsers.add_parser("ari", help="kernel-level ARI GPU vs CPU") + _add_common(ari) + ari.add_argument("--n-features", type=int, nargs="+") + ari.add_argument("--n-parts", type=int, nargs="+") + ari.add_argument("--n-objs", type=int, nargs="+") + ari.add_argument("--k", type=int, nargs="+") + ari.add_argument("--gpu-only", action="store_true") + ari.add_argument("--cpu-only", action="store_true") + + # scaling --------------------------------------------------------------- # + scaling = subparsers.add_parser("scaling", help="CPU n_jobs scaling") + _add_common(scaling) + scaling.add_argument("--features", type=int, nargs="+") + scaling.add_argument("--samples", type=int, nargs="+") + scaling.add_argument("--n-jobs", type=int, nargs="+") + scaling.add_argument("--pvalue-n-perms", type=int, default=None) + + return parser + + +def _resolve_grid(mode: str, args: argparse.Namespace, keys: list[str]) -> dict: + """Resolve grid axes: explicit args > preset > smoke defaults.""" + preset = get_preset(mode, args.preset) if args.preset else {} + smoke = get_preset(mode, "smoke") + resolved = {} + for key in keys: + attr = key.replace("-", "_") + val = getattr(args, attr, None) + if val: + resolved[key] = val + elif key in preset: + resolved[key] = preset[key] + else: + resolved[key] = smoke[key] + return resolved + + +def _require_gpu(cpu_only: bool) -> None: + """Fail fast (clear message) when a GPU mode is asked for without a GPU.""" + if not cpu_only and not gpu_available(): + msg = ( + "GPU benchmark requested but cupy / ccc_cuda_ext are unavailable.\n" + "Install the CUDA extension, or pass --cpu-only to run CPU-side only." + ) + raise SystemExit(msg) + + +def _maybe_profile(mode: str, args: argparse.Namespace, grid: dict) -> None: + if not args.profile: + return + import numpy as np + + from . import profiling + + if mode == "ari": + nf = grid["n_features"][0] + no = grid["n_objs"][0] + print("\n" + profiling.nsys_suggestion(nf, no, args.seed), file=sys.stderr) + return + + nf = grid["features"][0] + ns = grid["samples"][0] + cpu_only = getattr(args, "cpu_only", False) + gpu_only = getattr(args, "gpu_only", False) + + if not gpu_only: + from ccc.coef.impl import ccc as ccc_cpu + + np.random.seed(args.seed) + data = np.random.rand(nf, ns) + kwargs = {} + if mode == "coef": + kwargs = { + "pvalue_n_perms": args.pvalue_n_perms, + "return_parts": args.return_parts, + } + elif mode == "scaling": + kwargs = {"pvalue_n_perms": args.pvalue_n_perms} + prof = profiling.profile_cpu(ccc_cpu, data, n_jobs=1, **kwargs) + profiling.print_cpu_profile(prof) + + if not cpu_only and gpu_available(): + print("\n" + profiling.nsys_suggestion(nf, ns, args.seed), file=sys.stderr) + + +def main(argv: list[str] | None = None) -> int: + parser = build_parser() + args = parser.parse_args(argv) + + mode = args.mode + env = capture_environment() + writer = open_writer(args.output, args.format) + + try: + if mode == "coef": + if args.gpu_only and args.cpu_only: + raise SystemExit("--gpu-only and --cpu-only are mutually exclusive") + grid = _resolve_grid("coef", args, ["features", "samples", "n_jobs"]) + _require_gpu(args.cpu_only) + print(f"Running coef benchmark: {grid}", file=sys.stderr) + runners.run_coef( + writer=writer, + env=env, + features=grid["features"], + samples=grid["samples"], + n_jobs=grid["n_jobs"], + pvalue_n_perms=args.pvalue_n_perms, + return_parts=args.return_parts, + seed=args.seed, + repeats=args.repeats, + warmup=args.warmup, + gpu_only=args.gpu_only, + cpu_only=args.cpu_only, + ) + _maybe_profile("coef", args, grid) + + elif mode == "ari": + if args.gpu_only and args.cpu_only: + raise SystemExit("--gpu-only and --cpu-only are mutually exclusive") + grid = _resolve_grid("ari", args, ["n_features", "n_parts", "n_objs", "k"]) + _require_gpu(args.cpu_only) + print(f"Running ari benchmark: {grid}", file=sys.stderr) + runners.run_ari( + writer=writer, + env=env, + n_features=grid["n_features"], + n_parts=grid["n_parts"], + n_objs=grid["n_objs"], + k=grid["k"], + seed=args.seed, + repeats=args.repeats, + warmup=args.warmup, + gpu_only=args.gpu_only, + cpu_only=args.cpu_only, + ) + _maybe_profile("ari", args, grid) + + elif mode == "scaling": + grid = _resolve_grid("scaling", args, ["features", "samples", "n_jobs"]) + print(f"Running scaling benchmark: {grid}", file=sys.stderr) + runners.run_scaling( + writer=writer, + env=env, + features=grid["features"], + samples=grid["samples"], + n_jobs=grid["n_jobs"], + pvalue_n_perms=args.pvalue_n_perms, + seed=args.seed, + repeats=args.repeats, + warmup=args.warmup, + ) + _maybe_profile("scaling", args, grid) + finally: + writer.close() + + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/libs/ccc/bench/env.py b/libs/ccc/bench/env.py new file mode 100644 index 00000000..95582532 --- /dev/null +++ b/libs/ccc/bench/env.py @@ -0,0 +1,112 @@ +"""Environment capture for benchmark records. + +Collects GPU / driver / CUDA / CPU / package metadata so every benchmark record +is self-describing and comparable across machines and commits. Everything here +degrades gracefully: missing GPU or tools yield ``None`` fields rather than +raising. +""" + +import importlib.util +import platform +import subprocess + + +def gpu_available() -> bool: + """True when the CUDA extension is importable AND a CUDA device is accessible. + + Module importability alone is not proof of a usable GPU: a CUDA-enabled + environment on a device-less host (e.g. a CPU CI runner) imports cupy and + the extension fine but has no device. Probe the runtime for a device count + so callers fail fast with a clear message instead of a low-level CUDA error. + """ + if importlib.util.find_spec("ccc_cuda_ext") is None: + return False + if importlib.util.find_spec("cupy") is None: + return False + try: + import cupy as cp + + return cp.cuda.runtime.getDeviceCount() > 0 + except Exception: + return False + + +def _package_version() -> str | None: + try: + from importlib.metadata import version + + return version("cccgpu") + except Exception: + try: + import ccc + + return getattr(ccc, "__version__", None) + except Exception: + return None + + +def _cpu_model() -> str | None: + """Best-effort CPU model string (Linux /proc/cpuinfo, else platform).""" + try: + with open("/proc/cpuinfo") as fh: + for line in fh: + if line.startswith("model name"): + return line.split(":", 1)[1].strip() + except OSError: + pass + return platform.processor() or None + + +def _nvidia_smi_field(query: str) -> str | None: + try: + out = subprocess.run( + ["nvidia-smi", f"--query-gpu={query}", "--format=csv,noheader"], + capture_output=True, + text=True, + timeout=10, + check=False, + ) + except (OSError, subprocess.SubprocessError): + return None + if out.returncode != 0: + return None + first = out.stdout.strip().splitlines() + return first[0].strip() if first else None + + +def _gpu_info() -> dict: + """GPU name / driver / CUDA runtime via cupy, falling back to nvidia-smi.""" + info = {"gpu_name": None, "gpu_driver": None, "cuda_runtime": None} + if importlib.util.find_spec("cupy") is not None: + try: + import cupy as cp + + props = cp.cuda.runtime.getDeviceProperties(0) + name = props.get("name") + info["gpu_name"] = name.decode() if isinstance(name, bytes) else name + runtime = cp.cuda.runtime.runtimeGetVersion() + major, minor = divmod(runtime, 1000) + info["cuda_runtime"] = f"{major}.{minor // 10}" + except Exception: + pass + if info["gpu_name"] is None: + info["gpu_name"] = _nvidia_smi_field("name") + if info["gpu_driver"] is None: + info["gpu_driver"] = _nvidia_smi_field("driver_version") + return info + + +def capture_environment() -> dict: + """Return a metadata dict embedded in every benchmark record.""" + import os + + env = { + "package_version": _package_version(), + "python_version": platform.python_version(), + "platform": platform.platform(), + "cpu_model": _cpu_model(), + "cpu_count": os.cpu_count(), + "gpu_present": gpu_available(), + } + env.update(_gpu_info()) + return env diff --git a/libs/ccc/bench/output.py b/libs/ccc/bench/output.py new file mode 100644 index 00000000..8308f9d4 --- /dev/null +++ b/libs/ccc/bench/output.py @@ -0,0 +1,67 @@ +"""Incremental structured writers for benchmark records (JSON Lines / CSV). + +Records are flushed as they are produced so a long sweep that is interrupted +still leaves a parseable file with every completed case. +""" + +import csv +import json +import sys + + +class JSONLWriter: + """One JSON object per line, flushed after every record.""" + + def __init__(self, stream): + self._stream = stream + + def write(self, record: dict) -> None: + self._stream.write(json.dumps(record, default=str) + "\n") + self._stream.flush() + + def close(self) -> None: + if self._stream not in (sys.stdout, sys.stderr): + self._stream.close() + + +class CSVWriter: + """CSV with a header taken from the first record's keys.""" + + def __init__(self, stream): + self._stream = stream + self._writer = None + + def write(self, record: dict) -> None: + if self._writer is None: + self._writer = csv.DictWriter(self._stream, fieldnames=list(record.keys())) + self._writer.writeheader() + # Flatten anything non-scalar to a JSON string so the row stays 1-D. + row = { + k: (json.dumps(v, default=str) if isinstance(v, (dict, list)) else v) + for k, v in record.items() + } + self._writer.writerow(row) + self._stream.flush() + + def close(self) -> None: + if self._stream not in (sys.stdout, sys.stderr): + self._stream.close() + + +def open_writer(output_path: str | None, fmt: str): + """Return a writer for ``fmt`` ('jsonl'|'csv') writing to ``output_path``. + + ``output_path`` of None or ``-`` writes to stdout. + """ + if output_path in (None, "-"): + stream = sys.stdout + else: + # Kept open across incremental writes; closed via the writer's .close(). + stream = open(output_path, "w", newline="") # noqa: SIM115 + + if fmt == "csv": + return CSVWriter(stream) + if fmt == "jsonl": + return JSONLWriter(stream) + msg = f"Unknown output format: {fmt!r} (expected 'jsonl' or 'csv')" + raise ValueError(msg) diff --git a/libs/ccc/bench/presets.py b/libs/ccc/bench/presets.py new file mode 100644 index 00000000..dc11cb8b --- /dev/null +++ b/libs/ccc/bench/presets.py @@ -0,0 +1,53 @@ +"""Named benchmark grids ("presets"), data instead of commented-out parameter lists. + +``smoke`` grids finish in seconds (sanity / CI-adjacent). ``paper`` grids +reproduce the poster/README sweeps and can take a long time (the CPU reference +for the biggest cases is minutes per point) -- run them on the reference GPU box. +""" + +# coef / scaling grids use (features, samples, n_jobs); ari uses +# (n_features, n_parts, n_objs, k). + +COEF_PRESETS = { + "smoke": {"features": [50, 100], "samples": [100], "n_jobs": [1]}, + # Poster/README-style sweep: end-to-end GPU-vs-CPU over a feature grid. + "paper": { + "features": [500, 1000, 2000, 5000, 10000], + "samples": [1000], + "n_jobs": [24], + }, +} + +ARI_PRESETS = { + "smoke": {"n_features": [10, 50], "n_parts": [10], "n_objs": [300], "k": [10]}, + "paper": { + "n_features": [100, 200, 1000], + "n_parts": [20], + "n_objs": [300], + "k": [10], + }, +} + +SCALING_PRESETS = { + "smoke": {"features": [100], "samples": [500], "n_jobs": [1, 2]}, + "paper": {"features": [1000], "samples": [1000], "n_jobs": [1, 6, 12, 24]}, +} + +PRESETS_BY_MODE = { + "coef": COEF_PRESETS, + "ari": ARI_PRESETS, + "scaling": SCALING_PRESETS, +} + + +def get_preset(mode: str, name: str) -> dict: + """Return the grid dict for ``mode``/``name`` or raise a clear error.""" + presets = PRESETS_BY_MODE.get(mode) + if presets is None: + msg = f"No presets for mode {mode!r}" + raise KeyError(msg) + if name not in presets: + available = ", ".join(sorted(presets)) + msg = f"Unknown preset {name!r} for mode {mode!r} (available: {available})" + raise KeyError(msg) + return dict(presets[name]) diff --git a/libs/ccc/bench/profiling.py b/libs/ccc/bench/profiling.py new file mode 100644 index 00000000..c078d166 --- /dev/null +++ b/libs/ccc/bench/profiling.py @@ -0,0 +1,140 @@ +"""CPU category profiling + GPU profiler suggestion. + +The category map and attribution logic are ported from +``analysis/00-benchmark/run_profiling.py`` so the optimization track gets its +baseline from this one tool. ``--profile`` on a CPU run prints where time goes +(partitioning vs ARI vs coordination vs ...); on a GPU run it prints the +recommended external profiler (nsys) invocation, since kernel profiling stays +manual. +""" + +import cProfile +import pstats + +# Function -> category attribution (ported from analysis/00-benchmark). +FUNCTION_CATEGORIES = { + "adjusted_rand_index": "ARI", + "get_pair_confusion_matrix": "ARI", + "get_contingency_matrix": "ARI", + "get_parts": "Partitioning", + "run_quantile_clustering": "Partitioning", + "get_feature_parts": "Partitioning", + "get_range_n_clusters": "Partitioning", + "get_perc_from_k": "Partitioning", + "rank": "Ranking", + "cdist_parts_basic": "Coordination", + "compute_ccc": "Coordination", + "compute_coef": "Coordination", + "ccc": "Coordination", + "get_chunks": "Coordination", + "get_coords_from_index": "Coordination", + "get_feature_type_and_encode": "Coordination", +} + +_NUMPY_FUNCS = frozenset( + [ + "searchsorted", + "argsort", + "zeros", + "unique", + "full", + "ravel", + "dot", + "sum", + "max", + "argmax", + "floor", + "sqrt", + "ceil", + "round", + "array", + "arange", + "empty", + "copy", + ] +) + + +def categorize_function(func_name: str, filename: str) -> str: + """Categorize a profiled function by name and source file.""" + if func_name in FUNCTION_CATEGORIES: + return FUNCTION_CATEGORIES[func_name] + lower = filename.lower() + if func_name in _NUMPY_FUNCS or "numpy" in lower or "numba" in lower: + return "NumPy/Numba" + if "ccc" in lower: + return "Other CCC" + return "Other" + + +def category_breakdown(stats: pstats.Stats) -> tuple[list[dict], float]: + """Return per-category (time, calls, pct) totals and the CCC total time.""" + stats_dict = stats.stats + + total_time = 0.0 + for (filename, _line, func_name), value in stats_dict.items(): + if func_name == "ccc" and "impl.py" in filename: + total_time = value[3] + break + if total_time == 0.0: + total_time = max(v[3] for v in stats_dict.values()) + + totals: dict[str, dict] = {} + for (filename, _line, func_name), value in stats_dict.items(): + ncalls, tottime = value[0], value[2] + category = categorize_function(func_name, filename) + bucket = totals.setdefault(category, {"tottime": 0.0, "calls": 0}) + bucket["tottime"] += tottime + bucket["calls"] += ncalls + + rows = [ + { + "category": cat, + "tottime": data["tottime"], + "calls": data["calls"], + "pct": (data["tottime"] / total_time * 100.0) if total_time else 0.0, + } + for cat, data in totals.items() + ] + rows.sort(key=lambda r: r["tottime"], reverse=True) + return rows, total_time + + +def profile_cpu(ccc_fn, data, n_jobs: int = 1, **ccc_kwargs) -> dict: + """cProfile a single CPU ``ccc`` call and return the category breakdown.""" + profiler = cProfile.Profile() + profiler.enable() + ccc_fn(data, n_jobs=n_jobs, **ccc_kwargs) + profiler.disable() + + stats = pstats.Stats(profiler) + rows, total_time = category_breakdown(stats) + return {"total_time": total_time, "categories": rows} + + +def print_cpu_profile(profile: dict) -> None: + """Human-readable category breakdown to stdout.""" + print("\nCPU category profile:") + print("-" * 52) + print(f" {'Category':<16s} {'Time (s)':>10s} {'%':>7s} {'Calls':>12s}") + print("-" * 52) + for row in profile["categories"]: + print( + f" {row['category']:<16s} {row['tottime']:>10.4f} " + f"{row['pct']:>6.1f}% {int(row['calls']):>12,}" + ) + print("-" * 52) + print(f" {'TOTAL':<16s} {profile['total_time']:>10.4f}") + + +def nsys_suggestion(n_features: int, n_samples: int, seed: int) -> str: + """Return the recommended nsys command for GPU-side profiling.""" + return ( + "GPU kernel profiling is manual. Suggested Nsight Systems command:\n" + f" nsys profile -o ccc_gpu_f{n_features}_n{n_samples} \\\n" + f' python -c "import numpy as np; ' + f"from ccc.coef.impl_gpu import ccc; " + f"np.random.seed({seed}); " + f'ccc(np.random.rand({n_features}, {n_samples}))"\n' + "For kernel-level metrics use: ncu --set full " + ) diff --git a/libs/ccc/bench/runners.py b/libs/ccc/bench/runners.py new file mode 100644 index 00000000..fc5111fe --- /dev/null +++ b/libs/ccc/bench/runners.py @@ -0,0 +1,307 @@ +"""Benchmark runners: coef (end-to-end), ari (kernel), scaling (CPU n_jobs). + +Methodology: fixed seed, ``warmup`` untimed calls then ``repeats`` timed calls; +report min and mean wall time; GPU is synchronized before/after timing and its +memory pool is freed between cases. +""" + +import itertools +import sys +import time +from datetime import datetime, timezone + + +def _now_iso() -> str: + return datetime.now(timezone.utc).isoformat() + + +def _clean_gpu() -> None: + try: + import cupy as cp + + cp.get_default_memory_pool().free_all_blocks() + except Exception: + pass + + +def _gpu_sync() -> None: + import cupy as cp + + cp.cuda.Device().synchronize() + + +def _timed(fn, repeats: int, warmup: int, sync=None): + """Return (min_s, mean_s, last_result) for ``fn`` with warmup + repeats.""" + for _ in range(warmup): + fn() + if sync is not None: + sync() + times = [] + result = None + for _ in range(repeats): + t0 = time.perf_counter() + result = fn() + if sync is not None: + sync() + times.append(time.perf_counter() - t0) + return min(times), sum(times) / len(times), result + + +def _n_coefficients(n_features: int) -> int: + return n_features * (n_features - 1) // 2 + + +# --------------------------------------------------------------------------- # +# coef: end-to-end GPU vs CPU +# --------------------------------------------------------------------------- # +def run_coef( + *, + writer, + env: dict, + features: list[int], + samples: list[int], + n_jobs: list[int], + pvalue_n_perms: int | None, + return_parts: bool, + seed: int, + repeats: int, + warmup: int, + gpu_only: bool, + cpu_only: bool, + verbose: bool = True, +) -> list[dict]: + import numpy as np + + ccc_cpu = None + ccc_gpu = None + if not gpu_only: + from ccc.coef.impl import ccc as ccc_cpu + if not cpu_only: + from ccc.coef.impl_gpu import ccc as ccc_gpu + + records = [] + for nf, ns, nj in itertools.product(features, samples, n_jobs): + np.random.seed(seed) + data = np.random.rand(nf, ns) + kwargs = {"pvalue_n_perms": pvalue_n_perms, "return_parts": return_parts} + + gpu_min = gpu_mean = None + cpu_min = cpu_mean = None + if ccc_gpu is not None: + gpu_min, gpu_mean, _ = _timed( + lambda d=data, j=nj, kw=kwargs: ccc_gpu(d, n_jobs=j, **kw), + repeats, + warmup, + sync=_gpu_sync, + ) + _clean_gpu() + if ccc_cpu is not None: + cpu_min, cpu_mean, _ = _timed( + lambda d=data, j=nj, kw=kwargs: ccc_cpu(d, n_jobs=j, **kw), + repeats, + warmup, + ) + + speedup = cpu_min / gpu_min if (gpu_min and cpu_min and gpu_min > 0) else None + record = { + "mode": "coef", + "timestamp": _now_iso(), + "n_features": nf, + "n_samples": ns, + "n_jobs": nj, + "pvalue_n_perms": pvalue_n_perms, + "return_parts": return_parts, + "seed": seed, + "repeats": repeats, + "warmup": warmup, + "n_coefficients": _n_coefficients(nf), + "gpu_time_min_s": gpu_min, + "gpu_time_mean_s": gpu_mean, + "cpu_time_min_s": cpu_min, + "cpu_time_mean_s": cpu_mean, + "speedup": speedup, + **env, + } + writer.write(record) + records.append(record) + if verbose: + _print_coef(record) + return records + + +def _print_coef(r: dict) -> None: + parts = [f"coef f={r['n_features']} n={r['n_samples']} jobs={r['n_jobs']}"] + if r["gpu_time_min_s"] is not None: + parts.append(f"GPU={r['gpu_time_min_s']:.4f}s") + if r["cpu_time_min_s"] is not None: + parts.append(f"CPU={r['cpu_time_min_s']:.4f}s") + if r["speedup"] is not None: + parts.append(f"speedup={r['speedup']:.2f}x") + print(" " + " ".join(parts), file=sys.stderr) + + +# --------------------------------------------------------------------------- # +# ari: kernel-level GPU vs CPU +# --------------------------------------------------------------------------- # +def _pairwise_combinations(parts): + import numpy as np + + pairs = [] + n_slices = parts.shape[0] + for i in range(n_slices): + for j in range(i + 1, n_slices): + for row_i in parts[i]: + for row_j in parts[j]: + pairs.append((row_i, row_j)) + return np.array(pairs) + + +def run_ari( + *, + writer, + env: dict, + n_features: list[int], + n_parts: list[int], + n_objs: list[int], + k: list[int], + seed: int, + repeats: int, + warmup: int, + gpu_only: bool, + cpu_only: bool, + verbose: bool = True, +) -> list[dict]: + import numpy as np + + ari_ref = None + ari_int32 = None + if not gpu_only: + from ccc.sklearn.metrics import adjusted_rand_index as ari_ref + if not cpu_only: + import ccc_cuda_ext + + ari_int32 = ccc_cuda_ext.ari_int32 + + records = [] + for nf, npart, nobj, kk in itertools.product(n_features, n_parts, n_objs, k): + np.random.seed(seed) + parts = np.random.randint(0, kk, size=(nf, npart, nobj), dtype=np.int32) + n_feature_comp = nf * (nf - 1) // 2 + n_aris = n_feature_comp * npart * npart + + gpu_min = gpu_mean = None + cpu_min = cpu_mean = None + if ari_int32 is not None: + gpu_min, gpu_mean, _ = _timed( + lambda p=parts, a=nf, b=npart, c=nobj: ari_int32(p, a, b, c), + repeats, + warmup, + sync=_gpu_sync, + ) + _clean_gpu() + if ari_ref is not None: + pairs = _pairwise_combinations(parts) + + def _cpu_ref(pairs=pairs, ref=ari_ref): + return [ref(p0, p1) for p0, p1 in pairs] + + cpu_min, cpu_mean, _ = _timed(_cpu_ref, repeats, warmup) + + speedup = cpu_min / gpu_min if (gpu_min and cpu_min and gpu_min > 0) else None + record = { + "mode": "ari", + "timestamp": _now_iso(), + "n_features": nf, + "n_parts": npart, + "n_objs": nobj, + "k": kk, + "seed": seed, + "repeats": repeats, + "warmup": warmup, + "n_aris": n_aris, + "gpu_time_min_s": gpu_min, + "gpu_time_mean_s": gpu_mean, + "cpu_time_min_s": cpu_min, + "cpu_time_mean_s": cpu_mean, + "speedup": speedup, + **env, + } + writer.write(record) + records.append(record) + if verbose: + msg = [f"ari f={nf} parts={npart} objs={nobj} k={kk} n_aris={n_aris}"] + if gpu_min is not None: + msg.append(f"GPU={gpu_min:.4f}s") + if cpu_min is not None: + msg.append(f"CPU={cpu_min:.4f}s") + if speedup is not None: + msg.append(f"speedup={speedup:.2f}x") + print(" " + " ".join(msg), file=sys.stderr) + return records + + +# --------------------------------------------------------------------------- # +# scaling: CPU n_jobs scaling +# --------------------------------------------------------------------------- # +def run_scaling( + *, + writer, + env: dict, + features: list[int], + samples: list[int], + n_jobs: list[int], + pvalue_n_perms: int | None, + seed: int, + repeats: int, + warmup: int, + verbose: bool = True, +) -> list[dict]: + import numpy as np + + from ccc.coef.impl import ccc as ccc_cpu + + records = [] + for nf, ns in itertools.product(features, samples): + np.random.seed(seed) + data = np.random.rand(nf, ns) + + times = {} + for nj in sorted(n_jobs): + t_min, t_mean, _ = _timed( + lambda d=data, j=nj, pv=pvalue_n_perms: ccc_cpu( + d, n_jobs=j, pvalue_n_perms=pv + ), + repeats, + warmup, + ) + times[nj] = (t_min, t_mean) + + baseline = times[min(times)][0] + for nj in sorted(times): + t_min, t_mean = times[nj] + speedup = baseline / t_min if t_min > 0 else None + record = { + "mode": "scaling", + "timestamp": _now_iso(), + "n_features": nf, + "n_samples": ns, + "n_jobs": nj, + "baseline_n_jobs": min(times), + "pvalue_n_perms": pvalue_n_perms, + "seed": seed, + "repeats": repeats, + "warmup": warmup, + "cpu_time_min_s": t_min, + "cpu_time_mean_s": t_mean, + "speedup": speedup, + **env, + } + writer.write(record) + records.append(record) + if verbose: + sp = f"{speedup:.2f}x" if speedup is not None else "n/a" + print( + f" scaling f={nf} n={ns} jobs={nj} CPU={t_min:.4f}s speedup={sp}", + file=sys.stderr, + ) + return records diff --git a/pyproject.toml b/pyproject.toml index 594bc2c6..8a8902d8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -36,6 +36,9 @@ Issues = "https://github.com/pivlab/ccc-gpu/issues" [project.optional-dependencies] test = ["pytest", "pytest-cov"] +[project.scripts] +ccc-gpu-bench = "ccc.bench.cli:main" + [tool.scikit-build] # Configure scikit-build-core cmake.version = ">=4.0" @@ -62,8 +65,13 @@ wheel.platlib = true # Contains compiled extensions [tool.pytest.ini_options] minversion = "6.0" -addopts = "-ra -q" +addopts = "-ra -q --strict-markers" testpaths = ["tests"] +markers = [ + "gpu: test requires a CUDA GPU + the ccc_cuda_ext extension (auto-applied to tests/gpu; skipped when cupy/ccc_cuda_ext are unavailable)", + "slow: test is slow (e.g. 1M-element inputs); excluded from the default CI subset", + "network: test downloads data over the network; excluded from the default CI subset", +] # --------------------------------------------------------------------------- # # Ruff — single source of truth for Python lint + format. diff --git a/scripts/run_tests.sh b/scripts/run_tests.sh index 5d063028..a56498fe 100644 --- a/scripts/run_tests.sh +++ b/scripts/run_tests.sh @@ -4,16 +4,31 @@ # bash ./scripts/run_tests.sh [test_suite...] # Examples: # bash ./scripts/run_tests.sh all -# bash ./scripts/run_tests.sh python cpp -# bash ./scripts/run_tests.sh python +# bash ./scripts/run_tests.sh cpu gpu cpp +# bash ./scripts/run_tests.sh cpu +# +# Test suites map to the pytest markers registered in pyproject.toml +# (gpu / slow / network) and to the CUDA C++ gtests (cpp): +# cpu -> marker expression "not gpu and not slow and not network" (what CI runs) +# gpu -> marker expression "gpu" (requires a CUDA device + ccc_cuda_ext) +# slow -> marker expression "slow" (1M-element / heavy CPU cases) +# cpp -> the CUDA C++ gtests via CMake (-DCCC_BUILD_TESTS=ON) + ctest # Available test suites declare -A TEST_SUITES=( - ["python"]="Python tests" - ["cpp"]="C++ tests" - # Add new test suites here in the future - # ["rust"]="Rust tests" - # ["cuda"]="CUDA-specific tests" + ["cpu"]="CPU test subset (not gpu/slow/network)" + ["gpu"]="GPU tests (marker: gpu)" + ["slow"]="Slow CPU tests (marker: slow)" + ["cpp"]="CUDA C++ gtests (ctest)" +) + +# Research-only modules import undeclared heavy deps (requests/minepy/IPython) +# and are excluded from the published wheel; ignore them here too. +RESEARCH_IGNORES=( + --ignore=tests/test_giant.py + --ignore=tests/test_methods.py + --ignore=tests/test_plots.py + --ignore=tests/test_corr.py ) # Function to display usage @@ -27,26 +42,33 @@ usage() { exit 1 } -# Function to run Python tests -run_python_tests() { - echo -e "\033[34mRunning Python tests...\033[0m" - pytest -rs --color=yes ./tests/ --ignore ./tests/gpu/excluded +# CPU subset (the CI target). +run_cpu_tests() { + echo -e "\033[34mRunning CPU test subset (not gpu/slow/network)...\033[0m" + pytest -rs --color=yes tests/ \ + "${RESEARCH_IGNORES[@]}" \ + -m "not gpu and not slow and not network" -o addopts="" } -# Function to run C++ tests -run_cpp_tests() { - echo -e "\033[34mBuilding C++ tests...\033[0m" - # Clean up build directory - rm -rf build - # Build the CUDA extension module - cmake -S . -B build - cmake --build build -j $(nproc) +# GPU tests (needs a CUDA device + the built extension). +run_gpu_tests() { + echo -e "\033[34mRunning GPU tests (marker: gpu)...\033[0m" + pytest -rs --color=yes tests/ -m gpu -o addopts="" +} - echo -e "\033[34mRunning C++ tests...\033[0m" - for test in ./build/test_*; do - echo "Running $test..." - ./$test - done +# Slow CPU tests (1M-element / heavy parallel cases). +run_slow_tests() { + echo -e "\033[34mRunning slow CPU tests (marker: slow)...\033[0m" + pytest -rs --color=yes tests/ "${RESEARCH_IGNORES[@]}" -m slow -o addopts="" +} + +# CUDA C++ gtests via CMake + ctest. +run_cpp_tests() { + echo -e "\033[34mBuilding and running CUDA C++ gtests (ctest)...\033[0m" + rm -rf build-tests + cmake -S . -B build-tests -DCCC_BUILD_TESTS=ON -GNinja + cmake --build build-tests + ctest --test-dir build-tests --output-on-failure } # Check if no arguments provided @@ -78,13 +100,11 @@ fi for arg in "$@"; do case $arg in "all") - # Run all test suites for suite in "${!TEST_SUITES[@]}"; do run_${suite}_tests done ;; - "python"|"cpp") - # Run specific test suite + "cpu"|"gpu"|"slow"|"cpp") run_${arg}_tests ;; *) diff --git a/tests/README.md b/tests/README.md index 1b3bb94c..4a3408cd 100644 --- a/tests/README.md +++ b/tests/README.md @@ -1,13 +1,53 @@ # Unit tests +The suite is partitioned by pytest markers (registered in `pyproject.toml` with +`--strict-markers`): + +- `gpu` — requires a CUDA device and the compiled `ccc_cuda_ext` extension. + Auto-applied to everything under `tests/gpu/`. On a machine without + `cupy`/`ccc_cuda_ext` these tests are skipped (not collected) so a plain + `pytest tests/` still passes. +- `slow` — heavy CPU cases (e.g. 1M-element inputs); excluded from the default + CI subset. +- `network` — downloads data over the network (`test_giant.py`, the Titanic + dataset test); excluded from the default CI subset. + +Correctness tests contain **no** benchmarking: no wall-clock timing, no speedup +assertions, no log files. Performance measurement lives in the `ccc-gpu-bench` +CLI (`python -m ccc.bench`). + ## Run -These are the instructions to run the unit tests. It is assumed that you already -followed the steps to set up the environment and download the needed data, and that -your `PYTHONPATH` and `CM_ROOT_DIR` variables are adjusted appropriately. +```bash +# CPU subset — what CI runs (no GPU, no network, no slow cases): +pytest tests/ -m "not gpu and not slow and not network" + +# GPU tests (needs a CUDA device + the built extension): +pytest tests/ -m gpu + +# Slow CPU tests: +pytest tests/ -m slow + +# Everything (local, GPU box): +pytest tests/ + +# Convenience wrapper (installs the package, then runs a suite): +bash ./scripts/run_tests.sh cpu # or: gpu | slow | cpp | all +``` + +Note: the research-only modules (`test_giant.py`, `test_methods.py`, +`test_plots.py`, `test_corr.py`) import undeclared heavy dependencies +(`requests`, `minepy`, `IPython`) and are excluded from the wheel; ignore them +with `--ignore=...` when running a full local suite without those deps installed. + +## CUDA C++ tests (gtest / ctest) -Execute these commands to run the unit tests: +The native CUDA tests under `tests/cuda_ext/` are built behind an opt-in CMake +option (`CCC_BUILD_TESTS`, default OFF), so the default wheel build never fetches +googletest: ```bash -pytest -rs --color=yes tests/ +cmake -S . -B build-tests -DCCC_BUILD_TESTS=ON -GNinja +cmake --build build-tests +ctest --test-dir build-tests --output-on-failure ``` diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 00000000..661fa785 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,72 @@ +"""Shared pytest fixtures for the CCC test suite. + +Fixtures that are useful to both the CPU (`tests/`) and GPU (`tests/gpu/`) +suites live here. GPU-only helpers (e.g. ``clean_gpu_memory``) live in +``tests/gpu/conftest.py``. +""" + +import numpy as np +import pandas as pd +import pytest + + +def generate_categorical_data( + n_features, + n_objects, + n_categories=3, + categories=None, + str_length=None, + random_state=None, + feature_names=None, +): + """Generate a random categorical ``pandas.DataFrame`` of shape (n_objects, n_features). + + Parameters + ---------- + n_features : int + Number of features (columns). + n_objects : int + Number of objects (rows). + n_categories : int, optional + Number of unique categories when ``categories`` is None. + categories : list or None, optional + Explicit list of categories to sample from. If None, uses + ``range(n_categories)`` or random strings when ``str_length`` is given. + str_length : int or None, optional + If given (and ``categories`` is None), generate random uppercase-letter + string categories of this length. + random_state : int or None, optional + Seed for reproducibility. + feature_names : list or None, optional + Column names; defaults to ``feature_{i}``. + """ + if random_state is not None: + np.random.seed(random_state) + + if categories is None: + if str_length is not None: + letters = np.array(list("ABCDEFGHIJKLMNOPQRSTUVWXYZ")) + categories = [ + "".join(np.random.choice(letters, size=str_length)) + for _ in range(n_categories) + ] + else: + categories = list(range(n_categories)) + else: + n_categories = len(categories) + + random_indices = np.random.randint(0, n_categories, size=(n_objects, n_features)) + categorical_data = np.array( + [[categories[idx] for idx in row] for row in random_indices] + ) + + if feature_names is None: + feature_names = [f"feature_{i}" for i in range(n_features)] + + return pd.DataFrame(categorical_data, columns=feature_names) + + +@pytest.fixture +def categorical_data_generator(): + """Return the :func:`generate_categorical_data` helper for use in tests.""" + return generate_categorical_data diff --git a/tests/cuda_ext/test_ari_random.cu b/tests/cuda_ext/test_ari_random.cu index 7f45ec28..6a7b6074 100644 --- a/tests/cuda_ext/test_ari_random.cu +++ b/tests/cuda_ext/test_ari_random.cu @@ -313,6 +313,19 @@ TEST_P(PairwiseAriTest, RandomPartitions) { const auto params = GetParam(); + // The reference results are computed by a Python loop over every partition + // pair (n_feature_comp * n_parts^2). For large feature counts that loop is + // far too slow for a unit test (and the largest case would OOM). Skip those; + // large-input CUDA parity is exercised by the pytest test_pairwise_ari suite. + // (tracked: restructure-tests coverage backlog) + const uint64_t n_feature_comp = static_cast(params.n_features) * (params.n_features - 1) / 2; + const uint64_t n_ref_pairs = n_feature_comp * static_cast(params.n_parts) * params.n_parts; + if (n_ref_pairs > 100000) + { + GTEST_SKIP() << "Skipping large case: Python reference over " << n_ref_pairs + << " partition pairs is too slow for a unit test."; + } + // Generate test data auto parts = TestDataGenerator::generate_random_partitions( params.n_features, params.n_parts, params.n_objs, params.k); diff --git a/tests/gpu/conftest.py b/tests/gpu/conftest.py new file mode 100644 index 00000000..8c7348d4 --- /dev/null +++ b/tests/gpu/conftest.py @@ -0,0 +1,50 @@ +"""Environment-aware configuration for the GPU test suite. + +Every test under ``tests/gpu/`` is automatically tagged with the ``gpu`` marker +so it can be selected/deselected with ``-m gpu`` / ``-m "not gpu"``. + +On a machine without ``cupy`` or the compiled ``ccc_cuda_ext`` extension the GPU +modules cannot even be imported (they import cupy at module scope), so we tell +pytest to ignore them during collection. This keeps a plain ``pytest tests/`` +green on a CPU-only box instead of erroring out at import time. On a GPU box the +modules collect and run normally. +""" + +import importlib.util +from pathlib import Path + +import pytest + +_HERE = Path(__file__).parent + +# The GPU suite needs both cupy (for memory management / array transfer) and the +# compiled CUDA extension. If either is missing the tests are not collected. +_MISSING_GPU_DEPS = [ + name for name in ("cupy", "ccc_cuda_ext") if importlib.util.find_spec(name) is None +] +GPU_AVAILABLE = not _MISSING_GPU_DEPS + +# When GPU deps are unavailable, skip collecting the GPU test modules entirely so +# their module-level ``import cupy`` does not raise a collection error. +collect_ignore_glob = ["test_*.py"] if not GPU_AVAILABLE else [] + + +def pytest_collection_modifyitems(items): + """Auto-apply the ``gpu`` marker to every test collected under tests/gpu/.""" + for item in items: + item_path = Path(str(item.fspath)) + if item_path.parent == _HERE or _HERE in item_path.parents: + item.add_marker(pytest.mark.gpu) + + +@pytest.fixture(autouse=True) +def clean_gpu_memory(): + """Free the cupy default memory pool after each GPU test. + + Replaces the copy-pasted ``@clean_gpu_memory`` decorator that used to wrap + individual GPU tests. Applied automatically to every test in this directory. + """ + yield + import cupy as cp + + cp.get_default_memory_pool().free_all_blocks() diff --git a/tests/gpu/test_ari_gpu.py b/tests/gpu/test_ari_gpu.py index ad24ddd3..9a6bdce3 100644 --- a/tests/gpu/test_ari_gpu.py +++ b/tests/gpu/test_ari_gpu.py @@ -1,14 +1,18 @@ -import time +"""Kernel-level ARI parity: ``ccc_cuda_ext.ari_int32`` vs the CPU reference. + +The throughput/benchmark variant of the pairwise test was removed; its +measurement intent now lives in ``ccc-gpu-bench ari``. +""" import ccc_cuda_ext import numpy as np import pytest -from ccc.sklearn.metrics import ( - adjusted_rand_index, -) +from ccc.sklearn.metrics import adjusted_rand_index -# Test cases taken from sklearn.metrics.adjusted_rand_score +# Expected values are 2-decimal literals from the sklearn.metrics.adjusted_rand_score +# documentation, so a 1e-2 tolerance is used here (the exact kernel-vs-reference +# parity at 1e-5 is covered by test_pairwise_ari below). @pytest.mark.parametrize( "parts, expected_ari", [ @@ -27,19 +31,16 @@ def test_simple_ari_results(parts, expected_ari): def generate_pairwise_combinations(arr): pairs = [] - num_slices = arr.shape[0] # Number of 2D arrays in the 3D array - + num_slices = arr.shape[0] for i in range(num_slices): - for j in range(i + 1, num_slices): # Only consider pairs in different slices - for row_i in arr[i]: # Each row in slice i - for row_j in arr[j]: # Pairs with each row in slice j + for j in range(i + 1, num_slices): + for row_i in arr[i]: + for row_j in arr[j]: pairs.append([row_i, row_j]) - - # Convert list of pairs to a NumPy array return np.array(pairs) -# Test ari generation given a full 3D array of partitions +# Test ARI generation given a full 3D array of partitions against the CPU reference. @pytest.mark.parametrize( "n_features, n_parts, n_objs, k, seed", [ @@ -50,70 +51,16 @@ def generate_pairwise_combinations(arr): ], ) def test_pairwise_ari(n_features, n_parts, n_objs, k, seed): - # Set random seed for reproducibility np.random.seed(seed) parts = np.random.randint(0, k, size=(n_features, n_parts, n_objs), dtype=np.int32) - # Create test inputs n_feature_comp = n_features * (n_features - 1) // 2 n_aris = n_feature_comp * n_parts * n_parts ref_aris = np.zeros(n_aris, dtype=np.float32) - # Get partition pairs pairs = generate_pairwise_combinations(parts) for i, (part0, part1) in enumerate(pairs): - ari = adjusted_rand_index(part0, part1) - ref_aris[i] = ari - # Compute ARIs using CUDA - res_aris = ccc_cuda_ext.ari_int32(parts, n_features, n_parts, n_objs) - - # print(f"\nres_aris: {res_aris}, ref_aris: {ref_aris}") - assert np.allclose(res_aris, ref_aris) + ref_aris[i] = adjusted_rand_index(part0, part1) - -@pytest.mark.parametrize( - "n_features, n_parts, n_objs, k, seed", - [ - (100, 10, 300, 10, 42), - (100, 20, 300, 10, 42), - # (1000, 20, 300, 10, 42), - # (100000, 2, 10, 10, 42), - ], -) -def test_pairwise_ari_benchmark_features(n_features, n_parts, n_objs, k, seed): - # Set random seed for reproducibility - np.random.seed(seed) - - parts = np.random.randint(0, k, size=(n_features, n_parts, n_objs), dtype=np.int32) - # Create test inputs - n_feature_comp = n_features * (n_features - 1) // 2 - n_aris = n_feature_comp * n_parts * n_parts - ref_aris = np.zeros(n_aris, dtype=np.float32) - # Get partition pairs - pairs = generate_pairwise_combinations(parts) - - # Time CPU version - start_cpu = time.time() - for i, (part0, part1) in enumerate(pairs): - ari = adjusted_rand_index(part0, part1) - ref_aris[i] = ari - end_cpu = time.time() - cpu_time = end_cpu - start_cpu - - start_gpu = time.time() res_aris = ccc_cuda_ext.ari_int32(parts, n_features, n_parts, n_objs) - end_gpu = time.time() - gpu_time = end_gpu - start_gpu - assert np.allclose(res_aris, ref_aris) - - # Report benchmark results - print( - f"Testing with n_features={n_features}, n_parts={n_parts}, n_objs={n_objs}, k={k}, seed={seed}" - ) - print(f"GPU time: {gpu_time:.4f} seconds") - print(f"CPU time: {cpu_time:.4f} seconds") - speedup = cpu_time / gpu_time - print(f"Speedup: {speedup:.2f}x") - num_coefs = n_aris - print(f"Number of coefficients: {num_coefs}") diff --git a/tests/gpu/test_ccc_gpu.py b/tests/gpu/test_ccc_gpu.py index 20c78fb8..69521f99 100644 --- a/tests/gpu/test_ccc_gpu.py +++ b/tests/gpu/test_ccc_gpu.py @@ -1,307 +1,99 @@ -import os -import time -from typing import Any +"""GPU-vs-CPU parity for the end-to-end coefficient (numerical + categorical). + +These are pure correctness tests: no timing, no speedup assertions, no log +files. Performance measurement lives in the ``ccc-gpu-bench`` CLI. GPU memory is +cleaned between tests by the autouse ``clean_gpu_memory`` fixture in conftest. +""" import numpy as np import pandas as pd import pytest from ccc.coef.impl import ccc from ccc.coef.impl_gpu import ccc as ccc_gpu -from utils import clean_gpu_memory, generate_categorical_data - - -def setup_logging( - seed: int, - shape: tuple[int, int], - n_cpu_cores: int, - generate_logs: bool, -) -> dict[str, Any] | None: - """Setup logging infrastructure if logging is enabled. - - Args: - seed: Random seed for reproducibility - shape: Tuple of (n_features, n_samples) - n_cpu_cores: Number of CPU cores to use - generate_logs: Whether to generate log files - - Returns: - Dictionary containing logging information if logging is enabled, None otherwise - """ - if not generate_logs: - return None - - logs_dir = os.path.join("tests", "logs") - os.makedirs(logs_dir, exist_ok=True) - - base_filename = f"test_ccc_gpu_{seed}_f{shape[0]}_n{shape[1]}_c{n_cpu_cores}" - log_files = { - "log": os.path.join(logs_dir, f"{base_filename}.log"), - "gpu_results": os.path.join(logs_dir, f"{base_filename}_gpu_results.log"), - "cpu_results": os.path.join(logs_dir, f"{base_filename}_cpu_results.log"), - "input_data": os.path.join(logs_dir, f"{base_filename}_input_data.log"), - } - - print("Writing test output to:") - for name, path in log_files.items(): - print(f" - {name}: {path}") - - return {"files": log_files, "log_file": open(log_files["log"], "w")} - - -def log_test_info(log_file, shape: tuple[int, int], seed: int) -> None: - """Log basic test information.""" - print( - f"\nTesting with {shape[0]} features, {shape[1]} samples, seed {seed}", - file=log_file, - ) - - -def log_performance_metrics( - log_file, gpu_time: float, cpu_time: float, num_coefs: int -) -> None: - """Log performance metrics.""" - speedup = cpu_time / gpu_time - print(f"GPU time: {gpu_time:.4f} seconds", file=log_file) - print(f"CPU time: {cpu_time:.4f} seconds", file=log_file) - print(f"Speedup: {speedup:.2f}x", file=log_file) - print(f"Number of coefficients: {num_coefs}", file=log_file) - - -def analyze_differences( - c1: np.ndarray, - c2: np.ndarray, - shape: tuple[int, int], - log_file: Any | None = None, -) -> tuple[int, float, float, int]: - """Analyze differences between GPU and CPU results. - - Returns: - Tuple of (number of differences, max difference, percentage of differences, number of max differences) - """ - not_close_mask = ~np.isclose(c1, c2, rtol=1e-3, atol=1e-3) - not_close_indices = np.where(not_close_mask)[0] - not_close = len(not_close_indices) - - if log_file and not_close > 0: - log_differences(c1, c2, not_close_indices, shape, log_file) - - num_coefs = len(c1) - not_close_percentage = not_close / num_coefs * 100 - max_diff = np.max(np.abs(c1 - c2)) - max_diff_count = np.sum(np.abs(c1 - c2) == max_diff) if not_close > 0 else 0 - - if log_file: - log_statistics( - not_close, max_diff, max_diff_count, not_close_percentage, log_file - ) - - return not_close, max_diff, not_close_percentage, max_diff_count +# Parity tolerance contract (see openspec restructure-tests): GPU-vs-CPU +# end-to-end results must match within atol/rtol = 1e-6. +PARITY_ATOL = 1e-6 +PARITY_RTOL = 1e-6 -def log_differences( - c1: np.ndarray, - c2: np.ndarray, - not_close_indices: np.ndarray, - shape: tuple[int, int], - log_file: Any, -) -> None: - """Log detailed information about differences between results.""" - print("\nDifferences found:", file=log_file) - print( - "idx | GPU Result | CPU Result | Absolute Diff | Row i | Row j", - file=log_file, - ) - print("-" * 65, file=log_file) - - n_features = shape[0] - for diff_count, idx in enumerate(not_close_indices, 1): - i = int( - np.floor( - (2 * n_features - 1 - np.sqrt((2 * n_features - 1) ** 2 - 8 * idx)) / 2 - ) - ) - j = idx - i * n_features + (i * (i + 1)) // 2 - - print(f"\n[Diff #{diff_count}]", file=log_file) - print( - f"{idx:3d} | {c1[idx]:10.6f} | {c2[idx]:10.6f} | {abs(c1[idx] - c2[idx]):12.6f} | {i:5d} | {j:5d}", - file=log_file, - ) - print("-" * 65, file=log_file) - -def log_statistics( - not_close: int, - max_diff: float, - max_diff_count: int, - not_close_percentage: float, - log_file: Any, -) -> None: - """Log statistical information about the differences.""" - print(f"\nNumber of coefficients not close: {not_close}", file=log_file) - print(f"Max difference: {max_diff:.6f}", file=log_file) - print( - f"Number of coefficients with max difference: {max_diff_count}", - file=log_file, - ) - print( - f"Percentage of coefficients not close: {not_close_percentage}%", - file=log_file, - ) - - -@pytest.mark.parametrize( - "seed", [42] -) # More seeds for simple cases, only 42 for large cases +@pytest.mark.parametrize("seed", [42]) @pytest.mark.parametrize( - "shape, contain_singletons, max_not_close_percentage, generate_logs", + "shape, contain_singletons", [ - # Simple cases - ((10, 100), False, 0.0, False), - ((20, 200), False, 0.0, False), - ((30, 300), False, 0.0, False), - ((10, 100), True, 0.0, False), - ((20, 200), True, 0.0, False), - ((30, 300), True, 0.0, False), - # ((100, 100), 0.0, True), - # ((100, 1000), 0.0, True), - # Large cases - # ((1000, 100), 0.0, True), - # ((100, 1000), 0.008, False), # Skipped, too slow for a unit test - # ((1000, 1000), 0.0, False), - # ((2000, 1000), 0.0, False), - # ((3000, 1000), 0.0, True), - # ((4000, 1000), 0.0, True), - # ((5000, 100), 0.0, True), - # ((20000, 100), 0.0, False), - # Benchmark cases - # ((5000, 1000), 0.0, True), - # ((10000, 1000), 0.0, True), - # ((500, 1000), 0.0, True), - # ((1000, 1000), 0.0, True), - # ((2000, 1000), 0.0, True), - # ((4000, 1000), 0.0, True), - # ((6000, 1000), False, 0.0, True), - # ((12000, 1000), 0.0, True), - # ((16000, 1000), False, 0.0, True), - # ((20000, 1000), False, 0.0, True), - # ((8000, 1000), 0.0, True), - # ((12000, 1000), 0.0, True), - # ((56200, 755), False, 0.0, True), + ((10, 100), False), + ((20, 200), False), + ((30, 300), False), + ((10, 100), True), + ((20, 200), True), + ((30, 300), True), ], ) -@pytest.mark.parametrize("n_cpu_cores", [24]) -@clean_gpu_memory def test_ccc_gpu_with_numerical_input( seed: int, shape: tuple[int, int], contain_singletons: bool, - n_cpu_cores: int, - max_not_close_percentage: float, - generate_logs: bool, ): - """ - Test 2D CCC implementation with various data shapes and random seeds. - Combines both simple and large test cases. - - Args: - seed: Random seed for reproducibility - shape: Tuple of (n_features, n_samples) - n_cpu_cores: Number of CPU cores to use - max_not_close_percentage: Maximum allowed percentage of coefficients that can differ - generate_logs: Whether to generate detailed log files - """ - # Setup logging if enabled - logging_info = setup_logging(seed, shape, n_cpu_cores, generate_logs) - log_file = logging_info["log_file"] if logging_info else None - - # Generate test data + """GPU coefficients match the CPU reference for numerical input.""" np.random.seed(seed) - if log_file: - log_test_info(log_file, shape, seed) df = np.random.rand(*shape) - # If contain_singletons is True, set the first row to be a singleton, using value 0.0 if contain_singletons: + # Force a constant (singleton) feature to exercise the -2 marker path. df[0, :] = 0.0 - # Time GPU version - start_gpu = time.time() - # c1 = ccc_gpu(df) - # Catch exceptions - try: - c1 = ccc_gpu(df) - except Exception as e: - print(f"Error: {e}") - raise e - end_gpu = time.time() - gpu_time = end_gpu - start_gpu - - # Time CPU version - start_cpu = time.time() - c2 = ccc(df, n_jobs=n_cpu_cores) - end_cpu = time.time() - cpu_time = end_cpu - start_cpu - - # Log performance metrics - if log_file: - log_performance_metrics(log_file, gpu_time, cpu_time, len(c1)) - - # Analyze differences - not_close, max_diff, not_close_percentage, _ = analyze_differences( - c1, c2, shape, log_file - ) - - # Cleanup logging - if logging_info: - logging_info["log_file"].close() - - # Convert to DataFrame for better assertion hints - gpu_df = pd.DataFrame(c1) - gpu_df = gpu_df.astype(np.float64) - cpu_df = pd.DataFrame(c2) - pd.testing.assert_frame_equal(gpu_df, cpu_df, atol=1e-6, rtol=1e-6) + c_gpu = ccc_gpu(df) + c_cpu = ccc(df, n_jobs=2) - # Assert results using percentages. Useful if get_parts is implemented in GPU version in the future, in which - # case the parts generated will be slightly different due to floating point precision - # assert ( - # not_close_percentage <= max_not_close_percentage - # ), f"Results differ for shape={shape}, seed={seed}" + gpu_df = pd.DataFrame(c_gpu).astype(np.float64) + cpu_df = pd.DataFrame(c_cpu) + pd.testing.assert_frame_equal(gpu_df, cpu_df, atol=PARITY_ATOL, rtol=PARITY_RTOL) -@pytest.mark.parametrize( - "seed", [42] -) # More seeds for simple cases, only 42 for large cases +@pytest.mark.parametrize("seed", [42]) @pytest.mark.parametrize( "shape, n_categories, str_length", [ - # Simple cases ((10, 20), 10, 2), ((20, 200), 50, 3), ((30, 300), 200, 4), ((9, 10000), 500, 5), ], ) -@pytest.mark.parametrize("n_cpu_cores", [48]) -@clean_gpu_memory def test_ccc_gpu_with_categorical_input( seed: int, shape: tuple[int, int], n_categories: int, str_length: int, - n_cpu_cores: int, + categorical_data_generator, ): + """GPU coefficients match the CPU reference for categorical input.""" n_features, n_samples = shape - df = generate_categorical_data( + df = categorical_data_generator( n_features, n_samples, n_categories, str_length=str_length, random_state=seed ) - res_cpu = ccc(df, n_jobs=n_cpu_cores) + res_cpu = ccc(df, n_jobs=2) res_gpu = ccc_gpu(df) cpu_df = pd.DataFrame(res_cpu) gpu_df = pd.DataFrame(res_gpu.astype(np.float64)) - pd.testing.assert_frame_equal(gpu_df, cpu_df, atol=1e-6, rtol=1e-6) + pd.testing.assert_frame_equal(gpu_df, cpu_df, atol=PARITY_ATOL, rtol=PARITY_RTOL) + + +@pytest.mark.slow +@pytest.mark.parametrize("shape", [(1000, 1000)]) +def test_ccc_gpu_large_input_parity(shape: tuple[int, int]): + """Safety net: parity on a large grid (moved off the default fast path). + + The commented-out mega grids in the old test lived here as benchmark cases; + the full sweeps now belong to ``ccc-gpu-bench``. This single ``slow`` case + guards against large-input regressions in CI-excluded local runs. + """ + np.random.seed(42) + df = np.random.rand(*shape) + c_gpu = ccc_gpu(df) + c_cpu = ccc(df, n_jobs=-1) -# @clean_gpu_memory -# def test_ccc_gpu_with_mixed_input(): -# return + gpu_df = pd.DataFrame(c_gpu).astype(np.float64) + cpu_df = pd.DataFrame(c_cpu) + pd.testing.assert_frame_equal(gpu_df, cpu_df, atol=PARITY_ATOL, rtol=PARITY_RTOL) diff --git a/tests/gpu/test_ccc_gpu_return_parts.py b/tests/gpu/test_ccc_gpu_return_parts.py index 69f0ec26..c92851c0 100644 --- a/tests/gpu/test_ccc_gpu_return_parts.py +++ b/tests/gpu/test_ccc_gpu_return_parts.py @@ -1,170 +1,29 @@ -import os -import time -from typing import Any +"""GPU-vs-CPU parity for ``return_parts=True`` (partitions + max_parts). + +Pure correctness tests; timing/logging machinery removed (measurement lives in +``ccc-gpu-bench``). GPU memory is cleaned by the autouse fixture in conftest. +""" import numpy as np import pandas as pd import pytest from ccc.coef.impl import ccc from ccc.coef.impl_gpu import ccc as ccc_gpu -from utils import clean_gpu_memory - - -def setup_logging( - seed: int, - shape: tuple[int, int], - n_cpu_cores: int, - generate_logs: bool, -) -> dict[str, Any] | None: - """Setup logging infrastructure if logging is enabled. - - Args: - seed: Random seed for reproducibility - shape: Tuple of (n_features, n_samples) - n_cpu_cores: Number of CPU cores to use - generate_logs: Whether to generate log files - - Returns: - Dictionary containing logging information if logging is enabled, None otherwise - """ - if not generate_logs: - return None - - logs_dir = os.path.join("tests", "logs") - os.makedirs(logs_dir, exist_ok=True) - - base_filename = f"test_ccc_gpu_{seed}_f{shape[0]}_n{shape[1]}_c{n_cpu_cores}" - log_files = { - "log": os.path.join(logs_dir, f"{base_filename}.log"), - "gpu_results": os.path.join(logs_dir, f"{base_filename}_gpu_results.log"), - "cpu_results": os.path.join(logs_dir, f"{base_filename}_cpu_results.log"), - "input_data": os.path.join(logs_dir, f"{base_filename}_input_data.log"), - } - print("Writing test output to:") - for name, path in log_files.items(): - print(f" - {name}: {path}") - - return {"files": log_files, "log_file": open(log_files["log"], "w")} - - -def log_test_info(log_file, shape: tuple[int, int], seed: int) -> None: - """Log basic test information.""" - print( - f"\nTesting with {shape[0]} features, {shape[1]} samples, seed {seed}", - file=log_file, - ) - - -def log_performance_metrics( - log_file, gpu_time: float, cpu_time: float, num_coefs: int -) -> None: - """Log performance metrics.""" - speedup = cpu_time / gpu_time - print(f"GPU time: {gpu_time:.4f} seconds", file=log_file) - print(f"CPU time: {cpu_time:.4f} seconds", file=log_file) - print(f"Speedup: {speedup:.2f}x", file=log_file) - print(f"Number of coefficients: {num_coefs}", file=log_file) - - -def analyze_differences( - c1: np.ndarray, - c2: np.ndarray, - shape: tuple[int, int], - log_file: Any | None = None, -) -> tuple[int, float, float, int]: - """Analyze differences between GPU and CPU results. +PARITY_ATOL = 1e-6 +PARITY_RTOL = 1e-6 - Returns: - Tuple of (number of differences, max difference, percentage of differences, number of max differences) - """ - not_close_mask = ~np.isclose(c1, c2, rtol=1e-3, atol=1e-3) - not_close_indices = np.where(not_close_mask)[0] - not_close = len(not_close_indices) - - if log_file and not_close > 0: - log_differences(c1, c2, not_close_indices, shape, log_file) - - num_coefs = len(c1) - not_close_percentage = not_close / num_coefs * 100 - max_diff = np.max(np.abs(c1 - c2)) - max_diff_count = np.sum(np.abs(c1 - c2) == max_diff) if not_close > 0 else 0 - - if log_file: - log_statistics( - not_close, max_diff, max_diff_count, not_close_percentage, log_file - ) - - return not_close, max_diff, not_close_percentage, max_diff_count - - -def log_differences( - c1: np.ndarray, - c2: np.ndarray, - not_close_indices: np.ndarray, - shape: tuple[int, int], - log_file: Any, -) -> None: - """Log detailed information about differences between results.""" - print("\nDifferences found:", file=log_file) - print( - "idx | GPU Result | CPU Result | Absolute Diff | Row i | Row j", - file=log_file, - ) - print("-" * 65, file=log_file) - - n_features = shape[0] - for diff_count, idx in enumerate(not_close_indices, 1): - i = int( - np.floor( - (2 * n_features - 1 - np.sqrt((2 * n_features - 1) ** 2 - 8 * idx)) / 2 - ) - ) - j = idx - i * n_features + (i * (i + 1)) // 2 - - print(f"\n[Diff #{diff_count}]", file=log_file) - print( - f"{idx:3d} | {c1[idx]:10.6f} | {c2[idx]:10.6f} | {abs(c1[idx] - c2[idx]):12.6f} | {i:5d} | {j:5d}", - file=log_file, - ) - print("-" * 65, file=log_file) - - -def log_statistics( - not_close: int, - max_diff: float, - max_diff_count: int, - not_close_percentage: float, - log_file: Any, -) -> None: - """Log statistical information about the differences.""" - print(f"\nNumber of coefficients not close: {not_close}", file=log_file) - print(f"Max difference: {max_diff:.6f}", file=log_file) - print( - f"Number of coefficients with max difference: {max_diff_count}", - file=log_file, - ) - print( - f"Percentage of coefficients not close: {not_close_percentage}%", - file=log_file, - ) - -# Original CCC test in tests/test_coef.py def test_cm_return_parts_quadratic(): - # Prepare - np.random.seed(0) - # two features with a quadratic relationship + np.random.seed(0) feature0 = np.array([-4, -3, -2, -1, 0, 0, 1, 2, 3, 4]) feature1 = np.array([10, 9, 8, 7, 6, 6, 7, 8, 9, 10]) - # Run cm_value, max_parts, parts = ccc_gpu( feature0, feature1, internal_n_clusters=[2, 3], return_parts=True ) - # Validate assert np.isclose(round(cm_value, 2), 0.31) assert parts is not None @@ -179,24 +38,18 @@ def test_cm_return_parts_quadratic(): assert max_parts is not None assert hasattr(max_parts, "shape") assert max_parts.shape == (2,) - # the set of partitions that maximize ari is: - # - k == 3 for feature0 - # - k == 2 for feature1 + # the set of partitions that maximize ari is k==3 for feature0, k==2 for feature1 np.testing.assert_array_equal(max_parts, np.array([1, 0])) def test_cm_return_parts_linear(): - # Prepare - np.random.seed(0) - # two features on 100 objects with a linear relationship + np.random.seed(0) feature0 = np.random.rand(100) feature1 = feature0 * 5.0 - # Run cm_value, max_parts, parts = ccc_gpu(feature0, feature1, return_parts=True) - # Validate assert cm_value == 1.0 assert parts is not None @@ -207,105 +60,45 @@ def test_cm_return_parts_linear(): assert max_parts is not None assert hasattr(max_parts, "shape") assert max_parts.shape == (2,) - # even in this test we do not specify internal_n_clusters (so it goes from - # k=2 to k=10, nine partitions), k=2 for both features should already have - # the maximum value + # k=2 for both features already yields the maximum np.testing.assert_array_equal(max_parts, np.array([0, 0])) +@pytest.mark.parametrize("seed", [42]) @pytest.mark.parametrize( - "seed", [42] -) # More seeds for simple cases, only 42 for large cases -@pytest.mark.parametrize( - "shape, contain_singletons, generate_logs", + "shape, contain_singletons", [ - # Simple cases - ((10, 100), False, False), - ((20, 200), False, False), - ((30, 300), False, False), - ((10, 100), True, False), - ((20, 200), True, False), - ((30, 300), True, False), - ((100, 100), False, False), - ((100, 1000), False, False), - # # Large cases - # ((100, 2000), False, False), - # ((2000, 1000), False, False), + ((10, 100), False), + ((20, 200), False), + ((30, 300), False), + ((10, 100), True), + ((20, 200), True), + ((30, 300), True), + ((100, 100), False), + ((100, 1000), False), ], ) -@pytest.mark.parametrize("n_cpu_cores", [24]) -@clean_gpu_memory def test_ccc_gpu_with_numerical_input( seed: int, shape: tuple[int, int], contain_singletons: bool, - n_cpu_cores: int, - generate_logs: bool, ): - """ - Test 2D CCC implementation with various data shapes and random seeds. - Combines both simple and large test cases. - - Args: - seed: Random seed for reproducibility - shape: Tuple of (n_features, n_samples) - n_cpu_cores: Number of CPU cores to use - generate_logs: Whether to generate detailed log files - """ - # Setup logging if enabled - logging_info = setup_logging(seed, shape, n_cpu_cores, generate_logs) - log_file = logging_info["log_file"] if logging_info else None - - # Generate test data + """GPU return_parts output matches the CPU reference (coefs + parts + max_parts).""" np.random.seed(seed) - if log_file: - log_test_info(log_file, shape, seed) df = np.random.rand(*shape) - # If contain_singletons is True, set the first row to be a singleton, using value 0.0 if contain_singletons: df[0, :] = 0.0 - # Time GPU version - start_gpu = time.time() - # c1 = ccc_gpu(df) - # Catch exceptions - try: - c1, g_max_parts, g_parts = ccc_gpu(df, return_parts=True) - except Exception as e: - print(f"Error: {e}") - raise e - end_gpu = time.time() - gpu_time = end_gpu - start_gpu + c_gpu, g_max_parts, g_parts = ccc_gpu(df, return_parts=True) + c_cpu, c_max_parts, c_parts = ccc(df, n_jobs=2, return_parts=True) - # Time CPU version - start_cpu = time.time() - c2, c_max_parts, c_parts = ccc(df, n_jobs=n_cpu_cores, return_parts=True) - end_cpu = time.time() - cpu_time = end_cpu - start_cpu + gpu_df = pd.DataFrame(c_gpu).astype(np.float64) + cpu_df = pd.DataFrame(c_cpu) + pd.testing.assert_frame_equal(gpu_df, cpu_df, atol=PARITY_ATOL, rtol=PARITY_RTOL) - # Log performance metrics - if log_file: - log_performance_metrics(log_file, gpu_time, cpu_time, len(c1)) - - # Analyze differences - not_close, max_diff, not_close_percentage, _ = analyze_differences( - c1, c2, shape, log_file - ) - - # Cleanup logging - if logging_info: - logging_info["log_file"].close() - - # Convert to DataFrame for better assertion hints - gpu_df = pd.DataFrame(c1) - gpu_df = gpu_df.astype(np.float64) - cpu_df = pd.DataFrame(c2) - pd.testing.assert_frame_equal(gpu_df, cpu_df, atol=1e-6, rtol=1e-6) - - # Check if return_parts is correctly implemented + # Partitions must be identical (they are integer cluster labels). assert g_max_parts.shape == c_max_parts.shape assert g_parts.shape == c_parts.shape - for i in range(len(g_parts)): pd.testing.assert_frame_equal( pd.DataFrame(g_parts[i].astype(np.int16)), @@ -313,33 +106,22 @@ def test_ccc_gpu_with_numerical_input( check_exact=True, ) - # Validate max_parts: Both GPU and CPU choices should represent valid maxima - # This handles tie-breaking differences between implementations - for i in range(len(g_max_parts)): - gpu_parts = g_max_parts[i] - cpu_parts = c_max_parts[i] + # max_parts: where GPU and CPU disagree, both selections must be valid maxima + # of the ARI matrix (ties break differently between implementations). + from ccc.coef.impl import cdist_parts_basic, get_coords_from_index - # If they match exactly, no need for further validation - if np.array_equal(gpu_parts, cpu_parts): + for i in range(len(g_max_parts)): + gpu_choice = g_max_parts[i] + cpu_choice = c_max_parts[i] + if np.array_equal(gpu_choice, cpu_choice): continue - # For mismatches, verify both choices are valid maxima - # We need to compute the ARI matrix to check this - from ccc.coef.impl import cdist_parts_basic, get_coords_from_index - - # Get feature indices for this comparison feat_i, feat_j = get_coords_from_index(shape[0], i) - - # Compute ARI matrix for this feature pair ari_matrix = cdist_parts_basic(c_parts[feat_i], c_parts[feat_j]) max_ari = np.max(ari_matrix) - # Check GPU choice - gpu_ari = ari_matrix[gpu_parts[0], gpu_parts[1]] - # Check CPU choice - cpu_ari = ari_matrix[cpu_parts[0], cpu_parts[1]] - - # Both should be maximum values (within floating point tolerance) + gpu_ari = ari_matrix[gpu_choice[0], gpu_choice[1]] + cpu_ari = ari_matrix[cpu_choice[0], cpu_choice[1]] assert np.abs(gpu_ari - max_ari) < 1e-8, ( f"GPU choice at comparison {i} is not maximum: {gpu_ari} vs {max_ari}" ) @@ -347,42 +129,38 @@ def test_ccc_gpu_with_numerical_input( f"CPU choice at comparison {i} is not maximum: {cpu_ari} vs {max_ari}" ) - # Print info about the tie for debugging (optional) - ties = np.where(np.abs(ari_matrix - max_ari) < 1e-8) - num_ties = len(ties[0]) - if num_ties > 1: - print( - f"Tie detected at comparison {i} (features {feat_i} vs {feat_j}): {num_ties} positions with ARI={max_ari:.10f}" - ) - print(f" GPU chose: {tuple(gpu_parts)}, CPU chose: {tuple(cpu_parts)}") +@pytest.mark.parametrize("seed", [42]) +@pytest.mark.parametrize( + "shape, n_categories, str_length", + [ + ((10, 20), 10, 2), + ((20, 200), 50, 3), + ], +) +def test_ccc_gpu_with_categorical_input_return_parts( + seed: int, + shape: tuple[int, int], + n_categories: int, + str_length: int, + categorical_data_generator, +): + """Categorical ``return_parts`` on GPU matches the CPU reference. + + This path was previously commented out as known-broken; the fix-cuda-correctness + work made it pass, so it is a real parity test now (openspec restructure-tests + coverage gap 5.1). + """ + n_features, n_samples = shape + df = categorical_data_generator( + n_features, n_samples, n_categories, str_length=str_length, random_state=seed + ) + + c_gpu, g_max_parts, g_parts = ccc_gpu(df, return_parts=True) + c_cpu, c_max_parts, c_parts = ccc(df, n_jobs=2, return_parts=True) -# @pytest.mark.parametrize( -# "seed", [42] -# ) # More seeds for simple cases, only 42 for large cases -# @pytest.mark.parametrize( -# "shape, n_categories, str_length", -# [ -# # Simple cases -# ((10, 20), 10, 2), -# ((20, 200), 50, 3), -# # ((30, 300), 200, 4), # Failed for the number of categories -# # ((9, 10000), 500, 5), # Failed for the number of categories -# ], -# ) -# @pytest.mark.parametrize("n_cpu_cores", [48]) -# @clean_gpu_memory -# def test_ccc_gpu_with_categorical_input( -# seed: int, -# shape: Tuple[int, int], -# n_categories: int, -# str_length: int, -# n_cpu_cores: int, -# ): -# n_features, n_samples = shape -# df = generate_categorical_data( -# n_features, n_samples, n_categories, str_length=str_length, random_state=seed -# ) -# res_cpu = ccc(df, n_jobs=n_cpu_cores) -# res_gpu = ccc_gpu(df) -# assert np.allclose(res_cpu, res_gpu) + gpu_df = pd.DataFrame(np.asarray(c_gpu, dtype=np.float64)) + cpu_df = pd.DataFrame(np.asarray(c_cpu, dtype=np.float64)) + pd.testing.assert_frame_equal(gpu_df, cpu_df, atol=PARITY_ATOL, rtol=PARITY_RTOL) + assert g_parts.shape == c_parts.shape + assert g_max_parts.shape == c_max_parts.shape diff --git a/tests/gpu/test_coverage_gaps.py b/tests/gpu/test_coverage_gaps.py new file mode 100644 index 00000000..42354a15 --- /dev/null +++ b/tests/gpu/test_coverage_gaps.py @@ -0,0 +1,65 @@ +"""GPU coverage for known GPU/CPU divergence points (openspec restructure-tests §5.2). + +Closes the coverage gaps for: + * constant-feature matrices (NaN parity in the max reduction), + * too-few-objects error-path parity (both raise the same ValueError), + * mixed numerical + categorical DataFrames. + +Categorical ``return_parts`` parity (§5.1) lives in +``test_ccc_gpu_return_parts.py``. +""" + +import numpy as np +import pandas as pd +import pytest +from ccc.coef.impl import ccc as ccc_cpu +from ccc.coef.impl_gpu import ccc as ccc_gpu + +PARITY_ATOL = 1e-6 +PARITY_RTOL = 1e-6 + + +def test_constant_feature_matrix_nan_parity(): + """A constant (singleton) feature yields NaN for its comparisons on both + GPU and CPU, with identical NaN patterns and matching finite values.""" + np.random.seed(0) + data = np.random.rand(4, 100) + data[1, :] = 3.0 # constant feature -> -2 singleton marker + + g = np.asarray(ccc_gpu(data), dtype=np.float64) + c = np.asarray(ccc_cpu(data, n_jobs=2), dtype=np.float64) + + np.testing.assert_array_equal(np.isnan(g), np.isnan(c)) + finite = ~np.isnan(g) + np.testing.assert_allclose(g[finite], c[finite], atol=PARITY_ATOL, rtol=PARITY_RTOL) + + +def test_too_few_objects_error_parity(): + """Too-few-objects raises the same ValueError on GPU and CPU.""" + np.random.seed(123) + data = np.random.rand(10, 2) + + with pytest.raises(ValueError, match="too few objects"): + ccc_cpu(data, internal_n_clusters=3) + with pytest.raises(ValueError, match="too few objects"): + ccc_gpu(data, internal_n_clusters=3) + + +def test_mixed_numerical_and_categorical_dataframe_parity(): + """A DataFrame mixing numerical and categorical columns matches CPU.""" + np.random.seed(123) + numerical = np.random.rand(100) + median = np.percentile(numerical, 50) + categorical = np.full(numerical.shape[0], "", dtype=object) + categorical[numerical < median] = "l" + categorical[numerical >= median] = "u" + other = np.random.rand(100) + + df = pd.DataFrame({"num": numerical, "cat": categorical, "num2": other}) + + g = np.asarray(ccc_gpu(df), dtype=np.float64) + c = np.asarray(ccc_cpu(df, n_jobs=2), dtype=np.float64) + + np.testing.assert_array_equal(np.isnan(g), np.isnan(c)) + finite = ~np.isnan(g) + np.testing.assert_allclose(g[finite], c[finite], atol=PARITY_ATOL, rtol=PARITY_RTOL) diff --git a/tests/gpu/test_cuda_correctness.py b/tests/gpu/test_cuda_correctness.py index a003398d..9f65a235 100644 --- a/tests/gpu/test_cuda_correctness.py +++ b/tests/gpu/test_cuda_correctness.py @@ -26,14 +26,12 @@ import pytest from ccc.coef.impl import ccc as ccc_cpu from ccc.coef.impl_gpu import ccc as ccc_gpu -from utils import clean_gpu_memory # --------------------------------------------------------------------------- # 1. Input validation -> Python exception, no interpreter crash # --------------------------------------------------------------------------- -@clean_gpu_memory def test_compute_coef_shape_mismatch_raises_value_error(): # A valid int16 partitions array of shape (2, 1, 4)... parts = np.array([[[0, 0, 1, 1]], [[0, 0, 1, 2]]], dtype=np.int16) @@ -47,14 +45,12 @@ def test_compute_coef_shape_mismatch_raises_value_error(): ccc_cuda_ext.compute_coef(parts, 2, 1, 8) # n_objs=8 != shape[2]=4 -@clean_gpu_memory def test_compute_coef_zero_partitions_raises_value_error(): parts = np.zeros((2, 0, 4), dtype=np.int16) with pytest.raises(ValueError): ccc_cuda_ext.compute_coef(parts, 2, 0, 4) -@clean_gpu_memory def test_compute_coef_too_many_partitions_raises_value_error(): # max_parts stores partition indices as uint8, so n_partitions must be <= 255. parts = np.zeros((2, 256, 4), dtype=np.int16) @@ -62,7 +58,6 @@ def test_compute_coef_too_many_partitions_raises_value_error(): ccc_cuda_ext.compute_coef(parts, 2, 256, 4) -@clean_gpu_memory def test_invalid_input_does_not_crash_interpreter(): """A rejected call raises a Python exception and the process keeps working: a subsequent valid computation still succeeds.""" @@ -81,7 +76,6 @@ def test_invalid_input_does_not_crash_interpreter(): @pytest.mark.parametrize("k_list", [[20], [10, 20], [17, 24]]) -@clean_gpu_memory def test_pvalue_more_than_16_clusters_matches_cpu(k_list): """With > 16 clusters the GPU permutation path used to silently return ARI 0.0 for every permutation (MAX_CLUSTERS = 16), producing biased @@ -120,7 +114,6 @@ def test_pvalue_more_than_16_clusters_matches_cpu(k_list): # --------------------------------------------------------------------------- -@clean_gpu_memory def test_pvalue_categorical_feature_consistency_vs_cpu(): """A DataFrame mixing a numerical and categorical features exercises the categorical-marker (-1 -> ARI 0.0) semantics in the permutation null.""" @@ -161,7 +154,6 @@ def test_pvalue_categorical_feature_consistency_vs_cpu(): # --------------------------------------------------------------------------- -@clean_gpu_memory def test_pvalue_singleton_feature_nan_consistency_vs_cpu(): rs = np.random.RandomState(11) n = 100 diff --git a/tests/gpu/test_titanic_dataset.py b/tests/gpu/test_titanic_dataset.py index ccca7571..d3c01ba4 100644 --- a/tests/gpu/test_titanic_dataset.py +++ b/tests/gpu/test_titanic_dataset.py @@ -4,7 +4,9 @@ from ccc.coef.impl import ccc from ccc.coef.impl_gpu import ccc as ccc_gpu from scipy.spatial.distance import squareform -from utils import clean_gpu_memory + +# This test downloads the Titanic dataset over the network. +pytestmark = pytest.mark.network @pytest.fixture @@ -29,7 +31,6 @@ def print_correlation_matrix(correlations, title): print(" ".join(f"{x:8.4f}" for x in row)) -@clean_gpu_memory def test_ccc_gpu_with_titanic_dataset(titanic_data): """ Test the CCC (Categorical Correlation Coefficient) computation on the Titanic dataset. diff --git a/tests/gpu/utils.py b/tests/gpu/utils.py deleted file mode 100644 index c933bcfe..00000000 --- a/tests/gpu/utils.py +++ /dev/null @@ -1,98 +0,0 @@ -import functools - -import cupy as cp -import numpy as np -import pandas as pd - - -def clean_gpu_memory(func): - @functools.wraps(func) - def wrapper(*args, **kwargs): - try: - return func(*args, **kwargs) - finally: - mempool = cp.get_default_memory_pool() - mempool.free_all_blocks() - - return wrapper - - -def generate_categorical_data( - n_features, - n_objects, - n_categories=3, - categories=None, - str_length=None, - random_state=None, - feature_names=None, -): - """ - Generate random categorical data as a pandas DataFrame. - - Parameters: - ----------- - n_features : int - Number of features (columns) in the output DataFrame - n_objects : int - Number of objects (rows) in the output DataFrame - n_categories : int, optional (default=3) - Number of unique categories to generate if categories parameter is None - categories : list or None, optional (default=None) - List of categories to sample from. If None, will use range(n_categories) - or generate random strings if str_length is provided - str_length : int or None, optional (default=None) - If provided and categories is None, generate random string categories - of this length using uppercase letters - random_state : int or None, optional (default=None) - Seed for random number generation - feature_names : list or None, optional (default=None) - List of column names. If None, will use [f'feature_{i}' for i in range(n_features)] - - Returns: - -------- - pandas.DataFrame - DataFrame of shape (n_objects, n_features) containing random categorical data - - Examples: - -------- - >>> df = generate_categorical_data(2, 3, n_categories=3, str_length=2) - >>> print(df) - feature_0 feature_1 - 0 XY AB - 1 PQ XY - 2 AB PQ - """ - # Set random seed if provided - if random_state is not None: - np.random.seed(random_state) - - # Define categories if not provided - if categories is None: - if str_length is not None: - # Generate random string categories - letters = np.array(list("ABCDEFGHIJKLMNOPQRSTUVWXYZ")) - categories = [] - for _ in range(n_categories): - cat = "".join(np.random.choice(letters, size=str_length)) - categories.append(cat) - else: - categories = list(range(n_categories)) - else: - n_categories = len(categories) - - # Generate random indices - random_indices = np.random.randint(0, n_categories, size=(n_objects, n_features)) - - # Convert indices to categories - categorical_data = np.array( - [[categories[idx] for idx in row] for row in random_indices] - ) - - # Create feature names if not provided - if feature_names is None: - feature_names = [f"feature_{i}" for i in range(n_features)] - - # Create DataFrame - df = pd.DataFrame(categorical_data, columns=feature_names) - - return df diff --git a/tests/test_bench_cli.py b/tests/test_bench_cli.py new file mode 100644 index 00000000..24b08854 --- /dev/null +++ b/tests/test_bench_cli.py @@ -0,0 +1,104 @@ +"""CPU-only plumbing tests for the ``ccc-gpu-bench`` CLI. + +These exercise argument parsing and structured (JSONL/CSV) output only. They do +NOT assert on timings and never require a GPU (they use the CPU-only ``scaling`` +mode and ``coef --cpu-only``). Unmarked, so they run in the fast CI subset. +""" + +import json + +from ccc.bench import cli +from ccc.bench.env import capture_environment +from ccc.bench.presets import get_preset + + +def test_parser_builds_and_parses_coef(): + parser = cli.build_parser() + args = parser.parse_args( + ["coef", "--features", "10", "20", "--samples", "50", "--cpu-only"] + ) + assert args.mode == "coef" + assert args.features == [10, 20] + assert args.samples == [50] + assert args.cpu_only is True + assert args.format == "jsonl" + + +def test_capture_environment_has_expected_keys(): + env = capture_environment() + for key in ( + "package_version", + "python_version", + "platform", + "cpu_count", + "gpu_present", + "gpu_name", + ): + assert key in env + + +def test_presets_available(): + smoke = get_preset("coef", "smoke") + assert "features" in smoke and "samples" in smoke and "n_jobs" in smoke + + +def test_scaling_writes_valid_jsonl(tmp_path): + out = tmp_path / "scaling.jsonl" + rc = cli.main( + [ + "scaling", + "--features", + "10", + "--samples", + "50", + "--n-jobs", + "1", + "2", + "--repeats", + "1", + "--warmup", + "0", + "-o", + str(out), + ] + ) + assert rc == 0 + lines = out.read_text().strip().splitlines() + assert len(lines) == 2 # one record per n_jobs + records = [json.loads(line) for line in lines] + for rec in records: + assert rec["mode"] == "scaling" + assert rec["n_features"] == 10 + assert rec["n_samples"] == 50 + assert rec["cpu_time_min_s"] > 0 + assert "package_version" in rec # env metadata embedded + assert rec["seed"] == 42 + + +def test_coef_cpu_only_writes_valid_csv(tmp_path): + out = tmp_path / "coef.csv" + rc = cli.main( + [ + "coef", + "--features", + "10", + "--samples", + "50", + "--n-jobs", + "1", + "--cpu-only", + "--repeats", + "1", + "--warmup", + "0", + "--format", + "csv", + "-o", + str(out), + ] + ) + assert rc == 0 + text = out.read_text().strip().splitlines() + assert len(text) == 2 # header + 1 data row + assert "mode" in text[0] + assert "n_coefficients" in text[0] diff --git a/tests/test_coef.py b/tests/test_coef.py index ccf14617..98fa2056 100644 --- a/tests/test_coef.py +++ b/tests/test_coef.py @@ -1,5 +1,4 @@ import os -import time from concurrent.futures import ThreadPoolExecutor from random import shuffle from unittest.mock import patch @@ -22,8 +21,6 @@ from sklearn.metrics import adjusted_rand_score as ari from sklearn.preprocessing import minmax_scale -IN_GITHUB_ACTIONS = os.getenv("GITHUB_ACTIONS") == "true" - def test_get_perc_from_k_with_k_less_than_two(): assert get_perc_from_k(1) == [] @@ -497,11 +494,12 @@ def test_cm_x_and_y_are_pandas_dataframe(): assert "wrong combination" in str(e).lower() +@pytest.mark.slow def test_cm_integer_overflow_random(): # Prepare np.random.seed(0) - # two features on 100 objects with a linear relationship + # two features on 1,000,000 objects (large input exercises the 64-bit path) feature0 = np.random.rand(1000000) feature1 = np.random.rand(1000000) @@ -510,11 +508,12 @@ def test_cm_integer_overflow_random(): assert 0.0 <= cm_value <= 0.01 +@pytest.mark.slow def test_cm_integer_overflow_perfect_match(): # Prepare np.random.seed(0) - # two features on 100 objects with a linear relationship + # two features on 1,000,000 objects (large input exercises the 64-bit path) feature0 = np.random.rand(1000000) # Run @@ -1418,8 +1417,15 @@ def test_cm_numpy_array_input(): assert np.issubdtype(cm_value.dtype, float) +# The wall-clock speedup assertions that used to live in the n_jobs tests below +# were removed (flaky by nature); the timing measurement now lives in the +# `ccc-gpu-bench scaling` CLI mode. What remains here is a determinism check: +# parallel results must equal the single-thread result. Marked `slow` because +# they run large inputs through the parallel path. + + +@pytest.mark.slow @pytest.mark.skipif(os.cpu_count() < 2, reason="requires at least 2 cores") -@pytest.mark.skipif(IN_GITHUB_ACTIONS, reason="Test doesn't work in Github Actions.") def test_cm_numpy_array_input_with_n_jobs(): # Prepare np.random.seed(123) @@ -1427,18 +1433,11 @@ def test_cm_numpy_array_input_with_n_jobs(): # here I force data = np.random.rand(100, 1000) - # Run - start_time = time.time() + # Run: single-thread vs multi-thread must agree res0 = ccc(data, n_jobs=1) - elapsed_time_single_thread = time.time() - start_time - - start_time = time.time() res1 = ccc(data, n_jobs=2) - elapsed_time_multi_thread = time.time() - start_time # Validate - assert elapsed_time_multi_thread < 0.75 * elapsed_time_single_thread - assert res0 is not None assert isinstance(res0, np.ndarray) assert res0.shape == (int(data.shape[0] * (data.shape[0] - 1) / 2),) @@ -1446,8 +1445,8 @@ def test_cm_numpy_array_input_with_n_jobs(): np.testing.assert_array_equal(res0, res1) +@pytest.mark.slow @pytest.mark.skipif(os.cpu_count() < 2, reason="requires at least 2 cores") -@pytest.mark.skipif(IN_GITHUB_ACTIONS, reason="Test doesn't work in Github Actions.") def test_cm_two_features_input_with_n_jobs(): # Prepare np.random.seed(123) @@ -1456,25 +1455,18 @@ def test_cm_two_features_input_with_n_jobs(): x = np.random.rand(100000) y = np.random.rand(100000) - # Run - start_time = time.time() + # Run: single-thread vs multi-thread must agree res0 = ccc(x, y, n_jobs=1) - elapsed_time_single_thread = time.time() - start_time - - start_time = time.time() res1 = ccc(x, y, n_jobs=2) - elapsed_time_multi_thread = time.time() - start_time # Validate - assert elapsed_time_multi_thread < 0.75 * elapsed_time_single_thread - assert res0 is not None assert isinstance(res0, float) assert res0 == res1 +@pytest.mark.slow @pytest.mark.skipif(os.cpu_count() < 2, reason="requires at least 2 cores") -@pytest.mark.skipif(IN_GITHUB_ACTIONS, reason="Test doesn't work in Github Actions.") def test_cm_two_features_input_with_n_jobs_using_threads_for_partitioning(): # Prepare np.random.seed(123) @@ -1483,25 +1475,18 @@ def test_cm_two_features_input_with_n_jobs_using_threads_for_partitioning(): x = np.random.rand(100000) y = np.random.rand(100000) - # Run - start_time = time.time() + # Run: thread-based partitioning must agree with the single-thread result res0 = ccc(x, y, n_jobs=1) - elapsed_time_single_thread = time.time() - start_time - - start_time = time.time() res1 = ccc(x, y, n_jobs=2, partitioning_executor="thread") - elapsed_time_multi_thread = time.time() - start_time # Validate - assert elapsed_time_multi_thread < 0.75 * elapsed_time_single_thread - assert res0 is not None assert isinstance(res0, float) assert res0 == res1 +@pytest.mark.slow @pytest.mark.skipif(os.cpu_count() < 2, reason="requires at least 2 cores") -@pytest.mark.skipif(IN_GITHUB_ACTIONS, reason="Test doesn't work in Github Actions.") def test_cm_two_features_input_with_n_jobs_using_process_for_partitioning(): # Prepare np.random.seed(123) @@ -1510,20 +1495,11 @@ def test_cm_two_features_input_with_n_jobs_using_process_for_partitioning(): x = np.random.rand(1000000) y = np.random.rand(1000000) - # Run - start_time = time.time() + # Run: process-based partitioning must agree with the single-thread result res0 = ccc(x, y, n_jobs=1) - elapsed_time_single_thread = time.time() - start_time - - start_time = time.time() res1 = ccc(x, y, n_jobs=2, partitioning_executor="process") - elapsed_time_multi_thread = time.time() - start_time # Validate - # less stringent than with threads, because the overhead of using processes - # seems to be larger - assert elapsed_time_multi_thread < elapsed_time_single_thread - assert res0 is not None assert isinstance(res0, float) assert res0 == res1 diff --git a/tests/test_coef_pval.py b/tests/test_coef_pval.py index dabd6c00..2237ea5b 100644 --- a/tests/test_coef_pval.py +++ b/tests/test_coef_pval.py @@ -1,14 +1,9 @@ -import os -import time - import numpy as np import pandas as pd import pytest from ccc.coef import ccc from sklearn.preprocessing import minmax_scale -IN_GITHUB_ACTIONS = os.getenv("GITHUB_ACTIONS") == "true" - def test_cm_basic_pvalue_n_permutations_not_given(): # Prepare @@ -241,43 +236,10 @@ def test_cm_single_argument_is_matrix(): assert pvalue[2] > 0.10 -@pytest.mark.skipif(os.cpu_count() < 2, reason="requires at least 2 cores") -@pytest.mark.skipif(IN_GITHUB_ACTIONS, reason="Test doesn't work in Github Actions.") -def test_cm_large_n_objects_pvalue_permutations_is_parallelized(): - # Prepare - rs = np.random.RandomState(0) - n_runs = 5 - single_thread_times = [] - multi_thread_times = [] - - # two features on 100 objects with a linear relationship - feature0 = rs.rand(100000) - feature1 = rs.rand(100000) - - # Run multiple times to ensure consistency - for _ in range(n_runs): - # Single thread run - start_time = time.time() - _ = ccc(feature0, feature1, pvalue_n_perms=500, n_jobs=1) - single_thread_times.append(time.time() - start_time) - - # Multi thread run - start_time = time.time() - _ = ccc(feature0, feature1, pvalue_n_perms=500, n_jobs=2) - multi_thread_times.append(time.time() - start_time) - - # Print timing information for debugging - print(f"\nSingle thread times: {single_thread_times}") - print(f"Multi thread times: {multi_thread_times}") - print( - f"Average speedup: {np.mean(single_thread_times) / np.mean(multi_thread_times):.2f}x" - ) - - # Validate that parallel execution is consistently faster - for i in range(n_runs): - assert multi_thread_times[i] < single_thread_times[i], ( - f"Run {i + 1}: Multi-thread ({multi_thread_times[i]:.2f}s) not faster than single-thread ({single_thread_times[i]:.2f}s)" - ) +# NOTE: test_cm_large_n_objects_pvalue_permutations_is_parallelized was removed. +# It was a pure wall-clock benchmark (single- vs multi-thread p-value permutation +# timing on 100,000 objects) with no correctness assertion; its measurement intent +# now lives in the `ccc-gpu-bench scaling --pvalue-n-perms ...` CLI mode. def test_cm_return_parts_quadratic_pvalue(): diff --git a/tests/test_corr.py b/tests/test_corr.py index 4743e4d6..4380c88b 100644 --- a/tests/test_corr.py +++ b/tests/test_corr.py @@ -4,6 +4,12 @@ import numpy as np import pandas as pd +import pytest + +# ccc.corr is an analysis-only module (excluded from the published wheel and +# depends on `minepy`); skip cleanly when it or its deps are unavailable. +pytest.importorskip("ccc.corr") + from ccc import corr diff --git a/tests/test_giant.py b/tests/test_giant.py index 8ceadb60..bf33f3b9 100644 --- a/tests/test_giant.py +++ b/tests/test_giant.py @@ -9,8 +9,15 @@ "Skipping REST test on GIANT in non-Linux systems", allow_module_level=True ) +# ccc.giant is an analysis-only module (excluded from the published wheel and +# depends on `requests`); skip cleanly when it or its deps are unavailable. +pytest.importorskip("ccc.giant") + from ccc.giant import gene_exists, get_network, predict_tissue +# These tests hit the GIANT REST API over the network. +pytestmark = pytest.mark.network + # Gene mappings used in unit tests gene_mappings = pd.DataFrame( [ diff --git a/tests/test_methods.py b/tests/test_methods.py index b2c51451..3fba8c76 100644 --- a/tests/test_methods.py +++ b/tests/test_methods.py @@ -1,4 +1,10 @@ import numpy as np +import pytest + +# ccc.methods is an analysis-only module (excluded from the published wheel and +# depends on `minepy`); skip cleanly when it or its deps are unavailable. +pytest.importorskip("ccc.methods") + from ccc.methods import mic diff --git a/tests/test_plots.py b/tests/test_plots.py index dfa66b8e..23fc6803 100644 --- a/tests/test_plots.py +++ b/tests/test_plots.py @@ -10,6 +10,11 @@ import numpy as np import pandas as pd + +# ccc.plots is an analysis-only module (excluded from the published wheel and +# depends on matplotlib/seaborn/IPython); skip cleanly when unavailable. +pytest.importorskip("ccc.plots") + from ccc.plots import ( MyUpSet, jointplot,