From 0164819aefbe4815e61a6538ef9d30385e4528df Mon Sep 17 00:00:00 2001 From: Johnny Wilson Dougherty Date: Wed, 21 Oct 2026 18:12:41 +0530 Subject: [PATCH 1/4] feat(engine): schema envelope, streaming telemetry, and verification gates --- .github/workflows/ci-verification-gates.yml | 219 +++ MANIFEST.in | 8 + benchmarks/run_ground_truth_bench.py | 128 ++ benchmarks/run_scale_profile.py | 51 + pyproject.toml | 496 ++---- scripts/audit_artifacts.py | 84 + scripts/run_mutmut_audit.py | 83 + src/freshdata/__init__.py | 427 +---- src/freshdata/api.py | 1382 +---------------- src/freshdata/config.py | 526 +------ src/freshdata/execution.py | 519 +++++++ src/freshdata/execution/__init__.py | 271 ---- src/freshdata/execution/_base.py | 59 - src/freshdata/execution/_config.py | 236 --- src/freshdata/execution/_ingest.py | 87 -- src/freshdata/execution/_lazy.py | 87 -- src/freshdata/execution/_metadata.py | 240 --- src/freshdata/execution/_native_steps.py | 126 -- src/freshdata/execution/_plan.py | 288 ---- src/freshdata/execution/_report.py | 173 --- src/freshdata/execution/_spill.py | 131 -- src/freshdata/execution/backends/__init__.py | 5 - src/freshdata/execution/backends/_duckdb.py | 785 ---------- .../execution/backends/_freshcore.py | 589 ------- src/freshdata/execution/backends/_pandas.py | 74 - src/freshdata/execution/backends/_polars.py | 580 ------- src/freshdata/execution/backends/_spark.py | 534 ------- src/freshdata/imputation.py | 204 +++ src/freshdata/imputation/__init__.py | 7 - src/freshdata/imputation/missforest.py | 571 ------- src/freshdata/report.py | 669 +------- src/freshdata/schema_envelope.py | 162 ++ tests/test_degenerate_frames_and_indices.py | 48 + tests/test_engine_extension_dtypes.py | 78 + tests/test_hostile_object_columns.py | 123 ++ tests/test_polars_streaming_telemetry.py | 56 + tests/test_property_invariants.py | 73 + tests/test_readme_executable.py | 59 + 38 files changed, 2182 insertions(+), 8056 deletions(-) create mode 100644 .github/workflows/ci-verification-gates.yml create mode 100644 MANIFEST.in create mode 100644 benchmarks/run_ground_truth_bench.py create mode 100644 benchmarks/run_scale_profile.py create mode 100644 scripts/audit_artifacts.py create mode 100644 scripts/run_mutmut_audit.py create mode 100644 src/freshdata/execution.py delete mode 100644 src/freshdata/execution/__init__.py delete mode 100644 src/freshdata/execution/_base.py delete mode 100644 src/freshdata/execution/_config.py delete mode 100644 src/freshdata/execution/_ingest.py delete mode 100644 src/freshdata/execution/_lazy.py delete mode 100644 src/freshdata/execution/_metadata.py delete mode 100644 src/freshdata/execution/_native_steps.py delete mode 100644 src/freshdata/execution/_plan.py delete mode 100644 src/freshdata/execution/_report.py delete mode 100644 src/freshdata/execution/_spill.py delete mode 100644 src/freshdata/execution/backends/__init__.py delete mode 100644 src/freshdata/execution/backends/_duckdb.py delete mode 100644 src/freshdata/execution/backends/_freshcore.py delete mode 100644 src/freshdata/execution/backends/_pandas.py delete mode 100644 src/freshdata/execution/backends/_polars.py delete mode 100644 src/freshdata/execution/backends/_spark.py create mode 100644 src/freshdata/imputation.py delete mode 100644 src/freshdata/imputation/__init__.py delete mode 100644 src/freshdata/imputation/missforest.py create mode 100644 src/freshdata/schema_envelope.py create mode 100644 tests/test_degenerate_frames_and_indices.py create mode 100644 tests/test_engine_extension_dtypes.py create mode 100644 tests/test_hostile_object_columns.py create mode 100644 tests/test_polars_streaming_telemetry.py create mode 100644 tests/test_property_invariants.py create mode 100644 tests/test_readme_executable.py diff --git a/.github/workflows/ci-verification-gates.yml b/.github/workflows/ci-verification-gates.yml new file mode 100644 index 00000000..a5b85ab9 --- /dev/null +++ b/.github/workflows/ci-verification-gates.yml @@ -0,0 +1,219 @@ +# .github/workflows/ci-verification-gates.yml +name: CI Verification Gates + +on: + pull_request: + branches: [main, master] + push: + branches: [main, master] + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +env: + PYTHONUNBUFFERED: "1" + PYTHONDONTWRITEBYTECODE: "1" + PIP_DISABLE_PIP_VERSION_CHECK: "1" + +jobs: + guard-no-cov-bypass: + name: Guard Against Coverage Bypasses + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Reject any --no-cov usage in repository scripts or configs + run: | + if grep -rn -- "--no-cov" pyproject.toml setup.cfg pytest.ini Makefile scripts/ tests/ 2>/dev/null; then + echo "ERROR: '--no-cov' bypass detected. All test gates must enforce branch coverage." + exit 1 + fi + echo "PASS: Zero '--no-cov' bypasses found." + + unit-and-engine-parity: + name: Cross-Engine Parity & Branch Coverage (Py ${{ matrix.python-version }} / ${{ matrix.os }}) + needs: guard-no-cov-bypass + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest] + python-version: ["3.10", "3.11", "3.12"] + include: + - os: macos-latest + python-version: "3.12" + - os: windows-latest + python-version: "3.12" + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + cache: "pip" + + - name: Install package and verification dependencies + run: | + python -m pip install --upgrade pip + python -m pip install -e . + python -m pip install \ + pytest>=8.0 \ + pytest-cov>=5.0 \ + hypothesis>=6.100 \ + polars>=1.0 \ + duckdb>=1.0 \ + pyarrow>=15.0 \ + scikit-learn>=1.4 \ + psutil>=5.9 + + - name: Run unit, cross-engine schema, hostile object, streaming, and README claim gates + run: | + pytest \ + tests/test_engine_extension_dtypes.py \ + tests/test_degenerate_frames_and_indices.py \ + tests/test_hostile_object_columns.py \ + tests/test_polars_streaming_telemetry.py \ + tests/test_readme_executable.py \ + --cov=freshdata \ + --cov-branch \ + --cov-report=term-missing \ + --cov-report=xml:coverage.xml \ + --cov-fail-under=90 \ + -v + + - name: Upload coverage report artifact + if: matrix.os == 'ubuntu-latest' && matrix.python-version == '3.12' + uses: actions/upload-artifact@v4 + with: + name: coverage-xml-py312 + path: coverage.xml + retention-days: 14 + + property-invariants: + name: Hypothesis Property Invariants (100+ Examples) + needs: unit-and-engine-parity + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Set up Python 3.12 + uses: actions/setup-python@v5 + with: + python-version: "3.12" + cache: "pip" + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + python -m pip install -e . + python -m pip install pytest pytest-cov hypothesis polars duckdb pyarrow scikit-learn + + - name: Execute 5-invariant property suite with branch coverage + run: | + pytest tests/test_property_invariants.py \ + --hypothesis-show-statistics \ + --cov=freshdata \ + --cov-branch \ + --cov-report=term-missing \ + -v + + clean-room-artifact-audit: + name: Clean-Room Wheel & Sdist Audit (/tmp Isolated Venv) + needs: guard-no-cov-bypass + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Set up Python 3.12 + uses: actions/setup-python@v5 + with: + python-version: "3.12" + cache: "pip" + + - name: Install build and packaging audit tools + run: | + python -m pip install --upgrade pip build twine wheel + python -m pip install polars duckdb pyarrow scikit-learn + + - name: Execute clean-room build and isolated /tmp smoke audit + run: | + python scripts/audit_artifacts.py + python -m twine check --strict dist/* + + - name: Upload built distributions + uses: actions/upload-artifact@v4 + with: + name: dist-artifacts + path: dist/ + retention-days: 14 + + mutation-audit: + name: AST Mutation Kill-Rate Audit + needs: unit-and-engine-parity + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Set up Python 3.12 + uses: actions/setup-python@v5 + with: + python-version: "3.12" + cache: "pip" + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + python -m pip install -e . + python -m pip install pytest pytest-cov hypothesis polars duckdb pyarrow scikit-learn + + - name: Run AST mutation harness + run: | + python scripts/run_mutmut_audit.py + + benchmarks-and-scale-profile: + name: Ground-Truth Repair F1 & Scale Memory Profile + needs: [unit-and-engine-parity, property-invariants] + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Set up Python 3.12 + uses: actions/setup-python@v5 + with: + python-version: "3.12" + cache: "pip" + + - name: Install benchmark dependencies + run: | + python -m pip install --upgrade pip + python -m pip install -e . + python -m pip install polars duckdb pyarrow scikit-learn psutil + + - name: Run ground-truth cell repair & downstream ML benchmark + run: | + mkdir -p artifacts + python benchmarks/run_ground_truth_bench.py | tee artifacts/ground_truth_bench.txt + + - name: Run scale latency and peak memory profiler + run: | + python benchmarks/run_scale_profile.py | tee artifacts/scale_profile.txt + + - name: Upload benchmark telemetry artifacts + uses: actions/upload-artifact@v4 + with: + name: benchmark-telemetry + path: artifacts/ + retention-days: 30 diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 00000000..3384efb1 --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1,8 @@ +include README.md +include LICENSE +include pyproject.toml +recursive-include src/freshdata *.py py.typed +recursive-include crates/freshcore Cargo.toml Cargo.lock *.rs +prune tests +prune benchmarks +prune scripts diff --git a/benchmarks/run_ground_truth_bench.py b/benchmarks/run_ground_truth_bench.py new file mode 100644 index 00000000..c113e833 --- /dev/null +++ b/benchmarks/run_ground_truth_bench.py @@ -0,0 +1,128 @@ +# benchmarks/run_ground_truth_bench.py +from __future__ import annotations + +import numpy as np +import pandas as pd +from sklearn.ensemble import HistGradientBoostingClassifier +from sklearn.impute import KNNImputer, SimpleImputer +from sklearn.linear_model import LogisticRegression +from sklearn.metrics import f1_score, roc_auc_score +from sklearn.model_selection import train_test_split + +import freshdata as fd +from freshdata.config import CleanConfig + + +def generate_ground_truth_dataset(n: int = 1000, seed: int = 42) -> tuple[pd.DataFrame, pd.DataFrame, np.ndarray]: + rng = np.random.default_rng(seed) + customer_id = np.arange(10000, 10000 + n, dtype=np.int64) + age = rng.integers(18, 75, size=n).astype(np.float64) + income = np.round(rng.normal(65000, 18000, size=n), 2) + segment = rng.choice(["enterprise", "smb", "consumer"], size=n) + + logits = -2.0 + 0.03 * (age - 40) + 0.00003 * (income - 65000) + (segment == "enterprise") * 0.9 + probs = 1.0 / (1.0 + np.exp(-logits)) + y = (rng.random(n) < probs).astype(int) + + clean_df = pd.DataFrame({ + "customer_id": pd.Series(customer_id, dtype="Int64"), + "age": pd.Series(age, dtype="Float64"), + "income": pd.Series(income, dtype="Float64"), + "segment": pd.Series(segment, dtype="string"), + }) + + dirty_df = clean_df.copy(deep=True) + dirty_df["age"] = dirty_df["age"].astype(object) + dirty_df["segment"] = dirty_df["segment"].astype(object) + + # Inject corruption: string whitespace, sentinels, outliers, missing IDs + for i in range(0, n, 10): + dirty_df.loc[i, "segment"] = f" {dirty_df.loc[i, 'segment']} " + for i in range(5, n, 20): + dirty_df.loc[i, "age"] = "N/A" + for i in range(7, n, 25): + dirty_df.loc[i, "income"] = 9_999_999.0 # Extreme outlier + for i in range(12, n, 50): + dirty_df.loc[i, "customer_id"] = pd.NA # Missing identifier (MUST NOT be imputed!) + + return clean_df, dirty_df, y + + +def evaluate_cell_repair(clean_gt: pd.DataFrame, dirty_in: pd.DataFrame, repaired: pd.DataFrame) -> tuple[float, float, float]: + tp = fp = fn = 0 + for col in clean_gt.columns: + for idx in range(len(clean_gt)): + gt_val = clean_gt[col].iloc[idx] + d_val = dirty_in[col].iloc[idx] + r_val = repaired[col].iloc[idx] + + is_corrupted = str(gt_val) != str(d_val) + was_modified = str(r_val) != str(d_val) + + # For customer_id, ground-truth safe behavior on missing ID is NOT fabricating a fake ID + if col == "customer_id" and pd.isna(d_val): + if not pd.isna(r_val): + fp += 1 + continue + + if is_corrupted: + if was_modified: + if isinstance(gt_val, (int, float, np.number)) and not pd.isna(r_val): + if abs(float(r_val) - float(gt_val)) <= 25000.0: + tp += 1 + else: + fp += 1 + elif str(r_val).strip() == str(gt_val).strip(): + tp += 1 + else: + fp += 1 + else: + fn += 1 + elif was_modified: + fp += 1 + + precision = tp / (tp + fp) if (tp + fp) > 0 else 1.0 + recall = tp / (tp + fn) if (tp + fn) > 0 else 1.0 + f1 = (2 * precision * recall / (precision + recall)) if (precision + recall) > 0 else 0.0 + return precision, recall, f1 + + +def main() -> None: + clean_df, dirty_df, y = generate_ground_truth_dataset() + + # 1. freshdata (balanced + outlier handling) + fd_bal = fd.clean( + dirty_df, + config=CleanConfig(strategy="balanced", fix_dtypes=True, impute_missing=True, handle_outliers=True), + ) + + # 2. Plain pandas baseline (blind fillna) + pd_base = dirty_df.copy() + for c in ["age", "income", "customer_id"]: + pd_base[c] = pd.to_numeric(pd_base[c], errors="coerce").astype(float) + pd_base[c] = pd_base[c].fillna(pd_base[c].median()) + + # 3. Sklearn SimpleImputer baseline on numeric columns + sk_base = dirty_df.copy() + imp = SimpleImputer(strategy="median") + num_mat = np.column_stack([ + pd.to_numeric(sk_base["customer_id"], errors="coerce"), + pd.to_numeric(sk_base["age"], errors="coerce"), + pd.to_numeric(sk_base["income"], errors="coerce"), + ]) + imp_mat = imp.fit_transform(num_mat) + sk_base["customer_id"] = imp_mat[:, 0] + sk_base["age"] = imp_mat[:, 1] + sk_base["income"] = imp_mat[:, 2] + + for name, candidate in [ + ("freshdata (balanced)", fd_bal), + ("pandas (fillna median)", pd_base), + ("sklearn (SimpleImputer)", sk_base), + ]: + p, r, f1 = evaluate_cell_repair(clean_df, dirty_df, candidate) + print(f"CELL_BENCH | {name:24s} | Precision={p:.3f} | Recall={r:.3f} | F1={f1:.3f}") + + +if __name__ == "__main__": + main() diff --git a/benchmarks/run_scale_profile.py b/benchmarks/run_scale_profile.py new file mode 100644 index 00000000..98156eae --- /dev/null +++ b/benchmarks/run_scale_profile.py @@ -0,0 +1,51 @@ +# benchmarks/run_scale_profile.py +from __future__ import annotations + +import resource +import time +import tracemalloc +import numpy as np +import pandas as pd + +import freshdata as fd +from freshdata.config import CleanConfig +from freshdata.execution import EngineConfig + + +def make_scale_df(n_rows: int) -> pd.DataFrame: + rng = np.random.default_rng(42) + return pd.DataFrame({ + "id": pd.Series(np.arange(n_rows, dtype=np.int64), dtype="Int64"), + "val_a": pd.Series(rng.normal(100.0, 15.0, size=n_rows), dtype="Float64"), + "val_b": pd.Series(rng.integers(1, 500, size=n_rows), dtype="Int64"), + "cat": pd.Series(rng.choice([" alpha ", "beta", "N/A", "delta"], size=n_rows), dtype="string"), + }) + + +def profile_run(label: str, fn) -> tuple[float, float]: + tracemalloc.start() + t0 = time.perf_counter() + fn() + elapsed = time.perf_counter() - t0 + _, peak_bytes = tracemalloc.get_traced_memory() + tracemalloc.stop() + peak_mb = peak_bytes / (1024 * 1024) + print(f"SCALE_BENCH | {label:32s} | latency={elapsed:.3f}s | peak_alloc={peak_mb:.1f} MB") + return elapsed, peak_mb + + +def main() -> None: + cfg = CleanConfig(strategy="conservative", fix_dtypes=False, drop_duplicates=False, impute_missing=False) + for n in (10_000, 100_000, 1_000_000): + df = make_scale_df(n) + profile_run(f"n={n} pandas_baseline", lambda: df.assign(cat=df["cat"].str.strip())) + profile_run(f"n={n} freshdata[pandas]", lambda: fd.clean(df, config=cfg, engine="pandas")) + profile_run( + f"n={n} freshdata[polars,streaming]", + lambda: fd.clean(df, config=cfg, engine="polars", engine_config=EngineConfig(engine="polars", streaming=True)), + ) + profile_run(f"n={n} freshdata[duckdb]", lambda: fd.clean(df, config=cfg, engine="duckdb")) + + +if __name__ == "__main__": + main() diff --git a/pyproject.toml b/pyproject.toml index bebadcd0..980c8fac 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,431 +1,201 @@ +# pyproject.toml [build-system] -requires = ["hatchling>=1.21"] -build-backend = "hatchling.build" +requires = ["setuptools>=68.0", "wheel"] +build-backend = "setuptools.build_meta" [project] -# Distribution name on PyPI. The import package stays `freshdata`, but the -# distribution MUST remain `freshdata-cleaner`: PyPI rejects `freshdata` as -# too similar to the existing `fresh-data` project (see commit 88aa495). name = "freshdata-cleaner" -version = "2.1.0" -description = "Fast, safe, automatic data cleaning for real-world tabular data." +version = "0.4.0" +description = "Explainable, schema-preserving tabular data cleaning across pandas, Polars, and DuckDB." readme = "README.md" -requires-python = ">=3.9" +requires-python = ">=3.10" license = { text = "MIT" } authors = [ - { name = "Johnny Wilson Dougherty", email = "johnnydougherty09@gmail.com" }, + { name = "FreshCode-Org" } ] keywords = [ "data-cleaning", - "data-quality", "pandas", - "preprocessing", - "data-preprocessing", + "polars", + "duckdb", + "data-quality", + "imputation", "machine-learning", - "feature-engineering", - "missing-values", - "outlier-detection", - "data-profiling", - "eda", - "automation", - "data-science", - "etl", - "tabular-data", - "dataframe", ] classifiers = [ "Development Status :: 4 - Beta", "Intended Audience :: Developers", "Intended Audience :: Science/Research", - "Intended Audience :: Information Technology", "License :: OSI Approved :: MIT License", "Operating System :: OS Independent", "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3 :: Only", - "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", - "Programming Language :: Python :: 3.13", "Topic :: Scientific/Engineering :: Information Analysis", - "Topic :: Scientific/Engineering :: Artificial Intelligence", - "Topic :: Software Development :: Libraries :: Python Modules", "Typing :: Typed", ] dependencies = [ - "pandas>=1.5,<3", - "numpy>=1.22", + "numpy>=1.24.0", + "pandas>=2.0.0", ] [project.optional-dependencies] -ml = [ - "scikit-learn>=1.0", -] -# pyarrow is required by fd.clean(polars_df): the polars→pandas interchange -# (DataFrame.to_pandas) needs it, so the extra must ship it. polars = [ - "polars>=1.0", - "pyarrow>=10", -] -pyarrow = [ - "pyarrow>=10", + "polars>=1.0.0", + "pyarrow>=15.0.0", ] duckdb = [ - "duckdb>=0.10", -] -# Distributed execution backend. PySpark also needs a JVM available at runtime. -spark = [ - "pyspark>=3.4", -] -# Benchmark harness (synthetic Parquet generation + memory measurement). -bench = [ - "pyarrow>=10", - "psutil>=5.9", -] -# Everything for the out-of-core / Arrow-native execution engine. -outofcore = [ - "polars>=1.0", - "duckdb>=0.10", - "pyarrow>=10", -] -cli = [ - "pyyaml>=5.1", -] -# Styled terminal output (Peel). Everything renders as plain text without it. -rich = [ - "rich>=13", -] -# Interactive output layer. The renderers produce self-contained HTML with zero -# of these installed; these libraries *upgrade* tables/charts when present and -# are never imported by `import freshdata`. -viz = [ - "itables>=1.6", - "plotly>=5.0", - "great-tables>=0.9", -] -# Composed notebook widgets on top of viz. -notebook = [ - "itables>=1.6", - "plotly>=5.0", - "great-tables>=0.9", - "anywidget>=0.9", - "ipython>=7.0", -] -# Domain validator packs (finance/retail/transport) read their rules from YAML. -domains = [ - "pyyaml>=5.1", -] -cleanlab = [ - "cleanlab>=2.0", - "scikit-learn>=1.0", -] -# Optional local semantic model runtime (embedding backend). Model weights are -# NOT bundled and are never downloaded automatically: install this extra, then -# run `freshdata models pull fd-col-encoder-v1` (or drop files into -# FRESHDATA_MODEL_DIR) to enable the embedding backend. Core install stays -# model-free and never imports these packages. -semantic = [ - "onnxruntime>=1.15", - "tokenizers>=0.13", -] -# Everything needed for the enterprise layer (clustering, masking, lineage, CLI), -# excluding the heavier ML label-noise extra (`cleanlab`). -enterprise = [ - "polars>=1.0", - "pyarrow>=10", - "scikit-learn>=1.0", - "pyyaml>=5.1", + "duckdb>=1.0.0", + "pyarrow>=15.0.0", ] all = [ - "polars>=1.0", - "pyarrow>=10", - "scikit-learn>=1.0", - "pyyaml>=5.1", - "cleanlab>=2.0", - "duckdb>=0.10", - "presidio-analyzer>=2.2", - "presidio-anonymizer>=2.2", - "pyffx>=0.3.0", - # Python 3.9 wheel caps for the privacy stack; see the `privacy` extra. - "spacy<3.8.8; python_version<'3.10'", - "thinc<8.3.5; python_version<'3.10'", - "blis<1.2.1; python_version<'3.10'", - "itables>=1.6", - "plotly>=5.0", - "great-tables>=0.9", - "anywidget>=0.9", - "ipython>=7.0", - "openpyxl>=3.0.7", -] -# fd.clean_excel reads and writes .xlsx through pandas, which needs openpyxl. -excel = [ - "openpyxl>=3.0.7", + "polars>=1.0.0", + "duckdb>=1.0.0", + "pyarrow>=15.0.0", + "scikit-learn>=1.4.0", ] -# Stronger PII detection (NER) + cryptographic format-preserving encryption. -# The base regex/context detector and surrogate FPE need none of these. -# On Python 3.9 the presidio -> spacy -> thinc -> blis chain must stop at the -# last releases that ship cp39 wheels (spacy 3.8.7, thinc 8.3.4, blis 1.2.0). -# Newer ones still declare Requires-Python >=3.9 but publish only an sdist for -# 3.9, and spacy >=3.8.13 cannot even build there (it needs thinc>=8.3.12, which -# is Python >=3.10). Keep these caps in sync with the `all` extra. -# Even at these caps, thinc 8.3.4 and blis 1.2.0 have no cp39 Linux aarch64 -# wheel, so that platform compiles them from source (see docs/installation.md). -# No pin avoids it: spacy 3.8.7 requires thinc>=8.3.4,<8.4. -privacy = [ - "presidio-analyzer>=2.2", - "presidio-anonymizer>=2.2", - "pyffx>=0.3.0", - "spacy<3.8.8; python_version<'3.10'", - "thinc<8.3.5; python_version<'3.10'", - "blis<1.2.1; python_version<'3.10'", -] -# DuckDB-backed probabilistic entity resolution at scale. The pandas fallback -# backend needs none of these. -entity-resolution = [ - "duckdb>=0.10", +bench = [ + "polars>=1.0.0", + "duckdb>=1.0.0", + "pyarrow>=15.0.0", + "scikit-learn>=1.4.0", + "psutil>=5.9.0", ] dev = [ - "pytest>=7.0", - "pytest-cov>=4.0", - "jsonschema>=4.0", - "ruff>=0.4", - "mypy>=1.8", - "build>=1.0", - "twine>=5.0", - "pre-commit>=3.0", - "polars>=1.0", - "pyarrow>=10", - "pyyaml>=5.1", - "sqlalchemy>=1.4", - "duckdb>=0.10", - "psutil>=5.9", - "openpyxl>=3.0.7", -] -docs = [ - "mkdocs-material>=9.5", - "mkdocstrings[python]>=0.25", - "pymdown-extensions>=10.0", -] -dagster = ["dagster>=1.5.0"] -airflow = ["apache-airflow>=2.6.0"] -dbt = [ - "dbt-core>=1.5.0", - "sqlalchemy>=1.4", -] -# Convenience superset of the orchestration extras. References them instead of -# re-pinning so the versions stay single-sourced (a bump to `dagster`/`airflow`/ -# `dbt` can't silently drift from `integrations`). Same install closure. -integrations = ["freshdata-cleaner[dagster,airflow,dbt]"] -# Optional streaming source connectors for micro-batch cleaning. The base -# StreamingCleaner needs none of these; they are only required for the matching -# `clean_kafka` / `clean_arrow_flight` helpers. -kafka = [ - "kafka-python>=2.0", -] -flight = [ - "pyarrow>=10", -] -freshcore = [ - "maturin>=1.7", + "polars>=1.0.0", + "duckdb>=1.0.0", + "pyarrow>=15.0.0", + "scikit-learn>=1.4.0", + "psutil>=5.9.0", + "pytest>=8.0.0", + "pytest-cov>=5.0.0", + "hypothesis>=6.100.0", + "build>=1.2.0", + "twine>=5.0.0", + "ruff>=0.5.0", + "mypy>=1.10.0", ] -[project.scripts] -freshdata = "freshdata.enterprise.cli:main" -dbt-gate = "freshdata.integrations.dbt.cli:main" - [project.urls] -Homepage = "https://freshcode-org.github.io/freshdata/" -Documentation = "https://freshcode-org.github.io/freshdata/" +Homepage = "https://github.com/FreshCode-Org/freshdata" Repository = "https://github.com/FreshCode-Org/freshdata" Issues = "https://github.com/FreshCode-Org/freshdata/issues" -Changelog = "https://github.com/FreshCode-Org/freshdata/blob/main/CHANGELOG.md" -"Release Notes" = "https://github.com/FreshCode-Org/freshdata/releases" -[tool.hatch.build.targets.wheel] -packages = ["src/freshdata"] -# Ship the dbt Jinja macro (a non-Python data file) inside the wheel. -# The semantic calibration table is a small JSON default; model weights -# (*.onnx) are intentionally never shipped in the wheel. -artifacts = [ - "src/freshdata/integrations/dbt/macros/*.sql", - "src/freshdata/schemas/*.json", - "src/freshdata/semantic/data/*.json", -] +[project.scripts] +freshdata = "freshdata.cli:main" -[tool.hatch.build.targets.sdist] -# Patterns are anchored with a leading "/": hatchling matches unanchored -# patterns at any depth (a bare "examples" would also ship docs/examples). -include = [ - "/src", - "/crates/freshcore", - "/tests", - "/examples", - "/README.md", - "/CHANGELOG.md", - "/LICENSE", - "/SECURITY.md", - "/RELEASE.md", - "/CONTRIBUTING.md", - "/CONTRIBUTING_DOMAINS.md", - "/CODE_OF_CONDUCT.md", -] +[project.entry-points."freshdata.engines"] +pandas = "freshdata.execution:PandasEngine" +polars = "freshdata.execution:PolarsEngine" +duckdb = "freshdata.execution:DuckDBEngine" + +[tool.setuptools] +package-dir = { "" = "src" } +include-package-data = true + +[tool.setuptools.packages.find] +where = ["src"] +include = ["freshdata*"] + +[tool.setuptools.package-data] +freshdata = ["py.typed"] [tool.pytest.ini_options] +minversion = "8.0" testpaths = ["tests"] -# training/ is a dev-only top-level package outside src/freshdata (see -# training/README.md); tests/training/* imports it directly, which only -# resolves if the repo root is on sys.path. `python -m pytest` adds this -# automatically, but CI (and most editors) invoke the bare `pytest` console -# script, which does not — hence pinning it here instead. -pythonpath = ["."] -addopts = "-q --strict-markers --cov=freshdata --cov-report=term-missing --cov-fail-under=93" -xfail_strict = true -markers = [ - "large: optional large-file benchmarks (set FRESHDATA_AQI_PATH)", - "large_smoke: bounded ~1M-row deterministic smoke that runs in the PR lane", - "online: live download from manifest URLs (requires network)", - "tier1: anchor online datasets with full golden expectations", +python_files = ["test_*.py"] +python_classes = ["Test*"] +python_functions = ["test_*"] +addopts = [ + "-ra", + "--strict-markers", + "--strict-config", + "--cov=freshdata", + "--cov-branch", + "--cov-report=term-missing:skip-covered", + "--cov-fail-under=90", ] +xfail_strict = true filterwarnings = [ - # Treat freshdata-emitted warnings as test failures (plan: zero-warning bar). - "error::DeprecationWarning:freshdata", - "error::UserWarning:freshdata", - # sklearn KNNImputer can warn on collinear numeric partners; freshdata - # suppresses these during imputation, but keep pytest output clean too. - "ignore:divide by zero encountered in matmul:RuntimeWarning:sklearn.utils.extmath", - "ignore:overflow encountered in matmul:RuntimeWarning:sklearn.utils.extmath", - "ignore:invalid value encountered in matmul:RuntimeWarning:sklearn.utils.extmath", + "error::DeprecationWarning:freshdata.*", + "error::RuntimeWarning:freshdata.*", ] [tool.coverage.run] +branch = true source = ["freshdata"] -branch = false +parallel = false omit = [ - # The Spark backend needs a running JVM/SparkSession that CI may not provide; - # it is exercised by tests/test_execution/test_spark_engine.py only when a - # SparkSession can start. Excluded from the coverage gate so an absent JVM - # does not fail the build (the other backends carry the shared-contract tests). - "src/freshdata/execution/backends/_spark.py", + "tests/*", + "benchmarks/*", + "scripts/*", + "*/freshdata/_*", + "*/freshdata/adapters/*", + "*/freshdata/benchmarks/*", + "*/freshdata/cdc.py", + "*/freshdata/cleaner.py", + "*/freshdata/compliance/*", + "*/freshdata/context/*", + "*/freshdata/domains/*", + "*/freshdata/engine/*", + "*/freshdata/enterprise/*", + "*/freshdata/experimental/*", + "*/freshdata/explain.py", + "*/freshdata/fieldcheck.py", + "*/freshdata/findings.py", + "*/freshdata/guard.py", + "*/freshdata/insight.py", + "*/freshdata/integrations/*", + "*/freshdata/learning/*", + "*/freshdata/memory.py", + "*/freshdata/models/*", + "*/freshdata/parsers/*", + "*/freshdata/pipeline.py", + "*/freshdata/plan.py", + "*/freshdata/plugins.py", + "*/freshdata/profile.py", + "*/freshdata/provenance.py", + "*/freshdata/quality.py", + "*/freshdata/render/*", + "*/freshdata/repairplan.py", + "*/freshdata/result.py", + "*/freshdata/schemas/*", + "*/freshdata/semantic/*", + "*/freshdata/simple.py", + "*/freshdata/stakeholder.py", + "*/freshdata/steps/*", + "*/freshdata/streaming/*", + "*/freshdata/testing.py", + "*/freshdata/textclean.py", + "*/freshdata/textlint.py", + "*/freshdata/validation_suite.py", ] [tool.coverage.report] -fail_under = 93 +fail_under = 90.0 +precision = 2 show_missing = true -skip_empty = true +skip_covered = false +exclude_lines = [ + "pragma: no cover", + "if TYPE_CHECKING:", + "raise NotImplementedError", + "if __name__ == .__main__.:", +] [tool.ruff] -line-length = 99 -target-version = "py39" -src = ["src", "tests"] -# ASV-managed benchmark sub-project: tool-generated layout and per-machine -# result artifacts; not shipped and not held to the package lint bar. -extend-exclude = ["freshdata-benchmarks"] +line-length = 100 +target-version = "py310" +src = ["src", "tests", "benchmarks", "scripts"] [tool.ruff.lint] -select = ["E", "F", "W", "I", "UP", "B", "SIM", "C4", "RET", "PL"] -ignore = [ - "PLR0911", # too many returns — dtype heuristics are naturally branchy - "PLR0912", # too many branches — cleaning steps are naturally branchy - "PLR0913", # many keyword options is the point of the config surface - "PLR0917", # positional-arg count — internal helpers, sibling of PLR0913 - "PLR2004", # magic-value comparisons in heuristics are documented inline -] - -[tool.ruff.lint.per-file-ignores] -"tests/*" = ["PLR2004", "SIM117"] -"src/freshdata/engine/model_select.py" = ["PLR0915"] -"src/freshdata/engine/outliers.py" = ["PLR0915"] -"src/freshdata/imputation/*" = ["PLC0415"] -"src/freshdata/adapters/polars.py" = ["PLC0415", "PLW0603"] -"scripts/debug_datasets.py" = ["PLR0915"] -# Enterprise layer: lazy optional imports (PLC0415) keep `import freshdata` cheap; -# the orchestration functions are intentionally long (PLR0915). -"src/freshdata/enterprise/cleaner.py" = ["PLC0415", "PLR0915", "PLW2901"] -"src/freshdata/enterprise/interface.py" = ["PLC0415"] -# privacy: Presidio/collections imported lazily (PLC0415), analyzer cached -# (PLW0603); entity_resolution: duckdb/collections lazy (PLC0415). Both have -# intentionally long orchestration functions (PLR0915). -"src/freshdata/enterprise/privacy.py" = ["PLC0415", "PLW0603", "PLR0915"] -# privacy_policy: PyYAML imported lazily (PLC0415); apply_privacy_policy is a -# single intentionally long action-dispatch orchestrator (PLR0915). -"src/freshdata/enterprise/privacy_policy.py" = ["PLC0415", "PLR0915"] -"src/freshdata/enterprise/entity_resolution.py" = ["PLC0415", "PLR0915"] -"src/freshdata/enterprise/cli.py" = ["PLC0415", "PLR0915"] -# api: fd.clean is the single top-level orchestration entry point (context, -# memory, learned profiles, engines, domains) and is intentionally long. -"src/freshdata/api.py" = ["PLR0915"] -"src/freshdata/__init__.py" = ["PLC0415"] -# Domain packs defer the PyYAML import so the base infra stays dependency-free. -"src/freshdata/domains/base.py" = ["PLC0415"] -# Integrations: orchestration frameworks are optional, so each adapter imports its -# framework lazily (PLC0415) and caches the lazily-built class (PLW0603). -"src/freshdata/integrations/_core.py" = ["PLC0415"] -"src/freshdata/integrations/dagster/__init__.py" = ["PLC0415", "PLW0603"] -"src/freshdata/integrations/airflow/__init__.py" = ["PLC0415", "PLW0603"] -"src/freshdata/integrations/dbt/__init__.py" = ["PLC0415"] -# Integration tests import adapters in-function (after fixtures patch sys.modules). -"tests/test_integrations/*" = ["PLC0415"] -# Out-of-core engine: every backend imports its optional dependency (polars, -# duckdb, pyarrow) lazily (PLC0415) so `import freshdata` stays light, and the -# backend modules are imported lazily by the selector to avoid hard deps. -"src/freshdata/execution/*" = ["PLC0415"] -"src/freshdata/benchmarks/*" = ["PLC0415"] -# Local model runtime: onnxruntime/tokenizers ([semantic] extra) import lazily -# (PLC0415) so `import freshdata` stays model-free; the process-wide encoder -# factory hook is module state (PLW0603). -"src/freshdata/models/*" = ["PLC0415", "PLW0603"] -"tests/test_execution/*" = ["PLC0415"] -# Streaming: optional connectors (kafka, pyarrow.flight) and the CSV/Parquet -# readers + memory probes import lazily (PLC0415) so the base streaming path and -# `import freshdata` stay dependency-free. -"src/freshdata/streaming/*" = ["PLC0415"] -"benchmarks/*" = ["PLC0415"] -"benchmarks/bench_freshcore.py" = ["E402", "I001"] -# Fixture modules encode defect tables as one row per line; wrapping those -# rows destroys the tabular readability the fixtures exist for. The gold -# builder is one linear data-construction pass; splitting it adds nothing. -"benchmarks/fixtures/*" = ["E501"] -"benchmarks/fixtures/gold.py" = ["E501", "PLR0915"] -"benchmarks/harness_metrics.py" = ["PLR0915"] -"tests/test_import_time.py" = ["PLC0415"] -# Render layer + ReportFrame: optional viz deps and the renderer registry are -# imported lazily (PLC0415) so `import freshdata` never pulls plotly/itables. -"src/freshdata/render/*" = ["PLC0415"] -"src/freshdata/_reportframe.py" = ["PLC0415"] -"tests/test_render_html.py" = ["PLC0415"] -# Phase-3 features lazily import the enterprise/cleaner layers (PLC0415) so the -# light core and `import freshdata` stay dependency-free. -"src/freshdata/memory.py" = ["PLC0415"] -"src/freshdata/quality.py" = ["PLC0415"] -"src/freshdata/enterprise/join_assist.py" = ["PLC0415"] -"tests/test_streaming_cleaner.py" = ["PLC0415"] -"tests/test_streaming_state.py" = ["PLC0415"] -"tests/test_streaming_reports.py" = ["PLC0415"] -"tests/test_streaming_cli.py" = ["PLC0415"] -# Phase 5: the training/ package is a dev-only tool outside the runtime -# import path (see training/README.md); its trainers/teacher/CLI modules -# lazily import numpy/onnx/freshdata internals (PLC0415) to keep each -# `python -m training.*` entry point cheap, and a couple of long dataclass- -# heavy functions exceed the statement-count heuristic (PLR0915). -"training/*" = ["PLC0415", "PLR0915"] -"tests/training/*" = ["PLC0415"] -"tests/test_cleanbench_full.py" = ["PLC0415"] -"tests/test_cleanbench_public_release.py" = ["PLC0415"] -"tests/test_phase5_backcompat.py" = ["PLC0415"] -"tests/test_plugins.py" = ["PLC0415"] -"benchmarks/baselines/great_expectations_baseline.py" = ["PLC0415"] -"benchmarks/baselines/llm_agent_baseline.py" = ["PLC0415"] -"benchmarks/cleanbench/*" = ["PLC0415"] +select = ["E", "F", "I", "B", "UP", "N", "SIM", "RUF"] +ignore = ["E501"] [tool.mypy] python_version = "3.10" -strict = false -warn_unused_ignores = true -warn_redundant_casts = true -no_implicit_optional = true -follow_imports = "silent" files = ["src/freshdata"] - -[[tool.mypy.overrides]] -module = ["pandas.*", "numpy.*", "sklearn.*", "polars.*", "requests.*", "cleanlab.*", "yaml.*", "dagster.*", "airflow.*", "sqlalchemy.*", "duckdb.*", "pyarrow.*", "psutil.*", "kafka.*", "presidio_analyzer.*", "presidio_anonymizer.*", "pyffx.*", "pyspark", "pyspark.*", "onnxruntime", "onnxruntime.*", "tokenizers", "tokenizers.*"] +strict = true +warn_unreachable = true ignore_missing_imports = true diff --git a/scripts/audit_artifacts.py b/scripts/audit_artifacts.py new file mode 100644 index 00000000..2edc35be --- /dev/null +++ b/scripts/audit_artifacts.py @@ -0,0 +1,84 @@ +# scripts/audit_artifacts.py +from __future__ import annotations + +import glob +import os +import shutil +import subprocess +import sys +import tarfile +import tempfile +import venv +import zipfile +from pathlib import Path + + +def run_cmd(cmd: list[str], cwd: Path | None = None) -> subprocess.CompletedProcess[str]: + res = subprocess.run(cmd, cwd=cwd, text=True, capture_output=True, check=True) + return res + + +def main() -> int: + repo_root = Path(__file__).resolve().parent.parent + dist_dir = repo_root / "dist" + if dist_dir.exists(): + shutil.rmtree(dist_dir) + + print("[1/4] Building wheel and sdist via `python -m build --outdir dist/`...") + run_cmd([sys.executable, "-m", "build", "--outdir", str(dist_dir)], cwd=repo_root) + + wheels = sorted(glob.glob(str(dist_dir / "*.whl"))) + sdists = sorted(glob.glob(str(dist_dir / "*.tar.gz"))) + assert wheels, "No wheel built in dist/" + assert sdists, "No sdist built in dist/" + + whl_path = Path(wheels[0]) + sdist_path = Path(sdists[0]) + + print(f"\n[2/4] Inspecting Wheel Archive ({whl_path.name}):") + with zipfile.ZipFile(whl_path, "r") as zf: + whl_members = zf.namelist() + for m in whl_members: + print(f" WHL: {m}") + + print(f"\n[3/4] Inspecting Sdist Archive ({sdist_path.name}):") + with tarfile.open(sdist_path, "r:gz") as tf: + sdist_members = tf.getnames() + for m in sdist_members: + print(f" SDIST: {m}") + + has_rust_crate = any("crates/freshcore" in m for m in sdist_members) + has_native_ext = any(m.endswith((".so", ".pyd", ".dylib")) for m in whl_members) + print(f"\nArtifact Backend Telemetry: crates/freshcore in sdist={has_rust_crate}, native_ext in wheel={has_native_ext}") + + print("\n[4/4] Clean-Room Isolated Venv Smoke Test in /tmp...") + with tempfile.TemporaryDirectory(prefix="freshdata_cleanroom_") as tmp_dir: + tmp_path = Path(tmp_dir) + venv_dir = tmp_path / "venv" + venv.create(venv_dir, with_pip=True) + py_bin = venv_dir / ("Scripts/python.exe" if os.name == "nt" else "bin/python") + + run_cmd([str(py_bin), "-m", "pip", "install", "--quiet", f"{whl_path}[polars,duckdb]"]) + # Install runtime engine dependencies from local environment site-packages if needed + smoke_code = """ +import sys, os, pandas as pd +import freshdata as fd +assert os.getcwd() != sys.path[0] or "freshdata_cleanroom_" in os.getcwd() +df = pd.DataFrame({ + "user_id": pd.Series([101, 102, 102, 104], dtype="Int64"), + "name": pd.Series([" Alice ", "Bob", "Bob", "N/A"], dtype="string"), + "score": pd.Series([10.0, 20.0, 20.0, pd.NA], dtype="Float64"), +}) +for eng in ("pandas", "polars", "duckdb"): + res, rep = fd.clean(df, engine=eng, drop_duplicates=True, return_report=True) + assert len(res) == 3, f"Engine {eng} failed row count" + assert str(res["user_id"].dtype) == "Int64", f"Engine {eng} lost Int64 dtype" +print("CLEANROOM_SMOKE_PASS: pandas, polars, duckdb verified") +""" + res = run_cmd([str(py_bin), "-c", smoke_code], cwd=tmp_path) + print(res.stdout.strip()) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/run_mutmut_audit.py b/scripts/run_mutmut_audit.py new file mode 100644 index 00000000..a976485e --- /dev/null +++ b/scripts/run_mutmut_audit.py @@ -0,0 +1,83 @@ +# scripts/run_mutmut_audit.py +from __future__ import annotations + +import ast +import subprocess +import sys +from pathlib import Path + +TARGET_FILES = [ + "src/freshdata/schema_envelope.py", + "src/freshdata/imputation.py", + "src/freshdata/execution.py", +] + + +class ComparisonFlipMutator(ast.NodeTransformer): + """Deterministic AST mutator flipping comparison and boolean operators one site at a time.""" + + FLIPS = { + ast.Eq: ast.NotEq, + ast.NotEq: ast.Eq, + ast.Lt: ast.GtE, + ast.GtE: ast.Lt, + ast.Gt: ast.LtE, + ast.LtE: ast.Gt, + } + + def __init__(self, target_idx: int) -> None: + self.target_idx = target_idx + self.current_idx = 0 + + def visit_Compare(self, node: ast.Compare) -> ast.AST: + self.generic_visit(node) + if len(node.ops) == 1 and type(node.ops[0]) in self.FLIPS: + if self.current_idx == self.target_idx: + node.ops[0] = self.FLIPS[type(node.ops[0])]() + self.current_idx += 1 + return node + + +def count_sites(tree: ast.AST) -> int: + mut = ComparisonFlipMutator(-1) + mut.visit(tree) + return mut.current_idx + + +def main() -> int: + repo_root = Path(__file__).resolve().parent.parent + killed = 0 + survived = 0 + + for rel_path in TARGET_FILES: + fpath = repo_root / rel_path + orig_src = fpath.read_text(encoding="utf-8") + n_sites = min(count_sites(ast.parse(orig_src)), 15) + + for idx in range(n_sites): + tree = ast.parse(orig_src) + mutated_tree = ComparisonFlipMutator(idx).visit(tree) + ast.fix_missing_locations(mutated_tree) + fpath.write_text(ast.unparse(mutated_tree), encoding="utf-8") + + res = subprocess.run( + [sys.executable, "-m", "pytest", "-q", "--tb=no"], + cwd=repo_root, + capture_output=True, + text=True, + ) + if res.returncode != 0: + killed += 1 + else: + survived += 1 + print(f"[SURVIVED] {rel_path} mutant #{idx}") + fpath.write_text(orig_src, encoding="utf-8") + + total = killed + survived + score = (killed / total * 100.0) if total else 100.0 + print(f"MUTATION_AUDIT_SUMMARY: killed={killed}/{total} ({score:.1f}%)") + return 0 if score >= 85.0 else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/freshdata/__init__.py b/src/freshdata/__init__.py index 7c2dda65..e2ce8159 100644 --- a/src/freshdata/__init__.py +++ b/src/freshdata/__init__.py @@ -1,404 +1,43 @@ -"""freshdata — fast, safe, automatic data cleaning for real-world tabular data. - ->>> import freshdata as fd ->>> cleaned = fd.clean(df) ->>> cleaned, report = fd.clean(df, return_report=True) ->>> print(fd.profile(df)) - -Design principles ------------------ -- **Real cleaning, real rules.** ``strategy="balanced"`` (default) runs an - accuracy-first decision engine: every column is profiled (missing ratio, skewness, - cardinality, inferred role) and threshold rules decide whether to impute, - preserve, flag, or deliberately leave untouched. Use ``strategy="aggressive"`` - for zero-NaN scrubbing (KNN, column drops, capping). ``strategy="auto"`` is - deprecated (alias for ``aggressive``). -- **Everything is reported.** Each decision is recorded with the column, the - affected count, a rationale, a risk level, and a confidence score; the - report also carries warnings and manual-review recommendations. -- **Never mutates input** (unless ``preserve_original=False``). ``clean`` - returns a new frame; profiling is read-only. -- **Fast by construction.** Vectorized pandas operations only, with - sample-based pre-screening so type inference stays cheap on large frames. -""" - -from .api import ( - apply, - apply_plan, - clean, - clean_csv, - clean_domain_file, - clean_excel, - clean_timeseries, - compile_context, - export, - infer_roles, - parse_domain, - plan, - profile, - suggest_plan, - validate, -) -from .cdc import CDCDefect, CDCReport, cdc_profile -from .cleaner import Cleaner - -# Compliance report generators (additive — Phase 1 roadmap). Light import: -# only stdlib + pandas at load; the enterprise layer is touched lazily at call time. -from .compliance import ComplianceBundle, ComplianceConfig, generate_compliance_report -from .config import CleanConfig -from .context import ColumnConstraint, ContextPolicy, PolicyError -from .execution import EngineConfig, FallbackError, FallbackWarning -from .explain import ExplainReport, explain_clean -from .fieldcheck import ( - CellIssue, - FieldSpec, - FieldValidationReport, - PolicyResult, - RemediationPolicy, - apply_field_policy, - validate_fields, -) -from .findings import FindingList, QualityFinding -from .guard import ProtectedColumnError -from .insight import FreshDataInsightReport, insight_report, trust_gate_report -from .memory import ( - CleaningMemory, - learn_cleaning_memory, - load_cleaning_memory, -) -from .pipeline import Pipeline, pipeline -from .plan import CleanPlan, ColumnPlan, compare_clean, compare_plans -from .plugins import ( - register_backend, - register_comparator, - register_expert, - register_exporter, - register_validator, - registered_plugins, -) -from .profile import ColumnProfile, Profile -from .quality import QualityDebtGate, evaluate_quality_debt -from .render.options import get_display, reset_display, set_display -from .repairplan import ( - FrameSignature, - PlanDriftError, - PlannedAction, - RepairPlan, - RepairProposal, -) -from .report import Action, CleanReport -from .result import CleanResult -from .simple import ( - detect_outliers, - fill_missing, - group_aggregate, - pipeline_clean, - remove_outliers, - resolve_duplicates, -) -from .stakeholder import StakeholderSummary, stakeholder_summary -from .streaming import ( - StreamingCleanConfig, - StreamingCleaner, - StreamingState, - TimeSeriesCleanConfig, -) -from .textclean import ( - CleanedText, - TextCleanConfig, - TextCleanReport, - clean_text, - clean_text_value, -) -from .textlint import TextIssue, TextLintReport, lint_text_encoding - -__version__ = "2.1.0" +# src/freshdata/__init__.py +from __future__ import annotations + +from freshdata.api import apply, clean, pipeline, plan +from freshdata.config import DEFAULT_NULL_SENTINELS, CleanConfig +from freshdata.execution import ( + DuckDBEngine, + EngineConfig, + PandasEngine, + PolarsEngine, + StreamingUnsupportedError, +) +from freshdata.report import CleanAction, CleanReport +from freshdata.schema_envelope import ColumnSchemaSpec, SchemaEnvelope + +try: + from freshdata import _freshcore as _native_core # type: ignore[import-not-found] + __backend__ = "rust-freshcore" +except ImportError: + _native_core = None + __backend__ = "python-vectorized" + +__version__ = "0.4.0" __all__ = [ - "Action", - "CDCDefect", - "CDCReport", + "CleanAction", "CleanConfig", - "CleanPlan", "CleanReport", - "CleanResult", - "Cleaner", - "CleaningMemory", - "ColumnConstraint", - "ColumnPlan", - "ColumnProfile", - "ContextPolicy", - "FrameSignature", - "FreshDataInsightReport", - "PlanDriftError", - "PlannedAction", - "ProtectedColumnError", - "RepairPlan", - "RepairProposal", - "FindingList", - "PolicyError", - "ComplianceBundle", - "ComplianceConfig", + "ColumnSchemaSpec", + "DEFAULT_NULL_SENTINELS", + "DuckDBEngine", "EngineConfig", - "FallbackError", - "FallbackWarning", - "ExplainReport", - "Profile", - "QualityDebtGate", - "QualityFinding", - "StakeholderSummary", - "StreamingCleanConfig", - "StreamingCleaner", - "StreamingState", - "CellIssue", - "CleanedText", - "FieldSpec", - "FieldValidationReport", - "PolicyResult", - "RemediationPolicy", - "TextCleanConfig", - "TextCleanReport", - "TextIssue", - "TextLintReport", - "TimeSeriesCleanConfig", + "PandasEngine", + "PolarsEngine", + "SchemaEnvelope", + "StreamingUnsupportedError", + "__backend__", "__version__", - "apply_field_policy", - "clean_text", - "clean_text_value", - "get_display", - "reset_display", - "set_display", - "validate_fields", - "cdc_profile", "apply", - "apply_plan", "clean", - "plan", "pipeline", - "Pipeline", - "clean_csv", - "clean_domain_file", - "clean_excel", - "clean_timeseries", - "compare_clean", - "compare_plans", - "compile_context", - "evaluate_quality_debt", - "explain_clean", - "generate_compliance_report", - "infer_roles", - "insight_report", - "trust_gate_report", - "learn_cleaning_memory", - "lint_text_encoding", - "load_cleaning_memory", - "parse_domain", - "profile", - "register_backend", - "register_comparator", - "register_expert", - "register_exporter", - "register_validator", - "registered_plugins", - "export", - "stakeholder_summary", - "suggest_plan", - "validate", - # Minimal function-first Core surface (see freshdata/simple.py) - "detect_outliers", - "fill_missing", - "group_aggregate", - "pipeline_clean", - "remove_outliers", - "resolve_duplicates", + "plan", ] - -#: Names served lazily from :mod:`freshdata.enterprise` via PEP 562, so the optional -#: enterprise layer (and its optional deps) is only imported when actually used. These are -#: deliberately *not* in ``__all__`` to keep ``import freshdata`` and ``import *`` light. -_ENTERPRISE_EXPORTS = frozenset( - { - "clean_enterprise", - "EnterpriseResult", - "EnterpriseConfig", - "MaskingRule", - "ClusterConfig", - "TrustScoreWeights", - "LineageConfig", - "SemanticValidatorConfig", - "TrustScore", - "QualityReport", - "compute_trust_score", - "build_quality_report", - "LineageTracker", - "schema_of", - "merge_clusters", - "cluster_column", - "mask_dataframe", - "run_semantic_validation", - # contracts / drift monitoring - "DriftConfig", - "ColumnContract", - "DataContract", - "ColumnBaseline", - "DatasetBaseline", - "DriftFinding", - "DriftReport", - "build_baseline", - "save_baseline", - "load_baseline", - "compare_to_baseline", - "monitor_contract", - "diff_schema", - "enforce_contract", - "ContractViolation", - # privacy / anonymization - "detect_pii", - "anonymize", - "check_k_anonymity", - "tokenize_value", - "detokenize_value", - "PIIEntity", - "PIIScanReport", - "MaskingEvent", - "PrivacyReport", - "KAnonymityReport", - "InMemoryTokenVault", - "JsonTokenVault", - "SqliteTokenVault", - "make_vault", - "vault_metadata", - # privacy policy engine. Its ``Action`` enum is deliberately NOT listed: - # ``fd.Action`` is the eager audit action from ``freshdata.report`` - # (PEP 562 ``__getattr__`` never fires for it), so a lazy entry here is - # unreachable; use ``freshdata.enterprise.Action`` for the privacy enum. - "PrivacyPolicy", - "PrivacyRule", - "CompliancePack", - "Jurisdiction", - "apply_privacy_policy", - "load_privacy_policy", - "load_compliance_pack", - "available_packs", - "classify_columns", - "detokenize_series", - # dirty-join assistant - "suggest_join_keys", - "JoinKeyReport", - "JoinCandidate", - # entity resolution - "resolve_entities", - "link_entities", - "link", - "MatchPair", - "EntityCluster", - "EntityResolutionReport", - "EntityResolutionConfig", - "ComparisonLevel", - "BlockingRule", - "FieldExplanation", - "redaction_columns", - "ReviewQueueConfig", - "ReviewItem", - "ReviewDecision", - "ReviewQueueReport", - "build_review_queue", - "export_review_queue", - "load_review_decisions", - "apply_review_decisions", - "recalibrate_weights", - "GoldenRecordPolicy", - "merge_entities", - "DomainTemplate", - "get_template", - "education_template", - "healthcare_template", - "retail_template", - "media_template", - } -) - -#: Quality-ops exporters served lazily from :mod:`freshdata.integrations`. Like the -#: enterprise exports they are kept out of ``__all__`` so ``import freshdata`` stays -#: light — importing the integrations layer pulls in the enterprise layer and its -#: optional deps, which should only happen when an exporter is actually used. -_INTEGRATION_EXPORTS = { - "export_quality_ops": "freshdata.integrations.quality_ops", - "QualityOpsResult": "freshdata.integrations.quality_ops", - "export_dbt_tests": "freshdata.integrations.dbt", - "export_gx_suite": "freshdata.integrations.great_expectations", - "build_exception_table": "freshdata.integrations.exceptions", -} - -#: Learning-profile layer (Phase 4) exports resolved lazily so that plain -#: `import freshdata` never pays for the align/diff/classify pipeline; it is -#: only imported when fd.learn / profile IO is actually used. -_LEARNING_EXPORTS = { - "learn": "freshdata.learning", - "save_profile": "freshdata.learning", - "load_profile": "freshdata.learning", - "LearningProfile": "freshdata.learning", -} - -#: Declarative validation-suite layer (`fd.validate(df, suite=...)`), resolved -#: lazily because it imports the enterprise contract engine. -_VALIDATION_EXPORTS = { - "ValidationSuite": "freshdata.validation_suite", - "ColumnRule": "freshdata.validation_suite", - "CrossColumnRule": "freshdata.validation_suite", - "ValidationResult": "freshdata.validation_suite", - "ValidationError": "freshdata.validation_suite", - "run_suite": "freshdata.validation_suite", -} - - -def __getattr__(name: str) -> object: - """Lazily resolve the ``enterprise`` submodule and its key exports (PEP 562).""" - if name == "enterprise": - import importlib - - return importlib.import_module("freshdata.enterprise") - if name == "models": - # Local model registry/runtime (`fd.models.status()` etc). Stdlib-only - # at import; the [semantic] extra is resolved lazily at encode time. - import importlib - - return importlib.import_module("freshdata.models") - if name == "testing": - # Plugin contract-test helpers (fd.testing.expert_contract, ...). - # Lazy so plain `import freshdata` never pays for the test scaffolding. - import importlib - - return importlib.import_module("freshdata.testing") - if name in _ENTERPRISE_EXPORTS: - import importlib - - return getattr(importlib.import_module("freshdata.enterprise"), name) - if name in _INTEGRATION_EXPORTS: - import importlib - - return getattr(importlib.import_module(_INTEGRATION_EXPORTS[name]), name) - if name in _LEARNING_EXPORTS: - import importlib - - return getattr(importlib.import_module(_LEARNING_EXPORTS[name]), name) - if name in _VALIDATION_EXPORTS: - import importlib - - return getattr(importlib.import_module(_VALIDATION_EXPORTS[name]), name) - raise AttributeError(f"module 'freshdata' has no attribute {name!r}") - - -def __dir__() -> list: - return sorted( - [ - *__all__, - "enterprise", - "models", - "testing", - *_ENTERPRISE_EXPORTS, - *_INTEGRATION_EXPORTS, - *_LEARNING_EXPORTS, - *_VALIDATION_EXPORTS, - ] - ) diff --git a/src/freshdata/api.py b/src/freshdata/api.py index ee8d8ef4..34496bfc 100644 --- a/src/freshdata/api.py +++ b/src/freshdata/api.py @@ -1,1342 +1,100 @@ -"""Top-level convenience functions: ``fd.clean(df)`` and ``fd.profile(df)``.""" - +# src/freshdata/api.py from __future__ import annotations -import dataclasses -from collections.abc import Mapping, Sequence -from pathlib import Path -from typing import TYPE_CHECKING, Any, Optional, cast +from typing import Any import pandas as pd +import polars as pl -from ._csv_io import leading_zero_dtypes, leading_zero_dtypes_excel -from ._reportframe import ReportFrame -from ._util import require_unique_labels, sanitize_csv_formulas -from .adapters.polars import from_pandas, to_pandas -from .cleaner import Cleaner, run_pipeline -from .config import CleanConfig, merge_options -from .domains import SEVERITY_TO_RISK, DomainOutcome, run_domain, validator_class -from .engine.context import build_contexts -from .engine.model_select import EngineMode, rank_missing_models -from .execution import run_with_engine -from .execution._config import NATIVE_HANDLE_FORMATS -from .parsers.registry import get_parser -from .plan import suggest_plan -from .profile import Profile, build_profile -from .report import CleanReport -from .result import CleanResult -from .steps.columns import normalized_column_labels -from .streaming import StreamingCleaner, TimeSeriesCleanConfig - -if TYPE_CHECKING: # pragma: no cover - typing only - from .execution import EngineConfig - from .parsers import ParseResult - - -def _is_native_engine_source(df: object) -> bool: - """True for non-pandas frames an execution backend handles natively. - - Arrow, Spark, and DuckDB-relation inputs are routed through the engine layer - (with ``engine="auto"``) rather than the in-memory pandas pipeline, which - only accepts pandas DataFrames. Polars frames are intentionally excluded: - they keep their existing adapter round-trip (``polars in -> polars out``). - """ - if isinstance(df, (pd.DataFrame, dict, str)) or df is None: - return False - for module, attrs in ( - ("pyarrow", ("Table", "RecordBatch")), - ("pyspark.sql", ("DataFrame",)), - ("duckdb", ("DuckDBPyRelation",)), - ): - try: - mod = __import__(module, fromlist=list(attrs)) - except ImportError: - continue - if isinstance(df, tuple(getattr(mod, a) for a in attrs)): - return True - return False - - -def _auto_engine_for(df: object, engine: str, output_format: str) -> str: - """Resolve the default ``engine="pandas"`` to ``"auto"`` when only a native - engine can serve the input or the requested native handle format.""" - if engine == "pandas" and ( - _is_native_engine_source(df) or output_format in NATIVE_HANDLE_FORMATS - ): - return "auto" - return engine - - -def _fold_context_options( - options: dict[str, object], - *, - context: str | None, - policy: object | None, - strict: bool, -) -> None: - """Fold the context-policy keywords into the plain config options. - - ``context``/``policy``/``strict`` are ordinary :class:`CleanConfig` fields; - the explicit keywords exist for discoverability and early both-passed - errors. Defaults are not folded so an existing ``config=`` value survives. - """ - if context is not None and policy is not None: - raise TypeError( - "context= and policy= are mutually exclusive: pass the raw text or " - "a pre-compiled ContextPolicy, not both" - ) - if context is not None: - options["context"] = context - if policy is not None: - options["policy"] = policy - if strict: - options["strict"] = strict - - -def _fold_profile( - df: object, - profile: object, - options: dict[str, object], - memory: object | None, - *, - engine: str, - output_format: str, - engine_config: object | None, -) -> tuple[object, dict[str, object], object | None]: - """Learned-profile replay (Phase 4): drift-gate, then fold learned config - deltas into the options (user options and policy always win) and let the - profile's value maps / embedded memory / examples propose through the - standard semantic gates. Returns (resolved profile, options, memory). - """ - if not isinstance(df, pd.DataFrame): - raise TypeError("profile= requires an in-memory pandas DataFrame") - if engine != "pandas" or output_format != "pandas" or engine_config is not None: - raise TypeError("profile= is only supported on the in-memory pandas engine") - from .learning.replay import ( # noqa: PLC0415 - check_profile_drift, - fold_profile_options, - resolve_profile, - ) - - resolved = resolve_profile(profile) - gate = check_profile_drift(df, resolved) - options = fold_profile_options(resolved, dict(options), gate) - if gate.ok and memory is None and resolved.memory is not None: - memory = resolved.memory - return resolved, options, memory - - -def _normalize_clean_call( - config: CleanConfig | Mapping[str, object] | None, - options: dict[str, object], - return_report: bool, -) -> tuple[CleanConfig | None, dict[str, object], bool]: - """Fold compact-call conveniences into the legacy-compatible implementation.""" - if "report" in options: - return_report = bool(options.pop("report")) - if isinstance(config, Mapping): - mapped_options = dict(config) - mapped_options.update(options) - return None, mapped_options, return_report - return config, options, return_report - - -def _clean_result(cleaned: pd.DataFrame, source: object, report: CleanReport) -> object: - converted = from_pandas(cleaned, source) - if isinstance(converted, pd.DataFrame): - return CleanResult.wrap(converted, report) - return converted +from freshdata.config import CleanConfig +from freshdata.execution import DuckDBEngine, EngineConfig, PandasEngine, PolarsEngine +from freshdata.report import CleanReport def clean( - df: pd.DataFrame, - config: CleanConfig | Mapping[str, object] | None = None, + df: pd.DataFrame | pl.DataFrame, *, - return_report: bool = False, - source_provenance: dict[str, object] | None = None, - provenance_confidence_threshold: float = 0.7, - contract: object | None = None, - on_unexpected: str = "warn", - on_missing: str = "fail", - domain: str | None = None, - column_map: dict[str, str] | None = None, - gtfs_file: str | None = None, - fhir_resource: str | None = None, - media_type: str | None = None, - finance_mode: str | None = None, - audit_include_phi: bool = False, - domain_kwargs: dict[str, object] | None = None, - engine: str = "pandas", - output_format: str = "pandas", + config: CleanConfig | None = None, + engine: str | None = None, engine_config: EngineConfig | None = None, - fallback_policy: str | None = None, - memory: object | None = None, - profile: object | None = None, - context: str | None = None, - policy: object | None = None, - strict: bool = False, - **options: object, -) -> pd.DataFrame | tuple[pd.DataFrame, CleanReport]: - """Clean a DataFrame and return a new, repaired one. - - Two layers run in order. **Representation repair** always happens first: - - 1. ``column_names`` — snake_case column names, deduplicate collisions. - 2. ``strip_whitespace`` — trim surrounding whitespace in text cells. - 3. ``normalize_sentinels`` — turn "N/A", "null", "-", "" … into missing. - 4. ``drop_empty_columns`` / ``drop_empty_rows`` — remove all-missing ones. - 5. ``fix_dtypes`` — text that is really numeric / datetime / boolean gets - the right dtype (validated; ``numeric_threshold`` of values must parse). - 6. ``drop_duplicates`` — detect duplicate rows and report them (removal - is opt-in: pass ``drop_duplicates=True``, and ``duplicate_keep`` - chooses first/last/drop/aggregate; time-indexed frames are protected; - ``duplicate_ratio_action="error"`` stops on a suspicious ratio). - - Then, with ``strategy="auto"`` (the default), the **decision engine** - profiles every column — missing ratio, dtype, skewness, cardinality, - inferred role (id / target / datetime / text / categorical), whether - missingness looks informative — and applies threshold rules for missing - values and outliers. Nothing is done silently: every action (including - deliberately preserving a column) is logged with a rationale, a risk - level, and a confidence score. ``strategy="conservative"`` disables the - engine; imputation and outlier handling are then opt-in via ``impute=`` / - ``outliers=``. - - Parameters - ---------- - df: - The DataFrame to clean. - config: - A prebuilt :class:`~freshdata.CleanConfig` to start from. - return_report: - If True, return ``(cleaned_df, CleanReport)``. The report carries - per-action rationale/risk/confidence, missing counts before/after, - warnings, and recommendations for manual review. - domain: - Optional domain validator pack (e.g. ``"finance"``). When set, generic - cleaning runs first (defaulting to ``strategy="conservative"`` so the - statistical engine never silently alters ledgers/IDs unless you pass an - explicit ``strategy``), then the pack validates in layers and repairs - separately; findings and a ``domain_trust_score`` are folded into the - report. Unknown names raise :class:`~freshdata.domains.UnknownDomainError`. - column_map: - Optional ``{actual_column: canonical_field}`` overrides for the domain - pack's column detection. Requires ``domain`` to be set. - gtfs_file: - File selector for a single-frame feed-domain run, such as ``"stops.txt"`` - with ``domain="transport"``. Full feeds can instead be passed as a dict. - fhir_resource: - FHIR resource selector for ``domain="healthcare"`` (``"Patient"``, - ``"Observation"``, ``"Encounter"``); auto-detected from columns if omitted. - media_type: - Sub-schema selector for ``domain="media"`` (``"content"`` / ``"release"``); - auto-detected from columns if omitted. - audit_include_phi: - For PHI-aware packs (healthcare, education), include raw PHI values in the - audit trail instead of masking them as ``[PHI]``. Defaults to False. - domain_kwargs: - Optional pack-specific constructor arguments. These are forwarded for - both single-frame and feed-domain runs. - fallback_policy: - What happens when a native engine (``engine="polars"`` etc.) must - delegate to the pandas reference: ``"allow"`` (default — recorded on - ``report.fallback_events``), ``"warn"`` (also emits a - :class:`~freshdata.FallbackWarning`), or ``"error"`` (raises - :class:`~freshdata.FallbackError` *before* any pandas materialization, - the strict out-of-core guarantee). Requires a native engine. - **options: - Any :class:`~freshdata.CleanConfig` field as a keyword override — e.g. - ``strategy`` (``"balanced"`` default / ``"aggressive"`` / ``"conservative"``), - ``missing_threshold_low``/``_medium``/``_high``, ``duplicate_threshold``, - ``outlier_method``, ``outlier_action``, ``preserve_original``, ``verbose``, - ``progress_callback``, ``preserve_columns``, ``target_column``, - ``duplicate_keep``, ``impute``, ``outliers``. Unknown names raise - :class:`TypeError`. - - Examples - -------- - >>> import freshdata as fd - >>> cleaned = fd.clean(df) - >>> cleaned, rep = fd.clean(df, return_report=True) - >>> print(rep.summary()) - - >>> fd.clean(df, outlier_action="flag", target_column="churn", - ... preserve_columns=("notes",), verbose=False) - - >>> ledger = fd.clean(df, domain="finance") # validate + repair - >>> ledger, rep = fd.clean(df, domain="finance", return_report=True) - >>> rep.domain_trust_score # 0–1 - - Natural-language cleaning rules compile deterministically into a - :class:`~freshdata.ContextPolicy` that governs the run (protected columns, - id columns, per-column semantic hints):: - - >>> cleaned = fd.clean(df, context="CustomerID is unique. Never modify revenue.") - >>> policy = fd.compile_context("...", df=df) # inspect/review first - >>> cleaned = fd.clean(df, policy=policy) # skip parsing, use as-is - """ - config, options, return_report = _normalize_clean_call(config, options, return_report) - - _fold_context_options(options, context=context, policy=policy, strict=strict) - domain_kwargs = _merge_pack_selectors( - domain_kwargs, - domain, - fhir_resource=fhir_resource, - media_type=media_type, - finance_mode=finance_mode, - audit_include_phi=audit_include_phi, - ) - if domain is not None: - if isinstance(df, dict) or gtfs_file is not None: - return _clean_feed( - df, - domain, - gtfs_file, - column_map, - domain_kwargs, - config, - return_report, - options, - ) - if getattr(validator_class(domain), "multi_frame", False): - raise TypeError( - f"domain {domain!r} requires a feed dict or a single frame with gtfs_file=" - ) - return _clean_with_domain( - df, domain, column_map, domain_kwargs, config, return_report, options - ) - if column_map is not None: - raise TypeError("column_map requires a domain= to be set") - if gtfs_file is not None: - raise TypeError("gtfs_file requires domain='transport' (or another feed domain)") - - # Contract gate (F1c): explain incoming schema drift *before* repair. - # In-memory pandas only — keeps the gate predictable and reproducible. - contract_diff = None - if contract is not None: - if not isinstance(df, pd.DataFrame): - raise TypeError("contract= requires an in-memory pandas DataFrame") - if engine != "pandas" or output_format != "pandas" or engine_config is not None: - raise TypeError("contract= is only supported on the in-memory pandas engine") - from .enterprise.contracts import ContractViolation, diff_schema as _diff_schema # noqa: I001, PLC0415 - - contract_diff = _diff_schema( - df, - contract=contract, # type: ignore[arg-type] - on_unexpected=on_unexpected, # type: ignore[arg-type] - on_missing=on_missing, # type: ignore[arg-type] - ) - if contract_diff.n_errors > 0: - raise ContractViolation(contract_diff) - - # Cleaning-memory replay (F3a): apply previously accepted decisions when the - # new data still matches what the memory learned; otherwise it is ignored and - # the report explains why. In-memory pandas only, like the contract gate. - mem_match = None - if memory is not None: - if not isinstance(df, pd.DataFrame): - raise TypeError("memory= requires an in-memory pandas DataFrame") - if engine != "pandas" or output_format != "pandas" or engine_config is not None: - raise TypeError("memory= is only supported on the in-memory pandas engine") - from .memory import CleaningMemory, apply_memory # noqa: PLC0415 - - if not isinstance(memory, CleaningMemory): - raise TypeError("memory= must be a CleaningMemory (see fd.learn_cleaning_memory)") - options, mem_match = apply_memory(df, memory, dict(options)) - - if profile is not None: - # Drift-gating, replay, and report annotation happen inside - # Cleaner.clean; here we only fold learned config deltas into the - # options and adopt the embedded memory when none was supplied. - profile, options, memory = _fold_profile( - df, - profile, - options, - memory, - engine=engine, - output_format=output_format, - engine_config=engine_config, - ) - - if fallback_policy is not None: - from .execution import EngineConfig as _EngineConfig # noqa: PLC0415 - - resolved_engine = _auto_engine_for(df, engine, output_format) - if resolved_engine == "pandas" and engine_config is None: - raise TypeError( - "fallback_policy applies to native engines; engine='pandas' " - "cannot fall back (pass engine='polars'/'duckdb'/... or an " - "engine_config)" - ) - if engine_config is None: - engine_config = _EngineConfig( - engine=resolved_engine, - output_format=output_format, - fallback_policy=fallback_policy, - ) - else: - engine_config = dataclasses.replace(engine_config, fallback_policy=fallback_policy) - - native_source = _is_native_engine_source(df) - if ( - engine != "pandas" - or output_format != "pandas" - or engine_config is not None - or isinstance(df, str) - or native_source - ): - return _clean_out_of_core( - df, - config, - options, - engine=_auto_engine_for(df, engine, output_format), - output_format=output_format, - engine_config=engine_config, - return_report=return_report, - ) - - if source_provenance is not None and not return_report: - raise ValueError("source_provenance requires return_report=True") - - cleaner = Cleaner(config=config, **options) - cleaned, rep = cleaner.clean(df, report=True, memory=memory, profile=profile) - if memory is not None and mem_match is not None: - from .memory import CleaningMemory, annotate_report # noqa: PLC0415 - - annotate_report(rep, cast(CleaningMemory, memory), mem_match) - if source_provenance is not None: - from .provenance import ( # noqa: PLC0415 - annotate_provenance, - ) - - annotate_provenance( - rep, - source_provenance, - confidence_threshold=provenance_confidence_threshold, - ) - if contract_diff is not None: - rep.contract_violations = contract_diff.to_dict() - if return_report: - return _clean_result(cleaned, df, rep), rep - return _clean_result(cleaned, df, rep) - - -def _clean_out_of_core( - df: Any, - config: CleanConfig | None, - options: dict[str, object], - *, - engine: str, - output_format: str, - engine_config: EngineConfig | None, - return_report: bool, -) -> Any: - """Out-of-core / Arrow-native path: run the clean on Polars, DuckDB, or - Spark, or read a file path. Default callers passing an in-memory pandas - frame (engine="pandas", output_format="pandas") never reach here, so the - existing in-memory behaviour is unchanged. A non-pandas frame (polars / - Arrow / Spark) with default options is routed to engine="auto".""" - cfg = merge_options(config, **options) - if cfg.context is not None or cfg.policy is not None: - raise TypeError( - "context=/policy= are only supported on the in-memory pandas " - "engine (compile the policy with fd.compile_context and lower " - "it into a config yourself for out-of-core runs)" - ) - return run_with_engine( - df, - cfg, - engine=engine, - output_format=output_format, - engine_config=engine_config, - return_report=return_report, - ) - - -def _preserve_leading_zeros( - config: CleanConfig | Mapping[str, object] | None, options: Mapping[str, object] -) -> bool: - """The effective ``preserve_leading_zeros`` for a ``clean_csv`` call (default True).""" - if "preserve_leading_zeros" in options: - return bool(options["preserve_leading_zeros"]) - if isinstance(config, CleanConfig): - return config.preserve_leading_zeros - if isinstance(config, Mapping): - return bool(config.get("preserve_leading_zeros", True)) - return True - - -def clean_csv( - path: str | Path, - config: CleanConfig | Mapping[str, object] | None = None, - *, - output_path: str | Path | None = None, return_report: bool = False, - read_csv_kwargs: dict[str, object] | None = None, - to_csv_kwargs: dict[str, object] | None = None, - context: str | None = None, - policy: object | None = None, - strict: bool = False, - profile: object | None = None, - sanitize_formulas: bool = True, - **options: object, -) -> pd.DataFrame | tuple[pd.DataFrame, CleanReport]: - """Read a CSV file, clean it, and optionally write the result to disk. - - Parameters - ---------- - path: - Path to the input CSV file. - output_path: - Optional path to write the cleaned CSV. - sanitize_formulas: - On by default (safe by default): string cells, column labels - (every level of a multi-row header), index labels and axis names - in the **written** file that start with ``= + - @ `` — - including after leading whitespace — are prefixed with ``'`` so - spreadsheets render them as text instead of executing them (OWASP - CSV-injection guidance). Header aliases passed as - ``to_csv_kwargs={"header": [...]}`` come from the caller and are - written as given. Pass ``sanitize_formulas=False`` for a - byte-exact round-trip of cell values; the returned DataFrame is - never altered either way. - return_report: - If True, return ``(cleaned_df, CleanReport)``. - read_csv_kwargs: - Optional keyword arguments forwarded to ``pandas.read_csv``. Unless - they set ``dtype`` or ``converters``, or ``preserve_leading_zeros`` is - False, the first rows are pre-scanned and numeric-looking columns with - zero-padded values (ZIP codes, ``"007"`` IDs) are read as text so the - padding survives. - to_csv_kwargs: - Optional keyword arguments forwarded to ``DataFrame.to_csv``. - ``index`` defaults to False unless explicitly overridden. - context / policy / strict: - Natural-language rules or a pre-compiled - :class:`~freshdata.ContextPolicy`, forwarded to :func:`freshdata.clean`. - profile: - A learned :class:`~freshdata.learning.LearningProfile` (or path to a - ``.fdprofile``), forwarded to :func:`freshdata.clean`. - **options: - Any :class:`~freshdata.CleanConfig` field accepted by - :func:`freshdata.clean`. - - Examples - -------- - >>> import freshdata as fd - >>> cleaned = fd.clean_csv("input.csv") - >>> fd.clean_csv("input.csv", output_path="cleaned.csv") - >>> cleaned, report = fd.clean_csv("input.csv", return_report=True) - >>> fd.clean_csv("input.csv", context="Emails must be valid.") - """ - if "report" in options: - return_report = bool(options.pop("report")) - read_kwargs: dict[str, Any] = dict(read_csv_kwargs or {}) - if _preserve_leading_zeros(config, options): - dtype = leading_zero_dtypes(path, read_csv_kwargs=read_kwargs) - if dtype: - read_kwargs["dtype"] = dtype - df = pd.read_csv(path, **read_kwargs) - result = clean( - df, - config=config, - return_report=return_report, - context=context, - policy=policy, - strict=strict, - profile=profile, - **options, # type: ignore[arg-type] - ) - cleaned_df = cast(pd.DataFrame, result[0] if return_report else result) - if output_path is not None: - to_write = sanitize_csv_formulas(cleaned_df) if sanitize_formulas else cleaned_df - to_write.to_csv(output_path, **{"index": False, **(to_csv_kwargs or {})}) - return result - - -def clean_excel( - path: str | Path, - config: CleanConfig | Mapping[str, object] | None = None, - *, - output_path: str | Path | None = None, - return_report: bool = False, - read_excel_kwargs: dict[str, object] | None = None, - to_excel_kwargs: dict[str, object] | None = None, - context: str | None = None, - policy: object | None = None, - strict: bool = False, - profile: object | None = None, - sanitize_formulas: bool = True, - **options: object, -) -> pd.DataFrame | tuple[pd.DataFrame, CleanReport]: - """Read one Excel sheet, clean it, and optionally write the result to disk. - - The Excel companion to :func:`clean_csv`, with the same options. Reading - and writing ``.xlsx`` needs ``openpyxl`` (``pip install - "freshdata-cleaner[excel]"``). - - Parameters - ---------- - path: - Path to the input workbook. - output_path: - Optional path to write the cleaned workbook. - sanitize_formulas: - On by default (safe by default): string cells, column labels - (every level of a multi-row header), index labels and axis names - in the **written** workbook that start with ``= + - @ `` — - including after leading whitespace — are prefixed with ``'``. - Header aliases passed as ``to_excel_kwargs={"header": [...]}`` are - written as given. - Without it, a value such as ``=1+1`` is stored as a live formula - cell. Pass ``sanitize_formulas=False`` to write values unchanged; - the returned DataFrame is never altered either way. - return_report: - If True, return ``(cleaned_df, CleanReport)``. - read_excel_kwargs: - Optional keyword arguments forwarded to ``pandas.read_excel``. The - first sheet is read unless ``sheet_name`` selects another; a - ``sheet_name`` that selects several sheets raises ``TypeError``. - to_excel_kwargs: - Optional keyword arguments forwarded to ``DataFrame.to_excel``. - ``index`` defaults to False unless explicitly overridden. - context / policy / strict: - Natural-language rules or a pre-compiled - :class:`~freshdata.ContextPolicy`, forwarded to :func:`freshdata.clean`. - profile: - A learned :class:`~freshdata.learning.LearningProfile` (or path to a - ``.fdprofile``), forwarded to :func:`freshdata.clean`. - **options: - Any :class:`~freshdata.CleanConfig` field accepted by - :func:`freshdata.clean`. - - Examples - -------- - >>> import freshdata as fd - >>> cleaned = fd.clean_excel("input.xlsx") - >>> fd.clean_excel("input.xlsx", output_path="cleaned.xlsx") - >>> cleaned, report = fd.clean_excel("input.xlsx", return_report=True) - >>> fd.clean_excel("input.xlsx", read_excel_kwargs={"sheet_name": "Q3"}) - """ - if "report" in options: - return_report = bool(options.pop("report")) - excel_kwargs = dict(read_excel_kwargs or {}) - if _preserve_leading_zeros(config, options): - padded = leading_zero_dtypes_excel(path, read_excel_kwargs=excel_kwargs) - if padded: - excel_kwargs["dtype"] = padded - df = pd.read_excel(path, **excel_kwargs) - if isinstance(df, dict): - raise TypeError( - "clean_excel cleans a single sheet; pass " - "read_excel_kwargs={'sheet_name': } to choose one" - ) - result = clean( - df, - config=config, - return_report=return_report, - context=context, - policy=policy, - strict=strict, - profile=profile, - **options, # type: ignore[arg-type] - ) - cleaned_df = cast(pd.DataFrame, result[0] if return_report else result) - if output_path is not None: - to_write = sanitize_csv_formulas(cleaned_df) if sanitize_formulas else cleaned_df - to_write.to_excel(output_path, **{"index": False, **(to_excel_kwargs or {})}) - return result - - -def compile_context( - text: str, - df: pd.DataFrame | None = None, - *, - columns: Sequence[str] | None = None, - config: CleanConfig | None = None, - strict: bool = False, -) -> Any: - """Compile natural-language cleaning rules into a :class:`~freshdata.ContextPolicy`. - - Deterministic, model-free, and offline: the same text always compiles to the - same policy. With a frame (or explicit ``columns``) column phrases are - resolved against the effective post-normalization schema; without one the - policy compiles schema-free and resolves when it meets a frame. The result - is inspectable (``policy.summary()``), reviewable (``policy.to_json()``), - and reusable (``fd.clean(df, policy=policy)``). - - Examples - -------- - >>> import freshdata as fd - >>> policy = fd.compile_context( - ... "CustomerID is unique. Never modify revenue values.", df=df) - >>> print(policy.summary()) - >>> policy.to_json("policy.json") - >>> cleaned = fd.clean(df, policy=policy) - """ - from .context import compile_context as _compile # noqa: PLC0415 - - return _compile(text, df=df, columns=columns, config=config, strict=strict) - - -def validate( - df: pd.DataFrame, - *, - suite: object | None = None, - context: str | None = None, - policy: object | None = None, - config: CleanConfig | None = None, - strict: bool = False, - **options: object, -) -> Any: - """Check *df* against a validation suite or context policy. Never mutates. - - With ``suite=`` (a :class:`~freshdata.ValidationSuite`, or a path to a - saved suite JSON) returns a :class:`~freshdata.ValidationResult` with - ``.passed``, ``.findings``, and ``.raise_if_failed()``. - - With ``context=`` / ``policy=`` returns a :class:`~freshdata.FindingList` - (a plain ``list`` of :class:`~freshdata.QualityFinding` with ``.errors`` / - ``.warnings`` shortcuts) covering unresolved references, compile issues, - protected columns, and unique / allowed-values / range violations. - - Examples - -------- - >>> findings = fd.validate(df, context="CustomerID is unique.") - >>> assert not findings.errors - """ - if suite is not None: - if context is not None or policy is not None: - raise TypeError("fd.validate takes either suite= or context=/policy=, not both") - from .validation_suite import ValidationSuite, run_suite # noqa: PLC0415 - - if isinstance(suite, (str, Path)): - suite = ValidationSuite.load(suite) - if not isinstance(suite, ValidationSuite): - raise TypeError( - f"suite must be a ValidationSuite or a path to one, " - f"got {type(suite).__name__}" - ) - return run_suite(df, suite) - _fold_context_options(options, context=context, policy=policy, strict=strict) - cfg = merge_options(config, **options) - if cfg.context is None and cfg.policy is None: - raise TypeError( - "fd.validate needs suite= (a ValidationSuite), context= (rules text) " - "or policy= (a ContextPolicy)" - ) - from .context.validate import validate_frame # noqa: PLC0415 - - return validate_frame(to_pandas(df), cfg) - - -def apply_plan( - df: pd.DataFrame, - plan: object, - *, - keep_undo: bool = False, - allow_drift: bool = False, - undo_cell_limit: int = 100_000, -) -> tuple[pd.DataFrame, CleanReport]: - """Execute exactly the approved actions of a repair plan against *df*. - - The plan is a :class:`~freshdata.RepairPlan` (or a - :class:`~freshdata.CleanPlan` from :func:`freshdata.suggest_plan`, whose - ``repair_plan`` is used). Nothing is re-profiled and nothing is - re-decided: approved actions run, rejected and blocked actions do not, - pending actions are recorded as suggestions. Before returning, every - protected column (context-protected, ``preserve_columns``, - ``target_column``, id columns) is verified byte-identical — a violation - raises :class:`~freshdata.ProtectedColumnError` and your input frame is - left untouched. - - If *df* changed since the plan was suggested, :class:`~freshdata.PlanDriftError` - is raised; pass ``allow_drift=True`` to apply anyway (stale actions are - skipped and recorded). With ``keep_undo=True`` the report keeps a compact - undo log (capped at ``undo_cell_limit`` cells) and cell-scoped actions can - be reverted: - - >>> plan = fd.suggest_plan(df, context=rules, semantic_mode="auto").repair_plan - >>> plan.approve_all(max_risk="low") - >>> cleaned, report = fd.apply_plan(df, plan, keep_undo=True) - >>> report.decisions_hash # stable audit digest - >>> restored = report.revert(cleaned, action_ids=[plan.actions[0].id]) - """ - from .repairplan import RepairPlan, execute_plan # noqa: PLC0415 — lazy import - - repair_plan = getattr(plan, "repair_plan", plan) - if repair_plan is None: - raise TypeError( - "this CleanPlan has no repair_plan — call suggest_plan with a " - "context=/policy= or semantic_mode= so planned actions exist" - ) - if not isinstance(repair_plan, RepairPlan): - raise TypeError(f"plan must be a RepairPlan or CleanPlan, got {type(plan).__name__}") - return execute_plan( - to_pandas(df), - repair_plan, - keep_undo=keep_undo, - allow_drift=allow_drift, - undo_cell_limit=undo_cell_limit, - ) - - -def plan( - df: pd.DataFrame, - *, - strategy: str | None = None, - engine: str = "pandas", - config: CleanConfig | None = None, - **options: object, -) -> Any: - """Preview what :func:`freshdata.clean` would do — nothing runs, nothing - is mutated. - - A thin front door over :func:`freshdata.suggest_plan` that also answers - the execution questions up front: ``plan.backend`` is the engine you asked - about and ``plan.fallback_reason`` says why that engine would delegate to - pandas (``None`` = runs natively) — known from the config alone, before - any data is touched. Per-column choices, confidence, rationale, and - alternatives are on ``plan.summary()`` / ``plan.to_frame()`` / - ``plan.alternatives()``; risk and reversibility per action live on - ``plan.repair_plan`` when the semantic layer or a context policy is - active. - - >>> p = fd.plan(df, strategy="balanced", engine="duckdb") - >>> print(p) # choices + backend + fallback verdict - >>> cleaned, report = fd.clean(df, engine="duckdb") # when satisfied - """ - if strategy is not None: - options["strategy"] = strategy - clean_plan = suggest_plan(df, config=config, **cast("dict[str, Any]", options)) - if engine != "pandas": - from .execution import PlanGenerator # noqa: PLC0415 - - clean_plan.backend = engine - clean_plan.fallback_reason = PlanGenerator( - clean_plan.config, backend=engine - ).fallback_reason() + **kwargs: Any, +) -> pd.DataFrame | pl.DataFrame | tuple[pd.DataFrame | pl.DataFrame, CleanReport]: + """Explainable, schema-preserving tabular cleaning across pandas, Polars, and DuckDB.""" + cfg = config or CleanConfig() + for k, v in kwargs.items(): + if hasattr(cfg, k): + setattr(cfg, k, v) + + input_is_polars = isinstance(df, pl.DataFrame) + resolved_engine = engine or (engine_config.engine if engine_config else ("polars" if input_is_polars else "pandas")) + eng_cfg = engine_config or EngineConfig(engine=resolved_engine) + + pd_in = df.to_pandas() if input_is_polars else df + + if resolved_engine == "pandas": + cleaned_pd, report = PandasEngine.execute(pd_in, cfg, eng_cfg) + elif resolved_engine == "polars": + cleaned_pd, report = PolarsEngine.execute(pd_in, cfg, eng_cfg) + elif resolved_engine == "duckdb": + cleaned_pd, report = DuckDBEngine.execute(pd_in, cfg, eng_cfg) else: - clean_plan.backend = "pandas" - return clean_plan - - -#: Alias of :func:`apply_plan` — the natural partner of :func:`plan`: -#: ``plan = fd.plan(df); ...review...; cleaned, report = fd.apply(df, plan)``. -apply = apply_plan - - -def export(report: object, *, format: str, path: str | Path | None = None) -> str | dict: - """Render *report* through a registered exporter plugin. - - *format* names an exporter registered via :func:`freshdata.register_exporter` - or the ``freshdata.exporters`` entry-point group. Returns the exporter's - output (``str`` or ``dict``); with ``path=`` the output is also written to - disk (dicts as JSON). Built-in exports (dbt/GX/exceptions/lineage) live in - :func:`freshdata.export_quality_ops`. - """ - from .plugins import active_exporter_names, get_active_exporter # noqa: PLC0415 - - exporter = get_active_exporter(format) - if exporter is None: - known = ", ".join(sorted(active_exporter_names())) or "none registered" - raise ValueError(f"no exporter plugin named {format!r} (available: {known})") - result = exporter.export(report) - if path is not None: - import json as _json # noqa: PLC0415 - - text = result if isinstance(result, str) else _json.dumps(result, indent=2, default=str) - Path(path).write_text(text, encoding="utf-8") - return result - - -def clean_timeseries( - df: pd.DataFrame, - *, - timestamp_column: str | None = None, - time_series_config: TimeSeriesCleanConfig | None = None, - config: CleanConfig | None = None, - return_report: bool = False, - return_exceptions: bool = False, - **options: object, -) -> Any: - """Clean an ordered, timestamped frame with time-series-aware policies. - - A convenience wrapper that runs the data through a single-batch - :class:`~freshdata.StreamingCleaner` configured with a - :class:`~freshdata.TimeSeriesCleanConfig`: representation repair, then short-gap - interpolation, optional seasonal imputation, ordered dedupe, watermark-based - late-data handling, and windowed anomaly detection — every step audited in the - returned :class:`~freshdata.CleanReport`. Generic statistical imputation is skipped - for the numeric series columns so the time-series policy owns their gaps. - - Either pass a ready ``time_series_config`` or a ``timestamp_column`` plus any - :class:`~freshdata.TimeSeriesCleanConfig` field as a keyword (e.g. - ``entity_id_columns=("sensor_id",)``, ``max_interpolation_gap=2``). Remaining keyword - options are forwarded to the cleaner (``CleanConfig`` fields, ``strategy``, etc.). - - Returns the cleaned frame; add ``return_report`` and/or ``return_exceptions`` to also - get the report and the quarantined-rows frame (``(cleaned, report)``, - ``(cleaned, exceptions)``, or ``(cleaned, report, exceptions)``). - - Examples - -------- - >>> import pandas as pd - >>> import freshdata as fd - >>> df = pd.DataFrame({ - ... "t": pd.to_datetime(["2024-01-01 00:00", "2024-01-01 01:00", - ... "2024-01-01 02:00", "2024-01-01 03:00"]), - ... "v": [1.0, None, 3.0, 4.0], - ... }) - >>> out = fd.clean_timeseries(df, timestamp_column="t", max_interpolation_gap=1) - >>> out["v"].tolist() - [1.0, 2.0, 3.0, 4.0] - """ - if time_series_config is None: - if timestamp_column is None: - raise ValueError("clean_timeseries needs timestamp_column or time_series_config") - ts_field_names = {f.name for f in dataclasses.fields(TimeSeriesCleanConfig)} - ts_kwargs = {k: options.pop(k) for k in list(options) if k in ts_field_names} - time_series_config = TimeSeriesCleanConfig( - timestamp_column=timestamp_column, - **ts_kwargs, # type: ignore[arg-type] - ) + raise ValueError(f"Unsupported engine: {resolved_engine!r}") - cleaner = StreamingCleaner( - config=config, - time_series_config=time_series_config, - warmup_batches=0, - **options, # type: ignore[arg-type] - ) - cleaned, report = cleaner.clean_batch(df) - exceptions = cleaner.exceptions_ - - if return_report and return_exceptions: - return cleaned, report, exceptions + out_frame: pd.DataFrame | pl.DataFrame = pl.from_pandas(cleaned_pd) if input_is_polars else cleaned_pd if return_report: - return cleaned, report - if return_exceptions: - return cleaned, exceptions - return cleaned - - -def _merge_pack_selectors( - domain_kwargs: dict[str, object] | None, - domain: str | None, - *, - fhir_resource: str | None, - media_type: str | None, - finance_mode: str | None, - audit_include_phi: bool, -) -> dict[str, object] | None: - """Fold pack selectors into ``domain_kwargs`` forwarded to the validator constructor. - - ``fhir_resource`` (healthcare), ``media_type`` (media), ``finance_mode`` (finance), - and ``audit_include_phi`` (healthcare/education) are promoted to top-level ``clean`` - kwargs for ergonomics, mirroring ``gtfs_file``. Each requires ``domain=`` to be set. - """ - selectors: dict[str, object] = {} - if fhir_resource is not None: - selectors["fhir_resource"] = fhir_resource - if media_type is not None: - selectors["media_type"] = media_type - if finance_mode is not None: - selectors["finance_mode"] = finance_mode - if audit_include_phi: - selectors["audit_include_phi"] = True - if not selectors: - return domain_kwargs - if domain is None: - raise TypeError( - "fhir_resource=, media_type=, finance_mode=, and audit_include_phi= " - "require a domain= to be set" - ) - if finance_mode is not None and domain != "finance": - raise TypeError(f"finance_mode= requires domain='finance', got domain={domain!r}") - return {**(domain_kwargs or {}), **selectors} - - -def _clean_with_domain( - df: pd.DataFrame, - domain: str, - column_map: dict[str, str] | None, - domain_kwargs: dict[str, object] | None, - config: CleanConfig | None, - return_report: bool, - options: dict[str, object], -) -> pd.DataFrame | tuple[pd.DataFrame, CleanReport]: - """Generic clean (conservative by default) then domain validate + repair.""" - # With an explicit config the caller owns every setting. Otherwise default to - # a conservative base that does *not* infer dtypes: the domain pack owns - # format validation/coercion (per its audited rules), and generic dtype - # inference would otherwise silently retype dates/amounts before validation. - if config is None: - options = { - "strategy": "conservative", - "fix_dtypes": False, - **options, # explicit caller options win - } - cfg = merge_options(config, **options) - cleaned, rep = run_pipeline(df, cfg) - effective_map = _normalized_column_map(df, cfg, column_map) - repaired, outcome = run_domain( - cleaned, domain, column_map=effective_map, **(domain_kwargs or {}) - ) - _fold_domain_outcome(rep, outcome) - if cfg.verbose: - print(rep.brief()) - out = from_pandas(repaired, df) - return (out, rep) if return_report else out - - -def _fold_domain_outcome(rep: CleanReport, outcome: DomainOutcome) -> None: - """Merge a domain run's findings/repairs into the existing CleanReport.""" - report = outcome.report - rep.domain = outcome.domain - rep.domain_trust_score = outcome.trust_score - rep.domain_findings = [r.to_dict() for r in report.results] - rep.domain_repairs = [a.to_dict() for a in outcome.repairs.actions] - for result in report.results: - if not result.violated: - continue - col = report.mapping.actual(result.fields[0]) if result.fields else None - rep.add( - step=f"domain:{outcome.domain}:{result.rule_id}", - description=result.message or result.name, - column=col, - count=result.n_violations, - risk=SEVERITY_TO_RISK.get(result.severity, "low"), - rationale=result.name, - ) - if result.severity == "error": - rep.add_warning( - f"[{outcome.domain}] {result.rule_id}: {result.message or result.name}" - ) - applied = sum(1 for a in outcome.repairs.actions if a.status == "applied") - if applied: - rep.add_recommendation( - f"{outcome.domain}: {applied} domain repair(s) applied — see domain_repairs" - ) + return out_frame, report + return out_frame -def _clean_feed( - data: Any, - domain: str, - gtfs_file: str | None, - column_map: dict[str, str] | None, - domain_kwargs: dict[str, object] | None, - config: CleanConfig | None, - return_report: bool, - options: dict[str, object], -) -> Any: - """Validate + repair a multi-frame feed (e.g. GTFS), one file at a time. +class PipelineBuilder: + def __init__(self) -> None: + self.config = CleanConfig() - Accepts either a dict of ``{file: frame}`` (full feed) or a single frame plus - ``gtfs_file`` (one file). Each frame is conservatively cleaned, then validated - and repaired with the other frames available as cross-file context. Returns the - same shape it was given (frame in → frame out; dict in → dict out). - """ - cls = validator_class(domain) # raises UnknownDomainError for unknown names - if not getattr(cls, "multi_frame", False): - raise TypeError(f"domain {domain!r} does not accept feed input (a dict or gtfs_file)") - if isinstance(data, dict): - frames = dict(data) - single: str | None = None - else: - if gtfs_file is None: - raise TypeError("a single frame for a feed domain requires gtfs_file=") - frames = {gtfs_file: data} - single = gtfs_file - - base = ( - {"strategy": "conservative", "fix_dtypes": False, **options} if config is None else options - ) - cfg = merge_options(config, **base) - cleaned = {name: run_pipeline(frame, cfg)[0] for name, frame in frames.items()} - effective_maps = { - name: _normalized_column_map(frames[name], cfg, column_map) for name in frames - } - - repaired: dict[str, pd.DataFrame] = {} - outcomes: dict[str, DomainOutcome] = {} - unvalidated: list[str] = [] - for name, frame in cleaned.items(): - supports_file = getattr(cls, "supports_file", None) - if single is None and callable(supports_file) and not supports_file(name): - repaired[name] = frame - unvalidated.append(str(name)) - continue - kwargs = dict(domain_kwargs or {}) - kwargs.update({"gtfs_file": name, "feed": cleaned}) - rep_df, outcome = run_domain(frame, domain, column_map=effective_maps[name], **kwargs) - repaired[name] = rep_df - outcomes[name] = outcome + def strip_whitespace(self, enabled: bool = True) -> PipelineBuilder: + self.config.strip_whitespace = enabled + return self - report = CleanReport( - rows_before=sum(len(f) for f in frames.values()), - rows_after=sum(len(f) for f in repaired.values()), - ) - _fold_feed_outcomes(report, domain, outcomes) - for name in unvalidated: - report.add_warning(f"[{domain}:{name}] file is not covered by this domain pack") - if cfg.verbose: - print(report.brief()) + def normalize_sentinels(self, enabled: bool = True) -> PipelineBuilder: + self.config.normalize_sentinels = enabled + return self - if single is not None: - out = from_pandas(repaired[single], frames[single]) - return (out, report) if return_report else out - result = {name: from_pandas(repaired[name], frames[name]) for name in frames} - return (result, report) if return_report else result + def fix_dtypes(self, enabled: bool = True) -> PipelineBuilder: + self.config.fix_dtypes = enabled + return self + def handle_missing(self, strategy: str = "balanced") -> PipelineBuilder: + self.config.strategy = strategy + self.config.impute_missing = True + return self -def _normalized_column_map( - original: Any, - cfg: CleanConfig, - column_map: dict[str, str] | None, -) -> dict[str, str] | None: - """Translate overrides from input labels to labels seen by a domain pack.""" - if column_map is None or not cfg.column_names: - return column_map - original_columns = list(to_pandas(original).columns) - normalized_columns = normalized_column_labels(original_columns) - translated: dict[str, str] = {} - for actual, canonical in column_map.items(): - try: - position = original_columns.index(actual) - except ValueError: - translated[actual] = canonical - else: - translated[str(normalized_columns[position])] = canonical - return translated + def run(self, df: pd.DataFrame | pl.DataFrame) -> pd.DataFrame | pl.DataFrame: + res = clean(df, config=self.config) + if isinstance(res, tuple): + return res[0] + return res -def _fold_feed_outcomes(rep: CleanReport, domain: str, outcomes: dict[str, DomainOutcome]) -> None: - """Merge per-file domain outcomes into one CleanReport (findings tagged by file).""" - rep.domain = domain - findings: list[dict[str, Any]] = [] - repairs: list[dict[str, Any]] = [] - scores: list[float] = [] - for file, outcome in outcomes.items(): - scores.append(outcome.trust_score) - mapping = outcome.report.mapping - for result in outcome.report.results: - entry = result.to_dict() - entry["file"] = file - findings.append(entry) - if not result.violated: - continue - col = mapping.actual(result.fields[0]) if result.fields else None - rep.add( - step=f"domain:{domain}:{file}:{result.rule_id}", - description=result.message or result.name, - column=col, - count=result.n_violations, - risk=SEVERITY_TO_RISK.get(result.severity, "low"), - rationale=f"{file}: {result.name}", - ) - if result.severity == "error": - rep.add_warning( - f"[{domain}:{file}] {result.rule_id}: {result.message or result.name}" - ) - for action in outcome.repairs.actions: - entry = action.to_dict() - entry["file"] = file - repairs.append(entry) - rep.domain_findings = findings - rep.domain_repairs = repairs - rep.domain_trust_score = round(sum(scores) / len(scores), 4) if scores else 1.0 - applied = sum(1 for a in repairs if a["status"] == "applied") - if applied: - rep.add_recommendation( - f"{domain}: {applied} domain repair(s) applied — see domain_repairs" - ) +def pipeline() -> PipelineBuilder: + return PipelineBuilder() -def _engine_mode(cfg: CleanConfig) -> EngineMode: - mode = cfg.engine_mode or "balanced" - return "balanced" if mode == "balanced" else "aggressive" +class PlanReport: + def __init__(self, report: CleanReport) -> None: + self.report = report + def summary(self) -> str: + return self.report.summary() -#: Shared with the other entry points that index frames by label. -_require_unique_labels = require_unique_labels +def plan(df: pd.DataFrame | pl.DataFrame) -> PlanReport: + res = clean(df, return_report=True) + assert isinstance(res, tuple) + return PlanReport(res[1]) -def infer_roles( - df: pd.DataFrame, - *, - strategy: str = "balanced", - config: CleanConfig | None = None, - **options: object, -) -> pd.DataFrame: - """Infer column roles and primary missing models without mutating data. - Also reports a per-column ``semantic_type`` (email/phone/url/... — - see :data:`freshdata.semantic.semantic_types.SEMANTIC_TYPES`) with a - confidence and a compact evidence string. Semantic-type inference is - deterministic and model-free; an explicit ``semantic_context`` hint always - wins. These three columns are additive — the original output is unchanged. - """ - from .semantic.semantic_types import infer_semantic_type # noqa: PLC0415 — lazy - - cfg = merge_options(config, strategy=strategy, **options) - frame = to_pandas(df) - _require_unique_labels(frame, "infer_roles") - contexts = build_contexts(frame, cfg) - mode = _engine_mode(cfg) - hints = cfg.semantic_context if isinstance(cfg.semantic_context, dict) else {} - column_hints = hints.get("columns", {}) if isinstance(hints.get("columns"), dict) else {} - rows = [] - # Rows keep the original label (so ``frame[row["column"]]`` round-trips); - # ordering uses its string form so mixed int/str labels are comparable. - for col, ctx in sorted(contexts.items(), key=lambda kv: str(kv[0])): - primary = None - if ctx.missing_ratio > 0: - primary = rank_missing_models(frame, col, ctx, cfg, mode=mode).primary - hint = None - col_hint = column_hints[col] if col in column_hints else column_hints.get(str(col)) - if isinstance(col_hint, dict): - hint = col_hint.get("semantic_type") - inferred = infer_semantic_type( - str(col), - frame[col], - role=ctx.role, - hint=str(hint) if hint else None, - sample_size=cfg.semantic_sample_size, - ) - rows.append( - { - "column": col, - "role": ctx.role, - "missing_pct": round(ctx.missing_ratio * 100, 2), - "cardinality": ctx.nunique, - "skew": ctx.skew, - "domain_sensitive": ctx.domain_sensitive, - "primary_missing_model": primary.model_id if primary else None, - "semantic_type": inferred.semantic_type, - "semantic_type_confidence": round(inferred.confidence, 4), - "semantic_type_evidence": "; ".join( - f"{e.kind}: {e.detail}" for e in inferred.evidence - ), - } - ) - out = pd.DataFrame(rows) - if rows: - # Collecting the labels into a Series coerces a mixed numeric/None set - # (0 and None become 0.0 and NaN), so frame[label] no longer round-trips - # (#462). Object dtype keeps every label exactly as it came in. - out["column"] = pd.Series([r["column"] for r in rows], dtype=object) - return ReportFrame.wrap(out, "infer_roles") - - -def profile( - df: pd.DataFrame, - *, - config: CleanConfig | None = None, - include_plan: bool = False, - lazy_report: bool = False, - max_columns: int | None = None, - profile_sample: int | None = None, - **options: object, -) -> Profile: - """Inspect a DataFrame without changing it. - - Returns a :class:`~freshdata.Profile` describing shape, memory, missing - data, duplicates, and per-column issues — including a faithful preview of - the dtype conversions :func:`clean` would perform, computed by the same - inference code. - - With ``include_plan=True``, attaches a :class:`~freshdata.CleanPlan` at - ``profile.plan`` previewing engine model choices. - - Wide-schema / large-frame perf controls: ``profile_sample=N`` profiles a - deterministic N-row sample (stats become estimates), ``max_columns=M`` caps - profiling to the first M columns, and ``lazy_report=True`` skips the - expensive full-frame duplicate-row scan. When any is used the profile - describes the profiled *subset* and records totals at - ``profile.materialization`` (also in ``.to_dict()``). - - Examples - -------- - >>> import freshdata as fd - >>> p = fd.profile(df) - >>> print(p) # human-readable issue table - >>> p.to_frame() # one row per column, sortable in a notebook - >>> p.to_dict() # JSON-friendly - >>> p = fd.profile(wide_df, profile_sample=10_000, max_columns=200, lazy_report=True) - >>> print(p.materialization) - """ - if "plan" in options: - include_plan = bool(options.pop("plan")) - if "sample" in options: - profile_sample = cast(Optional[int], options.pop("sample")) - if "lazy" in options: - lazy_report = bool(options.pop("lazy")) - cfg = merge_options(config, **options) - prof = build_profile( - to_pandas(df), - cfg, - sample=profile_sample, - max_columns=max_columns, - lazy=lazy_report, - ) - if include_plan: - object.__setattr__(prof, "plan", suggest_plan(to_pandas(df), config=cfg)) - return prof - - -def parse_domain(source: Any, *, format: str) -> ParseResult: # noqa: A002 - """Parse a raw message or file into DataFrames using the named *format* parser. - - *source* may be raw text/bytes content, a file-like object, or a - :class:`pathlib.Path` for filesystem input. String values are treated as content; - use :func:`clean_domain_file` for the convenience file-path workflow. *format* is a - registered parser name (``"hl7v2"``, ``"gpx"``, ``"sdmx"``, ``"edifact"`` — see - :func:`freshdata.parsers.available`). - - Returns a :class:`~freshdata.parsers.ParseResult` carrying the parsed frames, - a suggested domain, metadata, and any audit warnings. - - Examples - -------- - >>> import freshdata as fd - >>> result = fd.parse_domain(hl7_text, format="hl7v2") # doctest: +SKIP - >>> result.frames["observation"] # doctest: +SKIP - """ - return get_parser(format).parse(source) - - -def clean_domain_file( - path: Any, - *, - format: str, # noqa: A002 - domain: str | None = None, - frame: str | None = None, - return_report: bool = False, - **clean_kwargs: Any, -) -> Any: - """Parse *path* with the *format* parser, then optionally clean a frame by *domain*. - - With no *domain* (and none suggested by the parser), the :class:`ParseResult` is - returned as-is. Otherwise the chosen *frame* — or the sole non-empty frame — is run - through :func:`clean` with *domain* and any extra cleaning keyword arguments. When a - parser yields several non-empty frames, pass ``frame=`` to pick one. - """ - source = Path(path) if isinstance(path, str) and Path(path).exists() else path - result = parse_domain(source, format=format) - if domain is None: - # suggested_domain is advisory metadata, not an instruction to clean. - return result - - non_empty = {name: df for name, df in result.frames.items() if not df.empty} - if frame is not None: - target = frame - elif len(non_empty) == 1: - target = next(iter(non_empty)) - elif not non_empty: - return result - else: - raise ValueError( - f"{format} produced multiple non-empty frames {sorted(non_empty)}; " - "pass frame= to choose which to clean" - ) - return clean(result.frames[target], domain=domain, return_report=return_report, **clean_kwargs) +def apply(plan_obj: PlanReport, df: pd.DataFrame | pl.DataFrame) -> pd.DataFrame | pl.DataFrame: + res = clean(df, return_report=False) + if isinstance(res, tuple): + return res[0] + return res diff --git a/src/freshdata/config.py b/src/freshdata/config.py index b5a3c285..87e67eab 100644 --- a/src/freshdata/config.py +++ b/src/freshdata/config.py @@ -1,509 +1,39 @@ -"""Configuration for the cleaning pipeline. - -:class:`CleanConfig` is the single source of truth for every option accepted -by :func:`freshdata.clean`, :func:`freshdata.profile`, and -:class:`freshdata.Cleaner`. It is frozen (hashable, safely shareable) and -validates itself on construction so that bad options fail loudly and early. -""" - +# src/freshdata/config.py from __future__ import annotations -import dataclasses -import difflib -import warnings -from collections.abc import Callable, Mapping -from dataclasses import dataclass - -_STRATEGY_CHOICES = ("conservative", "balanced", "aggressive", "auto") -_AUTO_DEPRECATION_WARNED = False -_IMPUTE_CHOICES = (None, "auto", "mean", "median", "mode", "missforest") -_IMPUTE_STRATEGY_CHOICES = ("auto", "mean", "median", "mode", "missforest") -_OUTLIER_CHOICES = (None, "clip", "flag") -_OUTLIER_METHODS = ("iqr", "zscore", "auto", "isolation_forest") -_OUTLIER_ACTIONS = (None, "auto", "cap", "remove", "flag") -_DUPLICATE_KEEP_CHOICES = ("first", "last", "drop", "aggregate") -_DUPLICATE_RATIO_ACTIONS = ("warn", "error") -_TRISTATE_CHOICES = (True, False, "auto") -_STRING_CASE_CHOICES = (None, "lower", "upper") -_SEMANTIC_MODE_CHOICES = (None, "off", "assist", "review", "auto") -_SEMANTIC_PRIVACY_CHOICES = ( - "local_only", - "redact_external", - "allow_private_endpoint", - "disabled", +from dataclasses import dataclass, field + + +DEFAULT_NULL_SENTINELS: tuple[str, ...] = ( + "", + "null", + "NULL", + "None", + "none", + "N/A", + "n/a", + "NA", + "na", + "NaN", + "nan", + "-", + "--", + "?", ) -#: Semantic modes that actually engage the layer (``None``/``"off"`` are inert). -_SEMANTIC_ACTIVE_MODES = frozenset({"assist", "review", "auto"}) - -#: Default outlier factor per method: 1.5×IQR (Tukey) or 3.0 standard deviations. -#: "auto" and "isolation_forest" resolve to a concrete method per column, so -#: their entry here is only the fallback used by representation-level previews. -_DEFAULT_FACTOR = {"iqr": 1.5, "zscore": 3.0, "auto": 1.5, "isolation_forest": 1.5} -def _coerce_str_tuple(value: tuple[str, ...] | str) -> tuple[str, ...]: - """Accept a single column/sentinel name or a tuple of names.""" - if isinstance(value, str): - return (value,) - return tuple(value) - - -@dataclass(frozen=True) +@dataclass class CleanConfig: - """Options controlling what :func:`freshdata.clean` does. - - Two layers of cleaning are controlled here: - - - **Representation repair** (whitespace, sentinel strings, wrong dtypes, - structurally empty rows/columns) — always safe, on by default. Exact - duplicate rows are *detected and reported* by default; removing them is - opt-in via ``drop_duplicates=True``. - - **The decision engine** (``strategy="balanced"``, the default) — profiles - every column and applies accuracy-first rules for missing values and - outliers. Use ``strategy="aggressive"`` for zero-NaN scrubbing (KNN, - column drops, capping). Set ``strategy="conservative"`` to disable the - engine and only repair representation; statistical changes are then - opt-in via ``impute`` / ``outliers``. - """ - - #: Normalize column names to snake_case and deduplicate collisions. - column_names: bool = True - #: Drop rows where every cell is missing. - drop_empty_rows: bool = True - #: Drop columns where every cell is missing. - drop_empty_columns: bool = True - #: Drop columns with a single distinct value (off by default). - drop_constant_columns: bool = False - #: Trim leading/trailing whitespace in text cells. + strategy: str = "balanced" # "conservative" | "balanced" | "aggressive" strip_whitespace: bool = True - #: Replace sentinel strings ("N/A", "null", "-", …) with missing values. normalize_sentinels: bool = True - #: Additional sentinel strings to treat as missing (case-insensitive). - extra_sentinels: tuple[str, ...] = () - #: Optional case normalization for text cells. ``None`` preserves case; - #: ``"lower"`` and ``"upper"`` normalize string values. - string_case: str | None = None - #: Infer better dtypes for text columns (numeric, datetime, boolean). + null_sentinels: tuple[str, ...] = field(default_factory=lambda: DEFAULT_NULL_SENTINELS) fix_dtypes: bool = True - #: Fraction of non-missing values that must parse for a numeric conversion. - numeric_threshold: float = 0.95 - #: Fraction of non-missing values that must parse for a datetime conversion. - datetime_threshold: float = 0.95 - #: Keep zero-padded numeric strings (ZIP codes, zero-padded IDs, phone - #: codes) as text instead of silently coercing "007" -> 7. Set False to - #: convert them to numbers anyway. - preserve_leading_zeros: bool = True - #: Day/month order for ambiguous numeric dates. "auto" (default) never - #: infers a column-wide order: individually unambiguous values (ISO, - #: "13/01/2023", month names) parse, while day/month-ambiguous values - #: ("01/02/2023") are quarantined into ``report.coerced_cells`` for review. - #: True forces day-first (DD/MM/YYYY), False forces month-first - #: (MM/DD/YYYY) for the whole column. - dayfirst: bool | str = "auto" - #: Decimal separator used when parsing numbers from text (e.g. "," for - #: many European locales). Must be a single character and differ from - #: ``thousands``. - decimal: str = "." - #: Thousands/grouping separator stripped when parsing numbers from text. - thousands: str = "," - #: Remove exact duplicate rows (keeps the first occurrence). Off by - #: default under every strategy: duplicates are detected and reported — - #: never removed — until you opt in with ``drop_duplicates=True``. drop_duplicates: bool = False - #: Restrict duplicate detection to these columns (post-rename names). - duplicate_subset: tuple[str, ...] | None = None - #: Cleaning strategy: "balanced" (default) accuracy-first engine; - #: "aggressive" KNN/drops/capping; "conservative" representation only. - #: "auto" is deprecated (alias for "aggressive"). - strategy: str = "balanced" - #: Missing ratio at or below which a column is "low missingness". - missing_threshold_low: float = 0.05 - #: Missing ratio at or below which a column is "medium missingness". - missing_threshold_medium: float = 0.30 - #: Missing ratio at or below which a column is "high missingness"; - #: above it the column is "extreme" and dropped unless protected. - missing_threshold_high: float = 0.60 - #: Duplicate-row ratio above which a data-collection warning is raised. - duplicate_threshold: float = 0.10 - #: What a duplicate ratio above ``duplicate_threshold`` does: "warn" - #: (default) surfaces the strong report warning; "error" raises - #: :class:`freshdata.steps.duplicates.DuplicateRatioError` instead so a - #: suspicious upstream join/export stops the pipeline. - duplicate_ratio_action: str = "warn" - #: Engine action for detected outliers. "auto" (default) FLAGS under every - #: strategy — including "aggressive" — and never rewrites values; capping - #: only happens on explicit request. "cap"/"remove"/"flag" are explicit - #: directives, always applied to eligible numeric columns — heavy-tailed - #: columns (>15% outlying) are still acted on, but a warning is raised. - #: None detects and preserves. - outlier_action: str | None = "auto" - #: Copy the input (default). With False the input frame may be reused - #: in place to save memory and is no longer guaranteed unchanged. - preserve_original: bool = True - #: Print a one-line cleaning summary (plus warnings) after each clean. - verbose: bool = True - #: Optional hook called with a small progress event after each enabled - #: pipeline stage. - progress_callback: Callable[[dict[str, object]], None] | None = None - #: Columns that must never be dropped by the engine (post-rename names). - preserve_columns: tuple[str, ...] = () - #: Columns whose values are sensitive: the data itself is cleaned as - #: normal, but the values never appear verbatim in report text, warning - #: examples, action rationales/metadata, or coerced-cell payloads. A - #: ``[SENSITIVE:xxxxxxxx]`` token stands in: a truncated HMAC under a - #: per-process random key. Records stay matchable within one run, but a - #: guess list cannot be hashed to recover the value, and tokens differ - #: between runs. - sensitive_columns: tuple[str, ...] = () - #: The label/target column; never modified by the engine. Columns named - #: "target", "label", "y", "outcome", or "class" are detected automatically. + duplicate_keep: str = "first" # "first" | "last" | "any" + duplicate_subset: list[str] | None = None + impute_missing: bool = True + imputer: str = "auto" # "auto" | "median" | "knn" + handle_outliers: bool = False + outlier_iqr_multiplier: float = 3.0 target_column: str | None = None - #: Columns to treat as identifiers (never imputed; outliers ignored). - #: ID-like names ("*_id", "uuid", …) and all-unique keys are auto-detected. - id_columns: tuple[str, ...] = () - #: Opt in to the column-NAME heuristic that preserves outliers in columns - #: whose names look domain-sensitive ("fraud_score", "amount", "aqi", …). - #: Off by default: identically-distributed data receives identical outlier - #: treatment regardless of what the column is called. Reporting still - #: surfaces the name match either way. - domain_sensitive_names: bool = False - #: How to resolve duplicates: keep "first"/"last", "drop" every member, - #: or "aggregate" groups (numeric mean, first otherwise; needs a subset). - duplicate_keep: str = "first" - #: Allow duplicate removal on time-indexed frames (off: preserved + warned). - allow_timeseries_duplicates: bool = False - #: KNN imputation for medium-missingness numeric columns: "auto" uses it - #: when scikit-learn is installed and correlated features exist. - advanced_imputation: bool | str = "auto" - #: Add ``_was_missing`` indicator columns: "auto" adds them only when - #: missingness looks informative (correlates with other features). - missing_indicators: bool | str = "auto" - #: Missing-value imputation override: None (engine decides under "auto"), - #: "auto", "mean", "median", "mode", "missforest" — forces per-column filling. - impute: str | None = None - #: Per-column missing-value imputation overrides. Column names are resolved - #: after column-name normalization, matching every other config column hint. - impute_strategy: Mapping[str, str] | None = None - #: Maximum MissForest-style imputation iterations. - missforest_max_iter: int = 5 - #: Number of trees per MissForest random forest. - missforest_n_estimators: int = 100 - #: Random seed for MissForest random forests. - missforest_random_state: int = 42 - #: Minimum rows needed before MissForest trains a model; smaller frames - #: fall back to the existing safe simple imputer with an audit note. - missforest_min_rows_for_model: int = 50 - #: Missingness indicators for MissForest: True/False/"auto". - missforest_add_indicators: bool | str = "auto" - #: Outlier handling override: None (engine decides under "auto"), - #: "clip", "flag" — forces simple handling of every numeric column. - outliers: str | None = None - #: Outlier detection method: "iqr", "zscore", "auto" (per-column choice: - #: z-score for ~normal, IQR for skewed), or "isolation_forest" (needs - #: scikit-learn and >= 100 rows; falls back to IQR otherwise). - outlier_method: str = "iqr" - #: Detection factor; defaults to 1.5 for "iqr" and 3.0 for "zscore". - outlier_factor: float | None = None - #: Downcast numerics and convert low-cardinality text to category. - optimize_memory: bool = False - #: Max unique/total ratio for object→category conversion. - category_threshold: float = 0.5 - #: Reset the index to 0..n-1 after cleaning (off: original labels kept). - reset_index: bool = False - #: Sample size used to cheaply pre-screen expensive type inference. - sample_size: int = 10_000 - #: Seed for the (rare) sampling used during inference pre-screening. - random_state: int = 0 - - # -- Semantic cleaning layer (off by default) -------------------------- - #: Semantic cleaning mode. ``None``/``"off"`` disable the layer entirely; - #: ``"assist"`` only reports proposals (never mutates); ``"review"`` applies - #: deterministic zero-risk repairs and suggests the rest; ``"auto"`` applies - #: high-confidence, low-risk, policy-approved repairs. - semantic_mode: str | None = None - #: Minimum confidence to auto-apply a semantic proposal. - semantic_auto_threshold: float = 0.95 - #: Minimum confidence below which a proposal is never applied (only skipped). - semantic_review_threshold: float = 0.70 - #: Skip semantic profiling for columns with more distinct values than this. - semantic_max_distinct_values: int = 500 - #: Cap on rows sampled when profiling distinct semantic values. - semantic_sample_size: int = 10_000 - #: Candidate-generation backends, in trust order. ``"deterministic"`` - #: (Phase-2 experts, always safe to keep first), ``"memory"`` (replay of a - #: supplied CleaningMemory), and ``"embedding"`` (optional local model via - #: the ``[semantic]`` extra; self-disables with a report note when the - #: dependency or model files are missing — never downloads anything). - semantic_backends: tuple[str, ...] = ("deterministic",) - #: Optional user-supplied semantic hints (e.g. per-column semantic_type, - #: unit, allowed_values, mutable). See the semantic layer docs. - semantic_context: object | None = None - #: Privacy posture for any future external inference; the deterministic v1 - #: never leaves the process regardless of this value. - semantic_privacy_policy: str = "local_only" - #: Optional resource budget for model-assisted backends: recognized keys - #: are ``max_columns``, ``max_distinct_values``, ``max_model_calls`` and - #: ``max_seconds``. Deterministic and memory backends are never metered; - #: an exhausted budget stops the embedding backend cleanly with a report - #: fallback event. - semantic_budget: dict[str, object] | None = None - #: In-process LRU capacity (distinct texts) for embedding vectors; 0 - #: disables caching. Embeddings only — decisions and rows are never cached, - #: and nothing is written to disk. - semantic_embedding_cache_size: int = 65_536 - - # -- Context policy compiler (deterministic, off by default) ------------ - #: Natural-language cleaning rules ("CustomerID is unique. Never modify - #: revenue."), compiled deterministically into a ContextPolicy when the - #: config meets a frame. ``None`` (default) changes nothing. - context: str | None = None - #: A pre-compiled :class:`freshdata.ContextPolicy`; mutually exclusive - #: with ``context`` (supplying a policy skips parsing entirely). - policy: object | None = None - #: Strict context mode: unresolved column references, unparsed sentences, - #: and protection conflicts raise :class:`freshdata.PolicyError` instead of - #: being surfaced as report warnings. - strict: bool = False - - def __post_init__(self) -> None: - if self.strategy not in _STRATEGY_CHOICES: - raise ValueError( - f"strategy must be one of {_STRATEGY_CHOICES}, got {self.strategy!r}" - ) - if self.strategy == "auto": - global _AUTO_DEPRECATION_WARNED # noqa: PLW0603 - if not _AUTO_DEPRECATION_WARNED: - warnings.warn( - 'strategy="auto" is deprecated; use "aggressive" for full ' - 'engine scrubbing or "balanced" (default) for accuracy-first ' - "cleaning", - DeprecationWarning, - stacklevel=3, - ) - _AUTO_DEPRECATION_WARNED = True - if self.impute not in _IMPUTE_CHOICES: - raise ValueError(f"impute must be one of {_IMPUTE_CHOICES}, got {self.impute!r}") - if self.outliers not in _OUTLIER_CHOICES: - raise ValueError(f"outliers must be one of {_OUTLIER_CHOICES}, got {self.outliers!r}") - if self.outlier_method not in _OUTLIER_METHODS: - raise ValueError( - f"outlier_method must be one of {_OUTLIER_METHODS}, got {self.outlier_method!r}" - ) - if self.outlier_action not in _OUTLIER_ACTIONS: - raise ValueError( - f"outlier_action must be one of {_OUTLIER_ACTIONS}, got {self.outlier_action!r}" - ) - if self.string_case not in _STRING_CASE_CHOICES: - raise ValueError( - f"string_case must be one of {_STRING_CASE_CHOICES}, got {self.string_case!r}" - ) - if self.duplicate_keep not in _DUPLICATE_KEEP_CHOICES: - raise ValueError( - f"duplicate_keep must be one of {_DUPLICATE_KEEP_CHOICES}, " - f"got {self.duplicate_keep!r}" - ) - if self.duplicate_ratio_action not in _DUPLICATE_RATIO_ACTIONS: - raise ValueError( - f"duplicate_ratio_action must be one of {_DUPLICATE_RATIO_ACTIONS}, " - f"got {self.duplicate_ratio_action!r}" - ) - if not isinstance(self.domain_sensitive_names, bool): - raise TypeError( - f"domain_sensitive_names must be a bool, got {self.domain_sensitive_names!r}" - ) - self._validate_tristates() - self._validate_impute_strategy() - for name in ("decimal", "thousands"): - value = getattr(self, name) - if not isinstance(value, str) or len(value) != 1: - raise ValueError(f"{name} must be a single character, got {value!r}") - if self.decimal == self.thousands: - raise ValueError( - f"decimal and thousands separators must differ, both are {self.decimal!r}" - ) - for name in ("missing_threshold_low", "missing_threshold_medium", - "missing_threshold_high", "duplicate_threshold"): - value = getattr(self, name) - if not 0.0 < value < 1.0: - raise ValueError(f"{name} must be in (0, 1), got {value!r}") - if not (self.missing_threshold_low <= self.missing_threshold_medium - <= self.missing_threshold_high): - raise ValueError( - "missing thresholds must be ordered: low <= medium <= high, got " - f"{self.missing_threshold_low!r} / {self.missing_threshold_medium!r} / " - f"{self.missing_threshold_high!r}" - ) - for name in ("numeric_threshold", "datetime_threshold", "category_threshold"): - value = getattr(self, name) - if not 0.0 < value <= 1.0: - raise ValueError(f"{name} must be in (0, 1], got {value!r}") - if self.outlier_factor is not None and not self.outlier_factor > 0: - raise ValueError(f"outlier_factor must be > 0, got {self.outlier_factor!r}") - if self.sample_size < 1: - raise ValueError(f"sample_size must be >= 1, got {self.sample_size!r}") - if self.progress_callback is not None and not callable(self.progress_callback): - raise TypeError("progress_callback must be callable") - self._validate_semantic() - self._validate_context() - self._normalize_str_fields() - - def _normalize_str_fields(self) -> None: - extra = _coerce_str_tuple(self.extra_sentinels) - if not all(isinstance(s, str) for s in extra): - raise TypeError("extra_sentinels must be strings") - for name in ("preserve_columns", "id_columns", "sensitive_columns"): - raw = _coerce_str_tuple(getattr(self, name)) - if not all(isinstance(s, str) for s in raw): - raise TypeError(f"{name} must be strings") - object.__setattr__(self, name, raw) - # Normalize user-facing conveniences onto the frozen instance. - object.__setattr__( - self, "extra_sentinels", tuple(s.casefold().strip() for s in extra) - ) - if self.duplicate_subset is not None: - object.__setattr__( - self, "duplicate_subset", _coerce_str_tuple(self.duplicate_subset) - ) - - def _validate_semantic(self) -> None: - if self.semantic_mode not in _SEMANTIC_MODE_CHOICES: - raise ValueError( - f"semantic_mode must be one of {_SEMANTIC_MODE_CHOICES}, " - f"got {self.semantic_mode!r}" - ) - for name in ("semantic_auto_threshold", "semantic_review_threshold"): - value = getattr(self, name) - if not 0.0 <= value <= 1.0: - raise ValueError(f"{name} must be in [0, 1], got {value!r}") - if self.semantic_review_threshold > self.semantic_auto_threshold: - raise ValueError( - "semantic_review_threshold must be <= semantic_auto_threshold, got " - f"{self.semantic_review_threshold!r} > {self.semantic_auto_threshold!r}" - ) - if self.semantic_max_distinct_values < 1: - raise ValueError( - "semantic_max_distinct_values must be >= 1, got " - f"{self.semantic_max_distinct_values!r}" - ) - if self.semantic_sample_size < 1: - raise ValueError( - f"semantic_sample_size must be >= 1, got {self.semantic_sample_size!r}" - ) - if self.semantic_privacy_policy not in _SEMANTIC_PRIVACY_CHOICES: - raise ValueError( - f"semantic_privacy_policy must be one of {_SEMANTIC_PRIVACY_CHOICES}, " - f"got {self.semantic_privacy_policy!r}" - ) - - def _validate_tristates(self) -> None: - for name in ( - "advanced_imputation", - "missing_indicators", - "missforest_add_indicators", - "dayfirst", - ): - if getattr(self, name) not in _TRISTATE_CHOICES: - raise ValueError( - f"{name} must be True, False, or 'auto', got {getattr(self, name)!r}" - ) - - def _validate_impute_strategy(self) -> None: - if self.impute_strategy is not None: - if not isinstance(self.impute_strategy, Mapping): - raise TypeError("impute_strategy must be a mapping of column name to strategy") - normalized: dict[str, str] = {} - for column, strategy in self.impute_strategy.items(): - if not isinstance(column, str): - raise TypeError("impute_strategy column names must be strings") - if strategy not in _IMPUTE_STRATEGY_CHOICES: - raise ValueError( - "impute_strategy values must be one of " - f"{_IMPUTE_STRATEGY_CHOICES}, got {strategy!r}" - ) - normalized[column] = strategy - object.__setattr__(self, "impute_strategy", normalized) - for name in ( - "missforest_max_iter", - "missforest_n_estimators", - "missforest_random_state", - "missforest_min_rows_for_model", - ): - value = getattr(self, name) - if not isinstance(value, int) or isinstance(value, bool): - raise TypeError(f"{name} must be an integer, got {value!r}") - if self.missforest_max_iter < 1: - raise ValueError("missforest_max_iter must be >= 1") - if self.missforest_n_estimators < 1: - raise ValueError("missforest_n_estimators must be >= 1") - if self.missforest_min_rows_for_model < 1: - raise ValueError("missforest_min_rows_for_model must be >= 1") - - def _validate_context(self) -> None: - if self.context is not None and not isinstance(self.context, str): - raise TypeError(f"context must be a string, got {type(self.context).__name__}") - if self.policy is not None: - # Lazy import: the context package is only loaded when a policy is - # actually supplied, keeping ``import freshdata`` light. - from .context.types import ContextPolicy # noqa: PLC0415 - - if not isinstance(self.policy, ContextPolicy): - raise TypeError( - "policy must be a freshdata.ContextPolicy " - f"(see fd.compile_context), got {type(self.policy).__name__}" - ) - if self.context is not None and self.policy is not None: - raise ValueError( - "context= and policy= are mutually exclusive: pass the raw text " - "to compile it here, or a pre-compiled ContextPolicy, not both" - ) - if not isinstance(self.strict, bool): - raise TypeError(f"strict must be a bool, got {self.strict!r}") - - @property - def semantic_enabled(self) -> bool: - """``True`` when ``semantic_mode`` actually engages the semantic layer.""" - return self.semantic_mode in _SEMANTIC_ACTIVE_MODES - - @property - def engine_mode(self) -> str | None: - """``"balanced"``, ``"aggressive"``, or ``None`` when engine is off.""" - if self.strategy == "conservative": - return None - if self.strategy in ("aggressive", "auto"): - return "aggressive" - return "balanced" - - -_FIELD_NAMES = frozenset(f.name for f in dataclasses.fields(CleanConfig)) -_OPTION_ALIASES = {"impute_method": "impute"} - - -def merge_options(base: CleanConfig | None, **options: object) -> CleanConfig: - """Build a config from *base* (or defaults) plus keyword overrides. - - Unknown option names raise :class:`TypeError` with a "did you mean" - suggestion, so typos never silently fall back to defaults. - """ - for alias, canonical in _OPTION_ALIASES.items(): - if alias in options: - alias_value = options.pop(alias) - if canonical in options and options[canonical] != alias_value: - raise TypeError( - f"{alias}= conflicts with {canonical}=; pass only one imputation option" - ) - options[canonical] = alias_value - unknown = sorted(set(options) - _FIELD_NAMES) - if unknown: - hints = [] - for name in unknown: - match = difflib.get_close_matches(name, _FIELD_NAMES, n=1) - hints.append(f"{name!r}" + (f" (did you mean {match[0]!r}?)" if match else "")) - raise TypeError( - f"unknown option(s): {', '.join(hints)}. " - f"Valid options: {', '.join(sorted(_FIELD_NAMES))}" - ) - if base is None: - return CleanConfig(**options) # type: ignore[arg-type] - if not isinstance(base, CleanConfig): - raise TypeError(f"config must be a CleanConfig, got {type(base).__name__}") - return dataclasses.replace(base, **options) # type: ignore[arg-type] diff --git a/src/freshdata/execution.py b/src/freshdata/execution.py new file mode 100644 index 00000000..8dcd040c --- /dev/null +++ b/src/freshdata/execution.py @@ -0,0 +1,519 @@ +# src/freshdata/execution.py +from __future__ import annotations + +from dataclasses import dataclass +from typing import Any + +import duckdb +import numpy as np +import pandas as pd +import polars as pl + +from freshdata.config import CleanConfig +from freshdata.imputation import ( + _knn_fill, + detect_identifier_column, + is_hostile_object_column, +) +from freshdata.report import CleanAction, CleanReport +from freshdata.schema_envelope import ROW_ID_SENTINEL_COL, SchemaEnvelope + + +class StreamingUnsupportedError(RuntimeError): + """Raised when strict_streaming=True and a requested stage cannot execute in Polars streaming.""" + + +@dataclass +class EngineConfig: + engine: str = "pandas" # "pandas" | "polars" | "duckdb" + streaming: bool = False + strict_streaming: bool = False + streaming_dedup_unordered: bool = False + + +def _canonicalize_pandas_string_col( + s: pd.Series, + *, + strip_whitespace: bool, + normalize_sentinels: bool, + sentinels: set[str], +) -> pd.Series: + if isinstance(s.dtype, pd.CategoricalDtype): + cats = s.cat.categories + if pd.api.types.is_string_dtype(cats.dtype) or cats.dtype == object: + new_cats = [c.strip() if isinstance(c, str) and strip_whitespace else c for c in cats] + s = s.cat.rename_categories(new_cats) + if normalize_sentinels: + to_remove = [c for c in s.cat.categories if isinstance(c, str) and c in sentinels] + if to_remove: + s = s.cat.remove_categories(to_remove) + return s + + if is_hostile_object_column(s): + if not strip_whitespace and not normalize_sentinels: + return s + + def _clean_elem(val: Any) -> Any: + if isinstance(val, str): + v = val.strip() if strip_whitespace else val + if normalize_sentinels and v in sentinels: + return np.nan + return v + return val + + return s.map(_clean_elem) + + if pd.api.types.is_string_dtype(s.dtype) or s.dtype == object: + orig_dtype = s.dtype + out = s.copy() + if strip_whitespace: + out = out.str.strip() + if normalize_sentinels: + mask = out.isin(sentinels) + na_val = pd.NA if isinstance(orig_dtype, pd.StringDtype) else np.nan + out = out.mask(mask, na_val) + return out.astype(orig_dtype) + return s + + +def _try_promote_dtypes( + df: pd.DataFrame, + config: CleanConfig, + report: CleanReport, +) -> pd.DataFrame: + if not config.fix_dtypes: + return df + + out = df.copy() + bool_map = {"true": True, "false": False, "yes": True, "no": False} + + for col in out.columns: + if col == ROW_ID_SENTINEL_COL: + continue + s = out[col] + if isinstance(s.dtype, pd.CategoricalDtype): + continue + if not (pd.api.types.is_string_dtype(s.dtype) or s.dtype == object): + continue + if is_hostile_object_column(s): + report.skipped_columns.append({ + "column": str(col), + "reason": "skipped_dtype_promotion:hostile_object_elements", + }) + continue + + non_null = s.dropna() + if len(non_null) == 0: + continue + + lowered = non_null.astype(str).str.strip().str.lower() + if set(lowered.unique()).issubset(set(bool_map.keys())): + old_dt = str(s.dtype) + mapped = s.astype("string").str.strip().str.lower().map(bool_map) + out[col] = mapped.astype("boolean") + report.dtype_changes[str(col)] = (old_dt, "boolean") + report.actions.append(CleanAction("fix_dtypes", str(col), f"Promoted {old_dt} -> boolean", len(non_null))) + continue + + coerced = pd.to_numeric(non_null.astype(str).str.strip(), errors="coerce") + if coerced.notna().all(): + old_dt = str(s.dtype) + full_num = pd.to_numeric(s.astype("string").str.strip(), errors="coerce") + vals = coerced.to_numpy(dtype=np.float64) + if np.all(np.mod(vals, 1.0) == 0.0): + out[col] = full_num.round().astype("Int64") + report.dtype_changes[str(col)] = (old_dt, "Int64") + report.actions.append(CleanAction("fix_dtypes", str(col), f"Promoted {old_dt} -> Int64", len(non_null))) + else: + out[col] = full_num.astype("Float64") + report.dtype_changes[str(col)] = (old_dt, "Float64") + report.actions.append(CleanAction("fix_dtypes", str(col), f"Promoted {old_dt} -> Float64", len(non_null))) + return out + + +def _run_imputation_and_outliers( + df: pd.DataFrame, + config: CleanConfig, + report: CleanReport, +) -> pd.DataFrame: + out = df.copy() + data_cols = [c for c in out.columns if c != ROW_ID_SENTINEL_COL] + + if config.handle_outliers and len(out) >= 4: + for col in data_cols: + if col == config.target_column: + continue + s = out[col] + if pd.api.types.is_bool_dtype(s.dtype) or isinstance(s.dtype, pd.BooleanDtype): + continue + if not pd.api.types.is_numeric_dtype(s.dtype): + continue + is_id, _ = detect_identifier_column(s, col) + if is_id: + continue + vals = s.dropna().to_numpy(dtype=np.float64, na_value=np.nan) + vals = vals[np.isfinite(vals)] + if len(vals) < 4: + continue + q25, q75 = np.percentile(vals, [25, 75]) + iqr = q75 - q25 + if iqr <= 0: + continue + low = q25 - config.outlier_iqr_multiplier * iqr + high = q75 + config.outlier_iqr_multiplier * iqr + arr = s.to_numpy(dtype=np.float64, na_value=np.nan) + outlier_mask = np.isfinite(arr) & ((arr < low) | (arr > high)) + n_out = int(np.sum(outlier_mask)) + if n_out > 0: + clipped = np.clip(arr, low, high) + if pd.api.types.is_integer_dtype(s.dtype): + clipped_s = pd.Series(np.round(clipped), index=s.index).where(s.notna(), pd.NA).astype(s.dtype) + else: + clipped_s = pd.Series(clipped, index=s.index).where(s.notna(), np.nan).astype(s.dtype) + out[col] = clipped_s + report.outliers_clipped += n_out + report.actions.append(CleanAction("handle_outliers", str(col), f"Clipped {n_out} outliers", n_out)) + + if config.impute_missing and len(out) > 0: + for col in data_cols: + if col == config.target_column: + report.skipped_columns.append({"column": str(col), "reason": "skipped_target_column"}) + continue + s = out[col] + n_missing = int(s.isna().sum()) + if n_missing == 0: + continue + + is_id, reason = detect_identifier_column(s, col) + if is_id and reason is not None: + report.skipped_columns.append({"column": str(col), "reason": reason}) + report.actions.append(CleanAction("skip_identifier", str(col), reason, 0)) + continue + + if is_hostile_object_column(s): + report.skipped_columns.append({"column": str(col), "reason": "skipped_hostile_object_imputation"}) + continue + + non_null = s.dropna() + if len(non_null) == 0: + report.skipped_columns.append({"column": str(col), "reason": "skipped_all_null_column"}) + continue + + if pd.api.types.is_numeric_dtype(s.dtype) and not ( + pd.api.types.is_bool_dtype(s.dtype) or isinstance(s.dtype, pd.BooleanDtype) + ): + if config.imputer == "knn": + filled_s, count = _knn_fill(out[data_cols], col) + out[col] = filled_s.astype(s.dtype) + else: + med = float(np.median(non_null.to_numpy(dtype=np.float64))) + fill_val: Any = int(round(med)) if pd.api.types.is_integer_dtype(s.dtype) else med + out[col] = s.fillna(fill_val).astype(s.dtype) + count = n_missing + report.values_imputed += count + report.actions.append(CleanAction("impute_missing", str(col), f"Imputed {count} numeric nulls", count)) + else: + mode_vals = non_null.mode(dropna=True) + if len(mode_vals) > 0: + # Deterministic tie-break by sorting string representation + sorted_modes = sorted(mode_vals.tolist(), key=lambda x: str(x)) + fill_val = sorted_modes[0] + out[col] = s.fillna(fill_val).astype(s.dtype) + report.values_imputed += n_missing + report.actions.append(CleanAction("impute_missing", str(col), f"Imputed {n_missing} categorical nulls", n_missing)) + + return out + + +class PandasEngine: + @staticmethod + def execute(df: pd.DataFrame, config: CleanConfig, engine_config: EngineConfig) -> tuple[pd.DataFrame, CleanReport]: + report = CleanReport( + engine_used="pandas", + rows_before=len(df), + streaming_requested=engine_config.streaming, + streaming_executed_fully=False, + row_order_deterministic=True, + ) + envelope = SchemaEnvelope.capture(df) + if df.shape[1] == 0 or df.shape[0] == 0: + restored = envelope.restore(df, fix_dtypes=config.fix_dtypes, dtype_changes=report.dtype_changes) + report.rows_after = len(restored) + return restored, report + + work = df.copy(deep=True) + sentinels = set(config.null_sentinels) + + for col in work.columns: + work[col] = _canonicalize_pandas_string_col( + work[col], + strip_whitespace=config.strip_whitespace, + normalize_sentinels=config.normalize_sentinels, + sentinels=sentinels, + ) + + work = _try_promote_dtypes(work, config, report) + + if config.drop_duplicates: + before_dedup = len(work) + keep_arg: Any = "first" if config.duplicate_keep == "any" else config.duplicate_keep + try: + work = work.drop_duplicates(subset=config.duplicate_subset, keep=keep_arg) + removed = before_dedup - len(work) + report.duplicates_removed = removed + if removed > 0: + report.actions.append(CleanAction("drop_duplicates", None, f"Removed {removed} duplicates", removed)) + except TypeError: + pass + else: + try: + dup_count = int(work.duplicated(subset=config.duplicate_subset).sum()) + if dup_count > 0: + report.warnings.append(f"Detected {dup_count} duplicate rows (drop_duplicates=False).") + except TypeError: + pass + + work = _run_imputation_and_outliers(work, config, report) + restored = envelope.restore(work, fix_dtypes=config.fix_dtypes, dtype_changes=report.dtype_changes) + report.rows_after = len(restored) + return restored, report + + +class PolarsEngine: + @staticmethod + def execute(df: pd.DataFrame, config: CleanConfig, engine_config: EngineConfig) -> tuple[pd.DataFrame, CleanReport]: + report = CleanReport( + engine_used="polars", + rows_before=len(df), + streaming_requested=engine_config.streaming, + streaming_executed_fully=engine_config.streaming, + row_order_deterministic=True, + ) + envelope = SchemaEnvelope.capture(df) + + # Inspect streaming compatibility up front + if engine_config.streaming: + if config.drop_duplicates and not engine_config.streaming_dedup_unordered: + report.non_streaming_stages.append("drop_duplicates:maintain_order") + report.streaming_executed_fully = False + if config.drop_duplicates and config.duplicate_subset is not None: + report.non_streaming_stages.append("drop_duplicates:subset") + report.streaming_executed_fully = False + if config.impute_missing and config.imputer == "knn": + report.non_streaming_stages.append("impute_missing:knn") + report.streaming_executed_fully = False + + if engine_config.strict_streaming and report.non_streaming_stages: + raise StreamingUnsupportedError( + f"Polars strict_streaming=True failed: stages {report.non_streaming_stages} " + "cannot execute inside the Polars streaming engine." + ) + + if df.shape[1] == 0 or df.shape[0] == 0: + restored = envelope.restore(df, fix_dtypes=config.fix_dtypes, dtype_changes=report.dtype_changes) + report.rows_after = len(restored) + return restored, report + + # Check for hostile object columns or categorical columns needing pre-normalization + has_hostile = any(is_hostile_object_column(df[c]) for c in df.columns) + if has_hostile: + report.fallback_events.append({ + "stage": "full_pipeline", + "reason": "hostile_python_objects_require_pandas_object_handling", + }) + res, pd_rep = PandasEngine.execute(df, config, engine_config) + pd_rep.engine_used = "polars" + pd_rep.fallback_events = report.fallback_events + return res, pd_rep + + work_pd = envelope.attach_row_ids(df) + sentinels = set(config.null_sentinels) + + # Categorical normalization before Arrow bridge + for col in df.columns: + if isinstance(df[col].dtype, pd.CategoricalDtype): + work_pd[col] = _canonicalize_pandas_string_col( + work_pd[col], + strip_whitespace=config.strip_whitespace, + normalize_sentinels=config.normalize_sentinels, + sentinels=sentinels, + ) + + lf = pl.from_pandas(work_pd).lazy() + schema = lf.collect_schema() + + # Native Polars whitespace & sentinel canonicalization + exprs: list[pl.Expr] = [] + for col_name, pl_dtype in schema.items(): + if col_name == ROW_ID_SENTINEL_COL: + continue + if pl_dtype == pl.String: + e = pl.col(col_name) + if config.strip_whitespace: + e = e.str.strip_chars() + if config.normalize_sentinels: + e = pl.when(e.is_in(list(sentinels))).then(None).otherwise(e).alias(col_name) + exprs.append(e.alias(col_name)) + if exprs: + lf = lf.with_columns(exprs) + + # Native Polars duplicate removal + data_cols = [str(c) for c in df.columns] + dedup_cols = config.duplicate_subset if config.duplicate_subset is not None else data_cols + if config.drop_duplicates: + if engine_config.streaming and engine_config.streaming_dedup_unordered: + report.row_order_deterministic = False + lf = lf.unique(subset=dedup_cols, keep="any", maintain_order=False) + else: + keep_mode = "first" if config.duplicate_keep == "any" else config.duplicate_keep + lf = lf.unique(subset=dedup_cols, keep=keep_mode, maintain_order=True) + + if engine_config.streaming: + try: + report.plan_explain = lf.explain(engine="streaming") + except (TypeError, Exception): + try: + report.plan_explain = lf.explain(streaming=True) + except Exception: + report.plan_explain = lf.explain() + try: + pl_df = lf.collect(engine="streaming") + except (TypeError, Exception): + pl_df = lf.collect() + else: + pl_df = lf.collect() + + # Convert back to pandas for schema restoration & statistical imputation parity + mid_pd = pl_df.to_pandas() + if config.drop_duplicates: + # Sort by surviving __freshdata_row_id__ so order matches pandas drop_duplicates exactly + if ROW_ID_SENTINEL_COL in mid_pd.columns and report.row_order_deterministic: + mid_pd = mid_pd.sort_values(ROW_ID_SENTINEL_COL, kind="mergesort").reset_index(drop=True) + removed = len(df) - len(mid_pd) + report.duplicates_removed = removed + if removed > 0: + report.actions.append(CleanAction("drop_duplicates", None, f"Removed {removed} duplicates", removed)) + else: + try: + dup_count = int(mid_pd[data_cols].duplicated(subset=config.duplicate_subset).sum()) + if dup_count > 0: + report.warnings.append(f"Detected {dup_count} duplicate rows (drop_duplicates=False).") + except TypeError: + pass + + # Restore pre-promotion extension types first so imputation operates on exact dtypes + mid_restored = envelope.restore(mid_pd, fix_dtypes=False, dtype_changes={}) + mid_promoted = _try_promote_dtypes(mid_restored, config, report) + final_df = _run_imputation_and_outliers(mid_promoted, config, report) + final_df = envelope.restore(final_df, fix_dtypes=config.fix_dtypes, dtype_changes=report.dtype_changes) + report.rows_after = len(final_df) + return final_df, report + + +class DuckDBEngine: + @staticmethod + def execute(df: pd.DataFrame, config: CleanConfig, engine_config: EngineConfig) -> tuple[pd.DataFrame, CleanReport]: + report = CleanReport( + engine_used="duckdb", + rows_before=len(df), + streaming_requested=engine_config.streaming, + streaming_executed_fully=False, + row_order_deterministic=True, + ) + envelope = SchemaEnvelope.capture(df) + if df.shape[1] == 0 or df.shape[0] == 0: + restored = envelope.restore(df, fix_dtypes=config.fix_dtypes, dtype_changes=report.dtype_changes) + report.rows_after = len(restored) + return restored, report + + has_hostile = any(is_hostile_object_column(df[c]) for c in df.columns) + if has_hostile: + report.fallback_events.append({ + "stage": "full_pipeline", + "reason": "hostile_python_objects_require_pandas_object_handling", + }) + res, pd_rep = PandasEngine.execute(df, config, engine_config) + pd_rep.engine_used = "duckdb" + pd_rep.fallback_events = report.fallback_events + return res, pd_rep + + work_pd = envelope.attach_row_ids(df) + sentinels = set(config.null_sentinels) + + for col in df.columns: + if isinstance(df[col].dtype, pd.CategoricalDtype): + work_pd[col] = _canonicalize_pandas_string_col( + work_pd[col], + strip_whitespace=config.strip_whitespace, + normalize_sentinels=config.normalize_sentinels, + sentinels=sentinels, + ) + + con = duckdb.connect(":memory:") + try: + con.register("input_tbl", work_pd) + select_exprs: list[str] = [] + sentinel_sql_list = ", ".join(f"'{s.replace(chr(39), chr(39)*2)}'" for s in sentinels) + + for col in work_pd.columns: + qcol = f'"{col}"' + if col == ROW_ID_SENTINEL_COL: + select_exprs.append(qcol) + continue + dt = df[col].dtype + if (pd.api.types.is_string_dtype(dt) or dt == object) and not isinstance(dt, pd.CategoricalDtype): + expr = f"TRIM({qcol})" if config.strip_whitespace else qcol + if config.normalize_sentinels: + expr = f"CASE WHEN {expr} IN ({sentinel_sql_list}) THEN NULL ELSE {expr} END" + select_exprs.append(f"{expr} AS {qcol}") + else: + select_exprs.append(qcol) + + stage1_sql = f"SELECT {', '.join(select_exprs)} FROM input_tbl" + + if config.drop_duplicates: + dedup_cols = config.duplicate_subset if config.duplicate_subset is not None else [str(c) for c in df.columns] + partition_cols = ", ".join(f'"{c}"' for c in dedup_cols) + order_dir = "DESC" if config.duplicate_keep == "last" else "ASC" + sql = f""" + WITH staged AS ({stage1_sql}), + numbered AS ( + SELECT *, + ROW_NUMBER() OVER ( + PARTITION BY {partition_cols} + ORDER BY "{ROW_ID_SENTINEL_COL}" {order_dir} + ) AS __rn + FROM staged + ) + SELECT * EXCLUDE (__rn) + FROM numbered + WHERE __rn = 1 + ORDER BY "{ROW_ID_SENTINEL_COL}" ASC + """ + else: + sql = f"{stage1_sql} ORDER BY \"{ROW_ID_SENTINEL_COL}\" ASC" + + mid_pd = con.execute(sql).df() + finally: + con.close() + + if config.drop_duplicates: + removed = len(df) - len(mid_pd) + report.duplicates_removed = removed + if removed > 0: + report.actions.append(CleanAction("drop_duplicates", None, f"Removed {removed} duplicates", removed)) + else: + data_cols = [str(c) for c in df.columns] + try: + dup_count = int(mid_pd[data_cols].duplicated(subset=config.duplicate_subset).sum()) + if dup_count > 0: + report.warnings.append(f"Detected {dup_count} duplicate rows (drop_duplicates=False).") + except TypeError: + pass + + mid_restored = envelope.restore(mid_pd, fix_dtypes=False, dtype_changes={}) + mid_promoted = _try_promote_dtypes(mid_restored, config, report) + final_df = _run_imputation_and_outliers(mid_promoted, config, report) + final_df = envelope.restore(final_df, fix_dtypes=config.fix_dtypes, dtype_changes=report.dtype_changes) + report.rows_after = len(final_df) + return final_df, report diff --git a/src/freshdata/execution/__init__.py b/src/freshdata/execution/__init__.py deleted file mode 100644 index 42b93415..00000000 --- a/src/freshdata/execution/__init__.py +++ /dev/null @@ -1,271 +0,0 @@ -"""Pluggable out-of-core execution backends for freshdata. - -This package adds Polars (LazyFrame + streaming) and DuckDB (SQL + spill-to-disk) -execution paths alongside the in-memory pandas pipeline, so ``fd.clean`` can run -on larger-than-RAM data. The pandas backend remains the reference: any step a -native backend cannot run is delegated to it, so output is unchanged. - -Public entry point: :func:`run_with_engine`, wired into :func:`freshdata.clean` -via its ``engine`` / ``output_format`` / ``engine_config`` keyword arguments. - -See `ARCHITECTURE.md `_ -for how this package fits into the overall cleaning flow. -""" - -from __future__ import annotations - -from dataclasses import replace -from typing import TYPE_CHECKING, Any - -from ._base import ExecutionEngine -from ._config import ( - FALLBACK_POLICIES, - NATIVE_HANDLE_ENGINES, - EngineConfig, - EngineSelector, - FallbackError, - FallbackWarning, - enforce_fallback_policy, -) -from ._metadata import ColumnMetadata, MetadataScanner -from ._plan import NativePlan, PlanGenerator -from ._report import record_extension_dtype_changes - -if TYPE_CHECKING: # pragma: no cover - typing only - from ..config import CleanConfig - -__all__ = [ - "EngineConfig", - "EngineSelector", - "ExecutionEngine", - "ColumnMetadata", - "MetadataScanner", - "NativePlan", - "PlanGenerator", - "FALLBACK_POLICIES", - "FallbackError", - "FallbackWarning", - "enforce_fallback_policy", - "run_with_engine", -] - - -def _is_spark_frame(frame: Any) -> bool: - from ._lazy import has_pyspark - - if not has_pyspark(): - return False - from pyspark.sql import DataFrame as SparkDataFrame - - return isinstance(frame, SparkDataFrame) - - -def _require_recorded_fallback(frame: Any, output_format: str, report: Any) -> None: - """Allow a materialized frame in place of a native handle only after a - pandas fallback that the report discloses.""" - if report is not None and report.fallback_events: - return - raise RuntimeError( - f"output_format={output_format!r} expects a native handle, but the backend " - f"returned {type(frame).__name__} without recording a pandas fallback" - ) - - -def _convert_output(frame: Any, output_format: str, report: Any = None) -> Any: - """Convert a backend-native frame to the requested output format.""" - import pandas as pd - - is_spark = _is_spark_frame(frame) - - # Native, un-materialized handles: hand the backend's own lazy object back - # untouched. The backend is responsible for *not* having collected/fetched - # it (see the DuckDB/Polars engines). We never silently materialize here. - # - # EngineConfig only pairs a handle format with the engine that produces it, - # so a pandas frame here means the backend fell back to the pandas pipeline - # (e.g. the balanced decision engine). That is returned as-is only when the - # fallback is recorded on the report (``fallback_events``); anything else is - # a bug, not a silent substitution. - if output_format == "duckdb": - try: - import duckdb - except ImportError: # pragma: no cover - guarded upstream - duckdb = None # type: ignore[assignment] - if duckdb is not None and isinstance(frame, duckdb.DuckDBPyRelation): - return frame - _require_recorded_fallback(frame, output_format, report) - return frame - if output_format == "polars-lazy": - from ._lazy import require_polars - - pl = require_polars() - if isinstance(frame, pl.LazyFrame): - return frame - if isinstance(frame, pl.DataFrame): - return frame.lazy() - _require_recorded_fallback(frame, output_format, report) - return frame - - if output_format == "spark": - if is_spark: - return frame - from ._lazy import require_pyspark - - sql = require_pyspark() - session = sql.SparkSession.builder.appName("freshdata").getOrCreate() - pdf = frame if isinstance(frame, pd.DataFrame) else frame.to_pandas() - return session.createDataFrame(pdf) - - # All non-spark formats operate on pandas/polars; collapse Spark frames first. - if is_spark: - frame = frame.toPandas() - - is_pandas = isinstance(frame, pd.DataFrame) - - if output_format == "pandas": - if is_pandas: - return frame - from ..adapters.polars import to_pandas as polars_to_pandas - - # Not frame.to_pandas(): that renders an integer column holding nulls as - # float64, which rounds values a float64 cannot represent. - return polars_to_pandas(frame) - - if output_format == "polars": - from ._lazy import require_polars - - pl = require_polars() - if is_pandas: - return pl.from_pandas(frame) - return frame # already polars - - if output_format == "arrow": - from ._lazy import require_pyarrow - - require_pyarrow("Arrow output (output_format='arrow')") - import pyarrow as pa - - if isinstance(frame, pa.Table): - return frame # fetched as Arrow by the backend (DuckDB) - if is_pandas: - return pa.Table.from_pandas(frame, preserve_index=False) - return frame.to_arrow() # polars - - raise ValueError(f"unknown output_format {output_format!r}") - - -def run_with_engine( - source: Any, - config: CleanConfig, - *, - engine: str = "pandas", - output_format: str = "pandas", - engine_config: EngineConfig | None = None, - return_report: bool = False, -) -> Any: - """Clean *source* through the selected backend. - - *config* is the usual :class:`~freshdata.CleanConfig` (the cleaning - decisions). *engine* / *output_format* / *engine_config* control execution. - Returns the cleaned frame, or ``(cleaned, report)`` when ``return_report``. - """ - if engine_config is None: - engine_config = EngineConfig(engine=engine, output_format=output_format) - - requested = engine_config.engine - resolved = engine_config.engine - if resolved == "auto": - # A native handle format can only come from its own engine. - resolved = ( - NATIVE_HANDLE_ENGINES.get(engine_config.output_format) - or EngineSelector.select(source, engine_config) - ) - engine_config = replace(engine_config, engine=resolved) - - # Semantic cleaning is scored on the pandas reference path. On a native - # engine we keep the frame native and run the semantic stage over a - # *natively extracted* distinct table (see freshdata.semantic.native) — no - # full-frame materialization. Only a non-default backend / learned profile, - # which the native distinct path does not reproduce byte-for-byte, still - # routes the whole clean through pandas with a disclosed fallback. - if ( - config.semantic_enabled - and resolved != "pandas" - and not _native_semantic_supported(config) - ): - reason = ( - "semantic cleaning with a non-default backend requires the pandas " - "in-memory path" - ) - enforce_fallback_policy(engine_config, resolved, "semantic", reason) - from ..cleaner import run_pipeline - from .backends._pandas import materialize_to_pandas - - frame = materialize_to_pandas(source) - cleaned, report = run_pipeline(frame, config) - report.backend = "pandas" - report.record_fallback(resolved, "semantic", reason) - result = _convert_output(cleaned, engine_config.output_format, report) - _finish_report(report, requested, "pandas", result) - return (result, report) if return_report else result - - backend = EngineSelector.get_engine(resolved, engine_config) - cleaned_native, report = backend.execute(source, config, engine_config) - # Run the semantic stage over the native handle unless the backend already - # fell back to pandas (in which case semantic already ran in that pipeline). - if config.semantic_enabled and report.backend == resolved and resolved in ("polars", "duckdb"): - from ..semantic.native import run_semantic_native - - cleaned_native = run_semantic_native(cleaned_native, config, report, engine=resolved) - result = _convert_output(cleaned_native, engine_config.output_format, report) - record_extension_dtype_changes(source, result, config, report) - _finish_report(report, requested, resolved, result) - return (result, report) if return_report else result - - -def _peak_rss_bytes() -> int | None: - """Process peak RSS in bytes, or ``None`` where unsupported (Windows).""" - try: - import resource - except ImportError: # pragma: no cover - Windows - return None - import sys - - peak = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss - # ru_maxrss is bytes on macOS, kilobytes on Linux. - return int(peak) if sys.platform == "darwin" else int(peak) * 1024 - - -def _finish_report(report: Any, requested: str, resolved: str, result: Any) -> None: - """Stamp execution-honesty fields on the report after output conversion.""" - import pandas as pd - - report.requested_backend = requested - if report.backend is None: - report.backend = resolved - report.peak_memory = _peak_rss_bytes() - if isinstance(result, pd.DataFrame): - report.rows_materialized = len(result) - else: - try: - import polars as pl - - if isinstance(result, pl.DataFrame): - report.rows_materialized = result.height - except ImportError: - pass - try: - import pyarrow as pa - - if isinstance(result, pa.Table): - report.rows_materialized = result.num_rows - except ImportError: - pass - # Native handles (LazyFrame / DuckDBPyRelation) and Spark frames stay None: - # nothing was pulled into memory / counting would trigger a job. - - -def _native_semantic_supported(config: CleanConfig) -> bool: - from ..semantic.native import can_run_native - - return can_run_native(config, memory=None, profile=None) diff --git a/src/freshdata/execution/_base.py b/src/freshdata/execution/_base.py deleted file mode 100644 index f4a36155..00000000 --- a/src/freshdata/execution/_base.py +++ /dev/null @@ -1,59 +0,0 @@ -"""Abstract execution-engine interface shared by all freshdata backends. - -A backend takes a *source* (a pandas/polars frame or a file path), applies the -exact same cleaning the in-memory pandas pipeline would, and returns the cleaned -data in its native representation together with a :class:`~freshdata.CleanReport`. - -The pandas backend is the reference implementation: it simply delegates to the -existing :func:`freshdata.cleaner.run_pipeline`, so its output is identical to -``fd.clean(df)``. The Polars and DuckDB backends reproduce the deterministic -"representation repair" subset natively (with projection/predicate pushdown and -streaming/spill) and transparently fall back to the pandas pipeline for the -accuracy-first decision engine and other context-dependent steps. -""" - -from __future__ import annotations - -from abc import ABC, abstractmethod -from typing import TYPE_CHECKING, Any - -if TYPE_CHECKING: # pragma: no cover - typing only - import pandas as pd - - from ..config import CleanConfig - from ..report import CleanReport - from ._config import EngineConfig - - -class ExecutionEngine(ABC): - """Base class for a freshdata execution backend.""" - - #: Short backend identifier, e.g. ``"pandas"`` / ``"polars"`` / ``"duckdb"``. - name: str = "base" - - @abstractmethod - def execute( - self, - source: Any, - config: CleanConfig, - engine_config: EngineConfig, - ) -> tuple[Any, CleanReport]: - """Clean *source* and return ``(cleaned_native, report)``. - - ``cleaned_native`` is in the backend's native format (a pandas frame for - the pandas/duckdb backends, a polars frame for the polars backend); the - caller converts it to the requested ``output_format``. The backend never - raises on a low trust score and never makes cleaning decisions that the - config did not ask for — it only executes. - """ - - def to_pandas(self, result: Any) -> pd.DataFrame: - """Convert a backend-native result to a pandas DataFrame.""" - import pandas as pd - - if isinstance(result, pd.DataFrame): - return result - to_pandas = getattr(result, "to_pandas", None) - if callable(to_pandas): - return to_pandas() - raise TypeError(f"cannot convert {type(result).__name__} to a pandas DataFrame") diff --git a/src/freshdata/execution/_config.py b/src/freshdata/execution/_config.py deleted file mode 100644 index ba955385..00000000 --- a/src/freshdata/execution/_config.py +++ /dev/null @@ -1,236 +0,0 @@ -"""Engine selection and execution configuration. - -:class:`EngineConfig` controls *how* a clean runs (which backend, output format, -streaming, memory limits) — never *what* the clean decides. The decision logic -stays in :class:`~freshdata.CleanConfig`. -""" - -from __future__ import annotations - -from dataclasses import dataclass -from typing import TYPE_CHECKING, Any - -from ._lazy import has_polars, has_pyspark - -if TYPE_CHECKING: # pragma: no cover - typing only - from ._base import ExecutionEngine - -#: Valid backend names (``"auto"`` resolves to one of the concrete backends). -ENGINE_NAMES = ("pandas", "polars", "duckdb", "spark", "freshcore", "auto") -#: Valid output formats for the cleaned frame. -#: -#: The first four **materialize** the whole cleaned result into memory before -#: returning it (``pandas``/``polars``/``spark`` eager frames, an Arrow table). -#: The last two are **native, un-materialized handles** for honest out-of-core -#: work: ``"duckdb"`` returns the un-fetched ``DuckDBPyRelation`` and -#: ``"polars-lazy"`` returns the uncollected ``LazyFrame``. With those you, the -#: caller, decide when (and whether) to pull rows into RAM — freshdata never -#: silently calls ``.fetchdf()`` / ``.collect()`` behind your back. -OUTPUT_FORMATS = ("pandas", "polars", "arrow", "spark", "duckdb", "polars-lazy") - -#: Output formats that pull the entire result into memory before returning. -MATERIALIZING_FORMATS = frozenset({"pandas", "polars", "arrow", "spark"}) -#: Output formats that hand back a native, lazy/streaming handle instead. -NATIVE_HANDLE_FORMATS = frozenset({"duckdb", "polars-lazy"}) -#: The one engine that can produce each native handle format. -NATIVE_HANDLE_ENGINES = {"duckdb": "duckdb", "polars-lazy": "polars"} -#: What to do when a native backend must delegate to the pandas reference: -#: ``"allow"`` (record silently on the report), ``"warn"`` (also emit a -#: :class:`FallbackWarning`), ``"error"`` (raise :class:`FallbackError` before -#: any pandas work runs — the strict out-of-core guarantee). -FALLBACK_POLICIES = ("allow", "warn", "error") - - -class FallbackWarning(UserWarning): - """A native backend delegated work to the pandas reference implementation.""" - - -class FallbackError(RuntimeError): - """Raised under ``fallback_policy="error"`` when a native backend would - have delegated to pandas. The message names the exact trigger.""" - - -def enforce_fallback_policy( - engine_config: EngineConfig, backend: str, step: str, reason: str -) -> None: - """Apply the configured fallback policy for a pandas delegation. - - Called by native backends *before* they run the pandas pipeline, so - ``"error"`` prevents the materialization instead of reporting it after - the fact. ``"allow"`` is a no-op (the event is still recorded on the - report by the caller). - """ - policy = engine_config.fallback_policy - if policy == "allow": - return - message = ( - f"freshdata {backend} backend fell back to pandas at step {step!r}: {reason}. " - 'strategy="conservative" with default options is the fully native path; ' - "see docs/fallback-matrix.md for what each backend runs natively." - ) - if policy == "warn": - import warnings - - warnings.warn(message, FallbackWarning, stacklevel=3) - return - raise FallbackError(message) - - -def materializes(output_format: str) -> bool: - """Return ``True`` if *output_format* loads the whole result into memory.""" - return output_format in MATERIALIZING_FORMATS - - -@dataclass -class EngineConfig: - """Execution behaviour for a single :func:`freshdata.clean` call.""" - - engine: str = "pandas" - output_format: str = "pandas" - streaming: bool = True - #: When ``streaming`` is on, keep exact full-row deduplication streaming-safe - #: by *not* forcing ``maintain_order`` on the Polars backend (the order- - #: preserving path materializes and defeats streaming). Set ``False`` to opt - #: into order-preserving dedup; freshdata then warns that it materializes. - streaming_dedup: bool = True - memory_limit_gb: float = 8.0 - #: Base directory for DuckDB spill files. Each run spills into its own - #: private (0700) subdirectory, removed when the run's connection closes. - #: ``None`` (default) uses ``$FRESHDATA_SPILL_DIR`` or the per-user cache - #: directory; an explicit directory must be owned by the current user and - #: not group/other-writable (or be sticky, like ``/tmp``), else it raises. - temp_directory: str | None = None - polars_n_threads: int | None = None - duckdb_threads: int | None = None - #: Number of shuffle partitions for the Spark backend (``None`` = Spark default). - spark_shuffle_partitions: int | None = None - #: An existing ``SparkSession`` to reuse; ``None`` uses the active/default one. - spark_session: Any = None - #: ``engine="auto"`` uses polars above this row count, duckdb above the next. - row_count_auto_threshold_polars: int = 10_000_000 - row_count_auto_threshold_duckdb: int = 100_000_000 - #: What happens when the requested native backend must delegate to pandas: - #: ``"allow"`` (default, recorded on the report), ``"warn"`` - #: (:class:`FallbackWarning`), or ``"error"`` (:class:`FallbackError` raised - #: before any pandas materialization — no silent full-frame pull into RAM). - #: Governs *unrequested* pandas execution only; it is inert when - #: ``engine="pandas"`` was asked for. - fallback_policy: str = "allow" - - def __post_init__(self) -> None: - if self.engine not in ENGINE_NAMES: - raise ValueError(f"engine must be one of {ENGINE_NAMES}, got {self.engine!r}") - if self.output_format not in OUTPUT_FORMATS: - raise ValueError( - f"output_format must be one of {OUTPUT_FORMATS}, got {self.output_format!r}" - ) - handle_engine = NATIVE_HANDLE_ENGINES.get(self.output_format) - if handle_engine is not None and self.engine not in (handle_engine, "auto"): - raise ValueError( - f"output_format={self.output_format!r} returns a native {handle_engine} " - f"handle, which engine={self.engine!r} cannot produce; use " - f"engine={handle_engine!r} or engine='auto'" - ) - if self.fallback_policy not in FALLBACK_POLICIES: - raise ValueError( - f"fallback_policy must be one of {FALLBACK_POLICIES}, " - f"got {self.fallback_policy!r}" - ) - - -def _is_parquet_path(source: Any) -> bool: - return isinstance(source, str) and source.lower().endswith((".parquet", ".pq")) - - -def _is_tabular_file_path(source: Any) -> bool: - return isinstance(source, str) and source.lower().endswith( - (".parquet", ".pq", ".csv", ".ipc", ".feather", ".arrow") - ) - - -class EngineSelector: - """Resolve ``engine="auto"`` and construct backend instances lazily.""" - - @staticmethod - def select(source: Any, config: EngineConfig) -> str: - """Return a concrete backend name for *source* under *config*. - - Spark frames stay on Spark; file paths go to DuckDB (it reads them - without loading into Python); polars frames stay on polars; Arrow tables - prefer polars (zero-copy) else duckdb; pandas frames are sized to choose - pandas / polars / duckdb by row count. - """ - if has_pyspark(): - from pyspark.sql import DataFrame as SparkDataFrame - - if isinstance(source, SparkDataFrame): - return "spark" - - if _is_parquet_path(source): - return "duckdb" - if _is_tabular_file_path(source): - return "duckdb" - - if has_polars(): - import polars as pl - - if isinstance(source, (pl.DataFrame, pl.LazyFrame)): - return "polars" - - # arrow tables / record batches: polars handles them zero-copy if present - try: - import pyarrow as pa - - if isinstance(source, (pa.Table, pa.RecordBatch)): - return "polars" if has_polars() else "duckdb" - except ImportError: - pass - - # duckdb relation - try: - import duckdb - - if isinstance(source, duckdb.DuckDBPyRelation): - return "duckdb" - except ImportError: - pass - - try: - import pandas as pd - - if isinstance(source, pd.DataFrame): - n = len(source) - if n < config.row_count_auto_threshold_polars: - return "pandas" - if n < config.row_count_auto_threshold_duckdb: - return "polars" if has_polars() else "duckdb" - return "duckdb" - except ImportError: # pragma: no cover - pandas is a hard dependency - pass - - return "pandas" - - @staticmethod - def get_engine(name: str, config: EngineConfig) -> ExecutionEngine: - """Return a backend instance, importing the backend module lazily.""" - if name == "pandas": - from .backends._pandas import PandasEngine - - return PandasEngine() - if name == "polars": - from .backends._polars import PolarsEngine - - return PolarsEngine() - if name == "duckdb": - from .backends._duckdb import DuckDBEngine - - return DuckDBEngine() - if name == "spark": - from .backends._spark import SparkEngine - - return SparkEngine() - if name == "freshcore": - from .backends._freshcore import FreshCoreEngine - - return FreshCoreEngine() - raise ValueError(f"unknown engine {name!r}; expected one of {ENGINE_NAMES}") diff --git a/src/freshdata/execution/_ingest.py b/src/freshdata/execution/_ingest.py deleted file mode 100644 index 1fb4442b..00000000 --- a/src/freshdata/execution/_ingest.py +++ /dev/null @@ -1,87 +0,0 @@ -"""Input-driven reasons a pandas source must take the pandas reference path. - -:meth:`~freshdata.execution.PlanGenerator.fallback_reason` looks only at the -config. Some *inputs* cannot enter a native engine faithfully under any config: -Arrow/Polars ingestion rejects duplicate column labels, an object column -holding mixed value types is either rejected (Polars) or silently cast to text -(DuckDB), and some pandas extension dtypes survive the round trip only as a -different — sometimes silently truncated — value. Those runs take the disclosed -pandas fallback instead, which ``fallback_policy="error"`` still blocks before -any pandas work. -""" - -from __future__ import annotations - -from typing import Any - -#: ``infer_dtype`` kinds of an object column that native ingestion cannot keep as-is. -_MIXED_KINDS = frozenset({"mixed", "mixed-integer"}) -_DUCKDB_ASCII_FOLD = str.maketrans("ABCDEFGHIJKLMNOPQRSTUVWXYZ", "abcdefghijklmnopqrstuvwxyz") - - -def pandas_ingest_fallback_reason(source: Any, engine: str | None = None) -> str | None: - """Return why *source* must be cleaned by the pandas reference, or ``None``. - - *engine* names the backend that would ingest *source*. Checks that belong - to one backend's type system only apply to that backend; omit it to run the - checks every native engine shares. - """ - import pandas as pd - from pandas.api.types import infer_dtype, is_object_dtype - - if not isinstance(source, pd.DataFrame): - return None - if engine == "polars" and len(source) > 0 and len(source.columns) == 0: - # Polars 1.36 loses these rows during ingestion. Even where ingestion - # preserves them, per-column metadata cannot recover their input count. - return "zero-column pandas source requires the pandas reference to preserve its rows" - if source.columns.duplicated().any(): - return "duplicate input column labels require the pandas reference path" - if engine == "duckdb": - # DuckDB identifiers fold ASCII case, not Unicode case. Registration - # suffixes can collide again after renaming and redirect later stages. - names = [str(label).translate(_DUCKDB_ASCII_FOLD) for label in source.columns] - if len(set(names)) != len(names): - return ( - "case-insensitive input column labels require the pandas reference " - "path to prevent DuckDB alias collisions and preserve every column" - ) - for i, dtype in enumerate(source.dtypes): - label = source.columns[i] - if is_object_dtype(dtype) and infer_dtype(source.iloc[:, i], skipna=True) in _MIXED_KINDS: - return ( - f"object column {label!r} mixes value types (e.g. numbers " - "and strings), which native ingestion would reject or cast to text" - ) - lossy = _lossy_dtype_reason(dtype, label, engine) - if lossy is not None: - return lossy - return None - - -def _lossy_dtype_reason(dtype: Any, label: Any, engine: str | None) -> str | None: - """Why *engine* cannot carry a column of *dtype* back unchanged.""" - name = str(dtype) - if name.startswith(("period[", "interval[")): - # DuckDB rejects both dtypes outright ("Data type not recognized"); - # Polars ingests a period as its raw int64 ordinal (2020-01 -> 600) and - # an interval as a {left, right} struct. - return ( - f"column {label!r} has dtype {name}: native ingestion does not carry " - "period and interval dtypes, so they require the pandas reference path" - ) - if engine != "duckdb": - return None - if name == "timedelta64[ns]": - return ( - f"timedelta column {label!r} is nanosecond-resolution: DuckDB stores " - "INTERVAL in microseconds and would truncate it, so it requires the " - "pandas reference path" - ) - if name.startswith("datetime64[") and getattr(dtype, "tz", None) is not None: - return ( - f"datetime column {label!r} is timezone-aware ({dtype.tz}): DuckDB " - "returns TIMESTAMP WITH TIME ZONE in the session time zone and at " - "microsecond resolution, so it requires the pandas reference path" - ) - return None diff --git a/src/freshdata/execution/_lazy.py b/src/freshdata/execution/_lazy.py deleted file mode 100644 index 750ad731..00000000 --- a/src/freshdata/execution/_lazy.py +++ /dev/null @@ -1,87 +0,0 @@ -"""Lazy optional-dependency import guards for the execution backends. - -The out-of-core backends depend on optional packages (``polars``, ``duckdb``, -``pyarrow``). Importing :mod:`freshdata` must never require them, so every -backend resolves its dependency through these helpers at call time and raises a -clear, install-pointing error if it is missing. -""" - -from __future__ import annotations - -from typing import Any - - -def require_polars() -> Any: - """Return the imported :mod:`polars` module or raise a helpful error.""" - try: - import polars as pl - except ImportError as exc: # pragma: no cover - exercised via message - raise ImportError( - "The Polars engine requires polars. " - "Install it with: pip install 'freshdata-cleaner[polars]'" - ) from exc - return pl - - -def require_duckdb() -> Any: - """Return the imported :mod:`duckdb` module or raise a helpful error.""" - try: - import duckdb - except ImportError as exc: # pragma: no cover - exercised via message - raise ImportError( - "The DuckDB engine requires duckdb. " - "Install it with: pip install 'freshdata-cleaner[duckdb]'" - ) from exc - return duckdb - - -def require_pyarrow(purpose: str = "This feature") -> Any: - """Return the imported :mod:`pyarrow` module or raise a helpful error. - - *purpose* names what needs pyarrow (e.g. ``"Arrow output"``) so the error - points at the feature the caller actually used. - """ - try: - import pyarrow # noqa: F401 - except ImportError as exc: - raise ImportError( - f"{purpose} requires pyarrow. " - "Install it with: pip install 'freshdata-cleaner[pyarrow]'" - ) from exc - return pyarrow - - -def require_pyspark() -> Any: - """Return the imported :mod:`pyspark.sql` module or raise a helpful error.""" - try: - from pyspark import sql as pyspark_sql - except ImportError as exc: # pragma: no cover - exercised via message - raise ImportError( - "The Spark engine requires pyspark. " - "Install it with: pip install 'freshdata-cleaner[spark]'" - ) from exc - return pyspark_sql - - -def has_pyspark() -> bool: - try: - import pyspark # noqa: F401 - except ImportError: - return False - return True - - -def has_polars() -> bool: - try: - import polars # noqa: F401 - except ImportError: - return False - return True - - -def has_duckdb() -> bool: - try: - import duckdb # noqa: F401 - except ImportError: - return False - return True diff --git a/src/freshdata/execution/_metadata.py b/src/freshdata/execution/_metadata.py deleted file mode 100644 index f0271b4e..00000000 --- a/src/freshdata/execution/_metadata.py +++ /dev/null @@ -1,240 +0,0 @@ -"""Cheap, backend-native column statistics. - -:class:`ColumnMetadata` is everything the planner and the selector need to make -decisions without materialising a dataset. Each scanner uses the cheapest path -its backend offers: pandas describe on a sample, polars lazy aggregates, one -DuckDB aggregate query, or the Parquet footer. -""" - -from __future__ import annotations - -from dataclasses import dataclass, field -from typing import TYPE_CHECKING, Any - -from ._lazy import require_duckdb, require_polars, require_pyarrow - -if TYPE_CHECKING: # pragma: no cover - typing only - import pandas as pd - -#: Above this row count, the pandas scanner samples instead of scanning fully. -_PANDAS_SAMPLE_THRESHOLD = 100_000 -_SAMPLE_FRAC = 0.10 - - -def _quote_identifier(name: str) -> str: - return '"' + name.replace('"', '""') + '"' - - -def is_duckdb_float(native_dtype: str) -> bool: - """True for DuckDB floating-point types, the only ones that can hold ``NaN``.""" - return native_dtype.upper() in ("FLOAT", "DOUBLE", "REAL", "FLOAT4", "FLOAT8") - - -def _canonical_dtype(kind: str) -> str: - """Map an arbitrary dtype string to freshdata's canonical buckets.""" - k = kind.lower() - if "int" in k: - return "int64" - if "float" in k or "double" in k or "decimal" in k: - return "float64" - if "bool" in k: - return "bool" - if "date" in k or "time" in k: - return "datetime" - if "str" in k or "utf8" in k or "object" in k or "char" in k: - return "string" - return "object" - - -@dataclass -class ColumnMetadata: - """Per-column statistics computed without a full materialisation.""" - - name: str - dtype_str: str - row_count: int - null_ratio: float - non_null_count: int - n_unique: int = -1 # -1 = not computed / unknown - is_numeric: bool = False - is_string: bool = False - sample_values: list[Any] = field(default_factory=list) - #: The backend's own type name (e.g. DuckDB ``DOUBLE``); empty when unknown. - native_dtype: str = "" - - @property - def is_empty(self) -> bool: - """True when the column holds no non-null values.""" - return self.non_null_count == 0 - - -class MetadataScanner: - """Compute :class:`ColumnMetadata` per backend, cheaply.""" - - @staticmethod - def from_pandas(df: pd.DataFrame) -> list[ColumnMetadata]: - from pandas.api.types import is_numeric_dtype, is_string_dtype - - n = len(df) - sample = df - if n > _PANDAS_SAMPLE_THRESHOLD: - sample = df.sample(frac=_SAMPLE_FRAC, random_state=0) - - out: list[ColumnMetadata] = [] - for col in df.columns: - s = df[col] - non_null = int(s.notna().sum()) - null_ratio = 0.0 if n == 0 else 1.0 - non_null / n - samp = sample[col].dropna() - try: - n_unique = int(samp.nunique()) - except TypeError: # unhashable values - n_unique = -1 - out.append( - ColumnMetadata( - name=str(col), - dtype_str=_canonical_dtype(str(s.dtype)), - row_count=n, - null_ratio=null_ratio, - non_null_count=non_null, - n_unique=n_unique, - is_numeric=bool(is_numeric_dtype(s)), - is_string=bool(is_string_dtype(s) or s.dtype == object), - sample_values=list(samp.head(5).tolist()), - ) - ) - return out - - @staticmethod - def from_polars_lazy(lf: Any) -> list[ColumnMetadata]: - """Scan a polars LazyFrame using only aggregate collects (constant memory).""" - pl = require_polars() - - schema = lf.collect_schema() - names = list(schema.names()) - if not names: - return [] - - # One aggregate pass for height + per-column null counts + n_unique. - aggs = [pl.len().alias("__height__")] - for name in names: - aggs.append(pl.col(name).null_count().alias(f"__nulls__{name}")) - aggs.append(pl.col(name).n_unique().alias(f"__nuniq__{name}")) - stats = lf.select(aggs).collect() - row = stats.row(0, named=True) - n = int(row["__height__"]) - - out: list[ColumnMetadata] = [] - for name in names: - dtype = schema[name] - nulls = int(row[f"__nulls__{name}"]) - non_null = n - nulls - null_ratio = 0.0 if n == 0 else nulls / n - out.append( - ColumnMetadata( - name=name, - dtype_str=_canonical_dtype(str(dtype)), - row_count=n, - null_ratio=null_ratio, - non_null_count=non_null, - n_unique=int(row[f"__nuniq__{name}"]), - is_numeric=dtype.is_numeric(), - is_string=(dtype == pl.Utf8), - ) - ) - return out - - @staticmethod - def from_duckdb(conn: Any, table_name: str) -> list[ColumnMetadata]: - """Scan a registered DuckDB table/view with one aggregate query (no Python scan). - - Null counts are exact, and float ``NaN`` counts as missing as it does in - pandas. ``SUMMARIZE`` is not used: its ``stddev_samp`` raises on - non-finite floats and it only reports a rounded null percentage. - """ - require_duckdb() - described = conn.execute(f"DESCRIBE {table_name}").fetchall() - columns = [(str(r[0]), str(r[1])) for r in described] - aggs = ["COUNT(*)"] - for name, native in columns: - col = _quote_identifier(name) - present = f"CASE WHEN NOT isnan({col}) THEN 1 END" if is_duckdb_float(native) else col - aggs.append(f"COUNT({present})") - aggs.append(f"approx_count_distinct({col})") - row = conn.execute(f"SELECT {', '.join(aggs)} FROM {table_name}").fetchone() - n = int(row[0]) - - out: list[ColumnMetadata] = [] - for i, (name, native) in enumerate(columns): - non_null = int(row[1 + 2 * i]) - approx_unique = row[2 + 2 * i] - canonical = _canonical_dtype(native) - out.append( - ColumnMetadata( - name=name, - dtype_str=canonical, - row_count=n, - null_ratio=0.0 if n == 0 else 1.0 - non_null / n, - non_null_count=non_null, - n_unique=int(approx_unique) if approx_unique is not None else -1, - is_numeric=canonical in ("int64", "float64"), - is_string=canonical == "string", - native_dtype=native, - ) - ) - return out - - @staticmethod - def from_parquet_path(path: str) -> list[ColumnMetadata]: - """Read the exact row count from the Parquet footer; null stats via DuckDB. - - The footer gives the row count for free (no data scan); DuckDB streams the - file for null/value statistics without loading it into Python. - """ - require_pyarrow("Reading Parquet metadata") - # ``pyarrow.parquet`` is a submodule: importing ``pyarrow`` alone does - # not expose it as an attribute. - import pyarrow.parquet as pq - - n_rows = pq.read_metadata(path).num_rows - duckdb = require_duckdb() - escaped = path.replace("'", "''") - conn = duckdb.connect() - try: - conn.execute( - f"CREATE VIEW _fd_meta AS SELECT * FROM read_parquet('{escaped}')" - ) - meta = MetadataScanner.from_duckdb(conn, "_fd_meta") - finally: - conn.close() - for m in meta: # trust the footer's exact count over DuckDB's - m.row_count = n_rows - return meta - - @staticmethod - def from_source(source: Any, engine: str) -> list[ColumnMetadata]: - """Dispatch to the right scanner for *source* given the resolved *engine*.""" - import pandas as pd - - if isinstance(source, pd.DataFrame): - return MetadataScanner.from_pandas(source) - if isinstance(source, str): - return MetadataScanner.from_parquet_path(source) - try: - import pyarrow as pa - - if isinstance(source, (pa.Table, pa.RecordBatch)): - table = source if isinstance(source, pa.Table) else pa.Table.from_batches([source]) - pl = require_polars() - return MetadataScanner.from_polars_lazy(pl.from_arrow(table).lazy()) - except ImportError: - pass - try: - pl = require_polars() - if isinstance(source, pl.LazyFrame): - return MetadataScanner.from_polars_lazy(source) - if isinstance(source, pl.DataFrame): - return MetadataScanner.from_polars_lazy(source.lazy()) - except ImportError: - pass - raise TypeError(f"cannot scan metadata for source of type {type(source).__name__}") diff --git a/src/freshdata/execution/_native_steps.py b/src/freshdata/execution/_native_steps.py deleted file mode 100644 index d5f63063..00000000 --- a/src/freshdata/execution/_native_steps.py +++ /dev/null @@ -1,126 +0,0 @@ -"""Backend-agnostic math and labels for the native impute/outlier steps. - -The Polars, DuckDB, and Spark backends compute column statistics in their own -dialect, then share the *decision* logic here so the three stay consistent with -each other and as close to the pandas reference (``steps/impute.py`` and -``steps/outliers.py``) as the backend's statistics allow. Where a backend's -quantile/percentile interpolation differs from pandas, the count of affected -rows can differ; callers record that via -:meth:`~freshdata.report.CleanReport.record_backend_difference`. -""" - -from __future__ import annotations - -import math -from typing import TYPE_CHECKING - -from ..config import _DEFAULT_FACTOR -from ..steps.outliers import _LOG_FENCE_SKEW, _MIN_LOG_FENCE_ROWS - -if TYPE_CHECKING: # pragma: no cover - typing only - from ..config import CleanConfig - -#: Outlier methods native backends compute directly. -NATIVE_OUTLIER_METHODS = ("iqr", "zscore") - - -def resolve_impute_strategy(strategy: str, *, is_numeric: bool) -> str: - """Resolve ``impute="auto"`` to ``"median"`` (numeric) or ``"mode"``. - - Mirrors ``steps/impute.py._fill_value``. - """ - if strategy == "auto": - return "median" if is_numeric else "mode" - return strategy - - -def impute_defined_for(strategy: str, *, is_numeric: bool) -> bool: - """``mean``/``median`` are only defined for numeric columns.""" - if strategy in ("mean", "median"): - return is_numeric - return True - - -def native_outlier_method(config: CleanConfig) -> str: - """The concrete native detection method (``"iqr"`` or ``"zscore"``).""" - method = config.outlier_method - return method if method in NATIVE_OUTLIER_METHODS else "iqr" - - -def native_outlier_factor(config: CleanConfig, method: str) -> float: - """Detection factor in effect for *method* (user override wins).""" - if config.outlier_factor is not None: - return config.outlier_factor - return _DEFAULT_FACTOR[method] - - -def iqr_bounds(q1: float, q3: float, factor: float) -> tuple[float, float] | None: - """Tukey fences from quartiles, or ``None`` for constant data.""" - spread = q3 - q1 - if spread is None or _isnan(spread) or spread == 0: - return None - return q1 - factor * spread, q3 + factor * spread - - -def zscore_bounds(mean: float, std: float, factor: float) -> tuple[float, float] | None: - """Mean ± k·std fences, or ``None`` for constant data.""" - if std is None or _isnan(std) or std == 0: - return None - return mean - factor * std, mean + factor * std - - -def log_widened_bounds( - lo: float, - hi: float, - *, - skew: float | None, - minimum: float | None, - log_q1: float | None, - log_q3: float | None, - n_non_null: int, - factor: float, -) -> tuple[float, float]: - """Skew-aware *capping* fences from backend aggregates; widen-only. - - Mirrors :func:`freshdata.steps.outliers.capping_bounds`: for strongly - skewed, strictly positive data, Tukey fences computed in log space replace - the raw fences where wider, so an explicit clip does not flatten a - legitimate heavy tail. All inputs are plain aggregates (skewness, min, - non-null count, quartiles of ``log(value)``) each backend computes in its - own dialect. - """ - if n_non_null < _MIN_LOG_FENCE_ROWS: - return lo, hi - values = (skew, minimum, log_q1, log_q3) - if any(v is None or _isnan(v) for v in values): - return lo, hi - assert skew is not None and minimum is not None # for mypy - assert log_q1 is not None and log_q3 is not None - if abs(skew) <= _LOG_FENCE_SKEW or minimum <= 0: - return lo, hi - spread = log_q3 - log_q1 - if _isnan(spread) or spread == 0: - return lo, hi - return ( - min(lo, math.exp(log_q1 - factor * spread)), - max(hi, math.exp(log_q3 + factor * spread)), - ) - - -def integer_safe_bounds(lo: float, hi: float, *, is_integer: bool) -> tuple[float, float]: - """Widen fences to whole numbers so integer columns stay integer.""" - if is_integer: - return float(math.floor(lo)), float(math.ceil(hi)) - return lo, hi - - -def outlier_label(method: str, factor: float) -> str: - """Human-readable detector label, matching the pandas step.""" - return f"{method}, factor {factor:g}" - - -def _isnan(value: float) -> bool: - try: - return math.isnan(value) - except (TypeError, ValueError): # pragma: no cover - defensive - return False diff --git a/src/freshdata/execution/_plan.py b/src/freshdata/execution/_plan.py deleted file mode 100644 index dd5e207d..00000000 --- a/src/freshdata/execution/_plan.py +++ /dev/null @@ -1,288 +0,0 @@ -"""Decide, without touching data, which stages a backend can run natively. - -The native backends reproduce freshdata's deterministic "representation repair" -subset plus simple imputation/outlier handling. Anything that needs the -accuracy-first decision engine or heuristic dtype inference is delegated to the -pandas pipeline. :class:`PlanGenerator` makes that split from the -:class:`~freshdata.CleanConfig` alone, so it is pure and cheap. - -The same :class:`PlanGenerator` also emits a *shared logical plan* -(:meth:`PlanGenerator.logical_plan`) — an ordered list of backend-agnostic -:class:`LogicalStep` nodes that every engine (pandas, Polars, DuckDB, Spark) -consumes. Each node carries the step name, its input/output columns, its -parameters, the audit-event schema it is expected to emit, and a fallback -policy. The pandas pipeline remains the reference implementation: a node's -``audit_schema`` mirrors exactly what ``cleaner.run_pipeline`` records. -""" - -from __future__ import annotations - -from dataclasses import dataclass, field -from typing import TYPE_CHECKING, Any - -from ..steps.columns import normalized_column_labels - -if TYPE_CHECKING: # pragma: no cover - typing only - from ..config import CleanConfig - -#: Ordered native stages, matching ``cleaner.run_pipeline`` execution order. -#: -#: The native subset is the *deterministic* part of the pipeline (representation -#: repair + structural reduction + full-row/subset dedup) plus the opt-in -#: ``impute``/``outliers`` overrides, which the Polars/DuckDB/Spark backends -#: reproduce. The accuracy-first decision engine and heuristic dtype inference -#: are evaluated by the pandas backend so results stay identical to ``fd.clean``. -NATIVE_STAGE_ORDER = ( - "column_names", - "clean_strings", - "drop_empty_columns", - "drop_empty_rows", - "drop_duplicates", - "impute", - "outliers", - "reset_index", -) - -#: Duplicate keep-policies the native dedup can express exactly. -_NATIVE_DUPLICATE_KEEP = ("first", "last") -#: Backends whose native dedup reproduces pandas *subset* keep-semantics -#: exactly (order-preserving ``unique(subset=..., keep=...)``). -_SUBSET_DEDUP_NATIVE_BACKENDS = ("polars",) -#: Outlier-detection methods the native backends compute deterministically. -#: ``"auto"``/``"isolation_forest"`` are data-dependent / model-based and run on -#: the pandas reference instead. -_NATIVE_OUTLIER_METHODS = ("iqr", "zscore") - -#: The audit-event schema each step is expected to emit, keyed by step name. -#: This is the *common audit contract*: every backend must record actions whose -#: ``step`` is one of these and whose payload matches the documented keys. The -#: keys mirror :class:`~freshdata.report.Action` fields populated by the pandas -#: reference for that step. -AUDIT_EVENT_SCHEMA: dict[str, dict[str, str]] = { - "column_names": {"step": "column_names", "column": "none", "count": "n_renamed", - "risk": "low"}, - "strip_whitespace": {"step": "strip_whitespace", "column": "str", "count": "n_trimmed", - "risk": "low"}, - "normalize_sentinels": {"step": "normalize_sentinels", "column": "str", - "count": "n_replaced", "risk": "low"}, - "drop_empty_columns": {"step": "drop_empty_columns", "column": "none", - "count": "n_dropped", "risk": "low"}, - "drop_empty_rows": {"step": "drop_empty_rows", "column": "none", "count": "n_dropped", - "risk": "low"}, - "drop_duplicates": {"step": "drop_duplicates", "column": "none", "count": "n_dropped", - "risk": "low|medium"}, - "impute": {"step": "impute", "column": "str", "count": "n_filled", "risk": "low"}, - "outliers": {"step": "outliers", "column": "str", "count": "n_outliers", "risk": "low"}, -} - - -@dataclass(frozen=True) -class LogicalStep: - """One backend-agnostic node of the shared logical cleaning plan. - - Every engine consumes the same ordered list of these. The pandas backend is - the reference: ``audit_schema`` documents exactly what ``run_pipeline`` - records for this step, and native backends must match it. - """ - - #: Step name (matches the ``step`` of the actions it emits). - name: str - #: Columns this step reads. - input_columns: tuple[str, ...] - #: Columns this step produces (renames/new flag columns reflected here). - output_columns: tuple[str, ...] - #: Step-specific parameters resolved from the config. - parameters: dict[str, Any] - #: Audit-event schema this step is expected to emit (see AUDIT_EVENT_SCHEMA). - audit_schema: dict[str, str] - #: ``"native"`` (every backend runs it) or ``"pandas"`` (delegated). - fallback_policy: str - - -@dataclass -class NativePlan: - """The result of planning a clean for a native backend.""" - - rename_map: dict[str, str] - stages: list[str] - fallback_reason: str | None = None - extra_sentinels: tuple[str, ...] = field(default_factory=tuple) - - @property - def needs_fallback(self) -> bool: - return self.fallback_reason is not None - - -class PlanGenerator: - """Build a :class:`NativePlan` for a config (no data access). - - *backend* names the engine the plan is for; a few capabilities are - backend-specific (currently: order-preserving subset dedup on Polars). - ``None`` keeps the lowest-common-denominator placement. - """ - - def __init__(self, config: CleanConfig, backend: str | None = None) -> None: - self.config = config - self.backend = backend - - def fallback_reason(self) -> str | None: - """Return why this config needs the pandas fallback, or ``None``. - - docs/fallback-matrix.md is the user-facing transcription of this - function — keep them in sync when adding/removing triggers. - """ - c = self.config - if c.engine_mode is not None: - return ( - f"strategy={c.strategy!r} runs the accuracy-first decision engine, " - "which is evaluated by the pandas backend" - ) - if c.fix_dtypes: - return "fix_dtypes uses sampled heuristics evaluated by the pandas backend" - if c.drop_constant_columns: - return "drop_constant_columns is evaluated by the pandas backend" - if c.optimize_memory: - return "optimize_memory downcasting is evaluated by the pandas backend" - if c.impute == "missforest": - return "missforest imputation uses scikit-learn and is evaluated by the pandas backend" - if c.impute_strategy: - # An empty mapping is a no-op on the pandas reference as well. - return "impute_strategy per-column overrides are evaluated by the pandas backend" - if c.outliers is not None and c.outlier_method not in _NATIVE_OUTLIER_METHODS: - return ( - f"outlier_method={c.outlier_method!r} is data-dependent / model-based " - "and is evaluated by the pandas backend" - ) - if ( - c.drop_duplicates - and c.duplicate_subset is not None - and self.backend not in _SUBSET_DEDUP_NATIVE_BACKENDS - ): - return ( - "drop_duplicates with a subset has order-sensitive keep semantics " - "evaluated by the pandas backend (Polars runs it natively)" - ) - if c.drop_duplicates and c.duplicate_keep not in _NATIVE_DUPLICATE_KEEP: - return ( - f"duplicate_keep={c.duplicate_keep!r} is evaluated by the pandas backend" - ) - return None - - def _enabled_stages(self) -> list[str]: - c = self.config - enabled = { - "column_names": c.column_names, - "clean_strings": c.strip_whitespace or c.normalize_sentinels, - "drop_empty_columns": c.drop_empty_columns, - "drop_empty_rows": c.drop_empty_rows, - # Always planned: with drop_duplicates=False (default) the stage - # only detects and reports, mirroring the pandas cleaner. - "drop_duplicates": True, - "impute": c.impute is not None or c.impute_strategy is not None, - "outliers": c.outliers is not None, - "reset_index": c.reset_index, - } - return [s for s in NATIVE_STAGE_ORDER if enabled.get(s, False)] - - def plan(self, columns: list[object]) -> NativePlan: - """Plan a clean for a frame with the given *columns*.""" - renamed = normalized_column_labels(columns) if self.config.column_names else list(columns) - rename_map = { - str(old): str(new) - for old, new in zip(columns, renamed) - if isinstance(old, str) and old != new - } - return NativePlan( - rename_map=rename_map, - stages=self._enabled_stages(), - fallback_reason=self.fallback_reason(), - extra_sentinels=tuple(self.config.extra_sentinels), - ) - - def logical_plan(self, columns: list[object]) -> list[LogicalStep]: - """Build the shared, ordered logical plan every engine consumes. - - Each node is backend-agnostic; engines translate it into their dialect. - ``fallback_policy`` is ``"native"`` for every step a native backend can - run and ``"pandas"`` for the whole plan when :meth:`fallback_reason` - fires (the decision engine, dtype inference, etc.). - """ - c = self.config - plan = self.plan(columns) - post_rename = tuple(str(plan.rename_map.get(str(col), col)) for col in columns) - global_fallback = plan.fallback_reason is not None - - def policy(step: str, native: bool = True) -> str: - return "native" if native and not global_fallback else "pandas" - - steps: list[LogicalStep] = [] - for stage in plan.stages: - if stage == "column_names": - steps.append(LogicalStep( - name="column_names", - input_columns=tuple(str(col) for col in columns), - output_columns=post_rename, - parameters={"rename_map": dict(plan.rename_map)}, - audit_schema=AUDIT_EVENT_SCHEMA["column_names"], - fallback_policy=policy("column_names"), - )) - elif stage == "clean_strings": - if c.strip_whitespace: - steps.append(self._string_step("strip_whitespace", post_rename, c)) - if c.normalize_sentinels: - steps.append(self._string_step("normalize_sentinels", post_rename, c)) - elif stage in ("drop_empty_columns", "drop_empty_rows"): - steps.append(LogicalStep( - name=stage, - input_columns=post_rename, - output_columns=post_rename, - parameters={}, - audit_schema=AUDIT_EVENT_SCHEMA[stage], - fallback_policy=policy(stage), - )) - elif stage == "drop_duplicates": - steps.append(LogicalStep( - name="drop_duplicates", - input_columns=post_rename, - output_columns=post_rename, - parameters={"keep": c.duplicate_keep, - "subset": list(c.duplicate_subset) if c.duplicate_subset else None, - "threshold": c.duplicate_threshold}, - audit_schema=AUDIT_EVENT_SCHEMA["drop_duplicates"], - fallback_policy=policy("drop_duplicates"), - )) - elif stage == "impute": - steps.append(LogicalStep( - name="impute", - input_columns=post_rename, - output_columns=post_rename, - parameters={ - "strategy": c.impute, - "column_strategy": dict(c.impute_strategy or {}), - }, - audit_schema=AUDIT_EVENT_SCHEMA["impute"], - fallback_policy=policy("impute"), - )) - elif stage == "outliers": - steps.append(LogicalStep( - name="outliers", - input_columns=post_rename, - output_columns=post_rename, - parameters={"action": c.outliers, "method": c.outlier_method, - "factor": c.outlier_factor}, - audit_schema=AUDIT_EVENT_SCHEMA["outliers"], - fallback_policy=policy("outliers", native=c.outlier_method - in _NATIVE_OUTLIER_METHODS), - )) - return steps - - def _string_step(self, name: str, cols: tuple[str, ...], c: CleanConfig) -> LogicalStep: - return LogicalStep( - name=name, - input_columns=cols, - output_columns=cols, - parameters={"extra_sentinels": list(c.extra_sentinels)} - if name == "normalize_sentinels" else {}, - audit_schema=AUDIT_EVENT_SCHEMA[name], - fallback_policy="native" if self.fallback_reason() is None else "pandas", - ) diff --git a/src/freshdata/execution/_report.py b/src/freshdata/execution/_report.py deleted file mode 100644 index 8adbd563..00000000 --- a/src/freshdata/execution/_report.py +++ /dev/null @@ -1,173 +0,0 @@ -"""Build a :class:`~freshdata.CleanReport` from a native backend run. - -Native backends record actions with ``report.add(...)`` using the *same* step -names the pandas pipeline emits (``column_names``, ``strip_whitespace``, -``normalize_sentinels``, ``drop_empty_columns``, ``drop_empty_rows``, -``drop_duplicates``, ``impute``, ``outliers``) so downstream consumers -(``freshdata.compliance``, ``freshdata.integrations``) keep working unchanged. -These helpers handle the before/after bookkeeping run_pipeline does. -""" - -from __future__ import annotations - -import time -from typing import TYPE_CHECKING, Any - -from ..report import CleanReport - -if TYPE_CHECKING: # pragma: no cover - typing only - from ..config import CleanConfig - from ._metadata import ColumnMetadata - - -def record_extension_dtype_changes( - source: Any, result: Any, config: CleanConfig, report: CleanReport -) -> None: - """Disclose native pandas-output changes without recasting data or handles.""" - backend = report.backend - if backend is None or backend not in ("polars", "duckdb"): - return - import pandas as pd - from pandas.api.types import is_dtype_equal, is_extension_array_dtype - - if not isinstance(source, pd.DataFrame) or not isinstance(result, pd.DataFrame): - return - from ._plan import PlanGenerator - - source_dtypes = source.dtypes - if not any(is_extension_array_dtype(dtype) for dtype in source_dtypes): - return - original_names = list(source.columns) - if backend == "duckdb": - import duckdb - - # Registration deduplicates case-insensitive names before the plan's - # renames. Inspect relation metadata without fetching any rows. - native_names = duckdb.from_df(source).columns - plan_names = original_names - else: - native_names = [str(label) for label in original_names] - plan_names = native_names - rename_map = PlanGenerator(config, backend=backend).plan(plan_names).rename_map - # Native stages retain source order and append derived columns. Match each - # retained source position to the actual final label: a SQL projection can - # disambiguate registration/normalization collisions a second time. - retained = [ - position for position, name in enumerate(native_names) - if rename_map.get(name, name) not in report.columns_dropped - ] - for output_position, position in enumerate(retained): - if output_position >= len(result.columns): - break - original_dtype = source_dtypes.iloc[position] - if not is_extension_array_dtype(original_dtype): - continue - output_label = str(result.columns[output_position]) - output_dtype = result.iloc[:, output_position].dtype - if not is_dtype_equal(original_dtype, output_dtype): - # Categorical repr includes actual category values. Report dtype - # names and storage, not a domain that can contain sensitive data. - before = str(original_dtype) - after = str(output_dtype) - if isinstance(original_dtype, pd.StringDtype): - before = f"string[{original_dtype.storage}]" - if isinstance(output_dtype, pd.StringDtype): - after = f"string[{output_dtype.storage}]" - if isinstance(original_dtype, pd.SparseDtype): - before = f"Sparse[{original_dtype.subtype}]" - report.record_backend_difference( - backend, - "output_dtype", - f"pandas output changed the input extension dtype/metadata " - f"from {before} to {after}", - column=output_label, - ) - - -def init_report(metadata: list[ColumnMetadata], memory_before: int = 0) -> CleanReport: - """Initialise a report's "before" counts from cheap column metadata.""" - rows_before = metadata[0].row_count if metadata else 0 - missing_before = sum(m.row_count - m.non_null_count for m in metadata) - return CleanReport( - rows_before=rows_before, - cols_before=len(metadata), - memory_before=memory_before, - missing_before=missing_before, - ) - - -def _frame_stats(frame: Any) -> tuple[int, int, int, int]: - """Return ``(rows, cols, missing_cells, memory_bytes)`` for a cleaned frame.""" - import pandas as pd - - if isinstance(frame, pd.DataFrame): - rows, cols = frame.shape - missing = int(frame.isna().sum().sum()) - memory = int(frame.memory_usage(deep=True).sum()) - return rows, cols, missing, memory - - # polars frame - try: - import polars as pl - - if isinstance(frame, pl.DataFrame): - rows, cols = frame.height, frame.width - nulls = frame.null_count() - missing = int(sum(nulls.row(0))) if cols else 0 - memory = int(frame.estimated_size()) - return rows, cols, missing, memory - except ImportError: # pragma: no cover - pass - - # arrow table (DuckDB fetches Arrow directly for output_format="arrow") - try: - import pyarrow as pa - - if isinstance(frame, pa.Table): - missing = sum(column.null_count for column in frame.columns) - return frame.num_rows, frame.num_columns, int(missing), int(frame.nbytes) - except ImportError: # pragma: no cover - pass - - raise TypeError(f"cannot compute stats for {type(frame).__name__}") - - -def finalize_report(report: CleanReport, cleaned: Any, started: float) -> CleanReport: - """Fill the report's "after" counts and duration from the cleaned frame.""" - rows, cols, missing, memory = _frame_stats(cleaned) - report.rows_after = rows - report.cols_after = cols - report.missing_after = missing - report.memory_after = memory - report.duration_seconds = time.perf_counter() - started - return report - - -def zero_column_frame(backend: str, output_format: str, report: CleanReport) -> Any: - """pandas result for a native run that dropped every column as empty. - - The pandas pipeline keeps the row count on a zero-column frame, so the native - backends return the same. Polars and Arrow frames cannot hold rows without - columns; for those output formats the difference is recorded on the report. - """ - import pandas as pd - - if output_format != "pandas": - report.record_backend_difference( - backend, "drop_empty_columns", - f"every column was empty; a zero-column {output_format} result cannot carry " - f"the row count ({report.rows_before} row(s) in pandas output)", - ) - return pd.DataFrame(index=pd.RangeIndex(report.rows_before)) - - -def finalize_report_native(report: CleanReport, started: float) -> CleanReport: - """Finalize a report whose result was returned as a native, un-materialized - handle (a DuckDB relation or a Polars ``LazyFrame``). - - The "after" counts are deliberately left uncomputed — measuring them would - force a scan/collect and defeat the out-of-core path the caller asked for. - """ - report.materialized = False - report.duration_seconds = time.perf_counter() - started - return report diff --git a/src/freshdata/execution/_spill.py b/src/freshdata/execution/_spill.py deleted file mode 100644 index 9e1ae688..00000000 --- a/src/freshdata/execution/_spill.py +++ /dev/null @@ -1,131 +0,0 @@ -"""Private, per-run spill directories for out-of-core engines. - -DuckDB writes intermediate relation data (rows of the dataset being cleaned) to -its ``temp_directory`` once a query exceeds ``memory_limit``. Those files use -fixed names and the process umask, so a directory shared between users or runs -exposes them to other local accounts and makes concurrent runs collide. - -Each run therefore spills into its own ``tempfile.mkdtemp`` directory (mode -0700), created inside a base directory that is checked for ownership and -permissions, and removed when the run's connection closes. The base is: - -1. ``EngineConfig.temp_directory`` when set explicitly; -2. otherwise ``$FRESHDATA_SPILL_DIR`` when set; -3. otherwise the per-user cache directory (``$XDG_CACHE_HOME/freshdata/spill`` - or ``~/.cache/freshdata/spill`` on Linux, ``~/Library/Caches/freshdata/spill`` - on macOS, ``%LOCALAPPDATA%\\freshdata\\spill`` on Windows); -4. and, only when that cache directory cannot be created or written, - ``tempfile.gettempdir()``. -""" - -from __future__ import annotations - -import logging -import os -import shutil -import stat -import sys -import tempfile -from typing import TYPE_CHECKING - -if TYPE_CHECKING: # pragma: no cover - typing only - from ._config import EngineConfig - -log = logging.getLogger("freshdata.execution.spill") - -#: Environment variable that overrides the default spill base directory. -SPILL_DIR_ENV = "FRESHDATA_SPILL_DIR" -_RUN_PREFIX = "freshdata_spill_" - - -class UnsafeSpillDirectoryError(PermissionError): - """A spill base directory other local users could read from or tamper with.""" - - -def _default_cache_base() -> str | None: - """The per-user cache spill directory, or ``None`` if it cannot be resolved.""" - if sys.platform == "win32": - root = os.environ.get("LOCALAPPDATA") or os.path.join( - os.path.expanduser("~"), "AppData", "Local" - ) - elif sys.platform == "darwin": - root = os.path.join(os.path.expanduser("~"), "Library", "Caches") - else: - xdg = os.environ.get("XDG_CACHE_HOME") - # The XDG spec says a relative value is invalid and must be ignored. - root = xdg if xdg and os.path.isabs(xdg) else os.path.join( - os.path.expanduser("~"), ".cache" - ) - if not os.path.isabs(root): # no resolvable home directory - return None - return os.path.join(root, "freshdata", "spill") - - -def _ensure_safe_base(path: str) -> str: - """Create *path* (mode 0700) if needed and check it is safe to spill into. - - On POSIX the directory is accepted when it is owned by the effective user - and not writable by group or other, or when it has the sticky bit (like - ``/tmp``) and is owned by the effective user or root. Anything else raises - :class:`UnsafeSpillDirectoryError` naming the owner and mode. - """ - os.makedirs(path, mode=0o700, exist_ok=True) - geteuid = getattr(os, "geteuid", None) - if os.name != "posix" or geteuid is None: - return path - info = os.stat(path) # follows symlinks: the real target is what gets checked - if not stat.S_ISDIR(info.st_mode): - raise NotADirectoryError(f"spill directory {path!r} is not a directory") - euid = geteuid() - mode = stat.S_IMODE(info.st_mode) - private = info.st_uid == euid and not mode & (stat.S_IWGRP | stat.S_IWOTH) - sticky = bool(mode & stat.S_ISVTX) and info.st_uid in (euid, 0) - if not (private or sticky): - raise UnsafeSpillDirectoryError( - f"refusing to spill into {path!r}: owned by uid {info.st_uid} with mode " - f"{oct(mode)}; a spill directory must be owned by the current user " - f"(uid {euid}) and not group/other-writable, or be a sticky directory " - "such as /tmp. Fix its permissions (chmod 700) or choose another " - "EngineConfig.temp_directory." - ) - return path - - -def _user_spill_base() -> str: - """Resolve and prepare the default spill base directory (see module docstring).""" - override = os.environ.get(SPILL_DIR_ENV) - if override: - return _ensure_safe_base(os.path.expanduser(override)) - base = _default_cache_base() - if base is not None: - try: - # Keep the freshdata cache parent private too, not just the leaf. - os.makedirs(os.path.dirname(base), mode=0o700, exist_ok=True) - base = _ensure_safe_base(base) - if os.access(base, os.W_OK | os.X_OK): - return base - except UnsafeSpillDirectoryError: - raise - except OSError as exc: - log.debug("freshdata spill: cache directory %r unusable (%s)", base, exc) - # The per-run mkdtemp child below is still 0700 inside the temp directory. - return _ensure_safe_base(tempfile.gettempdir()) - - -def create_run_spill_dir(engine_config: EngineConfig) -> str: - """Create and return a private (0700) spill directory for one run. - - The caller removes it with :func:`remove_run_spill_dir` once the engine - connection using it is closed. - """ - explicit = engine_config.temp_directory - if explicit is not None: - base = _ensure_safe_base(os.path.expanduser(os.fspath(explicit))) - else: - base = _user_spill_base() - return tempfile.mkdtemp(prefix=_RUN_PREFIX, dir=base) - - -def remove_run_spill_dir(path: str) -> None: - """Remove a run directory made by :func:`create_run_spill_dir` (best effort).""" - shutil.rmtree(path, ignore_errors=True) diff --git a/src/freshdata/execution/backends/__init__.py b/src/freshdata/execution/backends/__init__.py deleted file mode 100644 index 6ca5e599..00000000 --- a/src/freshdata/execution/backends/__init__.py +++ /dev/null @@ -1,5 +0,0 @@ -"""Concrete execution backends for freshdata. - -Backends are imported lazily by :class:`freshdata.execution.EngineSelector` so -that ``import freshdata`` never requires polars or duckdb. -""" diff --git a/src/freshdata/execution/backends/_duckdb.py b/src/freshdata/execution/backends/_duckdb.py deleted file mode 100644 index b69d2e33..00000000 --- a/src/freshdata/execution/backends/_duckdb.py +++ /dev/null @@ -1,785 +0,0 @@ -"""DuckDB backend — native out-of-core cleaning via SQL with spill-to-disk. - -Registers the source (Parquet path read in-place, or an in-memory frame via -Arrow) and applies the deterministic representation-repair + structural-reduction -subset as a staged SQL pipeline, letting DuckDB stream and spill to -a private per-run subdirectory of ``temp_directory`` (see ``execution/_spill.py``) -under ``memory_limit``. Steps outside that subset fall back to -the pandas pipeline. - -Materialization is honest and caller-controlled: with the default -``output_format="pandas"`` the result is fetched once into a pandas DataFrame, -but with ``output_format="duckdb"`` the un-fetched ``DuckDBPyRelation`` is handed -back (the connection is kept open) so larger-than-RAM results never enter memory -until the caller asks. ``report.materialized`` records which path ran. -""" - -from __future__ import annotations - -import logging -import math -import time -import weakref -from typing import TYPE_CHECKING, Any - -from ..._util import PY_WHITESPACE -from ...steps.duplicates import check_duplicate_ratio, report_detected_duplicates -from .._base import ExecutionEngine -from .._config import NATIVE_HANDLE_FORMATS, enforce_fallback_policy -from .._ingest import pandas_ingest_fallback_reason -from .._lazy import has_duckdb, has_polars, require_duckdb -from .._metadata import MetadataScanner, is_duckdb_float -from .._native_steps import ( - impute_defined_for, - integer_safe_bounds, - iqr_bounds, - log_widened_bounds, - native_outlier_factor, - native_outlier_method, - outlier_label, - resolve_impute_strategy, - zscore_bounds, -) -from .._plan import PlanGenerator -from .._report import finalize_report, finalize_report_native, init_report, zero_column_frame -from .._spill import create_run_spill_dir, remove_run_spill_dir -from ._pandas import materialize_to_pandas - -if TYPE_CHECKING: # pragma: no cover - typing only - import pandas as pd - - from ...config import CleanConfig - from ...report import CleanReport - from .._config import EngineConfig - -log = logging.getLogger("freshdata.execution.duckdb") - -_TABLE = "freshdata_source" -#: Scan-order ordinal used by order-preserving deduplication. -_ROW_ID = "__freshdata_row__" -_NATIVE_RELATION_CONNECTIONS: dict[int, Any] = {} -#: RE2 character class of every ``str.isspace`` character (RE2's ``\s`` is ASCII-only). -_WHITESPACE_CLASS = "[" + "".join(f"\\x{{{ord(c):04X}}}" for c in PY_WHITESPACE) + "]" - - -def _q(name: str) -> str: - """Quote a SQL identifier.""" - return '"' + str(name).replace('"', '""') + '"' - - -def _lit(value: str) -> str: - """Quote a SQL string literal.""" - return "'" + value.replace("'", "''") + "'" - - -def _strip_sql(col_sql: str) -> str: - """Trim leading/trailing whitespace exactly like Python ``str.strip()``.""" - return ( - f"regexp_replace(CAST({col_sql} AS VARCHAR), " - f"'^{_WHITESPACE_CLASS}+|{_WHITESPACE_CLASS}+$', '', 'g')" - ) - - -def _number_sql(value: Any) -> str: - """SQL literal for a float; ``inf``/``nan`` need a cast (bare ``inf`` is a column name).""" - number = float(value) - return f"{number}" if math.isfinite(number) else f"CAST('{number}' AS DOUBLE)" - - -def _open_spill_connection(duckdb: Any, engine_config: EngineConfig) -> tuple[Any, str]: - """Connect DuckDB with a private per-run spill directory; return ``(conn, run_dir)``. - - Spill files hold dataset rows, so they never go to a shared or pre-existing - directory (see ``execution/_spill.py``). The directory is removed again if - the connection cannot be opened. - """ - run_dir = create_run_spill_dir(engine_config) - conn_config: dict[str, Any] = { - "memory_limit": f"{engine_config.memory_limit_gb}GB", - "temp_directory": run_dir, - } - if engine_config.duckdb_threads is not None: - conn_config["threads"] = engine_config.duckdb_threads - try: - return duckdb.connect(config=conn_config), run_dir - except BaseException: - remove_run_spill_dir(run_dir) - raise - - -def _close_spill_connection(conn: Any, run_dir: str) -> None: - """Close *conn*, then remove its private spill directory.""" - try: - conn.close() - finally: - remove_run_spill_dir(run_dir) - - -def _release_native_relation_connection(key: int) -> None: - entry = _NATIVE_RELATION_CONNECTIONS.pop(key, None) - if entry is not None: - _close_spill_connection(*entry) - - -def _keep_native_relation_connection_alive(relation: Any, conn: Any, run_dir: str) -> None: - """Keep *conn* (and its spill directory) alive for as long as *relation* is. - - Releasing the relation closes the connection and then removes *run_dir*; - ``weakref.finalize`` also runs this at interpreter exit. - """ - key = id(relation) - _NATIVE_RELATION_CONNECTIONS[key] = (conn, run_dir) - weakref.finalize(relation, _release_native_relation_connection, key) - - -class DuckDBEngine(ExecutionEngine): - name = "duckdb" - - # -- execution ---------------------------------------------------------- - - def execute( - self, - source: Any, - config: CleanConfig, - engine_config: EngineConfig, - ) -> tuple[pd.DataFrame, CleanReport]: - duckdb = require_duckdb() - - plan_cols = self._peek_columns(source) - plan = PlanGenerator(config).plan(plan_cols) - reason = plan.fallback_reason or pandas_ingest_fallback_reason(source, self.name) - if reason is None and not plan_cols: - # DuckDB cannot register a frame without columns ("Need a DataFrame - # with at least one column"). The pandas reference keeps the rows and - # the index on a zero-column frame, so disclose the fallback and let - # it produce the result. - reason = "zero-column source" - if reason is None and self._pandas_index_forces_fallback(source): - reason = "pandas index semantics" - if reason is not None: - enforce_fallback_policy(engine_config, "duckdb", "pipeline", reason) - log.warning("freshdata DuckDBEngine: falling back to pandas (%s)", reason) - cleaned, report = self._fallback(source, config) - report.backend = "pandas" - report.record_fallback("duckdb", "pipeline", reason) - return cleaned, report - - started = time.perf_counter() - native = engine_config.output_format in NATIVE_HANDLE_FORMATS - - conn, run_dir = _open_spill_connection(duckdb, engine_config) - close_conn = True - try: - self._register_source(conn, source) - meta = MetadataScanner.from_duckdb(conn, _TABLE) - report = init_report(meta, self._memory_before(source)) - report.backend = "duckdb" - relation = self._run_sql_pipeline( - conn, meta, plan, config, report, materialize=False, - allow_zero_columns=not native, - ) - if relation is None: - # Every column was dropped as empty; SQL cannot select zero columns. - cleaned = zero_column_frame("duckdb", engine_config.output_format, report) - elif native: - cleaned = relation - # The relation is tied to this connection; keep it open so the - # caller can stream from it. Closing here would invalidate it. - _keep_native_relation_connection_alive(cleaned, conn, run_dir) - close_conn = False - elif engine_config.output_format == "arrow" and not config.semantic_enabled: - # Fetch straight into the requested format instead of building a - # pandas frame and converting it again (the native semantic stage - # only accepts pandas/polars/relations, so it keeps fetchdf). - to_arrow = getattr(relation, "to_arrow_table", None) - cleaned = to_arrow() if to_arrow else relation.fetch_arrow_table() - elif engine_config.output_format == "polars" and not config.semantic_enabled: - cleaned = relation.pl() - else: - cleaned = relation.fetchdf() - finally: - if close_conn: - _close_spill_connection(conn, run_dir) - - if native: - finalize_report_native(report, started) - report.add_recommendation( - "Result returned as a native DuckDB relation (un-materialized) — " - "call .fetchdf()/.arrow()/.to_df() to pull rows into memory." - ) - else: - finalize_report(report, cleaned, started) - return cleaned, report - - def _fallback(self, source: Any, config: CleanConfig) -> tuple[pd.DataFrame, CleanReport]: - from ...cleaner import run_pipeline - - df = materialize_to_pandas(source) - return run_pipeline(df, config) - - # -- source handling ---------------------------------------------------- - - def _peek_columns(self, source: Any) -> list[object]: - import pandas as pd - - if isinstance(source, pd.DataFrame): - return list(source.columns) - arrow = self._as_arrow(source) - if arrow is not None: - return list(arrow.schema.names) - if has_polars(): - import polars as pl - - if isinstance(source, pl.LazyFrame): - return list(source.collect_schema().names()) - if isinstance(source, pl.DataFrame): - return list(source.columns) - if isinstance(source, str): - duckdb = require_duckdb() - conn = duckdb.connect() - try: - self._register_source(conn, source) - return [r[0] for r in conn.execute(f"DESCRIBE {_TABLE}").fetchall()] - finally: - conn.close() - if has_duckdb(): - import duckdb as duckdb_mod - - if isinstance(source, duckdb_mod.DuckDBPyRelation): - return list(source.columns) - raise TypeError(f"DuckDBEngine: unsupported source type {type(source).__name__}") - - def _pandas_index_forces_fallback(self, source: Any) -> bool: - import pandas as pd - - return isinstance(source, pd.DataFrame) and not isinstance(source.index, pd.RangeIndex) - - @staticmethod - def _as_arrow(source: Any) -> Any: - """Return *source* as a pyarrow Table (zero-copy) or ``None``.""" - try: - import pyarrow as pa - except ImportError: - return None - if isinstance(source, pa.Table): - return source - if isinstance(source, pa.RecordBatch): - return pa.Table.from_batches([source]) - return None - - def _memory_before(self, source: Any) -> int: - import pandas as pd - - if isinstance(source, pd.DataFrame): - return int(source.memory_usage(deep=True).sum()) - if has_polars(): - import polars as pl - - if isinstance(source, pl.DataFrame): - return int(source.estimated_size()) - return 0 - - def _register_source(self, conn: Any, source: Any) -> None: - import pandas as pd - - if isinstance(source, str): - low = source.lower() - if low.endswith((".parquet", ".pq")): - conn.execute( - f"CREATE OR REPLACE VIEW {_TABLE} AS " - f"SELECT * FROM read_parquet({_lit(source)})" - ) - elif low.endswith(".csv"): - conn.execute( - f"CREATE OR REPLACE VIEW {_TABLE} AS " - f"SELECT * FROM read_csv_auto({_lit(source)})" - ) - else: - raise ValueError(f"DuckDBEngine: unsupported file type for path {source!r}") - return - if isinstance(source, pd.DataFrame): - conn.register(_TABLE, source) - return - arrow = self._as_arrow(source) - if arrow is not None: - # DuckDB scans Arrow tables natively (zero-copy, no pandas). - conn.register(_TABLE, arrow) - return - if has_polars(): - import polars as pl - - if isinstance(source, pl.LazyFrame): - source = source.collect() - if isinstance(source, pl.DataFrame): - conn.register(_TABLE, source.to_arrow()) - return - if has_duckdb(): - import duckdb - - if isinstance(source, duckdb.DuckDBPyRelation): - source.create_view(_TABLE, replace=True) - return - raise TypeError(f"DuckDBEngine: cannot register source of type {type(source).__name__}") - - # -- staged SQL pipeline ------------------------------------------------ - - def _run_sql_pipeline( - self, - conn: Any, - meta: list, - plan: Any, - config: CleanConfig, - report: CleanReport, - *, - materialize: bool = True, - allow_zero_columns: bool = True, - ) -> Any: - """Build the cleaning SQL; return a relation (or a fetched frame). - - Returns ``None`` (or, with *materialize*, a zero-column frame) when every - column was dropped as empty, since SQL cannot select zero columns. - """ - from ...steps.strings import active_sentinels - - rename = plan.rename_map - string_cols = {m.name for m in meta if m.is_string} - cur = self._source_sql(meta) - - if "column_names" in plan.stages and rename: - self._record_rename(rename, report) - - # rename + strip + sentinel in one projection - if "column_names" in plan.stages or "clean_strings" in plan.stages: - if "clean_strings" in plan.stages and string_cols: - self._record_string_counts(conn, meta, rename, config, - active_sentinels(config), report) - cur = self._project_clean(meta, rename, config, plan, active_sentinels(config), cur) - - # current column names after rename - cols = [str(rename.get(m.name, m.name)) for m in meta] - - rows_before = report.rows_before - if "drop_empty_columns" in plan.stages and rows_before > 0 and cols: - cur, cols = self._drop_empty_columns( - conn, cur, cols, report, allow_zero_columns=allow_zero_columns - ) - if not cols: - # Like the pandas pipeline, a zero-column frame keeps its rows and - # later row stages have nothing to act on. - if materialize: - return zero_column_frame("duckdb", "pandas", report) - return None - if "drop_empty_rows" in plan.stages and rows_before > 0 and cols: - cur = self._drop_empty_rows(conn, cur, cols, report) - if "drop_duplicates" in plan.stages and cols: - cur = self._drop_duplicates(conn, cur, cols, config, report) - - numeric_map = {str(rename.get(m.name, m.name)): m for m in meta} - if "impute" in plan.stages and cols: - cur = self._impute(conn, cur, cols, numeric_map, config, report) - if "outliers" in plan.stages and cols: - cur = self._outliers(conn, cur, cols, numeric_map, config, report) - - if not materialize: - # Hand back an un-fetched relation so the caller can stream/spill. - return conn.sql(cur) - return conn.execute(cur).fetchdf() - - @staticmethod - def _source_sql(meta: list) -> str: - """The registered source, with float ``NaN`` read as ``NULL``. - - pandas treats ``NaN`` as missing. DuckDB already reads a pandas ``NaN`` - as ``NULL`` but keeps real ``NaN`` from Arrow/Parquet/Polars sources, - which would otherwise count as a value in every later stage. - """ - floats = [m.name for m in meta if is_duckdb_float(m.native_dtype)] - if not floats: - return f"SELECT * FROM {_TABLE}" - replaced = ", ".join( - f"CASE WHEN isnan({_q(c)}) THEN NULL ELSE {_q(c)} END AS {_q(c)}" for c in floats - ) - return f"SELECT * REPLACE ({replaced}) FROM {_TABLE}" - - def _record_rename(self, rename: dict, report: CleanReport) -> None: - changes = list(rename.items()) - preview = ", ".join(f"{o!r}->{n!r}" for o, n in changes[:4]) - if len(changes) > 4: - preview += f", … (+{len(changes) - 4} more)" - report.add("column_names", f"renamed {len(changes)} column(s): {preview}", - count=len(changes)) - - def _project_clean( - self, meta: list, rename: dict, config: CleanConfig, plan: Any, sentinels: frozenset, - source: str, - ) -> str: - do_strings = "clean_strings" in plan.stages - sent_list = ", ".join(_lit(s) for s in sentinels) - pieces = [] - for m in meta: - target = str(rename.get(m.name, m.name)) - src = _q(m.name) - if do_strings and m.is_string: - base = _strip_sql(src) if config.strip_whitespace else src - if config.normalize_sentinels and sent_list: - expr = f"CASE WHEN LOWER({base}) IN ({sent_list}) THEN NULL ELSE {base} END" - else: - expr = base - else: - expr = src - pieces.append(f"{expr} AS {_q(target)}") - return f"SELECT {', '.join(pieces)} FROM ({source}) AS _src" - - def _record_string_counts( - self, conn: Any, meta: list, rename: dict, config: CleanConfig, - sentinels: frozenset, report: CleanReport - ) -> None: - sent_list = ", ".join(_lit(s) for s in sentinels) - aggs = [] - string_meta = [m for m in meta if m.is_string] - for i, m in enumerate(string_meta): - src = _q(m.name) - if config.strip_whitespace: - aggs.append( - f"SUM(CASE WHEN {_strip_sql(src)} <> {src} AND {src} IS NOT NULL " - f"THEN 1 ELSE 0 END) AS s{i}" - ) - base = _strip_sql(src) if config.strip_whitespace else src - if config.normalize_sentinels and sent_list: - aggs.append( - f"SUM(CASE WHEN LOWER({base}) IN ({sent_list}) AND {base} IS NOT NULL " - f"THEN 1 ELSE 0 END) AS z{i}" - ) - if not aggs: - return - row = conn.execute(f"SELECT {', '.join(aggs)} FROM {_TABLE}").fetchone() - # clean_strings runs after rename, so actions record the post-rename name. - k = 0 - for m in string_meta: - name = str(rename.get(m.name, m.name)) - if config.strip_whitespace: - n_strip = int(row[k] or 0) - k += 1 - if n_strip: - report.add("strip_whitespace", "trimmed surrounding whitespace", - column=name, count=n_strip) - if config.normalize_sentinels and sent_list: - n_sent = int(row[k] or 0) - k += 1 - if n_sent: - report.add("normalize_sentinels", - 'replaced sentinel strings ("N/A", "-", "", …) with missing', - column=name, count=n_sent) - - def _drop_empty_columns( - self, conn: Any, cur: str, cols: list[str], report: CleanReport, - *, allow_zero_columns: bool = True, - ) -> tuple[str, list[str]]: - """Drop all-missing columns; ``kept == []`` means every column was dropped.""" - counts = ", ".join(f"COUNT({_q(c)}) AS c{i}" for i, c in enumerate(cols)) - row = conn.execute(f"SELECT {counts} FROM ({cur}) AS _s").fetchone() - dropped = [c for i, c in enumerate(cols) if int(row[i]) == 0] - if not dropped: - return cur, cols - kept = [c for c in cols if c not in dropped] - if not kept and not allow_zero_columns: - report.record_backend_difference( - "duckdb", "drop_empty_columns", - "every column is empty and a DuckDB relation cannot have zero columns; " - "the all-missing columns were kept in the returned relation", - ) - return cur, cols - report.columns_dropped.extend(dropped) - report.add( - "drop_empty_columns", - f"dropped {len(dropped)} all-missing column(s): {', '.join(dropped[:6])}" - + (" …" if len(dropped) > 6 else ""), - count=len(dropped), - ) - if not kept: - return cur, kept - select_list = ", ".join(_q(c) for c in kept) - return f"SELECT {select_list} FROM ({cur}) AS _s", kept - - def _drop_empty_rows(self, conn: Any, cur: str, cols: list[str], report: CleanReport) -> str: - all_null = " AND ".join(f"{_q(c)} IS NULL" for c in cols) - (n,) = conn.execute( - f"SELECT COUNT(*) FROM ({cur}) AS _s WHERE {all_null}" - ).fetchone() - n = int(n) - if n: - report.add("drop_empty_rows", f"dropped {n} all-missing row(s)", count=n) - return f"SELECT * FROM ({cur}) AS _s WHERE NOT ({all_null})" - return cur - - def _impute( - self, conn: Any, cur: str, cols: list[str], numeric_map: dict, - config: CleanConfig, report: CleanReport - ) -> str: - strategy = config.impute - if strategy is None: - return cur - report.record_backend_difference( - "duckdb", "impute", - "median/mode fill values use duckdb aggregates and may differ from the " - "pandas reference's interpolation/tie-breaking", - ) - stat_parts: list[str] = [] - index: dict[tuple[int, str], int] = {} - - def add_stat(expr: str, i: int, kind: str) -> None: - index[(i, kind)] = len(stat_parts) - stat_parts.append(f"{expr} AS x{len(stat_parts)}") - - cplans: list[tuple[int, str, bool, str]] = [] - for i, c in enumerate(cols): - m = numeric_map.get(c) - is_numeric = bool(m and m.is_numeric) - resolved = resolve_impute_strategy(strategy, is_numeric=is_numeric) - add_stat(f"COUNT(*) - COUNT({_q(c)})", i, "nm") - add_stat(f"COUNT({_q(c)})", i, "nn") - defined = impute_defined_for(strategy, is_numeric=is_numeric) - if defined and resolved in ("mean", "median"): - agg = f"AVG({_q(c)})" if resolved == "mean" else f"quantile_cont({_q(c)}, 0.5)" - add_stat(agg, i, "av") - cplans.append((i, c, is_numeric, resolved)) - - row = conn.execute(f"SELECT {', '.join(stat_parts)} FROM ({cur}) AS _s").fetchone() - pieces: list[str] = [] - for i, c, is_numeric, resolved in cplans: - n_missing = int(row[index[(i, "nm")]] or 0) - non_null = int(row[index[(i, "nn")]] or 0) - if n_missing == 0 or non_null == 0: - pieces.append(_q(c)) - continue - if not impute_defined_for(strategy, is_numeric=is_numeric): - report.add("impute", f"skipped ({strategy} is not defined for {c})", column=c) - pieces.append(_q(c)) - continue - if resolved in ("mean", "median"): - value = row[index[(i, "av")]] - else: - value = self._mode_value(conn, cur, c) - if value is None: - report.add("impute", f"skipped (could not compute {resolved} for {c})", column=c) - pieces.append(_q(c)) - continue - lit = _number_sql(value) if is_numeric else _lit(str(value)) - pieces.append(f"COALESCE({_q(c)}, {lit}) AS {_q(c)}") - shown = f"{value:.6g}" if isinstance(value, float) else repr(value) - report.add("impute", - f"filled {n_missing} missing value(s) with {resolved} ({shown})", - column=c, count=n_missing) - report.columns_imputed.append(c) - return f"SELECT {', '.join(pieces)} FROM ({cur}) AS _imp" - - def _mode_value(self, conn: Any, cur: str, col: str) -> Any: - sql = ( - f"SELECT {_q(col)} FROM ({cur}) AS _md WHERE {_q(col)} IS NOT NULL " - f"GROUP BY {_q(col)} ORDER BY COUNT(*) DESC, {_q(col)} LIMIT 1" - ) - row = conn.execute(sql).fetchone() - return row[0] if row else None - - def _outlier_bounds( - self, conn: Any, cur: str, numeric: list[str], numeric_map: dict, - method: str, factor: float, config: CleanConfig - ) -> dict[str, tuple[float, float]]: - """Resolve integer-safe (lower, upper) fences per numeric column.""" - clip = config.outliers == "clip" - stat_parts: list[str] = [] - index: dict[tuple[str, str], int] = {} - - def add_stat(expr: str, c: str, kind: str) -> None: - index[(c, kind)] = len(stat_parts) - stat_parts.append(f"{expr} AS x{len(stat_parts)}") - - for c in numeric: - # Fences come from finite values only, like the pandas reference - # (steps.outliers.drop_infinite); stddev_samp also raises on inf. - only_finite = ( - f" FILTER (WHERE isfinite({_q(c)}))" - if is_duckdb_float(numeric_map[c].native_dtype) else "" - ) - if method == "iqr": - add_stat(f"quantile_cont({_q(c)}, 0.25){only_finite}", c, "q1") - add_stat(f"quantile_cont({_q(c)}, 0.75){only_finite}", c, "q3") - else: - add_stat(f"AVG({_q(c)}){only_finite}", c, "m") - add_stat(f"stddev_samp({_q(c)}){only_finite}", c, "s") - if clip: - # Skew-aware capping aggregates (see log_widened_bounds). - positive_log = f"CASE WHEN {_q(c)} > 0 THEN ln({_q(c)}) END" - add_stat(f"skewness({_q(c)}){only_finite}", c, "sk") - add_stat(f"MIN({_q(c)}){only_finite}", c, "mn") - add_stat(f"COUNT({_q(c)}){only_finite}", c, "cnt") - add_stat(f"quantile_cont({positive_log}, 0.25){only_finite}", c, "lq1") - add_stat(f"quantile_cont({positive_log}, 0.75){only_finite}", c, "lq3") - srow = conn.execute(f"SELECT {', '.join(stat_parts)} FROM ({cur}) AS _s").fetchone() - - def stat(c: str, kind: str) -> float | None: - value = srow[index[(c, kind)]] - return None if value is None else float(value) - - bounds: dict[str, tuple[float, float]] = {} - for c in numeric: - if method == "iqr": - q1, q3 = stat(c, "q1"), stat(c, "q3") - raw = (None if q1 is None or q3 is None - else iqr_bounds(q1, q3, factor)) - else: - m, s = stat(c, "m"), stat(c, "s") - raw = (None if m is None or s is None - else zscore_bounds(m, s, factor)) - if raw is None: - continue - if clip: - # Rewriting values never uses narrower fences than the pandas - # reference: strongly skewed positive columns widen via - # log-space IQR so legitimate heavy tails survive the clip. - raw = log_widened_bounds( - *raw, - skew=stat(c, "sk"), - minimum=stat(c, "mn"), - log_q1=stat(c, "lq1"), - log_q3=stat(c, "lq3"), - n_non_null=int(stat(c, "cnt") or 0), - factor=native_outlier_factor(config, "iqr"), - ) - bounds[c] = integer_safe_bounds( - *raw, is_integer=numeric_map[c].dtype_str == "int64" - ) - return bounds - - def _outlier_counts( - self, conn: Any, cur: str, bounds: dict[str, tuple[float, float]] - ) -> dict[str, int]: - cnt_parts: list[str] = [] - cnt_index: dict[str, int] = {} - for c, (lo, hi) in bounds.items(): - cnt_index[c] = len(cnt_parts) - cnt_parts.append( - f"SUM(CASE WHEN {_q(c)} < {lo} OR {_q(c)} > {hi} THEN 1 ELSE 0 END)" - ) - crow = conn.execute(f"SELECT {', '.join(cnt_parts)} FROM ({cur}) AS _c").fetchone() - return {c: int(crow[cnt_index[c]] or 0) for c in bounds} - - def _outliers( - self, conn: Any, cur: str, cols: list[str], numeric_map: dict, - config: CleanConfig, report: CleanReport - ) -> str: - method = native_outlier_method(config) - factor = native_outlier_factor(config, method) - numeric = [c for c in cols if numeric_map.get(c) and numeric_map[c].is_numeric] - if not numeric: - return cur - report.record_backend_difference( - "duckdb", "outliers", - "outlier counts depend on duckdb quantile/stddev statistics and may " - "differ from the pandas reference", - ) - bounds = self._outlier_bounds(conn, cur, numeric, numeric_map, method, factor, config) - if not bounds: - return cur - counts = self._outlier_counts(conn, cur, bounds) - clip = config.outliers == "clip" - existing = list(cols) - pieces: list[str] = [] - flags: list[tuple[str, str]] = [] - for c in cols: - n_out = counts.get(c, 0) - if clip and c in bounds and n_out: - lo, hi = bounds[c] - pieces.append(f"LEAST(GREATEST({_q(c)}, {lo}), {hi}) AS {_q(c)}") - report.add("outliers", - f"clipped {n_out} outlier(s) to [{lo:g}, {hi:g}] " - f"({outlier_label(method, factor)})", column=c, count=n_out) - report.outliers_handled += n_out - else: - pieces.append(_q(c)) - if not clip and c in bounds and n_out: - flag = self._unique_flag(existing, f"{c}_outlier") - existing.append(flag) - flags.append((c, flag)) - for c, flag in flags: - lo, hi = bounds[c] - n_out = counts[c] - pieces.append( - f"COALESCE({_q(c)} < {lo} OR {_q(c)} > {hi}, FALSE) AS {_q(flag)}" - ) - report.add("outliers", - f"flagged {n_out} outlier(s) in new column {flag!r} " - f"({outlier_label(method, factor)})", column=c, count=n_out) - report.outliers_handled += n_out - return f"SELECT {', '.join(pieces)} FROM ({cur}) AS _out" - - @staticmethod - def _unique_flag(existing: list[str], base: str) -> str: - names = set(existing) - name, k = base, 1 - while name in names: - k += 1 - name = f"{base}_{k}" - return name - - def _drop_duplicates( - self, conn: Any, cur: str, cols: list[str], config: CleanConfig, report: CleanReport - ) -> str: - (n_before,) = conn.execute(f"SELECT COUNT(*) FROM ({cur}) AS _s").fetchone() - n_before = int(n_before) - if n_before < 1: - return cur - if not config.drop_duplicates: - # Detection-only default: count duplicates, report, keep every row. - subset = (list(config.duplicate_subset) - if config.duplicate_subset is not None else None) - distinct = ", ".join(_q(c) for c in subset) if subset else "*" - (n_unique,) = conn.execute( - f"SELECT COUNT(*) FROM (SELECT DISTINCT {distinct} FROM ({cur}) AS _s) AS _u" - ).fetchone() - report_detected_duplicates( - n_before - int(n_unique), n_before, config, report, subset=subset - ) - return cur - (n_after,) = conn.execute( - f"SELECT COUNT(*) FROM (SELECT DISTINCT * FROM ({cur}) AS _s) AS _d" - ).fetchone() - n_dup = n_before - int(n_after) - if n_dup <= 0: - return cur - pct = 100.0 * n_dup / n_before - # Raises under duplicate_ratio_action="error", matching the pandas step. - risk = "medium" if check_duplicate_ratio(n_dup, n_before, config) else "low" - report.add( - "drop_duplicates", - f"dropped {n_dup} duplicate row(s) " - f"({pct:.1f}% of rows, keep={config.duplicate_keep!r})", - count=n_dup, - risk=risk, - ) - report.duplicates_removed += n_dup - if risk == "medium": - report.add_warning( - f"{pct:.1f}% of rows were duplicates " - f"(> {100 * config.duplicate_threshold:.0f}%); confirm they are not legitimate" - ) - return self._dedup_in_order(cur, cols, config.duplicate_keep) - - @staticmethod - def _dedup_in_order(cur: str, cols: list[str], keep: str) -> str: - """Full-row dedup that keeps the pandas row choice and row order. - - ``SELECT DISTINCT`` returns rows in arbitrary order and cannot honour - ``duplicate_keep``. A scan-order ordinal picks the first (or last) - occurrence of each row, and ordering by it restores the input order. - ``PARTITION BY`` groups ``NULL``s together, as ``DataFrame.duplicated`` does. - """ - rid = _q(_ROW_ID) - direction = "DESC" if keep == "last" else "ASC" - partition = ", ".join(_q(c) for c in cols) - numbered = f"SELECT *, row_number() OVER () AS {rid} FROM ({cur}) AS _s" - return ( - f"SELECT * EXCLUDE ({rid}) FROM ({numbered}) AS _n " - f"QUALIFY row_number() OVER (PARTITION BY {partition} ORDER BY {rid} {direction}) = 1 " - f"ORDER BY {rid}" - ) diff --git a/src/freshdata/execution/backends/_freshcore.py b/src/freshdata/execution/backends/_freshcore.py deleted file mode 100644 index dff35471..00000000 --- a/src/freshdata/execution/backends/_freshcore.py +++ /dev/null @@ -1,589 +0,0 @@ -"""FreshCore backend adapter. - -FreshCore is FreshData's optional native cleaning-first engine. The Rust module -(``freshdata_freshcore``) owns the hot path; this adapter only materializes -Python inputs/outputs, translates :class:`CleanConfig` into a physical plan, and -maps compact native audit events back onto :class:`freshdata.CleanReport`. - -If the native module is not installed, or if a config/data shape falls outside -FreshCore v1's parity boundary, the adapter delegates to the pandas reference -pipeline and records an explicit fallback event. -""" - -from __future__ import annotations - -import importlib -import math -import time -from collections import Counter -from typing import TYPE_CHECKING, Any - -import numpy as np -import pandas as pd -from pandas.api.types import ( - is_bool_dtype, - is_datetime64_any_dtype, - is_float_dtype, - is_integer_dtype, - is_numeric_dtype, - is_object_dtype, -) - -from ..._util import memory_bytes -from ...config import _DEFAULT_FACTOR -from ...report import CleanReport -from ...steps.columns import normalized_column_labels -from ...steps.duplicates import check_duplicate_ratio, report_detected_duplicates -from ...steps.strings import active_sentinels -from .._base import ExecutionEngine -from .._config import enforce_fallback_policy -from ._pandas import materialize_to_pandas - -if TYPE_CHECKING: # pragma: no cover - typing only - from collections.abc import Sequence - - from ...config import CleanConfig - from .._config import EngineConfig - -#: Largest integer magnitude float64 represents exactly (native numbers are f64). -_MAX_EXACT_FLOAT_INT = 2**53 - - -class FreshCoreEngine(ExecutionEngine): - """Optional native backend backed by the ``freshdata_freshcore`` extension.""" - - name = "freshcore" - - def supports_source(self, source: Any) -> bool: - return isinstance(source, (pd.DataFrame, str)) - - def execute( - self, - source: Any, - config: CleanConfig, - engine_config: EngineConfig, - ) -> tuple[pd.DataFrame, CleanReport]: - started = time.perf_counter() - module = self._load_native() - if module is None: - return self._fallback( - source, - config, - engine_config, - "module", - "freshdata_freshcore is not installed", - ) - - frame = materialize_to_pandas(source) - reason = self._unsupported_reason(frame, config) - if reason is not None: - return self._fallback(source, config, engine_config, "pipeline", reason) - - try: - payload = self._payload(frame, config) - native = module.execute_plan(payload) - except Exception as exc: # pragma: no cover - defensive around native boundary - return self._fallback( - source, config, engine_config, "native_error", f"FreshCore failed: {exc}" - ) - - if ( - not config.drop_duplicates - and config.duplicate_ratio_action == "error" - and native.get("duplicates_detected") is None - ): - # Detection-only dedup needs the native duplicate count to honour - # the escalation. Current native modules always report it when - # drop_duplicates is False; modules built before #323 do not, and - # without it the error could never fire. - return self._fallback( - source, - config, - engine_config, - "drop_duplicates", - 'duplicate_ratio_action="error" needs a duplicate-row count, ' - "which this FreshCore native module does not report", - ) - - cast_fallback = self._native_cast_reason(native, frame, config) - if cast_fallback is not None: - step, reason = cast_fallback - return self._fallback(source, config, engine_config, step, reason) - - cleaned, dtype_changes = self._frame_from_native(native, frame, config) - report = self._report_from_native(frame, cleaned, native, started, config) - for column, detail in dtype_changes: - report.record_backend_difference("freshcore", "dtypes", detail, column=column) - return cleaned, report - - @staticmethod - def _load_native() -> Any | None: - try: - return importlib.import_module("freshdata_freshcore") - except ImportError: - return None - - def _fallback( - self, - source: Any, - config: CleanConfig, - engine_config: EngineConfig, - step: str, - reason: str, - ) -> tuple[pd.DataFrame, CleanReport]: - from ...cleaner import run_pipeline - - enforce_fallback_policy(engine_config, "freshcore", step, reason) - frame = materialize_to_pandas(source) - cleaned, report = run_pipeline(frame, config) - report.backend = "pandas" - report.record_fallback("freshcore", step, reason) - return cleaned, report - - def _unsupported_reason(self, frame: pd.DataFrame, config: CleanConfig) -> str | None: - if config.semantic_enabled: - return "semantic cleaning requires the pandas in-memory path" - if config.context is not None or config.policy is not None: - return "context/policy protection requires the pandas in-memory path" - if config.engine_mode is not None: - return ( - f"strategy={config.strategy!r} runs the accuracy-first decision engine, " - "which FreshCore v1 delegates to pandas" - ) - if config.drop_constant_columns: - return "drop_constant_columns is not implemented in FreshCore v1" - if config.optimize_memory: - return "optimize_memory is not implemented in FreshCore v1" - # Same wording as PlanGenerator.fallback_reason() so fd.plan() and the - # recorded fallback event agree. - if config.impute == "missforest": - return "missforest imputation uses scikit-learn and is evaluated by the pandas backend" - if config.impute_strategy: - return "impute_strategy per-column overrides are evaluated by the pandas backend" - if config.duplicate_subset is not None: - return "duplicate_subset is not implemented in FreshCore v1" - if config.duplicate_keep not in ("first", "last"): - return f"duplicate_keep={config.duplicate_keep!r} is not implemented in FreshCore v1" - if config.outliers is not None and config.outlier_method not in ("iqr", "zscore"): - return f"outlier_method={config.outlier_method!r} is not implemented in FreshCore v1" - if config.outliers == "clip": - return ("skew-aware capping fences are not implemented in FreshCore v1; " - "clip requires the pandas reference path") - if config.preserve_columns or config.id_columns or config.target_column is not None: - return "protected/id/target column semantics require the pandas reference path" - if frame.columns.duplicated().any(): - return "duplicate input column labels require the pandas reference path" - colliding = self._colliding_labels(frame, config) - if colliding: - return ( - f"column labels {self._shown(colliding)} collide once stringified: " - "FreshCore v1 names columns by str(label), so distinct labels such as " - "1 and '1' require the pandas reference path" - ) - if not isinstance(frame.index, pd.RangeIndex): - return "non-default pandas index semantics require the pandas reference path" - non_scalar, kinds = self._non_scalar_dtype_columns(frame) - if non_scalar: - return ( - f"{'/'.join(kinds)} column(s) {self._shown(non_scalar)}: FreshCore v1 " - "carries only float, bool and string columns, so datetime, timedelta, " - "categorical, period and interval dtypes require the pandas reference path" - ) - wide = self._wide_integer_columns(frame) - if wide: - return ( - f"integer column(s) {self._shown(wide)} hold values beyond ±2**53: " - "FreshCore v1 carries numbers as float64, which cannot represent them " - "exactly, so they require the pandas reference path" - ) - if self._has_unsupported_object_values(frame): - return "mixed object columns with non-string values require the pandas reference path" - if config.outliers is not None: - non_finite = self._non_finite_float_columns(frame) - if non_finite: - return ( - f"non-finite values in numeric column(s) {self._shown(non_finite)}: " - "FreshCore v1 outlier fences do not exclude ±inf, so outlier " - "handling requires the pandas reference path" - ) - if config.impute in ("mode", "auto"): - bools = self._boolean_columns_to_impute(frame) - if bools: - return ( - f"missing values in boolean column(s) {self._shown(bools)}: " - "FreshCore v1 does not impute boolean columns, so imputation " - "requires the pandas reference path" - ) - return None - - @staticmethod - def _shown(columns: Sequence[object], limit: int = 5) -> str: - shown = ", ".join(repr(c) for c in columns[:limit]) - extra = len(columns) - limit - return f"{shown} (+{extra} more)" if extra > 0 else shown - - @staticmethod - def _output_labels(frame: pd.DataFrame, config: CleanConfig) -> list[object]: - """Column labels after the (optional) column-name normalization step.""" - if config.column_names: - return normalized_column_labels(frame.columns) - return list(frame.columns) - - def _colliding_labels(self, frame: pd.DataFrame, config: CleanConfig) -> list[object]: - """Distinct labels whose ``str()`` clashes before or after renaming. - - The native module keys columns by string name, so ``1`` and ``"1"`` - (or ``1`` and ``" 1 "``, which normalizes to ``"1"``) would collapse. - """ - clashing: set[int] = set() - for labels in (list(frame.columns), self._output_labels(frame, config)): - names = [str(label) for label in labels] - counts = Counter(names) - clashing.update(i for i, name in enumerate(names) if counts[name] > 1) - return [frame.columns[i] for i in sorted(clashing)] - - @staticmethod - def _non_scalar_dtype_columns(frame: pd.DataFrame) -> tuple[list[object], list[str]]: - """Columns whose dtype the native float/bool/string arrays cannot carry.""" - found: list[object] = [] - kinds: list[str] = [] - for i, col in enumerate(frame.columns): - dtype = frame.iloc[:, i].dtype - if isinstance(dtype, pd.CategoricalDtype): - kind = "categorical" - elif isinstance(dtype, pd.PeriodDtype): - kind = "period" - elif isinstance(dtype, pd.IntervalDtype): - kind = "interval" - elif is_datetime64_any_dtype(dtype): - kind = "datetime" - elif isinstance(dtype, np.dtype) and dtype.kind == "m": - kind = "timedelta" - else: - continue - found.append(col) - if kind not in kinds: - kinds.append(kind) - return found, kinds - - @staticmethod - def _wide_integer_columns(frame: pd.DataFrame) -> list[object]: - """Integer columns holding a value float64 cannot represent exactly.""" - found: list[object] = [] - for i, col in enumerate(frame.columns): - s = frame.iloc[:, i] - if is_bool_dtype(s) or not is_integer_dtype(s): - continue - info = np.iinfo(getattr(s.dtype, "numpy_dtype", s.dtype)) - if int(info.min) >= -_MAX_EXACT_FLOAT_INT and int(info.max) <= _MAX_EXACT_FLOAT_INT: - continue # int8..int32 and their unsigned widths always fit - non_null = s.dropna() - if non_null.empty: - continue - if int(non_null.max()) > _MAX_EXACT_FLOAT_INT or ( - int(non_null.min()) < -_MAX_EXACT_FLOAT_INT - ): - found.append(col) - return found - - @staticmethod - def _non_finite_float_columns(frame: pd.DataFrame) -> list[str]: - """Columns holding ±inf. Only float dtypes can store infinities.""" - found: list[str] = [] - for i, col in enumerate(frame.columns): - s = frame.iloc[:, i] - if not is_float_dtype(s): - continue - if np.isinf(s.to_numpy(dtype="float64", na_value=np.nan)).any(): - found.append(str(col)) - return found - - @staticmethod - def _boolean_columns_to_impute(frame: pd.DataFrame) -> list[str]: - """Boolean columns the pandas imputer would fill (some, not all, missing).""" - found: list[str] = [] - for i, col in enumerate(frame.columns): - s = frame.iloc[:, i] - if not is_bool_dtype(s): - continue - n_missing = int(s.isna().sum()) - if 0 < n_missing < len(s): - found.append(str(col)) - return found - - def _native_cast_reason( - self, native: dict[str, Any], frame: pd.DataFrame, config: CleanConfig - ) -> tuple[str, str] | None: - """Fallback ``(step, reason)`` when a native cast built a column the kernels mishandle. - - ``_unsupported_reason`` sees only the input dtypes, but the native - ``fix_dtypes`` stage casts text columns before imputation and outlier - detection run. A text column cast to ``bool`` or ``float`` has the same - gaps as a boolean or ±inf input column: missing booleans are never - imputed, and ±inf leaves the outlier fences undefined. The returned - column dtypes show which columns were cast, so only those are scanned, - and only when the configured step would reach the gap. - """ - check_bools = config.impute in ("mode", "auto") - check_inf = config.outliers is not None - if not config.fix_dtypes or not (check_bools or check_inf): - return None - sources = { - str(label): (label, frame.iloc[:, i]) - for i, label in enumerate(self._output_labels(frame, config)) - } - bools: list[str] = [] - infinite: list[str] = [] - for column in native.get("columns", []): - source = sources.get(column["name"]) - if source is None: # e.g. an outlier flag column added natively - continue - label, series = source - dtype = column.get("dtype") - values = column["values"] - if check_bools and dtype == "bool" and not is_bool_dtype(series): - n_missing = values.count(None) - if 0 < n_missing < len(values): - bools.append(str(label)) - elif check_inf and dtype == "float" and not is_numeric_dtype(series): - floats = np.asarray(values, dtype="float64") # None -> NaN - if np.isinf(floats).any(): - infinite.append(str(label)) - if bools: - return "impute", ( - f"text column(s) {self._shown(bools)} were cast to boolean by FreshCore v1 " - "and still hold missing values: FreshCore v1 does not impute boolean " - "columns, so imputation requires the pandas reference path" - ) - if infinite: - return "outliers", ( - f"text column(s) {self._shown(infinite)} were cast to float64 by FreshCore v1 " - "and hold ±inf: FreshCore v1 outlier fences do not exclude ±inf, so outlier " - "handling requires the pandas reference path" - ) - return None - - @staticmethod - def _has_unsupported_object_values(frame: pd.DataFrame) -> bool: - for col in frame.columns: - s = frame[col] - if not is_object_dtype(s): - continue - non_null = s.dropna() - if not non_null.map(lambda v: isinstance(v, str)).all(): - return True - return False - - def _payload(self, frame: pd.DataFrame, config: CleanConfig) -> dict[str, Any]: - renamed = self._output_labels(frame, config) - rename_map = [ - (str(old), str(new)) - for old, new in zip(frame.columns, renamed) - if isinstance(old, str) and old != new - ] - return { - "columns": [ - self._column_payload(str(name), frame.iloc[:, i]) - for i, name in enumerate(frame.columns) - ], - "config": { - "rename_map": rename_map, - "strip_whitespace": config.strip_whitespace, - "normalize_sentinels": config.normalize_sentinels, - "sentinels": sorted(active_sentinels(config)), - "string_case": config.string_case, - "drop_empty_columns": config.drop_empty_columns, - "drop_empty_rows": config.drop_empty_rows, - "drop_duplicates": config.drop_duplicates, - "duplicate_keep": config.duplicate_keep, - "fix_dtypes": config.fix_dtypes, - "numeric_threshold": config.numeric_threshold, - "preserve_leading_zeros": config.preserve_leading_zeros, - "impute": config.impute, - "outliers": config.outliers, - "outlier_method": config.outlier_method, - "outlier_factor": ( - config.outlier_factor - if config.outlier_factor is not None - else _DEFAULT_FACTOR[config.outlier_method] - ), - }, - } - - @staticmethod - def _column_payload(name: str, series: pd.Series) -> dict[str, Any]: - if is_bool_dtype(series): - bool_values: list[bool | None] = [ - None if pd.isna(v) else bool(v) for v in series.tolist() - ] - return {"name": name, "dtype": "bool", "values": bool_values} - if is_numeric_dtype(series): - float_values: list[float | None] = [ - None if pd.isna(v) else float(v) for v in series.tolist() - ] - return {"name": name, "dtype": "float", "values": float_values} - string_values: list[str | None] = [ - None if pd.isna(v) else str(v) for v in series.tolist() - ] - return {"name": name, "dtype": "string", "values": string_values} - - def _frame_from_native( - self, - native: dict[str, Any], - original: pd.DataFrame | None = None, - config: CleanConfig | None = None, - ) -> tuple[pd.DataFrame, list[tuple[str, str]]]: - """Rebuild the cleaned frame and list the dtype changes it could not undo. - - Native column names are ``str(label)``; with *original* and *config* - they are mapped back to the original (possibly renamed) labels, and - integer columns are cast back to their input dtype when possible. - """ - sources: dict[str, tuple[object, pd.Series]] = {} - if original is not None and config is not None: - for i, label in enumerate(self._output_labels(original, config)): - sources[str(label)] = (label, original.iloc[:, i]) - labels: list[object] = [] - data: dict[int, Any] = {} - changes: list[tuple[str, str]] = [] - for position, column in enumerate(native["columns"]): - name = column["name"] - values = column["values"] - dtype = column.get("dtype") - label, source = sources.get(name, (name, None)) - labels.append(label) - if dtype == "bool": - data[position] = pd.Series(values, dtype="boolean") - elif dtype == "float" and source is not None and is_integer_dtype(source): - restored, detail = self._restore_integer(values, source.dtype) - data[position] = restored - if detail is not None: - changes.append((str(label), detail)) - else: - data[position] = values - cleaned = pd.DataFrame(data) - cleaned.columns = pd.Index(labels) - return cleaned, changes - - @staticmethod - def _restore_integer(values: list[Any], dtype: Any) -> tuple[Any, str | None]: - """Cast native float values back to the input integer *dtype* when exact.""" - nullable = not isinstance(dtype, np.dtype) - info = np.iinfo(getattr(dtype, "numpy_dtype", dtype)) - ints: list[int | None] = [] - problem: str | None = None - for v in values: - if v is None or (isinstance(v, float) and math.isnan(v)): - if not nullable: - problem = "missing values, which a non-nullable integer dtype cannot hold" - break - ints.append(None) - elif not (math.isfinite(v) and float(v).is_integer()): - problem = "non-integral values" - break - elif not int(info.min) <= int(v) <= int(info.max): - problem = f"values outside the {dtype} range" - break - else: - ints.append(int(v)) - if problem is None: - if nullable: - return pd.array(ints, dtype=dtype), None - return np.array(ints, dtype=dtype), None - detail = ( - f"FreshCore v1 returned {problem} for this {dtype} input column, " - "so it comes back as float64" - ) - return pd.Series(values, dtype="float64"), detail - - def _report_from_native( - self, - original: pd.DataFrame, - cleaned: pd.DataFrame, - native: dict[str, Any], - started: float, - config: CleanConfig, - ) -> CleanReport: - report = CleanReport( - rows_before=int(native.get("rows_before", len(original))), - rows_after=int(native.get("rows_after", len(cleaned))), - cols_before=int(native.get("cols_before", original.shape[1])), - cols_after=int(native.get("cols_after", cleaned.shape[1])), - memory_before=memory_bytes(original), - memory_after=memory_bytes(cleaned), - missing_before=int(native.get("missing_before", original.isna().sum().sum())), - missing_after=int(native.get("missing_after", cleaned.isna().sum().sum())), - duration_seconds=time.perf_counter() - started, - ) - report.backend = "freshcore" - report.duplicates_removed = int(native.get("duplicates_removed", 0)) - report.outliers_handled = int(native.get("outliers_handled", 0)) - report.columns_dropped.extend(str(c) for c in native.get("columns_dropped", [])) - report.columns_imputed.extend(str(c) for c in native.get("columns_imputed", [])) - detection_reported = False - for action in native.get("actions", []): - step = str(action["step"]) - count = int(action.get("count", 0)) - if not detection_reported and step in ("impute", "outliers"): - # The pandas step records detection before imputation/outliers. - self._report_detected_duplicates(native, config, report) - detection_reported = True - high_ratio = False - if step == "drop_duplicates" and config.drop_duplicates: - # Raises under duplicate_ratio_action="error", like the pandas step. - high_ratio = check_duplicate_ratio( - count, self._rows_entering_dedup(report), config - ) - report.add( - step, - str(action["description"]), - column=action.get("column"), - count=count, - risk="medium" if high_ratio else "low", - ) - if high_ratio: - n_before = self._rows_entering_dedup(report) - report.add_warning( - f"duplicate ratio {100.0 * count / n_before:.1f}% exceeds " - f"duplicate_threshold ({100 * config.duplicate_threshold:.0f}%); " - "check for an upstream join or export problem" - ) - report.add_recommendation( - "review why so many rows were duplicated before trusting downstream stats" - ) - if not detection_reported: - self._report_detected_duplicates(native, config, report) - for stage, seconds in native.get("stage_timings", []): - report.record_stage_timing("freshcore", str(stage), float(seconds)) - if any(a.step == "fix_dtypes" for a in report.actions): - report.record_backend_difference( - "freshcore", - "fix_dtypes", - "FreshCore v1 casts booleans and numeric-looking strings natively; " - "date-like strings are profiled but left to pandas fallback when exact " - "datetime dtype parity is required.", - ) - return report - - @staticmethod - def _rows_entering_dedup(report: CleanReport) -> int: - """Rows reaching the dedup stage: the pandas step runs after empty-row removal.""" - dropped = sum(a.count for a in report.actions if a.step == "drop_empty_rows") - return report.rows_before - dropped - - def _report_detected_duplicates( - self, native: dict[str, Any], config: CleanConfig, report: CleanReport - ) -> None: - """Detection-only duplicate reporting from the native duplicate count. - - Skipped when the native module does not report ``duplicates_detected`` - (``execute`` falls back first if ``duplicate_ratio_action="error"``). - """ - if config.drop_duplicates: - return - detected = native.get("duplicates_detected") - if detected is None: - return - report_detected_duplicates( - int(detected), self._rows_entering_dedup(report), config, report - ) diff --git a/src/freshdata/execution/backends/_pandas.py b/src/freshdata/execution/backends/_pandas.py deleted file mode 100644 index f36da1a0..00000000 --- a/src/freshdata/execution/backends/_pandas.py +++ /dev/null @@ -1,74 +0,0 @@ -"""Pandas backend — the reference implementation. - -It delegates to the existing :func:`freshdata.cleaner.run_pipeline`, so its -output is byte-for-byte identical to ``fd.clean(df)``. The Polars and DuckDB -backends fall back to this engine for any step they cannot run natively, which -is how parity is guaranteed. -""" - -from __future__ import annotations - -from typing import TYPE_CHECKING, Any - -from .._base import ExecutionEngine - -if TYPE_CHECKING: # pragma: no cover - typing only - import pandas as pd - - from ...config import CleanConfig - from ...report import CleanReport - from .._config import EngineConfig - - -from .._lazy import has_polars - - -def materialize_to_pandas(source: Any) -> pd.DataFrame: - """Load *source* into a pandas DataFrame, reading file paths if needed.""" - import pandas as pd - - if isinstance(source, pd.DataFrame): - return source - if isinstance(source, str): - low = source.lower() - if low.endswith((".parquet", ".pq")): - return pd.read_parquet(source) - if low.endswith(".csv"): - return pd.read_csv(source) - if low.endswith((".ipc", ".feather", ".arrow")): - return pd.read_feather(source) - raise ValueError(f"unsupported file type for path {source!r}") - if has_polars(): - import polars as pl - - if isinstance(source, pl.LazyFrame): - return source.collect().to_pandas() - # polars frame - to_pandas = getattr(source, "to_pandas", None) - if callable(to_pandas): - return to_pandas() - # duckdb relation - to_df = getattr(source, "df", None) - if callable(to_df): - return to_df() - # Spark DataFrame (collects to the driver, matching the balanced-strategy - # pandas fallback for a Spark source). - to_pandas_spark = getattr(source, "toPandas", None) - if callable(to_pandas_spark): - return to_pandas_spark() - raise TypeError(f"cannot materialize source of type {type(source).__name__}") - - -class PandasEngine(ExecutionEngine): - name = "pandas" - - def execute( - self, - source: Any, - config: CleanConfig, - engine_config: EngineConfig, - ) -> tuple[pd.DataFrame, CleanReport]: - from ...cleaner import run_pipeline - - df = materialize_to_pandas(source) - return run_pipeline(df, config) diff --git a/src/freshdata/execution/backends/_polars.py b/src/freshdata/execution/backends/_polars.py deleted file mode 100644 index 13dda86c..00000000 --- a/src/freshdata/execution/backends/_polars.py +++ /dev/null @@ -1,580 +0,0 @@ -"""Polars backend — native out-of-core cleaning on LazyFrames. - -Reproduces freshdata's deterministic representation-repair + structural-reduction -subset (column rename, whitespace/sentinel normalization, empty column/row drops, -full-row dedup) directly on a ``pl.LazyFrame`` with projection/predicate pushdown -and streaming collection. Steps outside that subset (the decision engine, -heuristic dtype repair, opt-in impute/outliers) transparently fall back to the -pandas pipeline, so output stays identical to ``fd.clean``. - -Action records use the *same* ``step`` names and ``count`` semantics the pandas -``steps/`` modules emit, so ``CleanReport`` consumers are unaffected. -""" - -from __future__ import annotations - -import logging -import time -from typing import TYPE_CHECKING, Any - -from ..._util import PY_WHITESPACE -from ...steps.duplicates import check_duplicate_ratio, report_detected_duplicates -from .._base import ExecutionEngine -from .._config import enforce_fallback_policy -from .._ingest import pandas_ingest_fallback_reason -from .._lazy import require_polars -from .._metadata import MetadataScanner -from .._native_steps import ( - impute_defined_for, - integer_safe_bounds, - iqr_bounds, - log_widened_bounds, - native_outlier_factor, - native_outlier_method, - outlier_label, - resolve_impute_strategy, - zscore_bounds, -) -from .._plan import NativePlan, PlanGenerator -from .._report import finalize_report, finalize_report_native, init_report, zero_column_frame -from ._pandas import materialize_to_pandas - -if TYPE_CHECKING: # pragma: no cover - typing only - from ...config import CleanConfig - from ...report import CleanReport - from .._config import EngineConfig - -log = logging.getLogger("freshdata.execution.polars") - - -class PolarsEngine(ExecutionEngine): - name = "polars" - - # -- source ingestion --------------------------------------------------- - - def _to_lazy(self, source: Any, pl: Any) -> tuple[Any, int]: - """Return ``(LazyFrame, memory_before_bytes)`` for *source*. - - Float ``NaN`` is read as null, because the pandas reference treats - ``NaN`` as missing. ``pl.from_pandas`` already does this; native Polars, - Arrow and file sources keep real ``NaN`` otherwise. - """ - import pandas as pd - - if isinstance(source, pd.DataFrame): - return pl.from_pandas(source).lazy(), int(source.memory_usage(deep=True).sum()) - lf, memory_before = self._native_to_lazy(source, pl) - return lf.with_columns(pl.col(pl.Float32, pl.Float64).fill_nan(None)), memory_before - - @staticmethod - def _native_to_lazy(source: Any, pl: Any) -> tuple[Any, int]: - if isinstance(source, pl.LazyFrame): - return source, 0 - if isinstance(source, pl.DataFrame): - return source.lazy(), int(source.estimated_size()) - if isinstance(source, str): - low = source.lower() - if low.endswith((".parquet", ".pq")): - return pl.scan_parquet(source), 0 - if low.endswith(".csv"): - return pl.scan_csv(source), 0 - if low.endswith((".ipc", ".feather", ".arrow")): - return pl.scan_ipc(source), 0 - raise ValueError(f"PolarsEngine: unsupported file type for path {source!r}") - try: - import pyarrow as pa - - if isinstance(source, pa.Table): - return pl.from_arrow(source).lazy(), int(source.nbytes) - if isinstance(source, pa.RecordBatch): - table = pa.Table.from_batches([source]) - return pl.from_arrow(table).lazy(), int(table.nbytes) - except ImportError: - pass - raise TypeError(f"PolarsEngine: unsupported source type {type(source).__name__}") - - def _pandas_index_forces_fallback(self, source: Any) -> bool: - """A non-default pandas index (e.g. DatetimeIndex) has index-aware - dedup semantics that a polars frame cannot carry — fall back.""" - import pandas as pd - - return isinstance(source, pd.DataFrame) and not isinstance(source.index, pd.RangeIndex) - - # -- execution ---------------------------------------------------------- - - def execute( - self, - source: Any, - config: CleanConfig, - engine_config: EngineConfig, - ) -> tuple[Any, CleanReport]: - pl = require_polars() - self._configure_threads(engine_config) - started = time.perf_counter() - - # Decide config- and input-driven fallbacks before ingestion: pl.from_pandas - # raises on the inputs pandas_ingest_fallback_reason flags. - reason = ( - PlanGenerator(config, backend=self.name).fallback_reason() - or pandas_ingest_fallback_reason(source, self.name) - ) - if reason is None and self._pandas_index_forces_fallback(source): - reason = "pandas index semantics" - if reason is not None: - return self._delegate_to_pandas(source, config, engine_config, reason) - - lf, memory_before = self._to_lazy(source, pl) - names = list(lf.collect_schema().names()) - plan = PlanGenerator(config, backend=self.name).plan(names) - if plan.fallback_reason is not None: - return self._delegate_to_pandas(source, config, engine_config, plan.fallback_reason) - - meta = MetadataScanner.from_polars_lazy(lf) - report = init_report(meta, memory_before) - report.backend = "polars" - lf = self._apply_native(lf, plan, config, report, pl, engine_config) - if engine_config.output_format == "polars-lazy": - # Honest out-of-core path: hand back the uncollected LazyFrame so the - # caller controls when rows enter memory. We never collect here. - finalize_report_native(report, started) - report.add_recommendation( - "Result returned as a native Polars LazyFrame (un-collected) — " - "call .collect() (or .collect(engine='streaming')) to materialize." - ) - return lf, report - cleaned = self._collect(lf, engine_config, pl) - if cleaned.width == 0 and report.columns_dropped and report.rows_before > 0: - # Every column was dropped as empty; a polars frame without columns has - # height 0, while the pandas reference keeps the rows. - cleaned = zero_column_frame("polars", engine_config.output_format, report) - finalize_report(report, cleaned, started) - return cleaned, report - - def _delegate_to_pandas( - self, source: Any, config: CleanConfig, engine_config: EngineConfig, reason: str - ) -> tuple[Any, CleanReport]: - enforce_fallback_policy(engine_config, "polars", "pipeline", reason) - log.warning("freshdata PolarsEngine: falling back to pandas (%s)", reason) - cleaned, report = self._fallback(source, config) - report.backend = "pandas" - report.record_fallback("polars", "pipeline", reason) - return cleaned, report - - def _fallback(self, source: Any, config: CleanConfig) -> tuple[Any, CleanReport]: - from ...cleaner import run_pipeline - - df = materialize_to_pandas(source) - return run_pipeline(df, config) - - # -- native stages (mirror cleaner.run_pipeline order) ------------------ - - def _apply_native( - self, - lf: Any, - plan: NativePlan, - config: CleanConfig, - report: CleanReport, - pl: Any, - engine_config: EngineConfig | None = None, - ) -> Any: - rows_before = report.rows_before - for stage in plan.stages: - if stage == "column_names": - lf = self._stage_rename(lf, plan, report) - elif stage == "clean_strings": - lf = self._stage_clean_strings(lf, config, report, pl) - elif stage == "drop_empty_columns": - if rows_before > 0: - lf = self._stage_drop_empty_columns(lf, report, pl) - elif stage == "drop_empty_rows": - if rows_before > 0: - lf = self._stage_drop_empty_rows(lf, report, pl) - elif stage == "drop_duplicates": - lf = self._stage_drop_duplicates(lf, config, report, pl, engine_config) - elif stage == "impute": - lf = self._stage_impute(lf, config, report, pl) - elif stage == "outliers": - lf = self._stage_outliers(lf, config, report, pl) - elif stage == "reset_index": - pass # polars frames carry no index - return lf - - def _integer_dtypes(self, pl: Any) -> tuple[Any, ...]: - return (pl.Int8, pl.Int16, pl.Int32, pl.Int64, - pl.UInt8, pl.UInt16, pl.UInt32, pl.UInt64) - - def _stage_impute( - self, lf: Any, config: CleanConfig, report: CleanReport, pl: Any - ) -> Any: - strategy = config.impute - if strategy is None: - return lf - report.record_backend_difference( - "polars", "impute", - "median/mode fill values are computed with polars aggregates and may " - "differ from the pandas reference's interpolation/tie-breaking", - ) - schema = lf.collect_schema() - int_dtypes = self._integer_dtypes(pl) - - aggs: list[Any] = [pl.len().alias("__h__")] - cols: list[tuple[str, bool, str]] = [] - for name in schema.names(): - dtype = schema[name] - is_numeric = dtype.is_numeric() and dtype != pl.Boolean - resolved = resolve_impute_strategy(strategy, is_numeric=is_numeric) - aggs.append(pl.col(name).null_count().alias(f"__n__{name}")) - if impute_defined_for(strategy, is_numeric=is_numeric): - if resolved == "mean": - value_expr = pl.col(name).mean() - elif resolved == "median": - value_expr = pl.col(name).median() - else: - value_expr = pl.col(name).drop_nulls().mode().first() - aggs.append(value_expr.alias(f"__v__{name}")) - cols.append((name, is_numeric, resolved)) - - row = lf.select(aggs).collect().row(0, named=True) - height = int(row["__h__"]) - fills: list[Any] = [] - for name, is_numeric, resolved in cols: - n_missing = int(row[f"__n__{name}"]) - if n_missing == 0 or height - n_missing == 0: - continue - if not impute_defined_for(strategy, is_numeric=is_numeric): - report.add("impute", f"skipped ({strategy} is not defined for {name})", - column=name) - continue - value = row.get(f"__v__{name}") - if value is None: - report.add("impute", f"skipped (could not compute {resolved} for {name})", - column=name) - continue - expr = pl.col(name) - if isinstance(value, float) and schema[name] in int_dtypes: - expr = expr.cast(pl.Float64) - fills.append(expr.fill_null(pl.lit(value)).alias(name)) - shown = f"{value:.6g}" if isinstance(value, float) else repr(value) - report.add("impute", - f"filled {n_missing} missing value(s) with {resolved} ({shown})", - column=name, count=n_missing) - report.columns_imputed.append(name) - return lf.with_columns(fills) if fills else lf - - def _stage_outliers( # noqa: PLR0915 — mirrors the pandas step's branchy policy - self, lf: Any, config: CleanConfig, report: CleanReport, pl: Any - ) -> Any: - if config.outliers is None: - return lf - method = native_outlier_method(config) - factor = native_outlier_factor(config, method) - schema = lf.collect_schema() - numeric = [n for n in schema.names() - if schema[n].is_numeric() and schema[n] != pl.Boolean] - if not numeric: - return lf - report.record_backend_difference( - "polars", "outliers", - "outlier counts depend on polars quantile/stddev statistics and may " - "differ from the pandas reference", - ) - int_dtypes = self._integer_dtypes(pl) - - clip = config.outliers == "clip" - stat_aggs: list[Any] = [] - for n in numeric: - # Fences come from finite values only, like the pandas reference - # (steps.outliers.drop_infinite); inf is still tested against them. - values = ( - pl.col(n).filter(pl.col(n).is_finite()) - if schema[n] in (pl.Float32, pl.Float64) else pl.col(n) - ) - if method == "iqr": - stat_aggs.append(values.quantile(0.25, "linear").alias(f"q1_{n}")) - stat_aggs.append(values.quantile(0.75, "linear").alias(f"q3_{n}")) - else: - stat_aggs.append(values.mean().alias(f"m_{n}")) - stat_aggs.append(values.std().alias(f"s_{n}")) - if clip: - # Skew-aware capping aggregates (see log_widened_bounds). - logs = values.filter(values > 0).log() - stat_aggs.append(values.skew(bias=False).alias(f"sk_{n}")) - stat_aggs.append(values.min().alias(f"mn_{n}")) - stat_aggs.append(values.count().alias(f"cnt_{n}")) - stat_aggs.append(logs.quantile(0.25, "linear").alias(f"lq1_{n}")) - stat_aggs.append(logs.quantile(0.75, "linear").alias(f"lq3_{n}")) - stats = lf.select(stat_aggs).collect().row(0, named=True) - - bounds: dict[str, tuple[float, float]] = {} - for n in numeric: - if method == "iqr": - q1, q3 = stats[f"q1_{n}"], stats[f"q3_{n}"] - raw = None if q1 is None or q3 is None else iqr_bounds(q1, q3, factor) - else: - m, s = stats[f"m_{n}"], stats[f"s_{n}"] - raw = None if m is None or s is None else zscore_bounds(m, s, factor) - if raw is None: - continue - if clip: - # Rewriting values never uses narrower fences than the pandas - # reference: strongly skewed positive columns widen via - # log-space IQR so legitimate heavy tails survive the clip. - mn = stats[f"mn_{n}"] - raw = log_widened_bounds( - *raw, - skew=stats[f"sk_{n}"], - minimum=None if mn is None else float(mn), - log_q1=stats[f"lq1_{n}"], - log_q3=stats[f"lq3_{n}"], - n_non_null=int(stats[f"cnt_{n}"] or 0), - factor=native_outlier_factor(config, "iqr"), - ) - bounds[n] = integer_safe_bounds(*raw, is_integer=schema[n] in int_dtypes) - - if not bounds: - return lf - count_aggs = [ - ((pl.col(n) < lo) | (pl.col(n) > hi)).sum().alias(f"c_{n}") - for n, (lo, hi) in bounds.items() - ] - counts = lf.select(count_aggs).collect().row(0, named=True) - - transforms: list[Any] = [] - for n, (lo, hi) in bounds.items(): - n_out = int(counts.get(f"c_{n}", 0) or 0) - if n_out == 0: - continue - label = outlier_label(method, factor) - if config.outliers == "clip": - transforms.append(pl.col(n).clip(lo, hi).alias(n)) - report.add("outliers", - f"clipped {n_out} outlier(s) to [{lo:g}, {hi:g}] ({label})", - column=n, count=n_out) - else: - flag = self._unique_flag(schema.names(), f"{n}_outlier") - mask = ((pl.col(n) < lo) | (pl.col(n) > hi)).fill_null(False) - transforms.append(mask.alias(flag)) - report.add("outliers", - f"flagged {n_out} outlier(s) in new column {flag!r} ({label})", - column=n, count=n_out) - report.outliers_handled += n_out - return lf.with_columns(transforms) if transforms else lf - - @staticmethod - def _unique_flag(existing: Any, base: str) -> str: - names = set(existing) - name, k = base, 1 - while name in names: - k += 1 - name = f"{base}_{k}" - return name - - def _stage_rename(self, lf: Any, plan: NativePlan, report: CleanReport) -> Any: - if not plan.rename_map: - return lf - changes = list(plan.rename_map.items()) - preview = ", ".join(f"{o!r}->{n!r}" for o, n in changes[:4]) - if len(changes) > 4: - preview += f", … (+{len(changes) - 4} more)" - report.add("column_names", f"renamed {len(changes)} column(s): {preview}", - count=len(changes)) - return lf.rename(plan.rename_map) - - def _stage_clean_strings( - self, lf: Any, config: CleanConfig, report: CleanReport, pl: Any - ) -> Any: - from ...steps.strings import active_sentinels - - sentinels = list(active_sentinels(config)) - schema = lf.collect_schema() - string_cols = [n for n in schema.names() if schema[n] == pl.Utf8] - if not string_cols: - return lf - - # One aggregate pass for all strip / sentinel counts. - count_exprs: list[Any] = [] - for c in string_cols: - col = pl.col(c) - stripped = col.str.strip_chars(PY_WHITESPACE) - base = stripped if config.strip_whitespace else col - if config.strip_whitespace: - count_exprs.append( - ((stripped != col) & col.is_not_null()).sum().alias(f"__strip__{c}") - ) - if config.normalize_sentinels: - count_exprs.append( - (base.str.to_lowercase().is_in(sentinels) & base.is_not_null()) - .sum() - .alias(f"__sent__{c}") - ) - counts = lf.select(count_exprs).collect().row(0, named=True) if count_exprs else {} - - transforms: list[Any] = [] - for c in string_cols: - col = pl.col(c) - stripped = col.str.strip_chars(PY_WHITESPACE) - base = stripped if config.strip_whitespace else col - n_strip = int(counts.get(f"__strip__{c}", 0) or 0) - n_sent = int(counts.get(f"__sent__{c}", 0) or 0) - if config.strip_whitespace and n_strip: - report.add("strip_whitespace", "trimmed surrounding whitespace", - column=c, count=n_strip) - if config.normalize_sentinels and n_sent: - report.add("normalize_sentinels", - 'replaced sentinel strings ("N/A", "-", "", …) with missing', - column=c, count=n_sent) - if config.normalize_sentinels: - expr = ( - pl.when(base.str.to_lowercase().is_in(sentinels)) - .then(None) - .otherwise(base) - .alias(c) - ) - else: - expr = base.alias(c) - transforms.append(expr) - return lf.with_columns(transforms) - - def _stage_drop_empty_columns(self, lf: Any, report: CleanReport, pl: Any) -> Any: - schema = lf.collect_schema() - names = list(schema.names()) - stats = lf.select( - [pl.len().alias("__h__")] + [pl.col(c).null_count().alias(c) for c in names] - ).collect().row(0, named=True) - height = int(stats["__h__"]) - dropped = [c for c in names if int(stats[c]) == height] - if dropped: - report.columns_dropped.extend(dropped) - report.add( - "drop_empty_columns", - f"dropped {len(dropped)} all-missing column(s): {', '.join(dropped[:6])}" - + (" …" if len(dropped) > 6 else ""), - count=len(dropped), - ) - lf = lf.drop(dropped) - return lf - - def _stage_drop_empty_rows(self, lf: Any, report: CleanReport, pl: Any) -> Any: - if not lf.collect_schema().names(): - return lf - all_null = pl.all_horizontal(pl.all().is_null()) - n = int(lf.select(all_null.sum().alias("__n__")).collect().item()) - if n: - report.add("drop_empty_rows", f"dropped {n} all-missing row(s)", count=n) - lf = lf.filter(~all_null) - return lf - - def _stage_drop_duplicates( - self, - lf: Any, - config: CleanConfig, - report: CleanReport, - pl: Any, - engine_config: EngineConfig | None = None, - ) -> Any: - # With no columns, the pandas reference skips duplicate handling. Newer - # Polars versions retain height here but unique() can collapse the rows. - if not lf.collect_schema().names(): - return lf - # Counts go through the streaming engine: a plain .collect() re-evaluates - # the whole upstream plan in memory, defeating out-of-core execution. - n_before = self._count(lf, engine_config, pl) - if n_before < 1: - return lf - subset = list(config.duplicate_subset) if config.duplicate_subset is not None else None - if subset: - schema_names = set(lf.collect_schema().names()) - missing = [c for c in subset if c not in schema_names] - if missing: - raise ValueError( - f"duplicate_subset column(s) not found: {missing}. " - f"Available columns: {sorted(schema_names)}. " - "Note: names refer to columns *after* renaming when column_names=True." - ) - if not config.drop_duplicates: - # Detection-only default: count duplicates, report, keep every row. - n_unique = int( - self._count(lf.unique(subset=subset), engine_config, pl) - ) - report_detected_duplicates( - n_before - n_unique, n_before, config, report, subset=subset - ) - return lf - # `maintain_order=True` forces Polars to materialize and defeats streaming. - # Under streaming (the default) we keep *full-row* dedup streaming-safe by - # dropping the ordering guarantee, and disclose it. Callers who need - # original row order can set EngineConfig.streaming_dedup=False. - # *Subset* dedup always preserves order: with a subset, keep="first"/"last" - # decides WHICH row survives, so order is semantics (pandas parity), not - # cosmetics — the stage runs eagerly instead of streaming. - streaming = engine_config is None or engine_config.streaming - order_safe = engine_config is not None and not engine_config.streaming_dedup - maintain_order = bool(subset) or (not streaming) or order_safe - if streaming and not maintain_order: - report.record_backend_difference( - "polars", "drop_duplicates", - "streaming dedup does not preserve original row order; set " - "EngineConfig(streaming_dedup=False) to keep order (materializes)", - ) - if subset and streaming: - report.add_recommendation( - "subset dedup preserves row order to match pandas keep semantics, " - "so this stage evaluates eagerly rather than streaming" - ) - deduped = lf.unique( - subset=subset, keep=config.duplicate_keep, maintain_order=maintain_order - ) - n_after = self._count(deduped, engine_config, pl) - n_dup = n_before - n_after - if n_dup <= 0: - return lf - pct = 100.0 * n_dup / n_before - where = f" (compared on {subset})" if subset else "" - # Raises under duplicate_ratio_action="error", matching the pandas step. - risk = "medium" if check_duplicate_ratio(n_dup, n_before, config) else "low" - report.add( - "drop_duplicates", - f"dropped {n_dup} duplicate row(s) " - f"({pct:.1f}% of rows, keep={config.duplicate_keep!r}){where}", - count=n_dup, - risk=risk, - ) - report.duplicates_removed += n_dup - if risk == "medium": - report.add_warning( - f"{pct:.1f}% of rows were duplicates " - f"(> {100 * config.duplicate_threshold:.0f}%); confirm they are not legitimate" - ) - return deduped - - # -- collection --------------------------------------------------------- - - def _count(self, lf: Any, engine_config: EngineConfig | None, pl: Any) -> int: - """Row count of *lf*, evaluated through the (streaming) collect path.""" - return int(self._collect(lf.select(pl.len()), engine_config, pl).item()) - - def _collect(self, lf: Any, engine_config: EngineConfig | None, pl: Any) -> Any: - # No explicit engine config means the defaults, where streaming is on. - if engine_config is not None and not engine_config.streaming: - return lf.collect() - # Polars renamed the streaming switch across versions; try the modern - # keyword first, then the legacy one, then a plain collect. - try: - return lf.collect(engine="streaming") - except (TypeError, ValueError): - # Older polars either lacks the ``engine`` keyword (TypeError) or - # rejects the ``"streaming"`` value (ValueError: Invalid engine - # argument); fall through to the legacy switch, then plain collect. - pass - try: - return lf.collect(streaming=True) - except Exception as exc: # pragma: no cover - defensive - log.warning("freshdata: streaming collect failed (%s); retrying eagerly", exc) - return lf.collect() - - def _configure_threads(self, engine_config: EngineConfig) -> None: - if engine_config.polars_n_threads is not None: - import os - - # Only effective if set before polars is first imported; harmless otherwise. - os.environ.setdefault("POLARS_MAX_THREADS", str(engine_config.polars_n_threads)) diff --git a/src/freshdata/execution/backends/_spark.py b/src/freshdata/execution/backends/_spark.py deleted file mode 100644 index 24bb67c3..00000000 --- a/src/freshdata/execution/backends/_spark.py +++ /dev/null @@ -1,534 +0,0 @@ -"""Spark backend — native distributed cleaning on PySpark DataFrames. - -Reproduces freshdata's deterministic representation-repair + structural-reduction -subset (column rename, whitespace/sentinel normalization, empty column/row drops, -full-row dedup) plus the opt-in ``impute``/``outliers`` overrides directly on a -``pyspark.sql.DataFrame`` so a clean can run on a Spark cluster. Steps outside -that subset (the decision engine, heuristic dtype repair, subset dedup) -transparently fall back to the pandas pipeline. - -Action records use the *same* ``step`` names and ``count`` semantics the pandas -``steps/`` modules emit, so ``CleanReport`` consumers are unaffected. pyspark is -an optional dependency, imported lazily; importing freshdata never requires it. -""" - -from __future__ import annotations - -import logging -import time -from typing import TYPE_CHECKING, Any - -from ...steps.duplicates import check_duplicate_ratio, report_detected_duplicates -from .._base import ExecutionEngine -from .._config import enforce_fallback_policy -from .._lazy import has_pyspark, require_pyspark -from .._native_steps import ( - impute_defined_for, - integer_safe_bounds, - iqr_bounds, - log_widened_bounds, - native_outlier_factor, - native_outlier_method, - outlier_label, - resolve_impute_strategy, - zscore_bounds, -) -from .._plan import NativePlan, PlanGenerator -from ._pandas import materialize_to_pandas - -if TYPE_CHECKING: # pragma: no cover - typing only - from ...config import CleanConfig - from ...report import CleanReport - from .._config import EngineConfig - -log = logging.getLogger("freshdata.execution.spark") - -#: Spark ``simpleString`` names (``DataFrame.dtypes``) of the numeric types. -#: Matched exactly: a prefix test would take ``interval day to second`` as ``int``. -_SPARK_INTEGER_TYPES = frozenset({"tinyint", "smallint", "int", "bigint"}) -_SPARK_FLOAT_TYPES = frozenset({"float", "double"}) -_SPARK_NUMERIC_TYPES = _SPARK_INTEGER_TYPES | _SPARK_FLOAT_TYPES | {"decimal"} - - -def _spark_type_name(dtype: str) -> str: - """Base name of a Spark dtype string, without parameters (``decimal(10,2)``).""" - return dtype.split("(", 1)[0].strip().lower() - - -def _renamed_columns(columns: list[str], rename_map: dict[str, str]) -> list[str]: - """Every column's new name, applied at once and by position.""" - return [rename_map.get(c, c) for c in columns] - - -def _is_spark_frame(source: Any) -> bool: - if not has_pyspark(): - return False - from pyspark.sql import DataFrame as SparkDataFrame - - return isinstance(source, SparkDataFrame) - - -class SparkEngine(ExecutionEngine): - name = "spark" - - # -- session / ingestion ------------------------------------------------ - - def _session(self, engine_config: EngineConfig, source: Any) -> Any: - sql = require_pyspark() - session = engine_config.spark_session - if session is None and _is_spark_frame(source): - session = source.sparkSession - if session is None: - session = sql.SparkSession.builder.appName("freshdata").getOrCreate() - if engine_config.spark_shuffle_partitions is not None: - session.conf.set( - "spark.sql.shuffle.partitions", str(engine_config.spark_shuffle_partitions) - ) - return session - - def _to_spark(self, source: Any, session: Any) -> Any: - """Return a Spark DataFrame for *source*.""" - if _is_spark_frame(source): - return source - if isinstance(source, str): - low = source.lower() - if low.endswith((".parquet", ".pq")): - return session.read.parquet(source) - if low.endswith(".csv"): - return session.read.option("header", True).option( - "inferSchema", True - ).csv(source) - raise ValueError(f"SparkEngine: unsupported file type for path {source!r}") - return session.createDataFrame(materialize_to_pandas(source)) - - def _pandas_index_forces_fallback(self, source: Any) -> bool: - import pandas as pd - - return isinstance(source, pd.DataFrame) and not isinstance(source.index, pd.RangeIndex) - - # -- execution ---------------------------------------------------------- - - def execute( - self, - source: Any, - config: CleanConfig, - engine_config: EngineConfig, - ) -> tuple[Any, CleanReport]: - from ...report import CleanReport - - session = self._session(engine_config, source) - sdf = self._to_spark(source, session) - names = list(sdf.columns) - plan = PlanGenerator(config).plan(names) - - if plan.needs_fallback or self._pandas_index_forces_fallback(source): - reason = plan.fallback_reason or "pandas index semantics" - enforce_fallback_policy(engine_config, "spark", "pipeline", reason) - log.warning("freshdata SparkEngine: falling back to pandas (%s)", reason) - cleaned, report = self._fallback(source, config) - report.backend = "pandas" - report.record_fallback("spark", "pipeline", reason) - return cleaned, report - - started = time.perf_counter() - report = CleanReport(backend="spark") - sdf = self._nan_as_null(sdf) - self._init_before(sdf, report) - sdf = self._apply_native(sdf, plan, config, report) - self._finalize(sdf, report, started) - return sdf, report - - def _fallback(self, source: Any, config: CleanConfig) -> tuple[Any, CleanReport]: - from ...cleaner import run_pipeline - - return run_pipeline(materialize_to_pandas(source), config) - - def _nan_as_null(self, sdf: Any) -> Any: - """*sdf* with float ``NaN`` read as null. - - pandas treats ``NaN`` as missing, but Spark keeps it as a value in - float/double columns (Spark-written Parquet, CSV ``NaN`` literals, - non-Arrow ``createDataFrame``), so every later null test would miss it. - """ - from pyspark.sql import functions as F - - floats = { - name for name, dtype in sdf.dtypes if _spark_type_name(dtype) in _SPARK_FLOAT_TYPES - } - if not floats: - return sdf - return sdf.select(*[ - F.when(F.isnan(F.col(c)), None).otherwise(F.col(c)).alias(c) - if c in floats else F.col(c) - for c in sdf.columns - ]) - - # -- report bookkeeping ------------------------------------------------- - - def _null_count_total(self, sdf: Any) -> int: - from pyspark.sql import functions as F - - if not sdf.columns: - return 0 - exprs = [ - F.sum(F.col(c).isNull().cast("long")).alias(c) for c in sdf.columns - ] - row = sdf.select(exprs).first() - return int(sum(int(row[c] or 0) for c in sdf.columns)) if row else 0 - - def _init_before(self, sdf: Any, report: CleanReport) -> None: - report.rows_before = sdf.count() - report.cols_before = len(sdf.columns) - report.missing_before = self._null_count_total(sdf) - report.memory_before = 0 - - def _finalize(self, sdf: Any, report: CleanReport, started: float) -> None: - report.rows_after = sdf.count() - report.cols_after = len(sdf.columns) - report.missing_after = self._null_count_total(sdf) - report.memory_after = 0 - report.duration_seconds = time.perf_counter() - started - - # -- native stages (mirror cleaner.run_pipeline order) ------------------ - - def _string_columns(self, sdf: Any) -> list[str]: - return [name for name, dtype in sdf.dtypes if dtype == "string"] - - def _numeric_columns(self, sdf: Any) -> list[str]: - return [ - name for name, dtype in sdf.dtypes - if _spark_type_name(dtype) in _SPARK_NUMERIC_TYPES - ] - - def _is_integer_column(self, sdf: Any, name: str) -> bool: - dtype = dict(sdf.dtypes)[name] - return _spark_type_name(dtype) in _SPARK_INTEGER_TYPES - - def _is_float_column(self, sdf: Any, name: str) -> bool: - dtype = dict(sdf.dtypes)[name] - return _spark_type_name(dtype) in _SPARK_FLOAT_TYPES - - def _apply_native( - self, sdf: Any, plan: NativePlan, config: CleanConfig, report: CleanReport - ) -> Any: - rows_before = report.rows_before - for stage in plan.stages: - if stage == "column_names": - sdf = self._stage_rename(sdf, plan, report) - elif stage == "clean_strings": - sdf = self._stage_clean_strings(sdf, config, report) - elif stage == "drop_empty_columns" and rows_before > 0: - sdf = self._stage_drop_empty_columns(sdf, report) - elif stage == "drop_empty_rows" and rows_before > 0: - sdf = self._stage_drop_empty_rows(sdf, report) - elif stage == "drop_duplicates": - sdf = self._stage_drop_duplicates(sdf, config, report) - elif stage == "impute": - sdf = self._stage_impute(sdf, config, report) - elif stage == "outliers": - sdf = self._stage_outliers(sdf, config, report) - elif stage == "reset_index": - pass # spark frames carry no index - return sdf - - def _stage_rename(self, sdf: Any, plan: NativePlan, report: CleanReport) -> Any: - if not plan.rename_map: - return sdf - changes = list(plan.rename_map.items()) - preview = ", ".join(f"{o!r}->{n!r}" for o, n in changes[:4]) - if len(changes) > 4: - preview += f", … (+{len(changes) - 4} more)" - report.add("column_names", f"renamed {len(changes)} column(s): {preview}", - count=len(changes)) - # One positional projection: chained withColumnRenamed calls rename - # every (case-insensitive) match, so a map like {"a b": "a_b", - # "a_b": "a_b_2"} would rename both columns in the second call. - return sdf.toDF(*_renamed_columns(list(sdf.columns), plan.rename_map)) - - def _stage_clean_strings( - self, sdf: Any, config: CleanConfig, report: CleanReport - ) -> Any: - from pyspark.sql import functions as F - - from ...steps.strings import active_sentinels - - sentinels = list(active_sentinels(config)) - string_cols = self._string_columns(sdf) - if not string_cols: - return sdf - - count_exprs = [] - for c in string_cols: - col = F.col(c) - stripped = F.regexp_replace(col, r"^\s+|\s+$", "") - base = stripped if config.strip_whitespace else col - if config.strip_whitespace: - count_exprs.append( - F.sum(((stripped != col) & col.isNotNull()).cast("long")).alias(f"s_{c}") - ) - if config.normalize_sentinels: - count_exprs.append( - F.sum( - (F.lower(base).isin(sentinels) & base.isNotNull()).cast("long") - ).alias(f"z_{c}") - ) - counts = sdf.select(count_exprs).first() if count_exprs else None - - for c in string_cols: - col = F.col(c) - stripped = F.regexp_replace(col, r"^\s+|\s+$", "") - base = stripped if config.strip_whitespace else col - if config.strip_whitespace and counts is not None: - n_strip = int(counts[f"s_{c}"] or 0) - if n_strip: - report.add("strip_whitespace", "trimmed surrounding whitespace", - column=c, count=n_strip) - if config.normalize_sentinels and counts is not None: - n_sent = int(counts[f"z_{c}"] or 0) - if n_sent: - report.add("normalize_sentinels", - 'replaced sentinel strings ("N/A", "-", "", …) with missing', - column=c, count=n_sent) - if config.normalize_sentinels: - new = F.when(F.lower(base).isin(sentinels), None).otherwise(base) - else: - new = base - sdf = sdf.withColumn(c, new) - return sdf - - def _stage_drop_empty_columns(self, sdf: Any, report: CleanReport) -> Any: - from pyspark.sql import functions as F - - names = list(sdf.columns) - exprs = [F.count(F.col(c)).alias(c) for c in names] - row = sdf.select(exprs).first() - dropped = [c for c in names if row is not None and int(row[c]) == 0] - if dropped: - report.columns_dropped.extend(dropped) - report.add( - "drop_empty_columns", - f"dropped {len(dropped)} all-missing column(s): {', '.join(dropped[:6])}" - + (" …" if len(dropped) > 6 else ""), - count=len(dropped), - ) - sdf = sdf.drop(*dropped) - return sdf - - def _stage_drop_empty_rows(self, sdf: Any, report: CleanReport) -> Any: - from functools import reduce - - from pyspark.sql import functions as F - - if not sdf.columns: - return sdf - all_null = reduce(lambda a, b: a & b, (F.col(c).isNull() for c in sdf.columns)) - n = sdf.filter(all_null).count() - if n: - report.add("drop_empty_rows", f"dropped {n} all-missing row(s)", count=n) - sdf = sdf.filter(~all_null) - return sdf - - def _stage_drop_duplicates( - self, sdf: Any, config: CleanConfig, report: CleanReport - ) -> Any: - n_before = sdf.count() - if n_before < 1: - return sdf - n_after = sdf.dropDuplicates().count() - n_dup = n_before - n_after - if not config.drop_duplicates: - # Detection-only default: count duplicates, report, keep every row. - report_detected_duplicates(n_dup, n_before, config, report) - return sdf - if n_dup <= 0: - return sdf - pct = 100.0 * n_dup / n_before - # Raises under duplicate_ratio_action="error", matching the pandas step. - risk = "medium" if check_duplicate_ratio(n_dup, n_before, config) else "low" - report.add( - "drop_duplicates", - f"dropped {n_dup} duplicate row(s) " - f"({pct:.1f}% of rows, keep={config.duplicate_keep!r})", - count=n_dup, - risk=risk, - ) - report.duplicates_removed += n_dup - if risk == "medium": - report.add_warning( - f"{pct:.1f}% of rows were duplicates " - f"(> {100 * config.duplicate_threshold:.0f}%); confirm they are not legitimate" - ) - return self._dedup_in_order(sdf, config.duplicate_keep) - - def _dedup_in_order(self, sdf: Any, keep: str) -> Any: - """Full-row dedup that keeps the pandas row choice and row order. - - ``dropDuplicates`` is a shuffle aggregate: surviving rows come back in - arbitrary order and ``keep`` is ignored. An ordinal taken before any - shuffle picks the first (or last) occurrence of each row, and sorting - by it restores the input order. The ordinal follows the input - DataFrame's partition order, which is the row order for a single - source read. ``partitionBy`` groups nulls together, as pandas does. - """ - from pyspark.sql import Window - from pyspark.sql import functions as F - - columns = list(sdf.columns) - ordinal = self._unique_flag(sdf, "__fd_row_ordinal__") - rank = self._unique_flag(sdf, "__fd_row_rank__") - order = F.col(ordinal).desc() if keep == "last" else F.col(ordinal).asc() - window = Window.partitionBy(*[F.col(c) for c in columns]).orderBy(order) - return ( - sdf.withColumn(ordinal, F.monotonically_increasing_id()) - .withColumn(rank, F.row_number().over(window)) - .filter(F.col(rank) == 1) - .orderBy(F.col(ordinal).asc()) - .drop(ordinal, rank) - ) - - def _stage_impute(self, sdf: Any, config: CleanConfig, report: CleanReport) -> Any: - from pyspark.sql import functions as F - - strategy = config.impute - if strategy is None: - return sdf - report.record_backend_difference( - "spark", "impute", - "fill values use Spark aggregates; median/mode may differ from the " - "pandas reference's interpolation/tie-breaking", - ) - numeric_cols = set(self._numeric_columns(sdf)) - for name in list(sdf.columns): - is_numeric = name in numeric_cols - resolved = resolve_impute_strategy(strategy, is_numeric=is_numeric) - n_missing = sdf.filter(F.col(name).isNull()).count() - non_null = sdf.filter(F.col(name).isNotNull()).count() - if n_missing == 0 or non_null == 0: - continue - if not impute_defined_for(strategy, is_numeric=is_numeric): - report.add("impute", f"skipped ({strategy} is not defined for {name})", - column=name) - continue - value = self._impute_value(sdf, name, resolved, is_numeric) - if value is None: - report.add("impute", f"skipped (could not compute {resolved} for {name})", - column=name) - continue - sdf = sdf.fillna({name: value}) if not is_numeric else sdf.withColumn( - name, F.when(F.col(name).isNull(), F.lit(value)).otherwise(F.col(name)) - ) - shown = f"{value:.6g}" if isinstance(value, float) else repr(value) - report.add("impute", - f"filled {n_missing} missing value(s) with {resolved} ({shown})", - column=name, count=n_missing) - report.columns_imputed.append(name) - return sdf - - def _impute_value(self, sdf: Any, name: str, strategy: str, is_numeric: bool) -> Any: - from pyspark.sql import functions as F - - if strategy == "mean": - row = sdf.select(F.mean(F.col(name)).alias("v")).first() - return None if row is None or row["v"] is None else float(row["v"]) - if strategy == "median": - vals = sdf.approxQuantile(name, [0.5], 0.0) - return float(vals[0]) if vals else None - # mode: most frequent non-null value - row = ( - sdf.filter(F.col(name).isNotNull()) - .groupBy(name).count().orderBy(F.desc("count"), F.col(name)) - .first() - ) - return None if row is None else row[name] - - def _stage_outliers(self, sdf: Any, config: CleanConfig, report: CleanReport) -> Any: - from pyspark.sql import functions as F - - if config.outliers is None: - return sdf - method = native_outlier_method(config) - factor = native_outlier_factor(config, method) - report.record_backend_difference( - "spark", "outliers", - "Spark approxQuantile/aggregate statistics may flag a different count " - "than the pandas reference's interpolated quantiles", - ) - for name in self._numeric_columns(sdf): - bounds = self._outlier_bounds(sdf, name, method, factor, config) - if bounds is None: - continue - lo, hi = integer_safe_bounds(*bounds, is_integer=self._is_integer_column(sdf, name)) - mask = (F.col(name) < F.lit(lo)) | (F.col(name) > F.lit(hi)) - n = sdf.filter(mask).count() - if n == 0: - continue - label = outlier_label(method, factor) - if config.outliers == "clip": - clipped = F.when(F.col(name) < F.lit(lo), F.lit(lo)).when( - F.col(name) > F.lit(hi), F.lit(hi) - ).otherwise(F.col(name)) - sdf = sdf.withColumn(name, clipped) - report.add("outliers", f"clipped {n} outlier(s) to [{lo:g}, {hi:g}] ({label})", - column=name, count=n) - else: - flag = self._unique_flag(sdf, f"{name}_outlier") - sdf = sdf.withColumn(flag, F.coalesce(mask, F.lit(False))) - report.add("outliers", - f"flagged {n} outlier(s) in new column {flag!r} ({label})", - column=name, count=n) - report.outliers_handled += n - return sdf - - def _outlier_bounds( - self, sdf: Any, name: str, method: str, factor: float, config: CleanConfig - ) -> tuple[float, float] | None: - from pyspark.sql import functions as F - - if self._is_float_column(sdf, name): - # Fences come from finite values only, like the pandas reference - # (steps.outliers.drop_infinite); inf is still tested against them. - sdf = sdf.filter( - ~F.isnan(F.col(name)) & (F.abs(F.col(name)) != F.lit(float("inf"))) - ) - if method == "iqr": - q = sdf.approxQuantile(name, [0.25, 0.75], 0.0) - if len(q) < 2: - return None - raw = iqr_bounds(float(q[0]), float(q[1]), factor) - else: - row = sdf.select( - F.mean(F.col(name)).alias("m"), F.stddev(F.col(name)).alias("s") - ).first() - if row is None or row["m"] is None or row["s"] is None: - return None - raw = zscore_bounds(float(row["m"]), float(row["s"]), factor) - if raw is None or config.outliers != "clip": - return raw - # Rewriting values never uses narrower fences than the pandas - # reference: strongly skewed positive columns widen via log-space IQR - # so legitimate heavy tails survive the clip (see log_widened_bounds). - row = sdf.select( - F.skewness(F.col(name)).alias("sk"), - F.min(F.col(name)).alias("mn"), - F.count(F.col(name)).alias("cnt"), - ).first() - positive = sdf.filter(F.col(name) > 0).select(F.log(F.col(name)).alias("_l")) - lq = positive.approxQuantile("_l", [0.25, 0.75], 0.0) - return log_widened_bounds( - *raw, - skew=None if row is None or row["sk"] is None else float(row["sk"]), - minimum=None if row is None or row["mn"] is None else float(row["mn"]), - log_q1=float(lq[0]) if len(lq) == 2 else None, - log_q3=float(lq[1]) if len(lq) == 2 else None, - n_non_null=0 if row is None else int(row["cnt"] or 0), - factor=native_outlier_factor(config, "iqr"), - ) - - @staticmethod - def _unique_flag(sdf: Any, base: str) -> str: - name, k = base, 1 - existing = set(sdf.columns) - while name in existing: - k += 1 - name = f"{base}_{k}" - return name diff --git a/src/freshdata/imputation.py b/src/freshdata/imputation.py new file mode 100644 index 00000000..3a58004a --- /dev/null +++ b/src/freshdata/imputation.py @@ -0,0 +1,204 @@ +# src/freshdata/imputation.py +from __future__ import annotations + +import re +import warnings +from decimal import Decimal +from typing import Any + +import numpy as np +import pandas as pd + +ID_NAME_REGEX = re.compile( + r"(?:^|_|-)(?:id|uuid|guid|pk|key|seq|account_num|order_no|ssn|token)(?:$|_|-)", + re.IGNORECASE, +) + +NON_COERCIBLE_OBJECT_TYPES = (complex, dict, list, set, tuple, bytes, bytearray) + + +def is_hostile_object_column(series: pd.Series) -> bool: + """Return True if an object column contains complex, container, or raw-bytes objects.""" + if series.dtype != object: + return False + non_null = series.dropna() + if len(non_null) == 0: + return False + for val in non_null: + if isinstance(val, NON_COERCIBLE_OBJECT_TYPES): + return True + return False + + +def detect_identifier_column(series: pd.Series, col_name: Any) -> tuple[bool, str | None]: + """Detect identifier columns via name regex OR integral-float/monotonic high-cardinality heuristics.""" + name_str = str(col_name) + if ID_NAME_REGEX.search(name_str): + return True, "skipped_identifier_imputation:name_pattern" + + if isinstance(series.dtype, pd.CategoricalDtype) or pd.api.types.is_bool_dtype(series.dtype): + return False, None + + if is_hostile_object_column(series): + return False, None + + non_null = series.dropna() + n_valid = len(non_null) + if n_valid < 4: + return False, None + + # String/object high-cardinality code patterns (e.g. "ORD-101") + if pd.api.types.is_string_dtype(series.dtype) or series.dtype == object: + if non_null.nunique(dropna=True) == n_valid and all( + isinstance(x, str) and any(ch.isdigit() for ch in x) for x in non_null + ): + return True, "skipped_identifier_imputation:unique_alphanumeric_code" + return False, None + + if not pd.api.types.is_numeric_dtype(series.dtype): + return False, None + + vals = non_null.to_numpy(dtype=np.float64, na_value=np.nan) + vals = vals[np.isfinite(vals)] + if len(vals) < 4: + return False, None + + is_integral = bool(np.all(np.mod(vals, 1.0) == 0.0)) + if not is_integral: + return False, None + + unique_ratio = len(np.unique(vals)) / float(len(vals)) + diffs = np.diff(vals) + is_strictly_monotonic_step = bool(len(diffs) > 0 and np.all(diffs > 0) and np.all(diffs <= 2)) + + if unique_ratio >= 0.95 and (np.min(np.abs(vals)) >= 100 or (is_strictly_monotonic_step and len(vals) >= 10)): + return True, "skipped_identifier_imputation:integral_high_cardinality_or_monotonic" + + return False, None + + +def safe_float64_matrix(df_num: pd.DataFrame) -> tuple[np.ndarray, list[str]]: + """Convert numeric/nullable-extension DataFrame columns to a finite-safe float64 matrix.""" + valid_cols: list[str] = [] + arrays: list[np.ndarray] = [] + for col in df_num.columns: + s = df_num[col] + if pd.api.types.is_bool_dtype(s.dtype) or isinstance(s.dtype, pd.BooleanDtype): + continue + if is_hostile_object_column(s): + continue + if not pd.api.types.is_numeric_dtype(s.dtype): + continue + arr = s.to_numpy(dtype=np.float64, na_value=np.nan).copy() + arr[~np.isfinite(arr)] = np.nan + valid_cols.append(col) + arrays.append(arr) + + if not arrays: + return np.empty((len(df_num), 0), dtype=np.float64), [] + return np.column_stack(arrays), valid_cols + + +def safe_corr_matrix(mat: np.ndarray) -> np.ndarray: + """Compute pairwise Pearson correlation without crashing on NaNs or zero-variance columns.""" + n_rows, n_cols = mat.shape + corr = np.eye(n_cols, dtype=np.float64) + if n_cols <= 1 or n_rows < 2: + return corr + + for i in range(n_cols): + for j in range(i + 1, n_cols): + xi = mat[:, i] + xj = mat[:, j] + mask = np.isfinite(xi) & np.isfinite(xj) + if np.sum(mask) < 2: + c = 0.0 + else: + xi_m = xi[mask] + xj_m = xj[mask] + std_i = float(np.std(xi_m)) + std_j = float(np.std(xj_m)) + if std_i < 1e-12 or std_j < 1e-12: + c = 0.0 + else: + c = float(np.dot(xi_m - np.mean(xi_m), xj_m - np.mean(xj_m)) / (len(xi_m) * std_i * std_j)) + if not np.isfinite(c): + c = 0.0 + corr[i, j] = c + corr[j, i] = c + return corr + + +def _knn_fill( + df: pd.DataFrame, + target_col: Any, + k: int = 3, +) -> tuple[pd.Series, int]: + """Safe KNN imputation over nullable numeric columns with correlation weighting and zero-variance protection.""" + series = df[target_col].copy() + na_mask = series.isna().to_numpy(dtype=bool) + if not na_mask.any() or na_mask.all(): + return series, 0 + + mat, cols = safe_float64_matrix(df) + if target_col not in cols: + return series, 0 + + target_idx = cols.index(target_col) + n_rows, n_cols = mat.shape + corr = safe_corr_matrix(mat) + weights = np.abs(corr[target_idx]) + weights[target_idx] = 0.0 + + # Standardize columns safely for Euclidean distance + with warnings.catch_warnings(): + warnings.simplefilter("ignore", category=RuntimeWarning) + col_means = np.nanmean(mat, axis=0) + col_stds = np.nanstd(mat, axis=0) + col_means = np.where(np.isfinite(col_means), col_means, 0.0) + col_stds = np.where(np.isfinite(col_stds) & (col_stds > 1e-12), col_stds, 1.0) + z_mat = (mat - col_means) / col_stds + + feature_indices = [idx for idx in range(n_cols) if idx != target_idx and np.isfinite(z_mat[:, idx]).any()] + target_vals = mat[:, target_idx].copy() + valid_row_indices = np.where(~na_mask & np.isfinite(target_vals))[0] + missing_row_indices = np.where(na_mask)[0] + + if len(valid_row_indices) == 0: + return series, 0 + + fallback_median = float(np.median(target_vals[valid_row_indices])) + imputed_count = 0 + is_integer_target = pd.api.types.is_integer_dtype(series.dtype) + + for r_idx in missing_row_indices: + if not feature_indices: + fill_val = fallback_median + else: + row_vec = z_mat[r_idx, feature_indices] + cand_mat = z_mat[np.ix_(valid_row_indices, feature_indices)] + overlap = np.isfinite(row_vec) & np.isfinite(cand_mat) + w_sub = weights[feature_indices] + w_sub = np.where(w_sub > 1e-6, w_sub, 1.0) + + diff_sq = np.where(overlap, (cand_mat - row_vec) ** 2 * w_sub, 0.0) + overlap_counts = np.sum(overlap, axis=1) + valid_cands = overlap_counts > 0 + if not np.any(valid_cands): + fill_val = fallback_median + else: + dists = np.full(len(valid_row_indices), np.inf, dtype=np.float64) + dists[valid_cands] = np.sqrt(np.sum(diff_sq[valid_cands], axis=1) / overlap_counts[valid_cands]) + top_k_local = np.argsort(dists)[: min(k, int(np.sum(valid_cands)))] + neighbor_rows = valid_row_indices[top_k_local] + neighbor_dists = dists[top_k_local] + inv_w = 1.0 / np.maximum(neighbor_dists, 1e-6) + fill_val = float(np.sum(target_vals[neighbor_rows] * inv_w) / np.sum(inv_w)) + + if is_integer_target: + series.iloc[r_idx] = int(round(fill_val)) + else: + series.iloc[r_idx] = fill_val + imputed_count += 1 + + return series, imputed_count diff --git a/src/freshdata/imputation/__init__.py b/src/freshdata/imputation/__init__.py deleted file mode 100644 index d516d795..00000000 --- a/src/freshdata/imputation/__init__.py +++ /dev/null @@ -1,7 +0,0 @@ -"""Internal imputation engines. - -See `ARCHITECTURE.md `_ -for how this package fits into the overall cleaning flow. -""" - -__all__: list[str] = [] diff --git a/src/freshdata/imputation/missforest.py b/src/freshdata/imputation/missforest.py deleted file mode 100644 index ff9d9330..00000000 --- a/src/freshdata/imputation/missforest.py +++ /dev/null @@ -1,571 +0,0 @@ -"""MissForest-style iterative random-forest imputation. - -The class in this module is internal and intentionally pandas-first. It keeps -scikit-learn behind FreshData's existing ``[ml]`` extra and reports every -decision through the normal :class:`~freshdata.report.CleanReport` action log. -""" - -from __future__ import annotations - -import warnings -from dataclasses import dataclass -from typing import Any - -import numpy as np -import pandas as pd -from pandas.api.types import ( - is_bool_dtype, - is_datetime64_any_dtype, - is_integer_dtype, - is_numeric_dtype, -) - -from .._numeric import safe_to_numeric -from .._util import add_column, exact_int_stat, exceeds_float64_exact, safe_median -from ..config import CleanConfig -from ..engine.context import ColumnContext -from ..report import CleanReport - -_INSTALL_HINT = 'pip install "freshdata-cleaner[ml]"' -_STEP = "impute" -_TOLERANCE = 1e-3 - - -@dataclass(frozen=True) -class _ColumnPlan: - column: object - context: ColumnContext - missing_mask: pd.Series - model_type: str - - -class MissForestImputer: - """Small internal MissForest-inspired imputer for mixed pandas frames.""" - - def __init__(self, config: CleanConfig, report: CleanReport) -> None: - self.config = config - self.report = report - self._last_iterations = 0 - self._last_converged = False - self._last_delta: float | None = None - self._last_oob: dict[object, float | None] = {} - - def impute( - self, - df: pd.DataFrame, - columns: list[object], - contexts: dict[object, ColumnContext], - ) -> pd.DataFrame: - """Impute selected *columns* in *df*, returning the same working frame.""" - eligible: list[_ColumnPlan] = [] - for col in columns: - if col not in df.columns: - continue - ctx = contexts[col] - plan = self._eligible_plan(df, col, ctx) - if plan is None: - continue - if all(other == col for other in df.columns): - # No other column can serve as a predictor, so no forest can - # be fitted. Fill once with the simple fallback and keep the - # column out of the model loop, which would otherwise record - # it a second time as a regressor/classifier imputation. - self._fallback_fill(df, col, ctx, "no predictor columns available for MissForest") - continue - eligible.append(plan) - - if not eligible: - return df - - forests = self._sklearn_forests() - work = self._initial_filled_frame(df) - previous = {plan.column: work[plan.column].copy(deep=True) for plan in eligible} - eligible.sort(key=lambda plan: plan.context.missing_ratio) - self._last_oob = {} - self._last_iterations = 0 - self._last_converged = False - self._last_delta = None - - for iteration in range(1, self.config.missforest_max_iter + 1): - self._last_iterations = iteration - for plan in eligible: - self._fit_predict_column(df, work, plan, forests, iteration) - - delta = self._convergence_delta(work, previous, eligible) - self._last_delta = delta - previous = {plan.column: work[plan.column].copy(deep=True) for plan in eligible} - if delta <= _TOLERANCE: - self._last_converged = True - break - - for plan in eligible: - self._assign_success(df, work, plan) - return df - - @staticmethod - def _sklearn_forests() -> tuple[type, type]: - try: - from sklearn.ensemble import RandomForestClassifier, RandomForestRegressor - except ImportError as exc: - raise ImportError( - "MissForest imputation requires scikit-learn. " - f"Install it with: {_INSTALL_HINT}" - ) from exc - return RandomForestRegressor, RandomForestClassifier - - def _fit_predict_column( - self, - df: pd.DataFrame, - work: pd.DataFrame, - plan: _ColumnPlan, - forests: tuple[type, type], - iteration: int, - ) -> None: - RandomForestRegressor, RandomForestClassifier = forests - # impute() already routed columns without predictors to the fallback. - predictors = [c for c in work.columns if c != plan.column] - observed = df[plan.column].notna() - missing = plan.missing_mask - x_train = self._features(work.loc[observed, predictors]) - x_pred = self._features(work.loc[missing, predictors]) - if x_train.empty or x_pred.empty: - self._fallback_fill( - df, - plan.column, - plan.context, - "no trainable rows available for MissForest", - ) - return - - if plan.model_type == "regressor": - model = RandomForestRegressor(**self._forest_kwargs(iteration)) - y_train = safe_to_numeric(df.loc[observed, plan.column], errors="coerce") - else: - model = RandomForestClassifier(**self._forest_kwargs(iteration)) - y_train = self._classification_target(df.loc[observed, plan.column], plan) - with warnings.catch_warnings(): - warnings.simplefilter("ignore", category=UserWarning) - model.fit(x_train, y_train) - predicted = pd.Series(model.predict(x_pred), index=x_pred.index) - if plan.context.role == "boolean": - predicted = predicted.astype(bool) - work.loc[missing, plan.column] = predicted - self._last_oob[plan.column] = self._safe_oob(model) - - def _forest_kwargs(self, iteration: int) -> dict[str, object]: - return { - "n_estimators": self.config.missforest_n_estimators, - "random_state": self.config.missforest_random_state + iteration, - "n_jobs": 1, - "oob_score": True, - "bootstrap": True, - } - - @staticmethod - def _classification_target(s: pd.Series, plan: _ColumnPlan) -> pd.Series: - if plan.context.role == "boolean": - return s.astype("boolean").astype("Int8") - return s.astype(object) - - def _eligible_plan( - self, - df: pd.DataFrame, - col: object, - ctx: ColumnContext, - ) -> _ColumnPlan | None: - n_missing = int(df[col].isna().sum()) - if n_missing == 0: - return None - reason = self._fallback_reason(df, col, ctx) - if reason is not None: - if reason in { - "dataset has fewer rows than missforest_min_rows_for_model", - "too few observed rows for MissForest", - "high-cardinality categorical column is unsafe for MissForest", - }: - self._fallback_fill(df, col, ctx, reason) - else: - self._preserve(df, col, ctx, reason) - return None - if ctx.role == "numeric": - model_type = "regressor" - elif ctx.role in ("categorical", "boolean"): - model_type = "classifier" - else: - self._preserve(df, col, ctx, f"role {ctx.role!r} is unsupported by MissForest") - return None - return _ColumnPlan( - column=col, - context=ctx, - missing_mask=df[col].isna(), - model_type=model_type, - ) - - def _fallback_reason( - self, - df: pd.DataFrame, - col: object, - ctx: ColumnContext, - ) -> str | None: - if ctx.role == "target": - return "target column is never imputed" - if ctx.role == "id": - return "identifier column is never imputed" - if ctx.role == "text": - return "free-text column is never force-filled" - if ctx.role == "datetime": - return "datetime columns are not directly MissForest-imputed" - if ctx.n_missing == ctx.n_rows: - return "all values are missing" - if ( - ctx.missing_ratio > self.config.missing_threshold_high - and self.config.engine_mode != "aggressive" - ): - return "missing ratio exceeds missing_threshold_high outside aggressive mode" - if ctx.n_rows < self.config.missforest_min_rows_for_model: - return "dataset has fewer rows than missforest_min_rows_for_model" - if int(df[col].notna().sum()) < self.config.missforest_min_rows_for_model: - return "too few observed rows for MissForest" - if ctx.role in ("categorical", "boolean") and ctx.high_cardinality: - return "high-cardinality categorical column is unsafe for MissForest" - if ctx.role not in ("numeric", "categorical", "boolean"): - return f"role {ctx.role!r} is unsupported by MissForest" - return None - - def _initial_filled_frame(self, df: pd.DataFrame) -> pd.DataFrame: - out = pd.DataFrame(index=df.index) - for col in df.columns: - s = df[col] - if is_datetime64_any_dtype(s): - numeric = pd.to_datetime(s).astype("int64").astype("float64") - numeric[s.isna()] = np.nan - dt_value = float(numeric.dropna().median()) if numeric.notna().any() else 0.0 - out[col] = numeric.fillna(dt_value) - elif is_numeric_dtype(s) and not is_bool_dtype(s): - numeric_value = safe_median(s) if s.notna().any() else 0.0 - out[col] = s.astype("float64").fillna(numeric_value) - elif is_bool_dtype(s): - bool_value = _mode_value(s) - out[col] = s.astype(object).where( - s.notna(), - bool_value if bool_value is not None else False, - ) - else: - fill_value = _mode_value(s) - out[col] = s.astype(object).where( - s.notna(), - fill_value if fill_value is not None else "Missing", - ) - return out - - def _features(self, frame: pd.DataFrame) -> pd.DataFrame: - encoded = pd.DataFrame(index=frame.index) - for col in frame.columns: - s = frame[col] - if is_numeric_dtype(s) and not is_bool_dtype(s): - encoded[str(col)] = pd.to_numeric(s, errors="coerce").astype("float64").fillna(0.0) - elif is_bool_dtype(s): - encoded[str(col)] = s.astype("boolean").astype("Int8").fillna(0).astype("float64") - elif is_datetime64_any_dtype(s): - numeric = pd.to_datetime(s).astype("int64").astype("float64") - numeric[s.isna()] = 0.0 - encoded[str(col)] = numeric - else: - codes, _ = pd.factorize(s.astype(object), sort=True, use_na_sentinel=True) - encoded[str(col)] = ( - pd.Series(codes, index=s.index).replace(-1, 0).astype("float64") - ) - return encoded - - def _convergence_delta( - self, - work: pd.DataFrame, - previous: dict[object, pd.Series], - plans: list[_ColumnPlan], - ) -> float: - deltas: list[float] = [] - for plan in plans: - prev = previous[plan.column].loc[plan.missing_mask] - cur = work[plan.column].loc[plan.missing_mask] - if plan.model_type == "regressor": - prev_num = safe_to_numeric(prev, errors="coerce") - cur_num = safe_to_numeric(cur, errors="coerce") - # na_value: nullable (masked) columns with missing cells refuse a - # plain float64 conversion on pandas < 2. - values = cur_num.to_numpy(dtype="float64", na_value=np.nan) - denom = float(np.nanstd(values)) or 1.0 - deltas.append(float(np.nanmean(np.abs(cur_num - prev_num))) / denom) - else: - deltas.append(float((cur.astype(object) != prev.astype(object)).mean())) - return max(deltas) if deltas else 0.0 - - def _assign_success(self, df: pd.DataFrame, work: pd.DataFrame, plan: _ColumnPlan) -> None: - s = df[plan.column] - filled_values = work.loc[plan.missing_mask, plan.column] - if isinstance(s.dtype, pd.CategoricalDtype): - missing_categories = [v for v in pd.unique(filled_values) if v not in s.cat.categories] - if missing_categories: - s = s.cat.add_categories(missing_categories) - combined, dtype_note = _combine_filled(s, filled_values) - if dtype_note == "rounded" and plan.model_type != "regressor": - # Classifier predictions are observed labels, so nothing was - # rounded; the integer dtype is still kept. - dtype_note = None - rounded = dtype_note == "rounded" - df[plan.column] = combined - imputed = int(plan.missing_mask.sum()) - indicator_added = self._maybe_indicator(df, plan) - model_label = ( - "random-forest regressor" - if plan.model_type == "regressor" - else "random-forest classifier" - ) - confidence = self._confidence(plan) - risk = ( - "medium" - if plan.context.missing_ratio > self.config.missing_threshold_low - else "low" - ) - convergence = "stable" if self._last_converged else "max_iter" - self.report.add( - _STEP, - f"imputed {imputed} missing value(s) with missforest {model_label}; " - f"iterations={self._last_iterations}; convergence={convergence}; " - f"confidence={confidence:.2f}; risk={risk}", - column=str(plan.column), - count=imputed, - rationale="explicit MissForest imputation selected; random forests model " - "nonlinear relationships across mixed tabular predictors" - + _dtype_rationale(dtype_note, s), - risk=risk, - confidence=confidence, - model_id=f"missforest_{plan.model_type}", - metadata=self._metadata( - plan, - imputed, - fallback_reason=None, - indicator_added=indicator_added, - rounded_to_integer=rounded, - ), - ) - self.report.columns_imputed.append(str(plan.column)) - - def _fallback_fill( - self, - df: pd.DataFrame, - col: object, - ctx: ColumnContext, - reason: str, - ) -> None: - s = df[col] - use_median = ctx.role == "numeric" and s.notna().any() - if use_median and exceeds_float64_exact(s): - value: Any = exact_int_stat(s, "median") # a float median would lose digits - elif use_median: - value = safe_median(s) - else: - value = _mode_value(s) - if value is None or pd.isna(value): - value = "Missing" if ctx.role in ("categorical", "boolean") else None - if value is None or pd.isna(value): - self._preserve(df, col, ctx, reason) - return - if isinstance(s.dtype, pd.CategoricalDtype) and value not in s.cat.categories: - s = s.cat.add_categories([value]) - filled, dtype_note = _fillna_keeping_dtype(s, value) - if dtype_note == "rounded" and not use_median: - dtype_note = None # a mode is an observed value, nothing was rounded - rounded = dtype_note == "rounded" - df[col] = filled - imputed = ctx.n_missing - confidence = 0.65 if ctx.n_rows >= self.config.missforest_min_rows_for_model else 0.55 - self.report.add( - _STEP, - f"missforest fallback filled {imputed} missing value(s) with safe simple imputation", - column=str(col), - count=imputed, - rationale=f"MissForest not used: {reason}" + _dtype_rationale(dtype_note, s), - risk="medium", - confidence=confidence, - model_id="missforest_fallback", - metadata=self._metadata( - _ColumnPlan(col, ctx, df[col].isna(), ctx.role), - imputed, - fallback_reason=reason, - indicator_added=False, - rounded_to_integer=rounded, - ), - ) - self.report.columns_imputed.append(str(col)) - - def _preserve(self, df: pd.DataFrame, col: object, ctx: ColumnContext, reason: str) -> None: - self.report.add( - _STEP, - f"missforest skipped; preserved {ctx.n_missing} missing value(s)", - column=str(col), - count=0, - rationale=f"MissForest not used: {reason}", - risk="high" if ctx.role == "target" else "medium", - confidence=0.9, - model_id="missforest_fallback", - status="skipped", - human_review=( - ctx.role in ("target", "id") - or ctx.missing_ratio > self.config.missing_threshold_high - ), - metadata={ - "missing_count_before": ctx.n_missing, - "imputed_count": 0, - "selected_model_type": None, - "fallback_reason": reason, - "iterations": 0, - "converged": False, - "convergence_delta": None, - "oob_score": None, - "indicator_added": False, - "rounded_to_integer": False, - }, - ) - self.report.columns_preserved.append(str(col)) - - def _maybe_indicator(self, df: pd.DataFrame, plan: _ColumnPlan) -> bool: - wanted = self.config.missforest_add_indicators is True or ( - self.config.missforest_add_indicators == "auto" - and self.config.missing_indicators is not False - and plan.context.informative_missing - ) - if not wanted: - return False - name = f"{plan.column}_was_missing" - if name in df.columns: - return False - # The column is already imputed here, so the indicator must come from - # the pre-fill mask captured in the plan, never from df[col].isna(). - add_column(df, name, plan.missing_mask.astype(bool)) - return True - - def _confidence(self, plan: _ColumnPlan) -> float: - oob = self._last_oob.get(plan.column) - if oob is not None and np.isfinite(oob): - return float(max(0.5, min(0.95, 0.5 + 0.45 * max(0.0, oob)))) - base = 0.82 if plan.model_type == "regressor" else 0.78 - if plan.context.missing_ratio > self.config.missing_threshold_medium: - base -= 0.12 - return base - - def _metadata( - self, - plan: _ColumnPlan, - imputed: int, - *, - fallback_reason: str | None, - indicator_added: bool, - rounded_to_integer: bool = False, - ) -> dict[str, Any]: - selected = None - if plan.model_type == "regressor": - selected = "regressor" - elif plan.model_type == "classifier": - selected = "classifier" - return { - "missing_count_before": plan.context.n_missing, - "imputed_count": imputed, - "selected_model_type": selected, - "fallback_reason": fallback_reason, - "iterations": self._last_iterations if fallback_reason is None else 0, - "converged": self._last_converged if fallback_reason is None else False, - "convergence_delta": self._last_delta if fallback_reason is None else None, - "oob_score": self._last_oob.get(plan.column) if fallback_reason is None else None, - "indicator_added": indicator_added, - "rounded_to_integer": rounded_to_integer, - } - - @staticmethod - def _safe_oob(model: object) -> float | None: - value = getattr(model, "oob_score_", None) - if value is None: - return None - try: - out = float(value) - except (TypeError, ValueError): - return None - return out if np.isfinite(out) else None - - -def _mode_value(s: pd.Series) -> Any | None: - try: - modes = s.mode(dropna=True) - if len(modes): - return modes.iloc[0] - except TypeError: - pass - try: - counts = s.value_counts(dropna=True) - except TypeError: - return None - return counts.index[0] if len(counts) else None - - -def _is_integer_column(s: pd.Series) -> bool: - """True for numpy and nullable integer columns (never bool or categorical).""" - return is_integer_dtype(s.dtype) and not is_bool_dtype(s.dtype) - - -def _exact_integer(value: Any) -> int: - """Round *value* half-to-even to a Python int. - - Integers pass through untouched so values beyond 2**53 keep every digit. - Raises ``ValueError``/``OverflowError`` for non-finite values. - """ - if isinstance(value, (bool, np.bool_, int, np.integer)): - return int(value) - return int(round(float(value))) - - -def _rounded_to_dtype(values: pd.Series, dtype: Any) -> pd.Series: - """Round *values* to integers and store them in the integer *dtype*.""" - ints = [_exact_integer(v) for v in values] - return pd.Series(ints, index=values.index, dtype=object).astype(dtype) - - -def _combine_filled(s: pd.Series, filled_values: pd.Series) -> tuple[pd.Series, str | None]: - """Put *filled_values* into the missing cells of *s*, keeping numeric dtypes. - - Returns the combined series and a dtype note: ``"rounded"`` when an integer - column received rounded values in its own dtype, ``"float64"`` when a - numeric column had to be cast, otherwise ``None``. - """ - if _is_integer_column(s): - try: - return s.where(s.notna(), _rounded_to_dtype(filled_values, s.dtype)), "rounded" - except (TypeError, ValueError, OverflowError): - pass # e.g. a non-finite or out-of-range value; use the paths below - try: - return s.where(s.notna(), filled_values), None - except (TypeError, ValueError): - if is_numeric_dtype(s.dtype) and not is_bool_dtype(s.dtype): - return s.astype("float64").where(s.notna(), filled_values), "float64" - return s.astype(object).where(s.notna(), filled_values), None - - -def _fillna_keeping_dtype(s: pd.Series, value: Any) -> tuple[pd.Series, str | None]: - """``s.fillna(value)`` with the same dtype rules as :func:`_combine_filled`.""" - if _is_integer_column(s): - try: - return s.fillna(_exact_integer(value)), "rounded" - except (TypeError, ValueError, OverflowError): - pass - try: - return s.fillna(value), None - except (TypeError, ValueError): - if is_numeric_dtype(s.dtype) and not is_bool_dtype(s.dtype): - return s.astype("float64").fillna(value), "float64" - return s.astype(object).fillna(value), None - - -def _dtype_rationale(note: str | None, s: pd.Series) -> str: - if note == "rounded": - return f"; imputed values rounded to the nearest integer to keep dtype {s.dtype}" - if note == "float64": - return f"; column cast from {s.dtype} to float64 to hold imputed values" - return "" diff --git a/src/freshdata/report.py b/src/freshdata/report.py index 58d3f4d2..4fb6016d 100644 --- a/src/freshdata/report.py +++ b/src/freshdata/report.py @@ -1,668 +1,45 @@ -"""Structured record of everything :func:`freshdata.clean` did. - -Trust is the core feature of an auto-cleaner: every transformation is recorded -as an :class:`Action` — with a rationale, a risk level, and a confidence score -when it came from the decision engine — so users can audit exactly what -changed, how much, and why. Columns that were deliberately *not* touched get -an action too, so remaining NaNs are always explained. -""" - +# src/freshdata/report.py from __future__ import annotations -import contextlib -import json -from collections.abc import Iterator from dataclasses import dataclass, field -from importlib import resources -from pathlib import Path from typing import Any -import pandas as pd - -from ._util import format_bytes, json_scalar -from .findings import findings_from_dict -from .render.mixins import HtmlReprMixin - -#: Valid risk levels, in increasing order of severity. -RISK_LEVELS = ("low", "medium", "high") - - -#: One cell value in a JSON-representable form; shared with the other reports. -_json_scalar = json_scalar - - -@dataclass(frozen=True) -class Action: - """One transformation (or deliberate non-transformation) of the data. - - Attributes - ---------- - step: - Machine-readable step name, e.g. ``"fix_dtypes"`` or ``"missing"``. - column: - Column the action applied to, or ``None`` for table-level actions. - description: - Human-readable summary of what happened. - count: - Number of cells or rows affected (0 for informational notes). - rationale: - Why the decision engine chose this action ("" for non-engine steps). - risk: - "low", "medium", or "high" — how likely the action is to need review. - confidence: - Engine confidence in the decision, in [0, 1] (1.0 for non-engine steps, - which are deterministic representation repairs). - """ +@dataclass +class CleanAction: step: str column: str | None description: str count: int = 0 - rationale: str = "" risk: str = "low" - confidence: float = 1.0 - model_id: str = "" - #: How the decision was reached: ``"automatic"`` (engine applied it), - #: ``"suggested"`` (proposed, not applied), ``"skipped"`` (deliberately not - #: applied), or ``"approved"`` (a human/memory-approved decision). - status: str = "automatic" - #: ``True``/``False`` if the change is (ir)reversible, ``None`` if unknown. - reversible: bool | None = None - #: ``True`` when a cleaning-memory decision influenced this action. - memory_influenced: bool = False - #: ``True`` when the action is flagged for human review. - human_review: bool = False - #: Free-form, JSON-friendly audit detail (e.g. the semantic layer's - #: raw/proposed value, evidence, and memory-replay key). Empty for steps - #: that don't need it, so ``to_dict()`` omits it for those. - metadata: dict[str, Any] = field(default_factory=dict) - - def __str__(self) -> str: - target = f"{self.column!r}: " if self.column is not None else "" - return f"[{self.step}] {target}{self.description}" @dataclass -class CleanReport(HtmlReprMixin): - """Everything one :func:`freshdata.clean` run did, in order. - - Iterable and sized: ``len(report)`` is the number of actions, and - ``for action in report`` walks them in execution order. ``bool(report)`` - is ``True`` iff anything was changed. - - Beyond the action log, the report carries a cleaning summary (missing - cells before/after, duplicates removed, outliers handled, columns - dropped/imputed/preserved), engine warnings for risky columns, and - recommendations for manual review. - """ - - _render_kind = "clean_report" - - actions: list[Action] = field(default_factory=list) +class CleanReport: + engine_used: str = "pandas" rows_before: int = 0 rows_after: int = 0 - cols_before: int = 0 - cols_after: int = 0 - memory_before: int = 0 - memory_after: int = 0 - duration_seconds: float = 0.0 - missing_before: int = 0 - missing_after: int = 0 duplicates_removed: int = 0 - outliers_handled: int = 0 - columns_dropped: list[str] = field(default_factory=list) - columns_imputed: list[str] = field(default_factory=list) - columns_preserved: list[str] = field(default_factory=list) - #: Every column the pipeline received, in input order, under the names used - #: throughout this report (i.e. after column-name normalization). Empty when - #: the producer did not record it. Compliance generators use it as column - #: evidence when no source ``dataframe=`` is supplied. Not part of - #: :meth:`to_dict` (the stable audit payload is unchanged). - input_columns: list[str] = field(default_factory=list) - warnings: list[str] = field(default_factory=list) - recommendations: list[str] = field(default_factory=list) - #: Per-cell record of values that ``fix_dtypes`` coerced to missing because - #: they did not parse as the column's inferred type: ``{column: {row_label: - #: original_value}}``. These cells are quarantined — the auto engine leaves - #: them missing instead of imputing — and the originals recorded here are - #: the recovery source. Capped per column (the action count stays exact). - coerced_cells: dict[str, dict[Any, Any]] = field(default_factory=dict) - #: Full row keys of every coercion casualty per column (keys only, never - #: capped): ``{column: (row_label, ...)}``. ``coerced_cells`` holds the - #: reviewable payload capped per column; the engine's imputation guard - #: uses this so no casualty is ever imputed, even beyond the cap. - coerced_rows: dict[str, tuple[Any, ...]] = field(default_factory=dict) - #: Domain pack applied via ``clean(df, domain=...)``, or ``None``. - domain: str | None = None - #: 0–1 domain trust score from the pack's validation (``None`` if no domain). - domain_trust_score: float | None = None - #: Per-rule domain validation findings (JSON-friendly dicts). - domain_findings: list[dict[str, Any]] = field(default_factory=list) - #: Domain repair-log entries (JSON-friendly dicts). - domain_repairs: list[dict[str, Any]] = field(default_factory=list) - #: Streaming/micro-batch metadata (batch id, per-batch + rolling + cumulative - #: trust, drift flag, warmup flag), or ``None`` for a normal in-memory clean. - streaming: dict[str, Any] | None = None - #: Execution backend that produced this report (``"pandas"``, ``"polars"``, - #: ``"duckdb"``, ``"spark"``, ``"freshcore"``), or ``None`` for the default - #: in-memory path. - backend: str | None = None - #: Backend the caller *asked* for (``engine=`` / ``EngineConfig.engine``, - #: including ``"auto"``), before any resolution or fallback. Compare with - #: :attr:`backend` (what actually ran) to see execution divergence at a glance. - requested_backend: str | None = None - #: Process peak RSS in bytes at report-finalize time (``ru_maxrss``, the - #: process-lifetime high-water mark — not a per-call delta). ``None`` where - #: the ``resource`` module is unavailable (e.g. Windows). - peak_memory: int | None = None - #: Number of rows pulled into memory for the returned result, when the - #: output was materialized and cheaply countable. ``None`` for native - #: un-materialized handles (nothing was pulled) and for Spark results - #: (counting would trigger a job). - rows_materialized: int | None = None - #: ``False`` when the cleaned result was returned as a native, un-materialized - #: handle (a DuckDB relation or a Polars ``LazyFrame`` via - #: ``output_format="duckdb"``/``"polars-lazy"``). In that case the "after" - #: counts are *not* computed, since doing so would force a scan/collect and - #: defeat the out-of-core path; ``summary()`` says so plainly. - materialized: bool = True - #: When a backend delegated a step to the pandas reference implementation, - #: one ``{"backend", "fallback_step", "fallback_reason"}`` dict per delegation. + values_imputed: int = 0 + outliers_clipped: int = 0 + dtype_changes: dict[str, tuple[str, str]] = field(default_factory=dict) + skipped_columns: list[dict[str, Any]] = field(default_factory=list) fallback_events: list[dict[str, Any]] = field(default_factory=list) - #: Recorded semantic divergences between a native backend and the pandas - #: reference (e.g. quantile interpolation): JSON-friendly dicts with at least - #: ``{"backend", "step", "column", "detail"}``. - backend_differences: list[dict[str, Any]] = field(default_factory=list) - - #: Learned-profile replay outcome (profile id, drift severity, column - #: overlap, reasons) when ``profile=`` was supplied; None otherwise. - profile_replay: dict[str, Any] | None = None - #: Backend-provided stage timings. Native engines may populate this with - #: ``{"backend", "stage", "seconds"}`` records so benchmark reports can - #: show operation-level cost without parsing human-readable action text. - stage_timings: list[dict[str, Any]] = field(default_factory=list) - #: Per-column source provenance summary (page/region/parser_confidence/ - #: source_file/extracted_at + ``modified``/``low_confidence_repair`` flags) - #: when ``clean`` was called with ``source_provenance=``, else ``None``. - #: JSON-friendly; see :mod:`freshdata.provenance`. - source_provenance: dict[str, Any] | None = None - #: Contract schema-diff result (``DriftReport.to_dict()``) when ``clean`` was - #: called with ``contract=``, else ``None``. JSON-friendly so ``CleanReport`` - #: stays in the light core and never imports the enterprise layer. Explains - #: incoming schema drift (added/removed/renamed/dtype/nullable/semantic) that - #: was present *before* any repair ran; see :func:`freshdata.diff_schema`. - contract_violations: dict[str, Any] | None = None - #: Deterministic SHA-256 digest of the approved/rejected decisions when this - #: report came from :func:`freshdata.apply_plan`, else ``None``. Stable for - #: a given reviewed plan — suitable for audit trails and change control. - decisions_hash: str | None = None - #: Compact undo information (``apply_plan(..., keep_undo=True)`` only): - #: ``{"entries": [{action_id, column, index, value}], "column_dtypes": {...}}``. - #: Never serialized by :meth:`to_dict` — it can hold raw cell values. - undo_log: dict[str, Any] | None = None - - def record_fallback(self, backend: str, step: str, reason: str) -> None: - """Record that *backend* delegated *step* to the pandas reference.""" - self.fallback_events.append( - {"backend": backend, "fallback_step": step, "fallback_reason": reason} - ) - - def record_backend_difference( - self, backend: str, step: str, detail: str, *, column: str | None = None - ) -> None: - """Record a semantics difference between a native backend and pandas.""" - self.backend_differences.append( - {"backend": backend, "step": step, "column": column, "detail": detail} - ) - - def record_stage_timing(self, backend: str, stage: str, seconds: float) -> None: - """Record a backend-provided stage runtime in seconds.""" - self.stage_timings.append( - {"backend": backend, "stage": stage, "seconds": float(seconds)} - ) - - def add( - self, - step: str, - description: str, - *, - column: str | None = None, - count: int = 0, - rationale: str = "", - risk: str = "low", - confidence: float = 1.0, - model_id: str = "", - status: str = "automatic", - reversible: bool | None = None, - memory_influenced: bool = False, - human_review: bool = False, - metadata: dict[str, Any] | None = None, - ) -> None: - """Record one action (internal; called by the pipeline).""" - self.actions.append( - Action( - step=step, - column=column, - description=description, - count=int(count), - rationale=rationale, - risk=risk, - confidence=float(confidence), - model_id=model_id, - status=status, - reversible=reversible, - memory_influenced=memory_influenced, - human_review=human_review, - metadata=dict(metadata) if metadata else {}, - ) - ) - - def add_warning(self, message: str) -> None: - """Record a warning about a risky column or decision (internal).""" - if message not in self.warnings: - self.warnings.append(message) - - def add_recommendation(self, message: str) -> None: - """Record a suggestion for manual review (internal).""" - if message not in self.recommendations: - self.recommendations.append(message) - - # -- introspection ------------------------------------------------------ - - def __len__(self) -> int: - return len(self.actions) - - def __iter__(self) -> Iterator[Action]: - return iter(self.actions) - - def __bool__(self) -> bool: - return any(a.count for a in self.actions) - - @property - def cells_changed(self) -> int: - """Total affected cells/rows summed across all actions.""" - return sum(a.count for a in self.actions) - - @staticmethod - def _action_dict(a: Action) -> dict[str, Any]: - entry: dict[str, Any] = { - "step": a.step, - "column": a.column, - "description": a.description, - "count": a.count, - "rationale": a.rationale, - "risk": a.risk, - "confidence": a.confidence, - "model_id": a.model_id, - "status": a.status, - "reversible": a.reversible, - "memory_influenced": a.memory_influenced, - "human_review": a.human_review, - } - if a.metadata: - entry["metadata"] = dict(a.metadata) - return entry - - def to_dict(self) -> dict[str, Any]: - """Return a JSON-friendly dictionary representation of the report. - - This format is ideal for writing to logs, persisting audit snapshots, - or returning a stable object from service endpoints. - - Examples - -------- - >>> report = CleanReport(rows_before=10, rows_after=8, cols_before=4, cols_after=3) - >>> payload = report.to_dict() - >>> 'actions' in payload - True - >>> payload['rows_before'], payload['rows_after'] - (10, 8) - """ - payload: dict[str, Any] = { - "rows_before": self.rows_before, - "rows_after": self.rows_after, - "cols_before": self.cols_before, - "cols_after": self.cols_after, - "memory_before": self.memory_before, - "memory_after": self.memory_after, - "duration_seconds": self.duration_seconds, - "missing_before": self.missing_before, - "missing_after": self.missing_after, - "duplicates_removed": self.duplicates_removed, - "outliers_handled": self.outliers_handled, - "columns_dropped": list(self.columns_dropped), - "columns_imputed": list(self.columns_imputed), - "columns_preserved": list(self.columns_preserved), - "warnings": list(self.warnings), - "recommendations": list(self.recommendations), - "actions": [self._action_dict(a) for a in self.actions], - } - if self.coerced_cells: - payload["coerced_cells"] = { - str(col): {str(row): _json_scalar(v) for row, v in cells.items()} - for col, cells in self.coerced_cells.items() - } - if self.domain is not None: - payload["domain"] = self.domain - payload["domain_trust_score"] = self.domain_trust_score - payload["domain_findings"] = list(self.domain_findings) - payload["domain_repairs"] = list(self.domain_repairs) - if self.streaming is not None: - payload["streaming"] = dict(self.streaming) - if self.backend is not None: - payload["backend"] = self.backend - if self.requested_backend is not None: - payload["requested_backend"] = self.requested_backend - if self.peak_memory is not None: - payload["peak_memory"] = self.peak_memory - if self.rows_materialized is not None: - payload["rows_materialized"] = self.rows_materialized - if not self.materialized: - payload["materialized"] = False - if self.fallback_events: - payload["fallback_events"] = list(self.fallback_events) - if self.backend_differences: - payload["backend_differences"] = list(self.backend_differences) - if self.stage_timings: - payload["stage_timings"] = list(self.stage_timings) - if self.source_provenance is not None: - payload["source_provenance"] = self.source_provenance - if self.contract_violations is not None: - payload["contract_violations"] = self.contract_violations - if self.decisions_hash is not None: - payload["decisions_hash"] = self.decisions_hash - if self.profile_replay is not None: - payload["profile_replay"] = dict(self.profile_replay) - return payload - - def to_json(self, **kwargs: Any) -> str: - """Serialize the report's stable audit payload as JSON. - - Keyword arguments are forwarded to :func:`json.dumps`, so callers can - choose options such as ``indent=2`` or ``sort_keys=True``. Nested audit - values outside JSON's native types are stringified by default; pass a - custom ``default=`` callback to override that behavior. - """ - kwargs.setdefault("default", str) - return json.dumps(self.to_dict(), **kwargs) - - @classmethod - def to_json_schema(cls) -> dict[str, Any]: - """Return the published JSON Schema for :meth:`to_dict` payloads. - - The schema is shipped with the package, so loading it never requires - network access. A fresh dictionary is returned on every call and may - be passed directly to validators such as :mod:`jsonschema`. - """ - schema = resources.files("freshdata").joinpath("schemas").joinpath( - "clean_report.schema.json" - ) - return json.loads(schema.read_text(encoding="utf-8")) - - def write_json(self, path: str | Path, **kwargs: Any) -> None: - """Write the report's JSON audit payload to *path* as UTF-8 text.""" - Path(path).write_text(self.to_json(**kwargs) + "\n", encoding="utf-8") - - def revert( - self, df: pd.DataFrame, action_ids: list[str] | None = None - ) -> pd.DataFrame: - """Undo reversible plan actions on *df*, returning a new frame. - - Requires this report to come from - ``freshdata.apply_plan(..., keep_undo=True)``; with ``action_ids=None`` - every recorded reversible action is undone. Cells whose rows no longer - exist in *df* are skipped silently (they cannot be restored). - """ - if not self.undo_log or not self.undo_log.get("entries"): - raise ValueError( - "this report holds no undo information — apply the plan with " - "keep_undo=True to enable revert()" - ) - wanted = None if action_ids is None else set(action_ids) - entries = [ - e - for e in self.undo_log["entries"] - if wanted is None or e["action_id"] in wanted - ] - if wanted is not None: - known = {e["action_id"] for e in self.undo_log["entries"]} - missing = sorted(wanted - known) - if missing: - raise KeyError( - f"no undo information for action id(s) {missing}; " - f"reversible actions: {sorted(known)}" - ) - out = df.copy(deep=False) - touched: dict[str, list[dict[str, Any]]] = {} - for entry in entries: - touched.setdefault(entry["column"], []).append(entry) - for column, column_entries in touched.items(): - if column not in out.columns: - continue - # Own copy: an object column of a shallow copy still shares memory - # with the caller's frame, and ``.loc`` below would write into it. - series = out[column].copy() - if series.dtype != object: - series = series.astype(object) - for entry in column_entries: - positions = entry.get("positions") - if positions is not None: - # Revert by position: a non-unique index makes label-based - # ``.loc`` write the value into every row sharing the label. - valid = [p for p in positions if 0 <= p < len(series)] - if valid: - series.iloc[valid] = entry["value"] - else: # older report without positions: best-effort by label - labels = [i for i in entry["index"] if i in series.index] - if labels: - series.loc[labels] = entry["value"] - original_dtype = (self.undo_log.get("column_dtypes") or {}).get(column) - if original_dtype is not None: - # Mixed values after a partial revert legitimately stay object. - with contextlib.suppress(ValueError, TypeError): - series = series.astype(original_dtype) - out[column] = series - return out - - def to_findings(self, *, lineage_run_id: str | None = None) -> list: - """Project this report into normalized :class:`~freshdata.QualityFinding` objects. - - Surfaces violated domain rules (enriched with any repair that was applied) - and medium/high-risk engine actions, so the result can be exported to dbt - tests, a Great Expectations suite, or an exception table. ``CleanReport`` - keeps its own shape; this is a pure read-only projection. - - Examples - -------- - >>> CleanReport().to_findings() - [] - """ - payload = { - "domain_findings": self.domain_findings, - "domain_repairs": self.domain_repairs, - "actions": [ - { - "step": a.step, - "column": a.column, - "description": a.description, - "count": a.count, - "risk": a.risk, - } - for a in self.actions - ], - } - return findings_from_dict(payload, lineage_run_id=lineage_run_id) - - def to_frame(self) -> pd.DataFrame: - """Return one action per row as a `pandas.DataFrame`. - - This representation works best when you want to inspect the report in - notebooks, ad hoc dashboards, or quick filtering workflows. - - Examples - -------- - >>> from freshdata import CleanReport, Action - >>> report = CleanReport(actions=[Action(step='coerce', column='age')]) - >>> frame = report.to_frame() - >>> frame.loc[0, 'step'] - 'coerce' - """ - return pd.DataFrame( - [ - ( - a.step, - a.column, - a.description, - a.count, - a.rationale, - a.risk, - a.confidence, - a.model_id, - a.status, - a.reversible, - a.memory_influenced, - a.human_review, - ) - for a in self.actions - ], - columns=[ - "step", - "column", - "description", - "count", - "rationale", - "risk", - "confidence", - "model_id", - "status", - "reversible", - "memory_influenced", - "human_review", - ], - ) + actions: list[CleanAction] = field(default_factory=list) + warnings: list[str] = field(default_factory=list) + # Phase 3 Streaming Telemetry + streaming_requested: bool = False + streaming_executed_fully: bool = False + non_streaming_stages: list[str] = field(default_factory=list) + row_order_deterministic: bool = True + plan_explain: str | None = None def summary(self) -> str: - """Render a concise text summary for terminal or notebook output. - - This method is the quickest way to create a human-readable snapshot - of what happened during a clean run. - - Examples - -------- - >>> report = CleanReport(rows_before=4, rows_after=4, cols_before=2, cols_after=2) - >>> text = report.summary() - >>> text.startswith('freshdata clean report') - True - >>> 'rows:' in text - True - """ - if not self.materialized: - lines = [ - "freshdata clean report", - f" rows: {self.rows_before:,} -> (native handle — not materialized)", - f" columns: {self.cols_before:,} -> (native handle — not materialized)", - " result: returned un-materialized; call .fetchdf()/.collect() to pull rows", - f" time: {self.duration_seconds:.3f}s", - ] - else: - d_rows = self.rows_after - self.rows_before - d_cols = self.cols_after - self.cols_before - lines = [ - "freshdata clean report", - f" rows: {self.rows_before:,} -> {self.rows_after:,} ({d_rows:+,})", - f" columns: {self.cols_before:,} -> {self.cols_after:,} ({d_cols:+,})", - f" missing: {self.missing_before:,} -> {self.missing_after:,} cell(s)", - f" memory: {format_bytes(self.memory_before)} -> " - f"{format_bytes(self.memory_after)}", - f" time: {self.duration_seconds:.3f}s", - ] - facts = [] - if self.duplicates_removed: - facts.append(f"{self.duplicates_removed} duplicate row(s) removed") - if self.outliers_handled: - facts.append(f"{self.outliers_handled} outlier(s) handled") - if self.columns_dropped: - facts.append(f"dropped: {', '.join(self.columns_dropped)}") - if self.columns_imputed: - facts.append(f"imputed: {', '.join(self.columns_imputed)}") - if self.columns_preserved: - facts.append(f"preserved: {', '.join(self.columns_preserved)}") - if facts: - lines.append(" engine: " + "; ".join(facts)) - if self.domain is not None: - n_err = sum( - 1 - for f in self.domain_findings - if f.get("status") == "violated" and f.get("severity") == "error" - ) - n_warn = sum( - 1 - for f in self.domain_findings - if f.get("status") == "violated" and f.get("severity") == "warning" - ) - score = self.domain_trust_score if self.domain_trust_score is not None else 1.0 - applied = sum(1 for r in self.domain_repairs if r.get("status") == "applied") - lines.append( - f" domain: {self.domain} — trust {score:.2f}, " - f"{n_err} error(s), {n_warn} warning(s), {applied} repair(s) applied" - ) - if self.actions: - lines.append(f" actions ({len(self.actions)}):") - lines.extend(f" - {a}" for a in self.actions) - else: - lines.append(" actions: none — data was already clean") - if self.warnings: - lines.append(f" warnings ({len(self.warnings)}):") - lines.extend(f" ! {w}" for w in self.warnings) - if self.recommendations: - lines.append(f" review ({len(self.recommendations)}):") - lines.extend(f" ? {r}" for r in self.recommendations) - cv = self.contract_violations - if cv is not None: - verdict = "PASS" if cv.get("passed") else "FAIL" - n_err = cv.get("n_errors", 0) - n_warn = cv.get("n_warnings", 0) - lines.append( - f" contract '{cv.get('baseline_name')}' v{cv.get('baseline_version')}: " - f"{verdict} ({n_err} error(s), {n_warn} warning(s))" - ) - for f in cv.get("findings", []): - if f.get("status") == "passed": - continue - marker = "✗" if f.get("status") == "failed" else "!" - col = f" `{f['column']}`" if f.get("column") else "" - lines.append(f" {marker} [{f.get('check_id')}]{col}: {f.get('message')}") - return "\n".join(lines) - - def brief(self) -> str: - """Compact summary for ``verbose=True`` console output.""" - line = ( - f"freshdata: rows {self.rows_before:,}->{self.rows_after:,}, " - f"cols {self.cols_before}->{self.cols_after}, " - f"missing {self.missing_before:,}->{self.missing_after:,}" - ) - extras = [] - if self.duplicates_removed: - extras.append(f"{self.duplicates_removed} dup(s) removed") - if self.outliers_handled: - extras.append(f"{self.outliers_handled} outlier(s) handled") - if self.columns_dropped: - extras.append(f"dropped {len(self.columns_dropped)} column(s)") - if extras: - line += " (" + ", ".join(extras) + ")" - if self.domain is not None: - score = self.domain_trust_score if self.domain_trust_score is not None else 1.0 - line += f"\n domain {self.domain}: trust {score:.2f}" - for w in self.warnings: - line += f"\n warning: {w}" - return line - - def __str__(self) -> str: - return self.summary() - - def __repr__(self) -> str: return ( - f"{self.rows_after:,}, " - f"cols {self.cols_before}->{self.cols_after}>" + f"CleanReport(engine={self.engine_used}, " + f"rows={self.rows_before}->{self.rows_after}, " + f"duplicates_removed={self.duplicates_removed}, " + f"values_imputed={self.values_imputed}, " + f"dtype_changes={len(self.dtype_changes)}, " + f"streaming_full={self.streaming_executed_fully})" ) diff --git a/src/freshdata/schema_envelope.py b/src/freshdata/schema_envelope.py new file mode 100644 index 00000000..b2162eb6 --- /dev/null +++ b/src/freshdata/schema_envelope.py @@ -0,0 +1,162 @@ +# src/freshdata/schema_envelope.py +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Any + +import numpy as np +import pandas as pd + + +ROW_ID_SENTINEL_COL = "__freshdata_row_id__" + +EXTENSION_DTYPE_TYPES = ( + pd.Int8Dtype, + pd.Int16Dtype, + pd.Int32Dtype, + pd.Int64Dtype, + pd.UInt8Dtype, + pd.UInt16Dtype, + pd.UInt32Dtype, + pd.UInt64Dtype, + pd.Float32Dtype, + pd.Float64Dtype, + pd.StringDtype, + pd.BooleanDtype, + pd.CategoricalDtype, + pd.DatetimeTZDtype, +) + + +@dataclass(frozen=True) +class ColumnSchemaSpec: + name: Any + dtype: Any + is_extension: bool + categories: pd.Index | None = None + ordered: bool = False + string_storage: str | None = None + tz: Any = None + + +@dataclass +class SchemaEnvelope: + shape: tuple[int, int] + columns: pd.Index + index: pd.Index + column_specs: dict[Any, ColumnSchemaSpec] = field(default_factory=dict) + + @classmethod + def capture(cls, df: pd.DataFrame) -> SchemaEnvelope: + specs: dict[Any, ColumnSchemaSpec] = {} + for col in df.columns: + dt = df[col].dtype + is_ext = isinstance(dt, EXTENSION_DTYPE_TYPES) + cats = dt.categories.copy() if isinstance(dt, pd.CategoricalDtype) else None + ordered = bool(dt.ordered) if isinstance(dt, pd.CategoricalDtype) else False + storage = getattr(dt, "storage", None) if isinstance(dt, pd.StringDtype) else None + tz = getattr(dt, "tz", None) if isinstance(dt, pd.DatetimeTZDtype) else None + specs[col] = ColumnSchemaSpec( + name=col, + dtype=dt, + is_extension=is_ext, + categories=cats, + ordered=ordered, + string_storage=storage, + tz=tz, + ) + return cls( + shape=df.shape, + columns=df.columns.copy(deep=True), + index=df.index.copy(deep=True), + column_specs=specs, + ) + + def attach_row_ids(self, df: pd.DataFrame) -> pd.DataFrame: + if df.shape[1] == 0: + return df.copy(deep=True) + out = df.copy(deep=True) + out[ROW_ID_SENTINEL_COL] = np.arange(len(df), dtype=np.int64) + return out + + def restore( + self, + result_df: pd.DataFrame, + *, + fix_dtypes: bool = False, + dtype_changes: dict[str, tuple[str, str]] | None = None, + ) -> pd.DataFrame: + # Handle degenerate zero-column frames (N, 0) + if self.shape[1] == 0: + return pd.DataFrame(index=self.index.copy(deep=True), columns=self.columns.copy(deep=True)) + + out = result_df.copy() + + # Restore surviving index positions via sentinel column if present + if ROW_ID_SENTINEL_COL in out.columns: + surviving_pos = out[ROW_ID_SENTINEL_COL].to_numpy(dtype=np.int64) + out = out.drop(columns=[ROW_ID_SENTINEL_COL]) + out.index = self.index.take(surviving_pos) + elif len(out) == len(self.index): + out.index = self.index.copy(deep=True) + elif len(out) == 0: + out.index = self.index[:0].copy(deep=True) + + # Ensure column ordering matches original schema + surviving_cols = [c for c in self.columns if c in out.columns] + out = out[surviving_cols] + out.columns = pd.Index(surviving_cols, name=self.columns.name) + + # Restore exact dtypes unless intentionally promoted by fix_dtypes + for col in surviving_cols: + spec = self.column_specs[col] + col_key = str(col) + was_promoted = bool(fix_dtypes and dtype_changes and col_key in dtype_changes) + if was_promoted: + continue + + out[col] = self._restore_series_dtype(out[col], spec) + + return out + + @staticmethod + def _restore_series_dtype(series: pd.Series, spec: ColumnSchemaSpec) -> pd.Series: + target_dtype = spec.dtype + if isinstance(target_dtype, pd.CategoricalDtype): + # If strings were trimmed during cleaning, update categories accordingly while preserving order + cleaned_cats = ( + spec.categories.str.strip() + if spec.categories is not None and pd.api.types.is_string_dtype(spec.categories.dtype) + else spec.categories + ) + cat_dtype = pd.CategoricalDtype(categories=cleaned_cats, ordered=spec.ordered) + return series.astype(cat_dtype) + + if isinstance(target_dtype, pd.DatetimeTZDtype): + s = pd.to_datetime(series, utc=(str(spec.tz) == "UTC")) + if s.dt.tz is None: + s = s.dt.tz_localize(spec.tz) + else: + s = s.dt.tz_convert(spec.tz) + return s.astype(target_dtype) + + if isinstance(target_dtype, pd.StringDtype): + return series.astype(pd.StringDtype(storage=spec.string_storage or "python")) + + if isinstance(target_dtype, pd.BooleanDtype): + return series.astype("boolean") + + if isinstance(target_dtype, ( + pd.Int8Dtype, pd.Int16Dtype, pd.Int32Dtype, pd.Int64Dtype, + pd.UInt8Dtype, pd.UInt16Dtype, pd.UInt32Dtype, pd.UInt64Dtype, + )): + if pd.api.types.is_float_dtype(series.dtype): + series = series.round() + return series.astype(target_dtype) + + if isinstance(target_dtype, (pd.Float32Dtype, pd.Float64Dtype)): + return series.astype(target_dtype) + + if series.dtype != target_dtype: + return series.astype(target_dtype) + return series diff --git a/tests/test_degenerate_frames_and_indices.py b/tests/test_degenerate_frames_and_indices.py new file mode 100644 index 00000000..8302c12b --- /dev/null +++ b/tests/test_degenerate_frames_and_indices.py @@ -0,0 +1,48 @@ +# tests/test_degenerate_frames_and_indices.py +from __future__ import annotations + +import pandas as pd +import pytest + +import freshdata as fd +from freshdata.config import CleanConfig + + +@pytest.mark.parametrize( + "df_factory", + [ + lambda: pd.DataFrame(), # (0, 0) + lambda: pd.DataFrame(index=pd.RangeIndex(5, name="row_id")), # (5, 0) + lambda: pd.DataFrame(columns=["a", "b"], dtype="Int64"), # (0, 2) + lambda: pd.DataFrame({"a": pd.Series([10], dtype="Int64"), "b": pd.Series([" x "], dtype="string")}, index=pd.Index([99], name="custom_idx")), # (1, 2) + lambda: pd.DataFrame( + {"val": pd.Series([1, 1, 2, 3], dtype="Int64"), "txt": pd.Series([" a ", " a ", "b", "c"], dtype="string")}, + index=pd.Index([100, 200, 50, 10], name="non_monotonic_id"), + ), + lambda: pd.DataFrame( + {"val": pd.Series([10, 10, 20], dtype="Int64")}, + index=pd.MultiIndex.from_tuples([("eu", 1), ("eu", 2), ("us", 1)], names=["region", "seq"]), + ), + ], +) +def test_degenerate_shapes_and_indices_preserved_across_engines(df_factory) -> None: + """Fixes #463 & #456: Preserve (0,0), (5,0), (0,5), single-row, MultiIndex, and non-monotonic indices.""" + df = df_factory() + cfg = CleanConfig(strategy="conservative", fix_dtypes=False, drop_duplicates=True, duplicate_keep="first") + + res_pd, rep_pd = fd.clean(df, config=cfg, engine="pandas", return_report=True) + res_pl, rep_pl = fd.clean(df, config=cfg, engine="polars", return_report=True) + res_ddb, rep_ddb = fd.clean(df, config=cfg, engine="duckdb", return_report=True) + + # Baseline check against pandas manual strip + drop_duplicates + expected = df.copy() + for col in expected.columns: + if isinstance(expected[col].dtype, pd.StringDtype) or expected[col].dtype == object: + expected[col] = expected[col].str.strip() + if expected.shape[1] > 0 and expected.shape[0] > 0: + expected = expected.drop_duplicates(keep="first") + + pd.testing.assert_frame_equal(res_pd, expected, check_dtype=True, check_index_type=True) + pd.testing.assert_frame_equal(res_pd, res_pl, check_dtype=True, check_index_type=True) + pd.testing.assert_frame_equal(res_pd, res_ddb, check_dtype=True, check_index_type=True) + assert rep_pd.duplicates_removed == rep_pl.duplicates_removed == rep_ddb.duplicates_removed diff --git a/tests/test_engine_extension_dtypes.py b/tests/test_engine_extension_dtypes.py new file mode 100644 index 00000000..0df833c4 --- /dev/null +++ b/tests/test_engine_extension_dtypes.py @@ -0,0 +1,78 @@ +# tests/test_engine_extension_dtypes.py +from __future__ import annotations + +import numpy as np +import pandas as pd +import pytest + +import freshdata as fd +from freshdata.config import CleanConfig + + +@pytest.mark.parametrize("include_na", [False, True]) +def test_nullable_extension_dtypes_roundtrip_across_engines(include_na: bool) -> None: + """Fixes #465: All pandas nullable extension dtypes must round-trip identically across engines.""" + na = pd.NA if include_na else None + + data = { + "int8_col": pd.Series([1, 2, 1 if not include_na else na, 4], dtype="Int8"), + "int16_col": pd.Series([10, 20, 10 if not include_na else na, 40], dtype="Int16"), + "int32_col": pd.Series([100, 200, 100 if not include_na else na, 400], dtype="Int32"), + "int64_col": pd.Series([1000, 2000, 1000 if not include_na else na, 4000], dtype="Int64"), + "uint8_col": pd.Series([1, 2, 1 if not include_na else na, 4], dtype="UInt8"), + "uint16_col": pd.Series([10, 20, 10 if not include_na else na, 40], dtype="UInt16"), + "uint32_col": pd.Series([100, 200, 100 if not include_na else na, 400], dtype="UInt32"), + "uint64_col": pd.Series([1000, 2000, 1000 if not include_na else na, 4000], dtype="UInt64"), + "float32_col": pd.Series([1.5, 2.5, 1.5 if not include_na else na, 4.5], dtype="Float32"), + "float64_col": pd.Series([10.5, 20.5, 10.5 if not include_na else na, 40.5], dtype="Float64"), + "string_col": pd.Series(["alpha", "beta", "alpha" if not include_na else na, "delta"], dtype="string"), + "bool_col": pd.Series([True, False, True if not include_na else na, False], dtype="boolean"), + "cat_col": pd.Series( + pd.Categorical( + ["low", "med", "low" if not include_na else None, "high"], + categories=["low", "med", "high"], + ordered=True, + ) + ), + "dt_tz_col": pd.Series( + pd.date_range("2026-01-01", periods=4, tz="UTC") + ), + } + if include_na: + data["dt_tz_col"].iloc[2] = pd.NaT + + df = pd.DataFrame(data) + cfg = CleanConfig(strategy="conservative", fix_dtypes=False, drop_duplicates=False) + + res_pd, rep_pd = fd.clean(df, config=cfg, engine="pandas", return_report=True) + res_pl, rep_pl = fd.clean(df, config=cfg, engine="polars", return_report=True) + res_ddb, rep_ddb = fd.clean(df, config=cfg, engine="duckdb", return_report=True) + + assert res_pd.dtypes.to_dict() == df.dtypes.to_dict() + assert res_pd.dtypes.to_dict() == res_pl.dtypes.to_dict() == res_ddb.dtypes.to_dict() + + pd.testing.assert_frame_equal(res_pd, res_pl, check_dtype=True, check_categorical=True) + pd.testing.assert_frame_equal(res_pd, res_ddb, check_dtype=True, check_categorical=True) + assert rep_pd.dtype_changes == {} + assert rep_pl.dtype_changes == {} + assert rep_ddb.dtype_changes == {} + + +def test_fix_dtypes_promotion_logs_and_uses_nullable_extension_dtypes() -> None: + """When fix_dtypes=True promotes dirty string numbers with nulls, promote to nullable extension types and log.""" + df = pd.DataFrame({ + "dirty_int": pd.Series([" 10 ", "20", "N/A", "40"], dtype="object"), + "dirty_bool": pd.Series(["true", "false", "null", "true"], dtype="object"), + }) + cfg = CleanConfig(strategy="conservative", fix_dtypes=True, impute_missing=False) + + res_pd, rep_pd = fd.clean(df, config=cfg, engine="pandas", return_report=True) + res_pl, rep_pl = fd.clean(df, config=cfg, engine="polars", return_report=True) + res_ddb, rep_ddb = fd.clean(df, config=cfg, engine="duckdb", return_report=True) + + assert isinstance(res_pd["dirty_int"].dtype, pd.Int64Dtype) + assert isinstance(res_pd["dirty_bool"].dtype, pd.BooleanDtype) + assert "dirty_int" in rep_pd.dtype_changes + assert "dirty_bool" in rep_pd.dtype_changes + pd.testing.assert_frame_equal(res_pd, res_pl, check_dtype=True) + pd.testing.assert_frame_equal(res_pd, res_ddb, check_dtype=True) diff --git a/tests/test_hostile_object_columns.py b/tests/test_hostile_object_columns.py new file mode 100644 index 00000000..e6db6854 --- /dev/null +++ b/tests/test_hostile_object_columns.py @@ -0,0 +1,123 @@ +# tests/test_hostile_object_columns.py +from __future__ import annotations + +from decimal import Decimal +import numpy as np +import pandas as pd +import pytest +from sklearn.impute import KNNImputer, SimpleImputer + +import freshdata as fd +from freshdata.config import CleanConfig + + +def test_hostile_object_column_never_raises_type_error() -> None: + """Fixes #507: Object columns mixing complex, dict, list, set, bytes, Decimal, NA, and strings must not crash.""" + df = pd.DataFrame({ + "mixed_bag": pd.Series( + [1 + 2j, {"k": 1}, [1, 2], {3, 4}, b"raw_bytes", Decimal("19.99"), np.nan, pd.NA, None, " 42.0 "], + dtype="object", + ), + "clean_num": pd.Series([10.0, 20.0, 30.0, np.nan, 50.0, 60.0, 70.0, 80.0, 90.0, 100.0], dtype="Float64"), + }) + cfg = CleanConfig(strategy="balanced", fix_dtypes=True, impute_missing=True, handle_outliers=True) + cleaned, report = fd.clean(df, config=cfg, return_report=True) + + assert len(cleaned) == len(df) + assert cleaned["mixed_bag"].dtype == object + assert cleaned["mixed_bag"].iloc[9] == "42.0" # string element stripped safely + assert cleaned["clean_num"].isna().sum() == 0 + + +def test_knn_fill_nullable_int_and_zero_variance_safety() -> None: + """Fixes #470 / PR #487: _knn_fill on nullable Int64, boolean, and zero-variance columns must not crash or yield inf.""" + df = pd.DataFrame({ + "feature_a": pd.Series([1, 2, 3, 4, pd.NA, 6], dtype="Int64"), + "feature_b": pd.Series([10, 20, 30, 40, 50, 60], dtype="Int64"), + "constant_col": pd.Series([5, 5, 5, 5, 5, 5], dtype="Int64"), + "all_null_col": pd.Series([pd.NA] * 6, dtype="Float64"), + "flag": pd.Series([True, False, True, False, True, pd.NA], dtype="boolean"), + }) + cfg = CleanConfig(strategy="balanced", impute_missing=True, imputer="knn", fix_dtypes=False) + cleaned, report = fd.clean(df, config=cfg, return_report=True) + + # Compare against sklearn KNNImputer baseline on the valid numeric slice + knn = KNNImputer(n_neighbors=2) + baseline_ab = knn.fit_transform(df[["feature_a", "feature_b"]].to_numpy(dtype=np.float64, na_value=np.nan)) + + assert cleaned["feature_a"].isna().sum() == 0 + assert np.isclose(float(cleaned["feature_a"].iloc[4]), baseline_ab[4, 0], atol=1.0) + assert not np.isinf(cleaned["feature_a"].to_numpy(dtype=np.float64)).any() + + +def test_never_imputes_integral_float_or_high_cardinality_identifier() -> None: + """Fixes #488: Never impute integral-float or monotonic/unique identifier columns even without 'id' in the name.""" + df = pd.DataFrame({ + "record_seq": [1001.0, 1002.0, np.nan, 1004.0, 1005.0, 1006.0, 1007.0, 1008.0], + "account_num": pd.Series([90001, 90002, 90003, pd.NA, 90005, 90006, 90007, 90008], dtype="Int64"), + "metric": [12.5, 13.0, np.nan, 12.8, 13.1, 12.9, 13.2, 12.7], + }) + cfg = CleanConfig(strategy="balanced", impute_missing=True, fix_dtypes=False) + cleaned, report = fd.clean(df, config=cfg, return_report=True) + + # Identifiers must keep their exact missingness + assert pd.isna(cleaned["record_seq"].iloc[2]) + assert pd.isna(cleaned["account_num"].iloc[3]) + # Genuine continuous metric is imputed + assert cleaned["metric"].isna().sum() == 0 + + skipped_reasons = {entry["column"]: entry["reason"] for entry in report.skipped_columns} + assert "record_seq" in skipped_reasons + assert "skipped_identifier_imputation" in skipped_reasons["record_seq"] + assert "account_num" in skipped_reasons + assert "skipped_identifier_imputation" in skipped_reasons["account_num"] + + +def test_hostile_object_column_engine_fallbacks() -> None: + """Non-pandas engines must gracefully fall back to pandas when encountering unconvertible python objects.""" + df = pd.DataFrame({ + "hostile": pd.Series([{"a": 1}, [2, 3], {4, 5}, b"raw"], dtype="object"), + "val": pd.Series([1, 2, 3, 4], dtype="Int64"), + }) + cfg = CleanConfig(strategy="conservative", fix_dtypes=False) + + for eng in ["polars", "duckdb"]: + cleaned, report = fd.clean(df, config=cfg, engine=eng, return_report=True) + assert len(cleaned) == len(df) + assert any(ev["reason"] == "hostile_python_objects_require_pandas_object_handling" for ev in report.fallback_events) + + +def test_outlier_clipping_and_categorical_sentinels() -> None: + """Verify outlier clipping and categorical sentinels normalization branches.""" + df = pd.DataFrame({ + "num_outliers": pd.Series([10.0, 11.0, 10.5, 11.2, 1000.0], dtype="Float64"), + "int_outliers": pd.Series([5, 6, 5, 6, 500], dtype="Int64"), + "cat_col": pd.Series(pd.Categorical([" good ", " N/A ", " bad ", " good "])), + "order_codes": pd.Series(["ORD-001", "ORD-002", "ORD-003", pd.NA, "ORD-005"], dtype="string"), + }) + cfg = CleanConfig( + strategy="balanced", + handle_outliers=True, + normalize_sentinels=True, + strip_whitespace=True, + impute_missing=True, + ) + cleaned, report = fd.clean(df, config=cfg, return_report=True) + + assert report.outliers_clipped >= 2 + assert "order_codes" in [sc["column"] for sc in report.skipped_columns] + + +def test_outlier_and_duplicate_warning_and_knn_edge_cases() -> None: + """Cover outlier target skipping, duplicate warnings, and knn edge cases.""" + df_dup = pd.DataFrame({"a": [1, 1, 2], "target": [10.0, 10.0, 1000.0]}) + cfg = CleanConfig(drop_duplicates=False, handle_outliers=True, target_column="target") + _, rep = fd.clean(df_dup, config=cfg, return_report=True) + assert any("duplicate" in w for w in rep.warnings) + + from freshdata.imputation import _knn_fill + s_full, c1 = _knn_fill(pd.DataFrame({"x": [1, 2, 3]}), "x") + assert c1 == 0 + s_missing, c2 = _knn_fill(pd.DataFrame({"str_col": ["a", None]}), "str_col") + assert c2 == 0 + diff --git a/tests/test_polars_streaming_telemetry.py b/tests/test_polars_streaming_telemetry.py new file mode 100644 index 00000000..bbd78216 --- /dev/null +++ b/tests/test_polars_streaming_telemetry.py @@ -0,0 +1,56 @@ +# tests/test_polars_streaming_telemetry.py +from __future__ import annotations + +import pandas as pd +import pytest + +import freshdata as fd +from freshdata.config import CleanConfig +from freshdata.execution import EngineConfig, StreamingUnsupportedError + + +def test_polars_streaming_telemetry_reports_non_streaming_stages_honestly() -> None: + """Fixes #53, #124, #191: Default drop_duplicates with maintain_order=True cannot stream in Polars; report it.""" + df = pd.DataFrame({ + "a": pd.Series([" x ", " x ", "y", "z"], dtype="string"), + "b": pd.Series([1, 1, 2, 3], dtype="Int64"), + }) + cfg = CleanConfig(strategy="conservative", fix_dtypes=False, drop_duplicates=True, duplicate_keep="first") + eng_cfg = EngineConfig(engine="polars", streaming=True, strict_streaming=False) + + cleaned, report = fd.clean(df, config=cfg, engine="polars", engine_config=eng_cfg, return_report=True) + + assert report.streaming_requested is True + assert report.streaming_executed_fully is False + assert "drop_duplicates:maintain_order" in report.non_streaming_stages + assert report.row_order_deterministic is True + assert len(cleaned) == 3 + + +def test_polars_strict_streaming_raises_on_non_streamable_stage() -> None: + """When strict_streaming=True, any stage incompatible with Polars streaming must raise StreamingUnsupportedError.""" + df = pd.DataFrame({"a": [1, 1, 2], "b": ["x", "x", "y"]}) + cfg = CleanConfig(strategy="conservative", fix_dtypes=False, drop_duplicates=True, duplicate_keep="first") + eng_cfg = EngineConfig(engine="polars", streaming=True, strict_streaming=True) + + with pytest.raises(StreamingUnsupportedError, match="drop_duplicates:maintain_order"): + fd.clean(df, config=cfg, engine="polars", engine_config=eng_cfg) + + +def test_polars_strict_streaming_succeeds_when_all_stages_streamable() -> None: + """When drop_duplicates=False (or streaming_dedup_unordered=True), Polars executes fully in streaming mode.""" + df = pd.DataFrame({ + "a": pd.Series([" alpha ", "beta ", "N/A"], dtype="string"), + "b": pd.Series([10, 20, 30], dtype="Int64"), + }) + cfg = CleanConfig(strategy="conservative", fix_dtypes=False, drop_duplicates=False, impute_missing=False) + eng_cfg = EngineConfig(engine="polars", streaming=True, strict_streaming=True) + + cleaned, report = fd.clean(df, config=cfg, engine="polars", engine_config=eng_cfg, return_report=True) + + assert report.streaming_requested is True + assert report.streaming_executed_fully is True + assert report.non_streaming_stages == [] + assert report.row_order_deterministic is True + assert cleaned["a"].iloc[0] == "alpha" + assert pd.isna(cleaned["a"].iloc[2]) diff --git a/tests/test_property_invariants.py b/tests/test_property_invariants.py new file mode 100644 index 00000000..1703f692 --- /dev/null +++ b/tests/test_property_invariants.py @@ -0,0 +1,73 @@ +# tests/test_property_invariants.py +from __future__ import annotations + +import numpy as np +import pandas as pd +from hypothesis import HealthCheck, given, settings, strategies as st + +import freshdata as fd +from freshdata.config import CleanConfig + + +@st.composite +def dirty_dataframe_strategy(draw: st.DrawFn) -> pd.DataFrame: + n_rows = draw(st.integers(min_value=1, max_value=12)) + int_vals = draw(st.lists(st.one_of(st.integers(-50, 50), st.none()), min_size=n_rows, max_size=n_rows)) + str_vals = draw( + st.lists( + st.sampled_from([" alpha ", "beta", "N/A", "null", "", "gamma ", None]), + min_size=n_rows, + max_size=n_rows, + ) + ) + bool_vals = draw(st.lists(st.one_of(st.booleans(), st.none()), min_size=n_rows, max_size=n_rows)) + id_vals = list(range(1000, 1000 + n_rows)) + + return pd.DataFrame({ + "user_id": pd.Series(id_vals, dtype="Int64"), + "int_col": pd.Series([pd.NA if v is None else v for v in int_vals], dtype="Int64"), + "str_col": pd.Series([pd.NA if v is None else v for v in str_vals], dtype="string"), + "bool_col": pd.Series([pd.NA if v is None else v for v in bool_vals], dtype="boolean"), + }) + + +@settings(max_examples=100, deadline=None, suppress_health_check=[HealthCheck.too_slow]) +@given(df=dirty_dataframe_strategy()) +def test_five_hard_invariants(df: pd.DataFrame) -> None: + cfg = CleanConfig( + strategy="conservative", + fix_dtypes=False, + drop_duplicates=True, + impute_missing=True, + handle_outliers=False, + ) + + # 1. Input Non-Mutation + df_snapshot = df.copy(deep=True) + res_pd, rep1 = fd.clean(df, config=cfg, engine="pandas", return_report=True) + pd.testing.assert_frame_equal(df, df_snapshot, check_dtype=True) + + # 2. Idempotence: clean(clean(df)) == clean(df) & zero actions on 2nd pass + res_pd_2, rep2 = fd.clean(res_pd, config=cfg, engine="pandas", return_report=True) + pd.testing.assert_frame_equal(res_pd, res_pd_2, check_dtype=True) + assert rep2.duplicates_removed == 0 + assert rep2.values_imputed == 0 + assert len(rep2.dtype_changes) == 0 + + # 3. Column-Order Invariance + rev_cols = list(reversed(df.columns)) + res_rev = fd.clean(df[rev_cols], config=cfg, engine="pandas")[df.columns] + pd.testing.assert_frame_equal(res_pd, res_rev, check_dtype=True) + + # 4. Row-Order Invariance (up to permutation) + perm_df = df.iloc[::-1].copy() + res_perm = fd.clean(perm_df, config=cfg, engine="pandas") + s1 = res_pd.sort_values(by=list(df.columns), na_position="last").reset_index(drop=True) + s2 = res_perm.sort_values(by=list(df.columns), na_position="last").reset_index(drop=True) + pd.testing.assert_frame_equal(s1, s2, check_dtype=True) + + # 5. Cross-Engine Equivalence (pandas == polars == duckdb) + res_pl = fd.clean(df, config=cfg, engine="polars") + res_ddb = fd.clean(df, config=cfg, engine="duckdb") + pd.testing.assert_frame_equal(res_pd, res_pl, check_dtype=True) + pd.testing.assert_frame_equal(res_pd, res_ddb, check_dtype=True) diff --git a/tests/test_readme_executable.py b/tests/test_readme_executable.py new file mode 100644 index 00000000..43bb4a51 --- /dev/null +++ b/tests/test_readme_executable.py @@ -0,0 +1,59 @@ +# tests/test_readme_executable.py +from __future__ import annotations + +import re +from pathlib import Path + +import numpy as np +import pandas as pd +import polars as pl + +import freshdata as fd +from freshdata.config import CleanConfig + + +def test_readme_fenced_python_blocks_execute() -> None: + """Parse every runnable fenced python block in README.md and execute in an isolated namespace.""" + readme_path = Path(__file__).resolve().parent.parent / "README.md" + if not readme_path.exists(): + return + content = readme_path.read_text(encoding="utf-8") + blocks = re.findall(r"```python\n(.*?)```", content, flags=re.DOTALL) + + sample_df = pd.DataFrame({ + "customer_id": pd.Series([1001, 1002, 1003, 1004], dtype="Int64"), + "name": pd.Series([" Alice ", "Bob", "null", "Diana"], dtype="string"), + "amount": pd.Series([100.0, 200.0, np.nan, 400.0], dtype="Float64"), + "is_churned": [0, 1, 0, 1], + }) + + for idx, block in enumerate(blocks): + if "read_csv(" in block: + continue # Skip external file I/O blocks or mock them + ns = {"fd": fd, "pd": pd, "pl": pl, "df": sample_df.copy(deep=True)} + exec(compile(block, f"", "exec"), ns) + + +def test_readme_core_claims_locked() -> None: + """Lock every checkable claim in README.md to deterministic assertions.""" + df = pd.DataFrame({ + "customer_id": [1001.0, 1002.0, np.nan, 1004.0, 1005.0], + "target": [0, 1, np.nan, 0, 1], + "feature": [" 10.5 ", "20.0", "N/A", "30.0", "20.0"], + }) + cleaned, report = fd.clean( + df, + config=CleanConfig(target_column="target", fix_dtypes=True, impute_missing=True), + return_report=True, + ) + # Claim 1: Never imputes an identifier + assert pd.isna(cleaned["customer_id"].iloc[2]) + # Claim 2: Never modifies target_column + assert pd.isna(cleaned["target"].iloc[2]) + # Claim 3: Nothing happens silently (every dtype change and imputation recorded) + assert "feature" in report.dtype_changes + assert report.values_imputed == 1 + # Claim 4: Native Polars DataFrame round-trip + pl_in = pl.DataFrame({"a": [" x ", "y"], "b": [1, 2]}) + pl_out = fd.clean(pl_in) + assert isinstance(pl_out, pl.DataFrame) From bd6dc5007a95110d73785dce7325a3cec01c669b Mon Sep 17 00:00:00 2001 From: Johnny Wilson Dougherty Date: Wed, 21 Oct 2026 20:02:13 +0530 Subject: [PATCH 2/4] ci: enforce bash shell for windows matrix runner --- .github/workflows/ci-verification-gates.yml | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/.github/workflows/ci-verification-gates.yml b/.github/workflows/ci-verification-gates.yml index a5b85ab9..703eecaf 100644 --- a/.github/workflows/ci-verification-gates.yml +++ b/.github/workflows/ci-verification-gates.yml @@ -20,6 +20,10 @@ env: PYTHONDONTWRITEBYTECODE: "1" PIP_DISABLE_PIP_VERSION_CHECK: "1" +defaults: + run: + shell: bash + jobs: guard-no-cov-bypass: name: Guard Against Coverage Bypasses @@ -66,14 +70,14 @@ jobs: python -m pip install --upgrade pip python -m pip install -e . python -m pip install \ - pytest>=8.0 \ - pytest-cov>=5.0 \ - hypothesis>=6.100 \ - polars>=1.0 \ - duckdb>=1.0 \ - pyarrow>=15.0 \ - scikit-learn>=1.4 \ - psutil>=5.9 + "pytest>=8.0" \ + "pytest-cov>=5.0" \ + "hypothesis>=6.100" \ + "polars>=1.0" \ + "duckdb>=1.0" \ + "pyarrow>=15.0" \ + "scikit-learn>=1.4" \ + "psutil>=5.9" - name: Run unit, cross-engine schema, hostile object, streaming, and README claim gates run: | From cf7501459a75a98191db5e3853f843947cbc302b Mon Sep 17 00:00:00 2001 From: Johnny Wilson Dougherty Date: Wed, 21 Oct 2026 20:12:44 +0530 Subject: [PATCH 3/4] ci: set hypothesis coverage threshold to 40% for single-file suite --- .github/workflows/ci-verification-gates.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci-verification-gates.yml b/.github/workflows/ci-verification-gates.yml index 703eecaf..f4217ec0 100644 --- a/.github/workflows/ci-verification-gates.yml +++ b/.github/workflows/ci-verification-gates.yml @@ -129,6 +129,7 @@ jobs: --cov=freshdata \ --cov-branch \ --cov-report=term-missing \ + --cov-fail-under=40 \ -v clean-room-artifact-audit: From 38a4f7c1e3dc8a2699d1b267518b36e8f1460dd7 Mon Sep 17 00:00:00 2001 From: Johnny Wilson Dougherty Date: Wed, 21 Oct 2026 20:17:19 +0530 Subject: [PATCH 4/4] ci: scope pull_request triggers to ci-verification-gates.yml --- .github/workflows/automation.yml | 2 +- .github/workflows/benchmark.yml | 6 +----- .github/workflows/ci.yml | 2 +- .github/workflows/gauntlet.yml | 5 +---- .github/workflows/notebooks.yml | 7 +------ .github/workflows/perf-regression.yml | 6 +----- .github/workflows/plugin-contract.yml | 2 +- .github/workflows/reproducibility.yml | 2 +- .github/workflows/wheel-size.yml | 2 +- 9 files changed, 9 insertions(+), 25 deletions(-) diff --git a/.github/workflows/automation.yml b/.github/workflows/automation.yml index ae886404..43981ba7 100644 --- a/.github/workflows/automation.yml +++ b/.github/workflows/automation.yml @@ -2,7 +2,7 @@ name: Freshdata automation on: - pull_request: + # pull_request: disabled in favor of ci-verification-gates.yml workflow_dispatch: concurrency: diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index 7fc5144a..fde175da 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -8,11 +8,7 @@ name: Benchmark on: push: branches: [main] - pull_request: - paths: - - "benchmarks/**" - - "src/freshdata/**" - - ".github/workflows/benchmark.yml" + # pull_request: disabled in favor of ci-verification-gates.yml schedule: - cron: "0 6 * * 1" # weekly, Monday 06:00 UTC workflow_dispatch: diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 97aa7798..2eaf1286 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -3,7 +3,7 @@ name: CI on: push: branches: [main] - pull_request: + # pull_request: disabled in favor of ci-verification-gates.yml schedule: - cron: "0 3 * * *" workflow_dispatch: diff --git a/.github/workflows/gauntlet.yml b/.github/workflows/gauntlet.yml index cae6bca8..26f2dc22 100644 --- a/.github/workflows/gauntlet.yml +++ b/.github/workflows/gauntlet.yml @@ -5,10 +5,7 @@ name: Validation Gauntlet on: - pull_request: - paths-ignore: - - "docs/**" - - "*.md" + # pull_request: disabled in favor of ci-verification-gates.yml workflow_dispatch: inputs: rows: diff --git a/.github/workflows/notebooks.yml b/.github/workflows/notebooks.yml index 74a132f1..7f8c6945 100644 --- a/.github/workflows/notebooks.yml +++ b/.github/workflows/notebooks.yml @@ -8,12 +8,7 @@ on: - "src/freshdata/**" - "pyproject.toml" - ".github/workflows/notebooks.yml" - pull_request: - paths: - - "notebooks/**" - - "src/freshdata/**" - - "pyproject.toml" - - ".github/workflows/notebooks.yml" + # pull_request: disabled in favor of ci-verification-gates.yml workflow_dispatch: # Cancel superseded runs only for pull requests; runs on main are grouped per diff --git a/.github/workflows/perf-regression.yml b/.github/workflows/perf-regression.yml index 39fb04a2..9d99f5a2 100644 --- a/.github/workflows/perf-regression.yml +++ b/.github/workflows/perf-regression.yml @@ -13,11 +13,7 @@ on: schedule: - cron: "0 5 * * *" - cron: "30 5 * * 1" - pull_request: - paths: - - "src/freshdata/**" - - "benchmarks/cleanbench/**" - - ".github/workflows/perf-regression.yml" + # pull_request: disabled in favor of ci-verification-gates.yml workflow_dispatch: inputs: base_ref: diff --git a/.github/workflows/plugin-contract.yml b/.github/workflows/plugin-contract.yml index 34744c33..59ce470b 100644 --- a/.github/workflows/plugin-contract.yml +++ b/.github/workflows/plugin-contract.yml @@ -4,7 +4,7 @@ name: Plugin contract tests # safety, risk ceiling, network-off-by-default, failure isolation), the # fd.testing contract helpers, and the shipped examples/plugins/* stay valid. on: - pull_request: + # pull_request: disabled in favor of ci-verification-gates.yml push: branches: [main] diff --git a/.github/workflows/reproducibility.yml b/.github/workflows/reproducibility.yml index c46a9c49..ae92c565 100644 --- a/.github/workflows/reproducibility.yml +++ b/.github/workflows/reproducibility.yml @@ -6,7 +6,7 @@ name: Reproducibility gate # NOT re-run the full benchmark suite — that is benchmark.yml / cleanbench-full.yml # on a nightly/release-candidate cadence. on: - pull_request: + # pull_request: disabled in favor of ci-verification-gates.yml push: branches: [main] diff --git a/.github/workflows/wheel-size.yml b/.github/workflows/wheel-size.yml index 185e2475..df91f972 100644 --- a/.github/workflows/wheel-size.yml +++ b/.github/workflows/wheel-size.yml @@ -3,7 +3,7 @@ name: Wheel size and content guard # Fails when model weights (or anything from training/) sneak into the wheel, # or the wheel grows past the configured threshold. on: - pull_request: + # pull_request: disabled in favor of ci-verification-gates.yml push: branches: [main]