ml_filter/pyproject.toml at master · Modalities/ml_filter · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
[project]
name = "ml-filter"
version = "0.1.0"
requires-python = ">=3.10,<3.12"
description = "ML Filter, a python framework for filtering websites containing educational content."
dependencies = [
    "hydra-core",
    "requests",
    "tqdm",
    "datasets",
    "pydantic",
    "transformers==4.57.0",
    "flash-attn==2.8.3",
    "click",
    "click_pathlib",
    "Jinja2",
    "scikit-learn",
    # this is only needed for MN5 as it has Keras 3 installed which is incompatible with the latest version of transformers.
    # When running mn_filters the transformers generally checks if keras is installed and if so if it is the correct version.
    # This throws an error as the version of keras is not compatible with the transformers version.
    # If Keras is not installed this check is just skipped.
    "tf-keras==2.18.0",
    "deepl",
    "openai",
    "statsmodels",
    "krippendorff",
    "seaborn",
    "python-dotenv",
    "jq",
    "tabulate",
    "datatrove==0.5.0",
    "orjson",
    "dotenv",
    "lightning",
    "h5py",
    "pydantic_settings",
    "accelerate~=1.6.0",


]

[project.optional-dependencies]
linting = ["pre-commit"]
tests = ["pytest", "pytest-cov", "pytest-dotenv"]
install_helper = ["ninja"]
cuda126 = [
  "torch==2.6.0+cu126",
  "torchvision==0.21.0+cu126",
  "torchaudio==2.6.0+cu126",
  "xformers==0.0.29.post3"
]

[project.scripts]
ml_filter = "ml_filter.__main__:main"

[build-system]
requires = ["setuptools >= 61.0.0"]
build-backend = "setuptools.build_meta"

[tool.black]
target-version = ["py310"]
line-length = 120

[tool.isort]
profile = "black"
line_length = 120
src_paths = ["src", "tests"]

[tool.ruff]
line-length = 120

[tool.pytest.ini_options]
addopts = "--cov=src --cov-report term --cov-report html"

[tool.coverage.run]
branch = true

[tool.coverage.report]
# Regexes for lines to exclude from consideration
exclude_also = [
    # Don't complain about missing debug-only code:
    "def __repr__",
    "if self\\.debug",

    # Don't complain if tests don't hit defensive assertion code:
    "raise AssertionError",
    "raise NotImplementedError",

    # Don't complain if non-runnable code isn't run:
    "if 0:",
    "if __name__ == .__main__.:",

    # Don't complain about abstract methods, they aren't run:
    "@(abc\\.)?abstractmethod",
]


ignore_errors = true

[tool.coverage.html]
directory = "coverage_html_report"