Skip to content

Commit 42e67df

Browse files
SamoraHunterSamoraHunter
committed
Refactor install script and update project dependencies
- Merge `install_ts.sh` into `install.sh` with improved argument parsing and help output. - Add proxy support (`-p`) and robust Python interpreter detection to `install.sh`. - Optimize pip installation by pre-installing heavy dependencies (TensorFlow, PyTorch) and increasing the resolver backtrack limit. - Update `pyproject.toml` dependencies: - Upgrade core libraries (Pandas >=2.2, NumPy <2.0). - Relax constraints for ML frameworks (TensorFlow, PyTorch). - Add new AutoML and ML libraries (AutoGluon, FLAML, FastAI, TabPFN). - Update time-series extras (aeon, gluonts). - Remove obsolete `install_ts.sh`. - Update `requirements.txt` with newer TensorFlow and PyTorch versions.
1 parent 65f439d commit 42e67df

4 files changed

Lines changed: 110 additions & 88 deletions

File tree

‎install.sh‎

Lines changed: 64 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -4,13 +4,7 @@
44
ENV_NAME="ml_grid_env"
55
EXTRAS="[test]"
66
INSTALL_TYPE="standard"
7-
8-
# Check for 'ts' argument for time-series installation
9-
if [ "$1" == "ts" ]; then
10-
ENV_NAME="ml_grid_ts_env"
11-
EXTRAS="[test,ts]"
12-
INSTALL_TYPE="time-series"
13-
fi
7+
PROXY_MODE=false
148

159
# Function to print error message and exit
1610
print_error_and_exit() {
@@ -19,15 +13,54 @@ print_error_and_exit() {
1913
exit 1
2014
}
2115

22-
# Determine whether to use python or python3
23-
if command -v python3 &>/dev/null; then
16+
show_help() {
17+
echo "Usage: ./install_ml_grid.sh [OPTIONS] [ts]"
18+
echo ""
19+
echo "Arguments:"
20+
echo " ts Install time-series variant (ml_grid_ts_env with [test,ts] extras)"
21+
echo ""
22+
echo "Options:"
23+
echo " -h, --help Show this help message"
24+
echo " -p, --proxy Install with proxy support"
25+
}
26+
27+
# Parse command line arguments
28+
while [[ $# -gt 0 ]]; do
29+
case $1 in
30+
-p|--proxy) PROXY_MODE=true; shift;;
31+
-h|--help) show_help; exit 0;;
32+
ts)
33+
ENV_NAME="ml_grid_ts_env"
34+
EXTRAS="[test,ts]"
35+
INSTALL_TYPE="time-series"
36+
shift;;
37+
*) echo "Unknown option: $1"; show_help; exit 1;;
38+
esac
39+
done
40+
41+
echo "Detecting Python interpreter..."
42+
43+
if command -v python3.10 &>/dev/null; then
44+
PYTHON_CMD="python3.10"
45+
elif command -v python3.11 &>/dev/null; then
46+
PYTHON_CMD="python3.11"
47+
elif command -v python3 &>/dev/null; then
2448
PYTHON_CMD="python3"
2549
elif command -v python &>/dev/null; then
2650
PYTHON_CMD="python"
2751
else
28-
print_error_and_exit "Python is not installed. Please install it and try again."
52+
print_error_and_exit "Python is not installed."
2953
fi
3054

55+
# Debug output
56+
PYTHON_PATH=$(command -v $PYTHON_CMD)
57+
PYTHON_VERSION=$($PYTHON_CMD --version 2>&1)
58+
59+
echo "Using Python command: $PYTHON_CMD"
60+
echo "Resolved path: $PYTHON_PATH"
61+
echo "Python version: $PYTHON_VERSION"
62+
echo ""
63+
3164
# Check if virtual environment exists
3265
if [ ! -d "$ENV_NAME" ]; then
3366
# Create virtual environment
@@ -41,15 +74,32 @@ source "$ENV_NAME/bin/activate" || print_error_and_exit "Failed to activate virt
4174

4275
echo "Virtual environment activated."
4376

44-
# Upgrade pip
45-
echo "Upgrading pip..."
46-
pip install 'setuptools<70.0.0' wheel || print_error_and_exit "Failed to upgrade pip."
77+
echo "Upgrading pip, setuptools, and wheel..."
78+
pip_base_args=()
79+
if [ "$PROXY_MODE" = true ]; then
80+
pip_base_args+=("--trusted-host" "dh-cap02" "-i" "http://dh-cap02:8008/mirrors/ml_binary_classification_gridsearch_hyperOpt")
81+
fi
82+
83+
python -m pip install --upgrade pip 'setuptools<70.0.0' wheel "${pip_base_args[@]}" \
84+
|| print_error_and_exit "Failed to upgrade pip."
85+
86+
# Increase the resolver backtrack depth to handle the large, complex dependency
87+
# graph (TensorFlow + PyTorch + AutoGluon + FastAI + aeon[all_extras] etc.).
88+
# PIP_RESOLVER_MAX_BACKTRACK is honoured by pip >= 23.3.
89+
export PIP_RESOLVER_MAX_BACKTRACK=10000
4790

91+
# Pre-install heavy dependencies to simplify graph resolution
92+
echo "Pre-installing heavy dependencies..."
93+
python -m pip install "tensorflow==2.20.0" "torch==2.10.0" "nvidia-cuda-nvcc-cu12" "h2o>=3.46.0.5" "scikit-learn>=1.5.2,<1.6" "${pip_base_args[@]}" || print_error_and_exit "Failed to pre-install frameworks."
4894

4995
# Install the project in editable mode along with testing dependencies.
5096
# This reads all dependencies from pyproject.toml.
5197
echo "Installing project dependencies ($EXTRAS)..."
52-
pip install -e ."$EXTRAS" || print_error_and_exit "Failed to install project dependencies."
98+
pip_install_args=("--no-build-isolation" "-e" ".$EXTRAS")
99+
if [ "$PROXY_MODE" = true ]; then
100+
pip_install_args+=("--trusted-host" "dh-cap02" "-i" "http://dh-cap02:8008/mirrors/ml_binary_classification_gridsearch_hyperOpt" "--retries" "5" "--timeout" "60")
101+
fi
102+
pip install "${pip_install_args[@]}" || print_error_and_exit "Failed to install project dependencies."
53103

54104
# Add kernel spec for Jupyter
55105
echo "Registering Jupyter kernel..."

‎install_ts.sh‎

Lines changed: 0 additions & 38 deletions
This file was deleted.

‎pyproject.toml‎

Lines changed: 44 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -22,48 +22,58 @@ classifiers = [
2222
# Core runtime dependencies for the project
2323
dependencies = [
2424
# Core data and ML libraries
25-
"pandas", #==2.0.3
25+
"pandas>=2.2.0,<2.4.0",
2626
"polars", #==1.9.0
27-
"numpy==1.26.4",
28-
"scikit-learn==1.5.2",
29-
"imbalanced-learn==0.12.4",
27+
"numpy>=1.26.4,<2.0.0",
28+
"scikit-learn>=1.5.2,<1.6",
29+
"imbalanced-learn>=0.12.4",
3030

31-
# ML frameworks
32-
"tensorflow==2.17.0",
33-
"torch>=2.0.0", # Updated: removed strict version pin
31+
# ML frameworks and dependencies
32+
"keras>=3.0.0",
33+
"tensorflow>=2.16.0", # Relaxed to allow resolution
34+
"torch>=2.2.0", # Relaxed to allow resolution
3435
"torchvision>=0.15.0", # Updated: compatible with torch 2.x
35-
"tab_transformer_pytorch==0.3.0",
36-
"lightgbm==4.5.0",
37-
"xgboost==2.1.1",
38-
"catboost==1.2.7",
39-
"h2o==3.46.0.5",
40-
"scikeras==0.13.0",
36+
"tab_transformer_pytorch>=0.3.0",
37+
"lightgbm>=4.5.0",
38+
"xgboost>=2.1.1",
39+
"catboost>=1.2.7",
40+
"h2o>=3.46.0.5",
41+
"scikeras>=0.13.0",
42+
"autogluon.tabular",
43+
"tpot==0.12.2",
44+
"FLAML",
45+
"autokeras",
46+
"spacy>=3.8.0", # Force modern spacy for pydantic v2 compatibility
47+
"fastai",
48+
"ray>=2.43.0,<2.53.0",
4149

4250
# Hyperparameter optimization and feature tools
4351
"hyperopt",#==0.2.7",
44-
"scikit-optimize==0.10.2",
45-
"shap==0.46.0",
46-
"pyimpetus==4.1.2",
47-
"fitter==1.7.1",
48-
"fuzzysearch==0.7.3",
52+
"scikit-optimize>=0.10.2",
53+
"shap>=0.46.0",
54+
"pyimpetus>=4.1.2",
55+
"fitter>=1.7.1",
56+
"fuzzysearch>=0.7.3",
4957
# "simbsig", # Commented out: conflicts with torch 2.x requirement
58+
"tabpfn",
5059

5160
# Visualization and notebook integration
52-
"matplotlib==3.9.2",
53-
"seaborn==0.13.2",
54-
"upsetplot==0.9.0",
55-
"ipykernel==6.29.5",
56-
"ipywidgets==8.1.5",
57-
"tabulate==0.9.0",
58-
"pydotplus==2.0.2",
59-
"graphviz==0.20.3",
61+
"matplotlib>=3.9.0",
62+
"seaborn>=0.13.2",
63+
"upsetplot>=0.9.0",
64+
"ipykernel>=6.29.5",
65+
"ipython>=8.0.0",
66+
"ipywidgets>=8.1.5",
67+
"tabulate>=0.9.0",
68+
"pydotplus>=2.0.2",
69+
"graphviz>=0.20.3",
6070

6171
# Other utilities
62-
"statsmodels==0.14.4",
63-
"pyarrow==18.1.0",
64-
"pyyaml==6.0.2",
65-
"lxml==5.3.0",
66-
"setuptools<70.0.0", # Added for pkg_resources, required by hyperopt in Py3.12+
72+
"statsmodels>=0.14.4",
73+
"pyarrow>=18.1.0",
74+
"pyyaml>=6.0.2",
75+
"lxml>=5.3.0",
76+
"pydantic>=2.5.0", # Force modern pydantic
6777

6878
# Dev
6979
"black",
@@ -77,19 +87,19 @@ dependencies = [
7787

7888
[project.optional-dependencies]
7989
test = [
80-
"pytest",
90+
"pytest>=7.0.0",
8191
"nbmake==1.4.6",
8292
]
8393
ts = [
8494
# Time-series specific libraries
85-
"aeon[dl]==0.11.1",
95+
"aeon[all_extras]>=1.2.0", # Updated for compatibility with pandas>=2.2
8696
# Removed tensorflow-addons: deprecated and incompatible with Python 3.12
8797
"tsfresh",
8898
"tensorflow_probability",
8999
"keras-self-attention==0.51.0",
90100
"prophet==1.1.3",
91101
"pmdarima==2.0.3",
92-
"gluonts==0.13.2",
102+
"gluonts>=0.14.0",
93103
]
94104
docs = [
95105
"sphinx",

‎requirements.txt‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -625,7 +625,7 @@ tensorboard-data-server==0.7.2
625625
# tensorboard
626626
tensorboard-plugin-wit==1.8.1
627627
# via -r requirements.in
628-
tensorflow==2.17.0
628+
tensorflow==2.20.0
629629
# via -r requirements.in
630630
tensorflow-addons==0.23.0; sys_platform == "linux"
631631
tensorflow-addons==0.21.0; sys_platform == "win32"
@@ -651,7 +651,7 @@ threadpoolctl==3.5.0
651651
# scikit-learn
652652
tomli==2.0.2
653653
# via build
654-
torch==1.12.1
654+
torch==2.10.0
655655
# via
656656
# -r requirements.in
657657
# simbsig

0 commit comments

Comments
 (0)