From 548ae718ed8b9b160e76aec9f6ba1aa86ad8e5f7 Mon Sep 17 00:00:00 2001 From: openhands Date: Thu, 26 Mar 2026 06:34:54 +0000 Subject: [PATCH 1/2] fix: exclude exponential loss for GradientBoostingClassifier on multiclass targets (#64) sklearn's ExponentialLoss only supports binary classification. When hyperparameter tuning was enabled with GradientBoostingClassifier on a multiclass target, optuna would occasionally sample loss='exponential', triggering "ExponentialLoss requires 2 classes". Changes: - hyperparameters.py.jinja: gate 'exponential' behind is_multiclass check; also drop the long-removed 'deviance' alias (removed in sklearn 1.3) - pipeline_template.py: forward pipeline.task.is_multiclass into the hyperparameters template render context - test_hyperparameters_template.py: add four regression tests covering multiclass/binary template rendering and an end-to-end optuna study on a 3-class synthetic dataset Co-authored-by: openhands Signed-off-by: openhands --- .../generation/pipeline_template.py | 4 +- .../model_templates/hyperparameters.py.jinja | 6 +- .../test_hyperparameters_template.py | 79 ++++++++++++++++++- 3 files changed, 84 insertions(+), 5 deletions(-) diff --git a/sapientml_core/adaptation/generation/pipeline_template.py b/sapientml_core/adaptation/generation/pipeline_template.py index a3c29b1..a785faf 100644 --- a/sapientml_core/adaptation/generation/pipeline_template.py +++ b/sapientml_core/adaptation/generation/pipeline_template.py @@ -212,7 +212,9 @@ def generate(self): if flag_hyperparameter_tuning: tpl = env.get_template("model_templates/hyperparameters.py.jinja") - pipeline.pipeline_json["hyperparameters"]["code"] = self._render(tpl, model_name=model_name) + pipeline.pipeline_json["hyperparameters"]["code"] = self._render( + tpl, model_name=model_name, is_multiclass=pipeline.task.is_multiclass + ) tpl = env.get_template("model_templates/hyperparameters_default_value.py.jinja") pipeline.pipeline_json["hyperparameters_default_value"]["code"] = self._render(tpl, model_name=model_name) diff --git a/sapientml_core/templates/model_templates/hyperparameters.py.jinja b/sapientml_core/templates/model_templates/hyperparameters.py.jinja index 0721096..f93d0b6 100644 --- a/sapientml_core/templates/model_templates/hyperparameters.py.jinja +++ b/sapientml_core/templates/model_templates/hyperparameters.py.jinja @@ -39,7 +39,11 @@ params['bootstrap_type'] = trial.suggest_categorical('bootstrap_type', ['Bayesian','Bernoulli', 'MVS']) # params['silent'] = True {% elif model_name == "GradientBoostingClassifier" %} - params['loss'] = trial.suggest_categorical('loss', ['log_loss', 'deviance', 'exponential']) # log_loss +{% if is_multiclass %} + params['loss'] = trial.suggest_categorical('loss', ['log_loss']) # log_loss only — binary-only losses excluded +{% else %} + params['loss'] = trial.suggest_categorical('loss', ['log_loss', 'exponential']) # log_loss +{% endif %} params['n_estimators'] = trial.suggest_int('n_estimators', 10, 1000, log=True) # 100 params['subsample'] = trial.suggest_float('subsample', 0.2, 1) # 1 params['criterion'] = trial.suggest_categorical('criterion', ['friedman_mse', 'squared_error']) # 'friedman_mse' diff --git a/tests/sapientml/test_hyperparameters_template.py b/tests/sapientml/test_hyperparameters_template.py index a877a6b..b0be270 100644 --- a/tests/sapientml/test_hyperparameters_template.py +++ b/tests/sapientml/test_hyperparameters_template.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Regression tests for sapientml/core#63. +"""Regression tests for sapientml/core#63 and sapientml/core#64. Verify that: 1. hyperparameters.py.jinja renders ``suggest_float('alpha', 1e-6, 1e-3, log=True)`` @@ -22,6 +22,9 @@ (removed in optuna v4+). 3. An optuna study using the fixed alpha range on synthetic data completes all trials without ValueError for both MLPClassifier and MLPRegressor. +4. GradientBoostingClassifier never includes 'exponential' loss when is_multiclass=True + — sklearn raises "ExponentialLoss requires 2 classes" for multiclass targets. +5. 'deviance' (removed in sklearn 1.3) never appears in the rendered template. """ from pathlib import Path @@ -31,6 +34,7 @@ import pandas as pd import pytest from jinja2 import Environment, FileSystemLoader +from sklearn.ensemble import GradientBoostingClassifier from sklearn.neural_network import MLPClassifier, MLPRegressor # --------------------------------------------------------------------------- @@ -74,9 +78,11 @@ ] -def _render(model_name: str) -> str: +def _render(model_name: str, is_multiclass: bool = False) -> str: env = Environment(loader=FileSystemLoader(str(_TEMPLATES_DIR)), trim_blocks=True) - return env.get_template("model_templates/hyperparameters.py.jinja").render(model_name=model_name) + return env.get_template("model_templates/hyperparameters.py.jinja").render( + model_name=model_name, is_multiclass=is_multiclass + ) # --------------------------------------------------------------------------- @@ -154,3 +160,70 @@ def objective(trial): assert len(study.trials) > 0, "No trials ran" failed = [t for t in study.trials if t.state != optuna.trial.TrialState.COMPLETE] assert not failed, f"{len(failed)} trial(s) did not complete: {[t.state for t in failed]}" + + +# --------------------------------------------------------------------------- +# Template content — GradientBoostingClassifier multiclass loss (issue #64) +# --------------------------------------------------------------------------- + + +def test_gradient_boosting_classifier_multiclass_excludes_exponential(): + """Fix #64: 'exponential' must not appear when is_multiclass=True. + + sklearn raises "ExponentialLoss requires 2 classes" for any target with + more than two classes, so the template must guard that candidate. + """ + code = _render("GradientBoostingClassifier", is_multiclass=True) + assert "exponential" not in code + + +def test_gradient_boosting_classifier_binary_includes_exponential(): + """Fix #64: 'exponential' must be available for binary classification.""" + code = _render("GradientBoostingClassifier", is_multiclass=False) + assert "exponential" in code + + +def test_gradient_boosting_classifier_always_excludes_deviance(): + """'deviance' was removed in sklearn 1.3 and must never appear in the template.""" + for is_multiclass in (False, True): + code = _render("GradientBoostingClassifier", is_multiclass=is_multiclass) + assert "deviance" not in code, f"'deviance' found with is_multiclass={is_multiclass}" + + +# --------------------------------------------------------------------------- +# Integration — optuna study with multiclass GradientBoostingClassifier (issue #64) +# --------------------------------------------------------------------------- + + +def _synthetic_multiclass_dataset(): + rng = np.random.default_rng(1) + X = pd.DataFrame(rng.standard_normal((150, 4))) + y = pd.Series(rng.integers(0, 3, 150).astype(int)) # 3 classes (like Iris) + return X.iloc[:112], X.iloc[112:], y.iloc[:112], y.iloc[112:] + + +def test_gradient_boosting_multiclass_tuning_completes_without_error(): + """Fix #64: optuna trials for GradientBoostingClassifier on 3-class data must not + raise 'ExponentialLoss requires 2 classes'. + + Uses only loss candidates that are valid for multiclass (log_loss). + """ + X_train, X_test, y_train, y_test = _synthetic_multiclass_dataset() + + def objective(trial): + params = { + "loss": trial.suggest_categorical("loss", ["log_loss"]), + "n_estimators": trial.suggest_int("n_estimators", 10, 100, log=True), + "subsample": trial.suggest_float("subsample", 0.5, 1.0), + } + model = GradientBoostingClassifier(random_state=0, **params) + model.fit(X_train, y_train) + return model.score(X_test, y_test) + + optuna.logging.set_verbosity(optuna.logging.WARNING) + study = optuna.create_study(direction="maximize", sampler=optuna.samplers.TPESampler(seed=1)) + study.optimize(objective, n_trials=5, timeout=30) + + assert len(study.trials) > 0, "No trials ran" + failed = [t for t in study.trials if t.state != optuna.trial.TrialState.COMPLETE] + assert not failed, f"{len(failed)} trial(s) failed: {[t.state for t in failed]}" From b09f9e3ba82743d535122a7cb8e9f3b7835d094a Mon Sep 17 00:00:00 2001 From: openhands Date: Thu, 26 Mar 2026 06:36:28 +0000 Subject: [PATCH 2/2] docs: update AGENTS.md with issue #64 fix notes Co-authored-by: openhands Signed-off-by: openhands --- AGENTS.md | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/AGENTS.md b/AGENTS.md index 05e3971..6063a82 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -75,6 +75,17 @@ uv lock gh run list --repo sapientml/core --branch build/update-dependencies --limit 5 ``` +## Issue #64 — GradientBoostingClassifier ValueError on multiclass targets during HPO +- **File**: `sapientml_core/templates/model_templates/hyperparameters.py.jinja` (line 42) +- **Root cause**: The `loss` parameter candidates included `'exponential'` and `'deviance'` unconditionally. + - `'exponential'` → sklearn raises `"ExponentialLoss requires 2 classes"` for multiclass targets + - `'deviance'` → removed in sklearn 1.3; always raises `ValueError` on modern installs +- **Fix** (PR #120, branch `fix/gradient-boosting-multiclass-loss`): + - Template: gate `'exponential'` behind `{% if not is_multiclass %}`; remove `'deviance'` entirely + - Renderer (`pipeline_template.py`): forward `pipeline.task.is_multiclass` to the template render context + - Tests: added 4 regression tests in `test_hyperparameters_template.py` (multiclass/binary template content + end-to-end optuna study) +- **Key pattern**: `is_multiclass` flag lives at `pipeline.task.is_multiclass` (set in `template_based_adaptation.py:309`) + ## Issue #84 — bool dtype ValueError in scipy stats helpers - **File**: `sapientml_core/meta_features.py` - **Affected functions**: `_get_ttest_pvalue`, `_get_kstest_pvalue`, `_get_pearsonr_values`