Optimize metric calculation and H2O prediction robustness

- `ml_grid/util/project_score_save.py`: Explicitly convert prediction targets to numpy arrays before calculating metrics (MCC, F1, etc.). This bypasses expensive pandas overhead in scikit-learn's `np.unique` checks.
- `ml_grid/model_classes/H2OBaseClassifier.py`:
    - Implement "lazy loading" for H2O models in `predict` and `predict_proba` to reduce redundant API calls to the H2O cluster.
    - Add retry logic and fallback handling for `java.lang.NullPointerException` crashes in the H2O backend during prediction.

Author: SamoraHunter

ml_grid/model_classes/H2OBaseClassifier.py

@@ -559,8 +559,9 @@ def predict(self, X: pd.DataFrame) -> np.ndarray:
         # Ensure H2O is running
         self._ensure_h2o_is_running()
 
-        # Ensure the model is loaded (critical for cross-validation)
-        self._ensure_model_is_loaded()
+        # OPTIMIZATION: Lazy load model. Only check if we don't have the object.
+        if self.model_ is None:
+            self._ensure_model_is_loaded()
 
         try:
             # --- ROBUSTNESS FIX for java.lang.NullPointerException ---
@@ -592,20 +593,27 @@ def predict(self, X: pd.DataFrame) -> np.ndarray:
         try:
             predictions = self.model_.predict(test_h2o)
         except Exception as e:
-            # --- FIX: Catch H2O backend crashes (NPE) during prediction and fallback ---
-            if "java.lang.NullPointerException" in str(e):
-                self.logger.warning(
-                    f"H2O backend crashed with NPE during predict(). Returning dummy predictions. Details: {e}"
-                )
-                # Fallback: predict the first class (usually 0)
-                dummy_val = (
-                    self.classes_[0]
-                    if self.classes_ is not None and len(self.classes_) > 0
-                    else 0
-                )
-                return np.full(len(X), dummy_val)
+            # If prediction failed, it might be because the model was unloaded/GC'd on server.
+            # Try reloading and predicting again.
+            self.logger.debug(f"Prediction failed ({e}), attempting to reload model...")
+            try:
+                self._ensure_model_is_loaded()
+                predictions = self.model_.predict(test_h2o)
+            except Exception as e2:
+                # --- FIX: Catch H2O backend crashes (NPE) during prediction and fallback ---
+                if "java.lang.NullPointerException" in str(e):
+                    self.logger.warning(
+                        f"H2O backend crashed with NPE during predict(). Returning dummy predictions. Details: {e}"
+                    )
+                    # Fallback: predict the first class (usually 0)
+                    dummy_val = (
+                        self.classes_[0]
+                        if self.classes_ is not None and len(self.classes_) > 0
+                        else 0
+                    )
+                    return np.full(len(X), dummy_val)
 
-            raise RuntimeError(f"H2O prediction failed: {e}")
+                raise RuntimeError(f"H2O prediction failed: {e2}")
 
         # Extract predictions
         pred_df = predictions.as_data_frame(use_multi_thread=False)
@@ -665,8 +673,9 @@ def predict_proba(self, X: pd.DataFrame) -> np.ndarray:
         # Ensure H2O is running
         self._ensure_h2o_is_running()
 
-        # Ensure the model is loaded
-        self._ensure_model_is_loaded()
+        # OPTIMIZATION: Lazy load model.
+        if self.model_ is None:
+            self._ensure_model_is_loaded()
 
         # Create H2O frame with explicit column names
         try:
@@ -687,20 +696,26 @@ def predict_proba(self, X: pd.DataFrame) -> np.ndarray:
         try:
             predictions = self.model_.predict(test_h2o)
         except Exception as e:
-            # --- FIX: Catch H2O backend crashes (NPE) during prediction and fallback ---
-            if "java.lang.NullPointerException" in str(e):
-                self.logger.warning(
-                    f"H2O backend crashed with NPE during predict_proba(). Returning dummy probabilities. Details: {e}"
-                )
-                # Fallback: uniform probabilities
-                n_classes = (
-                    len(self.classes_)
-                    if self.classes_ is not None and len(self.classes_) > 0
-                    else 2
-                )
-                return np.full((len(X), n_classes), 1.0 / n_classes)
+            # Retry logic for unloaded models
+            self.logger.debug(f"Prediction failed ({e}), attempting to reload model...")
+            try:
+                self._ensure_model_is_loaded()
+                predictions = self.model_.predict(test_h2o)
+            except Exception as e2:
+                # --- FIX: Catch H2O backend crashes (NPE) during prediction and fallback ---
+                if "java.lang.NullPointerException" in str(e):
+                    self.logger.warning(
+                        f"H2O backend crashed with NPE during predict_proba(). Returning dummy probabilities. Details: {e}"
+                    )
+                    # Fallback: uniform probabilities
+                    n_classes = (
+                        len(self.classes_)
+                        if self.classes_ is not None and len(self.classes_) > 0
+                        else 2
+                    )
+                    return np.full((len(X), n_classes), 1.0 / n_classes)
 
-            raise RuntimeError(f"H2O prediction failed: {e}")
+                raise RuntimeError(f"H2O prediction failed: {e2}")
 
         # Extract probabilities (drop the 'predict' column)
         prob_df = predictions.drop("predict").as_data_frame(use_multi_thread=False)

ml_grid/util/project_score_save.py

@@ -187,9 +187,21 @@ def update_score_log(
             column_list = _get_score_log_columns(list(global_params.metric_list.keys()))
             line = pd.DataFrame(data=None, columns=column_list)
 
+            # --- OPTIMIZATION: Pre-process targets for faster metric calculation ---
+            # Convert to numpy arrays to avoid pandas overhead in sklearn metrics
+            y_test_np = y_test.values if hasattr(y_test, "values") else y_test
+            best_pred_np = best_pred_orig.values if hasattr(best_pred_orig, "values") else best_pred_orig
+
+            # Attempt to convert to integers (e.g. "0"/"1" strings from H2O) for faster np.unique
+            try:
+                y_test_np = y_test_np.astype(int)
+                best_pred_np = best_pred_np.astype(int)
+            except (ValueError, TypeError):
+                pass
+
             # best_pred_orig = grid.best_estimator_.predict(X_test_orig)
             try:
-                auc = metrics.roc_auc_score(y_test, best_pred_orig)
+                auc = metrics.roc_auc_score(y_test_np, best_pred_np)
             except Exception as e:
                 logger.warning(f"Could not calculate AUC score: {e}")
                 logger.debug(f"y_test unique values: {y_test.unique()!s}")
@@ -198,11 +210,11 @@ def update_score_log(
                 )
                 auc = np.nan
 
-            mcc = matthews_corrcoef(y_test, best_pred_orig)
-            f1 = f1_score(y_test, best_pred_orig, average="binary")
-            precision = precision_score(y_test, best_pred_orig, average="binary")
-            recall = recall_score(y_test, best_pred_orig, average="binary")
-            accuracy = accuracy_score(y_test, best_pred_orig)
+            mcc = matthews_corrcoef(y_test_np, best_pred_np)
+            f1 = f1_score(y_test_np, best_pred_np, average="binary")
+            precision = precision_score(y_test_np, best_pred_np, average="binary")
+            recall = recall_score(y_test_np, best_pred_np, average="binary")
+            accuracy = accuracy_score(y_test_np, best_pred_np)
 
             # get info from current settings iter...local_param_dict ml_grid_object
             for key in ml_grid_object.local_param_dict: