Optimize TF/GPU init and fix nested parallelism in grid search

- Prevent repeated TensorFlow/GPU initialization overhead by adding a global `_TF_INITIALIZED` flag in `grid_search_cross_validate.py`.
- Detect execution within worker processes (daemon) and force `grid_n_jobs=1` to prevent CPU oversubscription caused by nested parallelism (outer hyperopt loop + inner cross-validation loop).

Author: SamoraHunter

ml_grid/pipeline/grid_search_cross_validate.py

@@ -1,5 +1,6 @@
 import time
 import logging
+import multiprocessing
 import warnings
 from typing import Any, Dict, List, Optional, Union
 
@@ -46,6 +47,9 @@
 from ml_grid.util.bayes_utils import is_skopt_space
 from skopt.space import Categorical
 
+# Global flag to ensure TensorFlow/GPU setup runs only once per process
+_TF_INITIALIZED = False
+
 
 class grid_search_crossvalidate:
 
@@ -102,26 +106,42 @@ def __init__(
 
         self.sub_sample_parameter_val = sub_sample_parameter_val
 
-        grid_n_jobs = self.global_params.grid_n_jobs
+        # --- OPTIMIZATION: Detect Nested Parallelism ---
+        # If running inside a worker process (daemon), force n_jobs=1 to prevent
+        # oversubscription (outer loop parallel * inner loop parallel).
+        if multiprocessing.current_process().daemon:
+            self.global_params.grid_n_jobs = 1
+            grid_n_jobs = 1
+        else:
+            grid_n_jobs = self.global_params.grid_n_jobs
 
         # Configure GPU usage and job limits for specific models
         is_gpu_model = (
             "keras" in method_name.lower()
             or "xgb" in method_name.lower()
             or "catboost" in method_name.lower()
         )
+
+        global _TF_INITIALIZED
         if is_gpu_model:
             grid_n_jobs = 1
-            try:
-                gpu_devices = tf.config.experimental.list_physical_devices("GPU")
-                if gpu_devices:
-                    for device in gpu_devices:
-                        tf.config.experimental.set_memory_growth(device, True)
-                else:
-                    # Explicitly set CPU as the visible device for TensorFlow to avoid CUDA init errors
-                    tf.config.set_visible_devices([], "GPU")
-            except Exception as e:
-                self.logger.warning(f"Could not configure GPU for TensorFlow: {e}")
+            # --- OPTIMIZATION: One-time TF/GPU Setup ---
+            if not _TF_INITIALIZED:
+                try:
+                    gpu_devices = tf.config.experimental.list_physical_devices("GPU")
+                    if gpu_devices:
+                        for device in gpu_devices:
+                            try:
+                                tf.config.experimental.set_memory_growth(device, True)
+                            except RuntimeError:
+                                pass  # Memory growth must be set before GPUs have been initialized
+                    else:
+                        # Explicitly set CPU as the visible device for TensorFlow to avoid CUDA init errors
+                        tf.config.set_visible_devices([], "GPU")
+                except Exception as e:
+                    self.logger.warning(f"Could not configure GPU for TensorFlow: {e}")
+                finally:
+                    _TF_INITIALIZED = True
 
         self.metric_list = self.global_params.metric_list