"use_gpus" is never set, so remove. So since the sims explicitly want procs matched to GPUs we really ought to assign sims to those resource sets first. Only if no GPU rsets exist do we fallback to non-gpu sets, producing a correct wait upon allocation. Thanks Claude.

Author: jlnav

libensemble/tests/functionality_tests/test_mpi_gpu_settings.py

@@ -39,7 +39,7 @@
 
 # Do not change these lines - they are parsed by run-tests.sh
 # TESTSUITE_COMMS: mpi local
-# TESTSUITE_NPROCS: 3 6
+# TESTSUITE_NPROCS: 4 7
 
 import os
 import sys
@@ -88,7 +88,7 @@
         "out": [("priority", float), ("resource_sets", int), ("x", float, n)],
         "give_all_with_same_priority": False,
         "async_return": False,
-        "initial_batch_size": nworkers - 1,
+        "initial_batch_size": nworkers,
         "user": {
             "max_resource_sets": nworkers,  # Any sim created can req. 1 worker up to all.
             "lb": np.array([-3, -2]),

libensemble/tools/alloc_support.py

@@ -4,7 +4,7 @@
 
 from libensemble.message_numbers import EVAL_GEN_TAG, EVAL_SIM_TAG
 from libensemble.resources.resources import Resources
-from libensemble.resources.scheduler import InsufficientFreeResources, InsufficientResourcesError, ResourceScheduler
+from libensemble.resources.scheduler import InsufficientResourcesError, ResourceScheduler
 from libensemble.utils.misc import extract_H_ranges
 
 logger = logging.getLogger(__name__)
@@ -76,13 +76,13 @@ def assign_resources(self, rsets_req, use_gpus=None, user_params=[]):
         """
         rset_team = None
         if self.resources is not None:
-            # Try schedule to non-gpu rsets first
-            if use_gpus is None:
+            # When GPUs exist and use_gpus not explicitly set, try GPU rsets first
+            if use_gpus is None and self.sched.resources.total_num_gpu_rsets > 0:
                 try:
-                    rset_team = self.sched.assign_resources(rsets_req, use_gpus=False, user_params=user_params)
+                    rset_team = self.sched.assign_resources(rsets_req, use_gpus=True, user_params=user_params)
                     return rset_team
-                except (InsufficientFreeResources, InsufficientResourcesError):
-                    pass
+                except InsufficientResourcesError:
+                    pass  # More rsets requested than GPU rsets exist - fall back to any
 
             rset_team = self.sched.assign_resources(rsets_req, use_gpus, user_params)
         return rset_team
@@ -160,11 +160,6 @@ def _req_resources_sim(self, libE_info, user_params, H, H_rows):
             )
         else:
             num_rsets_req = 1
-        if "use_gpus" in H.dtype.names:
-            if np.any(H[H_rows]["use_gpus"]):
-                use_gpus = True
-            else:
-                use_gpus = False
         if "num_gpus" in H.dtype.names:
             gpus_per_rset = self.resources.resource_manager.gpus_per_rset
             num_rsets_req_for_gpus = AllocSupport._convert_rows_to_rsets(