Fixed user in alloc_fs to be the combination of gen_specs and alloc_specs["user"]. Updated alloc_f docstrings to reflect where the options can be. Other docstring updates. Adjust tests to have the relevant alloc options in gen_specs instead. some mypy adjusts.

Author: jlnav

.pre-commit-config.yaml

@@ -37,3 +37,4 @@ repos:
   rev: v1.19.1
   hooks:
     - id: mypy
+      exclude: ^libensemble/utils/(launcher|loc_stack|runners|pydantic|output_directory)\.py$|^libensemble/tests/regression_tests/support\.py$|^libensemble/tests/functionality_tests/

docs/tutorials/calib_cancel_tutorial.rst

@@ -151,11 +151,9 @@ The allocation function used in this example is the *only_persistent_gens* funct
 
     alloc_specs = {
         "alloc_f": alloc_f,
-        "user": {
-            "init_sample_size": init_sample_size,
-            "async_return": True,
-            "active_recv_gen": True,
-        },
+        "initial_batch_size": init_sample_size,
+        "async_return": True,
+        "active_recv_gen": True,
     }
 
 **async_return** tells the allocation function to return results to the generator as soon

libensemble/alloc_funcs/fast_alloc.py

@@ -7,7 +7,7 @@ def give_sim_work_first(W, H, sim_specs, gen_specs, alloc_specs, persis_info, li
     to evaluate in the simulation function. The fields in ``sim_specs["in"]``
     are given. If all entries in `H` have been given a be evaluated, a worker
     is told to call the generator function, provided this wouldn't result in
-    more than ``alloc_specs["user"]["num_active_gen"]`` active generators.
+    more than ``gen_specs["num_active_gens"]`` or ``alloc_specs["user"]["num_active_gens"]`` active generators.
 
     This fast_alloc variation of give_sim_work_first is useful for cases that
     simply iterate through H, issuing evaluations in order and, in particular,
@@ -23,7 +23,8 @@ def give_sim_work_first(W, H, sim_specs, gen_specs, alloc_specs, persis_info, li
     if libE_info["sim_max_given"] or not libE_info["any_idle_workers"]:
         return {}, persis_info
 
-    user = alloc_specs.get("user", {})
+    user = {**gen_specs, **alloc_specs.get("user", {})}
+
     manage_resources = libE_info["use_resource_sets"]
 
     support = AllocSupport(W, manage_resources, persis_info, libE_info)

libensemble/alloc_funcs/fast_alloc_and_pausing.py

@@ -28,13 +28,16 @@ def give_sim_work_first(W, H, sim_specs, gen_specs, alloc_specs, persis_info, li
     if libE_info["sim_max_given"] or not libE_info["any_idle_workers"]:
         return {}, persis_info
 
+    user = {**gen_specs, **alloc_specs.get("user", {})}
     manage_resources = libE_info["use_resource_sets"]
     support = AllocSupport(W, manage_resources, persis_info, libE_info)
     Work = {}
     gen_count = support.count_gens()
 
     if gen_specs["user"].get("single_component_at_a_time"):
-        assert alloc_specs["user"]["batch_mode"], "Must be in batch mode when using 'single_component_at_a_time'"
+        assert (
+            alloc_specs["user"]["batch_mode"] or gen_specs["batch_mode"]
+        ), "Must be in batch mode when using 'single_component_at_a_time'"
     if len(H) != persis_info["H_len"]:
         # Something new is in the history.
         persis_info["need_to_give"].update(H["sim_id"][persis_info["H_len"] :].tolist())
@@ -119,13 +122,13 @@ def give_sim_work_first(W, H, sim_specs, gen_specs, alloc_specs, persis_info, li
             break
 
     while len(idle_gen_workers):
-        if gen_count < alloc_specs["user"].get("num_active_gens", gen_count + 1):
+        if gen_count < user.get("num_active_gens", gen_count + 1):
             lw = persis_info["last_worker"]
 
             last_size = persis_info.get("last_size")
             if len(H):
                 # Don't give gen instances in batch mode if points are unfinished
-                if alloc_specs["user"].get("batch_mode") and not all(
+                if (alloc_specs["user"].get("batch_mode") or gen_specs.get("batch_mode")) and not all(
                     np.logical_or(H["sim_ended"][last_size:], H["paused"][last_size:])
                 ):
                     break
@@ -142,7 +145,7 @@ def give_sim_work_first(W, H, sim_specs, gen_specs, alloc_specs, persis_info, li
             persis_info["last_worker"] = i
             persis_info["last_size"] = len(H)
 
-        elif gen_count >= alloc_specs["user"].get("num_active_gens", gen_count + 1):
+        elif gen_count >= user.get("num_active_gens", gen_count + 1):
             idle_gen_workers = []
 
     return Work, persis_info

libensemble/alloc_funcs/give_sim_work_first.py

@@ -18,14 +18,14 @@ def give_sim_work_first(
     """
     Decide what should be given to workers. This allocation function gives any
     available simulation work first, and only when all simulations are
-    completed or running does it start (at most ``alloc_specs["user"]["num_active_gens"]``)
+    completed or running does it start (at most ``gen_specs["num_active_gens"]`` or ``alloc_specs["user"]["num_active_gens"]``)
     generator instances.
 
-    Allows for a ``alloc_specs["user"]["batch_mode"]`` where no generation
+    Allows for a ``gen_specs["batch_mode"]`` or ``alloc_specs["user"]["batch_mode"]`` where no generation
     work is given out unless all entries in ``H`` are returned.
 
     Can give points in highest priority, if ``"priority"`` is a field in ``H``.
-    If ``alloc_specs["user"]["give_all_with_same_priority"]`` is set to True, then
+    If ``gen_specs["give_all_with_same_priority"]`` or ``alloc_specs["user"]["give_all_with_same_priority"]`` is set to True, then
     all points with the same priority value are given as a batch to the sim.
 
     Workers performing sims will be assigned resources given in H["resource_sets"]
@@ -40,7 +40,8 @@ def give_sim_work_first(
         `test_uniform_sampling.py <https://github.com/Libensemble/libensemble/blob/develop/libensemble/tests/functionality_tests/test_uniform_sampling.py>`_ # noqa
     """
 
-    user = alloc_specs.get("user", {})
+    user = {**gen_specs, **alloc_specs.get("user", {})}
+
     if "cancel_sims_time" in user:
         # Cancel simulations that are taking too long
         rows = np.where(np.logical_and.reduce((H["sim_started"], ~H["sim_ended"], ~H["cancel_requested"])))[0]
@@ -52,11 +53,8 @@ def give_sim_work_first(
     if libE_info["sim_max_given"] or not libE_info["any_idle_workers"]:
         return {}, persis_info
 
-    # Initialize options - check gen_specs first
-    batch_give = gen_specs.get("give_all_with_same_priority", user.get("give_all_with_same_priority", False))
-    num_active_gens = gen_specs.get("num_active_gens", user.get("num_active_gens", 1))
-    batch_mode = gen_specs.get("batch_mode", user.get("batch_mode", False))
-
+    # Initialize alloc_specs["user"] as user.
+    batch_give = user.get("give_all_with_same_priority", False)
     gen_in = gen_specs.get("in", [])
 
     manage_resources = libE_info["use_resource_sets"]
@@ -79,11 +77,11 @@ def give_sim_work_first(
     else:
         for wid in support.avail_worker_ids(gen_workers=True):
             # Allow at most num_active_gens active generator instances
-            if gen_count >= num_active_gens:
+            if gen_count >= user.get("num_active_gens", gen_count + 1):
                 break
 
             # Do not start gen instances in batch mode if workers still working
-            if batch_mode and not support.all_sim_ended(H):
+            if user.get("batch_mode") and not support.all_sim_ended(H):
                 break
 
             # Give gen work

libensemble/alloc_funcs/persistent_aposmm_alloc.py

@@ -21,7 +21,8 @@ def persistent_aposmm_alloc(W, H, sim_specs, gen_specs, alloc_specs, persis_info
     if libE_info["sim_max_given"] or not libE_info["any_idle_workers"]:
         return {}, persis_info
 
-    init_sample_size = gen_specs["user"]["initial_sample_size"]
+    user = {**gen_specs, **alloc_specs.get("user", {})}
+    init_sample_size = user["initial_batch_size"]
     manage_resources = libE_info["use_resource_sets"]
     support = AllocSupport(W, manage_resources, persis_info, libE_info)
     gen_count = support.count_persis_gens()

libensemble/alloc_funcs/start_only_persistent.py

@@ -7,12 +7,12 @@
 def only_persistent_gens(W, H, sim_specs, gen_specs, alloc_specs, persis_info, libE_info):
     """
     This allocation function will give simulation work if possible, but
-    otherwise start up to ``alloc_specs["user"]["num_active_gens"]``
+    otherwise start up to ``gen_specs["num_active_gens"]`` or ``alloc_specs["user"]["num_active_gens"]``
     persistent generators (defaulting to one).
 
     By default, evaluation results are given back to the generator once
     all generated points have been returned from the simulation evaluation.
-    If ``alloc_specs["user"]["async_return"]`` is set to True, then any
+    If ``gen_specs["async_return"]`` or ``alloc_specs["user"]["async_return"]`` is set to True, then any
     returned points are given back to the generator.
 
     If any workers are marked as zero_resource_workers, then these will only
@@ -25,7 +25,7 @@ def only_persistent_gens(W, H, sim_specs, gen_specs, alloc_specs, persis_info, l
 
     To be provided in calling script: E.g., ``alloc_specs["user"]["async_return"] = True``
 
-    initial_batch_size: int, optional
+    init_sample_size: int, optional
         Initial sample size - always return in batch. Default: 0
 
     num_active_gens: int, optional
@@ -56,21 +56,20 @@ def only_persistent_gens(W, H, sim_specs, gen_specs, alloc_specs, persis_info, l
         return {}, persis_info
 
     # Initialize alloc_specs["user"] as user.
-    user = alloc_specs.get("user", {})
+    user = {**gen_specs, **alloc_specs.get("user", {})}
+
     manage_resources = libE_info["use_resource_sets"]
 
-    active_recv_gen = gen_specs.get("active_recv_gen", user.get("active_recv_gen", False))
-    initial_batch_size = gen_specs.get("initial_batch_size", user.get("initial_batch_size", 0))
-    batch_give = gen_specs.get("give_all_with_same_priority", user.get("give_all_with_same_priority", False))
+    active_recv_gen = user.get("active_recv_gen", False)  # Persistent gen can handle irregular communications
+    initial_batch_size = user.get("initial_batch_size", 0)  # Always batch return until this many evals complete
+    batch_give = user.get("give_all_with_same_priority", False)
 
     support = AllocSupport(W, manage_resources, persis_info, libE_info)
     gen_count = support.count_persis_gens()
     Work = {}
 
     # Asynchronous return to generator
-    async_return = (
-        gen_specs.get("async_return", user.get("async_return", False)) and sum(H["sim_ended"]) >= initial_batch_size
-    )
+    async_return = user.get("async_return", False) and sum(H["sim_ended"]) >= initial_batch_size
 
     if gen_count < persis_info.get("num_gens_started", 0):
         # When a persistent worker is done, trigger a shutdown (returning exit condition of 1)
@@ -96,7 +95,7 @@ def only_persistent_gens(W, H, sim_specs, gen_specs, alloc_specs, persis_info, l
     # Now the give_sim_work_first part
     points_to_evaluate = ~H["sim_started"] & ~H["cancel_requested"]
     avail_workers = support.avail_worker_ids(persistent=False, zero_resource_workers=False, gen_workers=False)
-    if gen_specs.get("alt_type", user.get("alt_type")):
+    if user.get("alt_type"):
         avail_workers = list(
             set(support.avail_worker_ids(persistent=False, zero_resource_workers=False))
             | set(support.avail_worker_ids(persistent=EVAL_SIM_TAG, zero_resource_workers=False))
@@ -108,7 +107,7 @@ def only_persistent_gens(W, H, sim_specs, gen_specs, alloc_specs, persis_info, l
         sim_ids_to_send = support.points_by_priority(H, points_avail=points_to_evaluate, batch=batch_give)
 
         try:
-            if gen_specs.get("alt_type", user.get("alt_type")):
+            if user.get("alt_type"):
                 Work[wid] = support.sim_work(
                     wid, H, sim_specs["in"], sim_ids_to_send, persis_info.get(wid), persistent=True
                 )
@@ -124,7 +123,7 @@ def only_persistent_gens(W, H, sim_specs, gen_specs, alloc_specs, persis_info, l
         avail_workers = support.avail_worker_ids(persistent=False, zero_resource_workers=True, gen_workers=True)
 
         for wid in avail_workers:
-            if gen_count < gen_specs.get("num_active_gens", user.get("num_active_gens", 1)):
+            if gen_count < user.get("num_active_gens", 1):
                 # Finally, start a persistent generator as there is nothing else to do.
                 try:
                     Work[wid] = support.gen_work(

libensemble/gen_funcs/persistent_sampling_var_resources.py

@@ -25,11 +25,11 @@
 ]
 
 
-def _get_user_params(user_specs):
+def _get_user_params(gen_specs):
     """Extract user params"""
-    b = user_specs["initial_batch_size"]
-    ub = user_specs["ub"]
-    lb = user_specs["lb"]
+    b = gen_specs["initial_batch_size"]
+    ub = gen_specs["user"]["ub"]
+    lb = gen_specs["user"]["lb"]
     n = len(lb)  # dimension
     return b, n, lb, ub
 
@@ -43,7 +43,7 @@ def uniform_sample(_, persis_info, gen_specs, libE_info):
         `test_uniform_sampling_with_variable_resources.py <https://github.com/Libensemble/libensemble/blob/develop/libensemble/tests/functionality_tests/test_uniform_sampling_with_variable_resources.py>`_
     """  # noqa
 
-    b, n, lb, ub = _get_user_params(gen_specs["user"])
+    b, n, lb, ub = _get_user_params(gen_specs)
     rng = persis_info["rand_stream"]
     ps = PersistentSupport(libE_info, EVAL_GEN_TAG)
     tag = None
@@ -76,7 +76,7 @@ def uniform_sample_with_var_gpus(_, persis_info, gen_specs, libE_info):
         `test_GPU_variable_resources.py <https://github.com/Libensemble/libensemble/blob/develop/libensemble/tests/regression_tests/test_GPU_variable_resources.py>`_
     """  # noqa
 
-    b, n, lb, ub = _get_user_params(gen_specs["user"])
+    b, n, lb, ub = _get_user_params(gen_specs)
     rng = persis_info["rand_stream"]
     ps = PersistentSupport(libE_info, EVAL_GEN_TAG)
     tag = None
@@ -111,7 +111,7 @@ def uniform_sample_with_procs_gpus(_, persis_info, gen_specs, libE_info):
         `test_GPU_variable_resources.py <https://github.com/Libensemble/libensemble/blob/develop/libensemble/tests/regression_tests/test_GPU_variable_resources.py>`_
     """  # noqa
 
-    b, n, lb, ub = _get_user_params(gen_specs["user"])
+    b, n, lb, ub = _get_user_params(gen_specs)
     rng = persis_info["rand_stream"]
     ps = PersistentSupport(libE_info, EVAL_GEN_TAG)
     tag = None
@@ -137,7 +137,7 @@ def uniform_sample_with_var_priorities(_, persis_info, gen_specs, libE_info):
     resource sets and priorities are requested for each point.
     """
 
-    b, n, lb, ub = _get_user_params(gen_specs["user"])
+    b, n, lb, ub = _get_user_params(gen_specs)
     rng = persis_info["rand_stream"]
     ps = PersistentSupport(libE_info, EVAL_GEN_TAG)
 
@@ -175,7 +175,7 @@ def uniform_sample_diff_simulations(_, persis_info, gen_specs, libE_info):
         `test_GPU_variable_resources_multi_task.py <https://github.com/Libensemble/libensemble/blob/develop/libensemble/tests/regression_tests/test_GPU_variable_resources_multi_task.py>`_
     """  # noqa
 
-    b, n, lb, ub = _get_user_params(gen_specs["user"])
+    b, n, lb, ub = _get_user_params(gen_specs)
     rng = persis_info["rand_stream"]
     ps = PersistentSupport(libE_info, EVAL_GEN_TAG)
     tag = None
@@ -209,7 +209,7 @@ def uniform_sample_with_sim_gen_resources(_, persis_info, gen_specs, libE_info):
         `test_GPU_variable_resources.py <https://github.com/Libensemble/libensemble/blob/develop/libensemble/tests/regression_tests/test_GPU_variable_resources.py>`_
     """  # noqa
 
-    b, n, lb, ub = _get_user_params(gen_specs["user"])
+    b, n, lb, ub = _get_user_params(gen_specs)
     rng = persis_info["rand_stream"]
     ps = PersistentSupport(libE_info, EVAL_GEN_TAG)
     tag = None

libensemble/gen_funcs/sampling.py

@@ -19,7 +19,7 @@
 @output_data([("x", float, 2)])  # default: can be overwritten in gen_specs
 def uniform_random_sample(_, persis_info, gen_specs):
     """
-    Generates ``gen_specs["user"]["gen_batch_size"]`` points uniformly over the domain
+    Generates ``gen_specs["batch_size"]`` points uniformly over the domain
     defined by ``gen_specs["user"]["ub"]`` and ``gen_specs["user"]["lb"]``.
 
     .. seealso::
@@ -29,7 +29,7 @@ def uniform_random_sample(_, persis_info, gen_specs):
     lb = gen_specs["user"]["lb"]
 
     n = len(lb)
-    b = gen_specs["user"]["gen_batch_size"]
+    b = gen_specs["batch_size"]
 
     H_o = np.zeros(b, dtype=gen_specs["out"])
 
@@ -40,7 +40,7 @@ def uniform_random_sample(_, persis_info, gen_specs):
 
 def uniform_random_sample_with_variable_resources(_, persis_info, gen_specs):
     """
-    Generates ``gen_specs["user"]["gen_batch_size"]`` points uniformly over the domain
+    Generates ``gen_specs["batch_size"]`` points uniformly over the domain
     defined by ``gen_specs["user"]["ub"]`` and ``gen_specs["user"]["lb"]``.
 
     Also randomly requests a different number of resource sets to be used in each evaluation.
@@ -56,7 +56,7 @@ def uniform_random_sample_with_variable_resources(_, persis_info, gen_specs):
     max_rsets = gen_specs["user"]["max_resource_sets"]
 
     n = len(lb)
-    b = gen_specs["user"]["gen_batch_size"]
+    b = gen_specs["batch_size"]
 
     H_o = np.zeros(b, dtype=gen_specs["out"])
 
@@ -84,7 +84,7 @@ def uniform_random_sample_with_var_priorities_and_resources(H, persis_info, gen_
     n = len(lb)
 
     if len(H) == 0:
-        b = gen_specs["user"]["initial_batch_size"]
+        b = gen_specs["batch_size"]
 
         H_o = np.zeros(b, dtype=gen_specs["out"])
         for i in range(0, b):
@@ -119,7 +119,7 @@ def uniform_random_sample_obj_components(H, persis_info, gen_specs):
 
     n = len(lb)
     m = gen_specs["user"]["components"]
-    b = gen_specs["user"]["gen_batch_size"]
+    b = gen_specs["batch_size"]
 
     H_o = np.zeros(b * m, dtype=gen_specs["out"])
     for i in range(0, b):
@@ -143,7 +143,7 @@ def uniform_random_sample_cancel(_, persis_info, gen_specs):
     lb = gen_specs["user"]["lb"]
 
     n = len(lb)
-    b = gen_specs["user"]["gen_batch_size"]
+    b = gen_specs["batch_size"]
 
     H_o = np.zeros(b, dtype=gen_specs["out"])
     for i in range(b):
@@ -158,7 +158,7 @@ def uniform_random_sample_cancel(_, persis_info, gen_specs):
 @output_data([("x", float, (1,))])
 def latin_hypercube_sample(_, persis_info, gen_specs):
     """
-    Generates ``gen_specs["user"]["gen_batch_size"]`` points in a Latin
+    Generates ``gen_specs["batch_size"]`` points in a Latin
     hypercube sample over the domain defined by ``gen_specs["user"]["ub"]`` and
     ``gen_specs["user"]["lb"]``.
 
@@ -170,7 +170,7 @@ def latin_hypercube_sample(_, persis_info, gen_specs):
     lb = gen_specs["user"]["lb"]
 
     n = len(lb)
-    b = gen_specs["user"]["gen_batch_size"]
+    b = gen_specs["batch_size"]
 
     H_o = np.zeros(b, dtype=gen_specs["out"])
 

libensemble/gen_funcs/uniform_or_localopt.py

@@ -15,7 +15,7 @@
 
 def uniform_or_localopt(H, persis_info, gen_specs, libE_info):
     """
-    This generation function returns ``gen_specs["user"]["gen_batch_size"]`` uniformly
+    This generation function returns ``gen_specs["batch_size"]`` uniformly
     sampled points when called in nonpersistent mode (i.e., when
     ``libE_info["persistent"]`` isn't ``True``).  Otherwise, the generation
     function starts a persistent nlopt local optimization run.
@@ -31,7 +31,7 @@ def uniform_or_localopt(H, persis_info, gen_specs, libE_info):
         ub = gen_specs["user"]["ub"]
         lb = gen_specs["user"]["lb"]
         n = len(lb)
-        b = gen_specs["user"]["gen_batch_size"]
+        b = gen_specs["batch_size"]
 
         H_o = np.zeros(b, dtype=gen_specs["out"])
         for i in range(0, b):