Merge pull request #1253 from Libensemble/release/v_1.2.1

shuds13 · web-flow · commit 7c5937344516 · 2024-02-23T11:05:00.000-06:00
Release/v 1.2.1
diff --git a/.codecov.yml b/.codecov.yml
@@ -0,0 +1,4 @@
+ignore:
+  - "libensemble/tools/parse_args.py"
+  - "libensemble/tools/forkable_pdb.py"
+  - "libensemble/sim_funcs/executor_hworld.py"
diff --git a/.github/workflows/basic.yml b/.github/workflows/basic.yml
@@ -167,4 +167,4 @@ jobs:
         runs-on: ubuntu-latest
         steps:
         - uses: actions/checkout@v4
-        - uses: crate-ci/typos@v1.18.1
+        - uses: crate-ci/typos@v1.18.2
diff --git a/.github/workflows/extra.yml b/.github/workflows/extra.yml
@@ -250,4 +250,4 @@ jobs:
         runs-on: ubuntu-latest
         steps:
         - uses: actions/checkout@v4
-        - uses: crate-ci/typos@v1.18.1
+        - uses: crate-ci/typos@v1.18.2
diff --git a/.wci.yml b/.wci.yml
@@ -16,8 +16,8 @@ description: |
 language: Python
 
 release:
-  version: 1.2.0
-  date: 2024-02-08
+  version: 1.2.1
+  date: 2024-02-23
 
 documentation:
   general: https://libensemble.readthedocs.io
diff --git a/CHANGELOG.rst b/CHANGELOG.rst
@@ -8,6 +8,24 @@ GitHub issues are referenced, and can be viewed with hyperlinks on the `github r
 
 .. _`github releases page`: https://github.com/Libensemble/libensemble/releases
 
+Release 1.2.1
+--------------
+
+:Date: February 23, 2024
+
+* Fix documentation bug where pydantic models do not display correctly.  #1249
+* Improve internal efficiency. #1243 / #1249
+
+:Note:
+
+* Tests were run on Linux and MacOS with Python versions 3.9, 3.10, 3.11, 3.12
+* Heterogeneous workflows tested on Aurora (ALCF), Frontier (OLCF), Polaris, and Perlmutter (NERSC).
+* Tests were also run on Bebop and Improv LCRC systems.
+
+:Known Issues:
+
+* See known issues section in the documentation.
+
 Release 1.2.0
 --------------
 
diff --git a/docs/data_structures/gen_specs.rst b/docs/data_structures/gen_specs.rst
@@ -23,7 +23,7 @@ Can be constructed and passed to libEnsemble as a Python class or a dictionary.
 
         gen_specs = GenSpecs(
             gen_f=gen_random_sample,
-            out=[("x", float, (1,))],
+            outputs=[("x", float, (1,))],
             user={
                 "lower": np.array([-3]),
                 "upper": np.array([3]),
diff --git a/docs/data_structures/persis_info.rst b/docs/data_structures/persis_info.rst
@@ -38,7 +38,6 @@ Examples:
        :linenos:
        :start-at:     for wid in support.avail_worker_ids():
        :end-before:         # Give sim work if possible
-       :emphasize-lines: 3-4
        :caption: libensemble/alloc_funcs/fast_alloc.py
 
   .. tab-item:: Tracking running generators
diff --git a/docs/data_structures/sim_specs.rst b/docs/data_structures/sim_specs.rst
@@ -23,7 +23,7 @@ Can be constructed and passed to libEnsemble as a Python class or a dictionary.
         sim_specs = SimSpecs(
             sim_f=sim_find_sine,
             inputs=["x"],
-            out=[("y", float)],
+            outputs=[("y", float)],
             user={"batch": 1234},
         )
         ...
diff --git a/docs/nitpicky b/docs/nitpicky
@@ -13,6 +13,7 @@ py:class numpy.typing.NDArray
 py:class numpy._typing._generic_alias.ScalarType
 py:class numpy._typing._dtype_like._DTypeDict
 py:class numpy._typing._dtype_like._SupportsDType
+py:class numpy._typing._array_like._ScalarType_co
 
 # Pending on python docs links issue #11975
 py:class list
@@ -34,6 +35,8 @@ py:class int
 py:class <class 'dict'>
 py:class <class 'int'>
 py:class +ScalarType
+py:class +_ScalarType_co
+py:class collections.abc.Sequence
 
 # Internal paths that are verified importable but Sphinx can't find
 py:class libensemble.resources.platforms.Aurora
diff --git a/install/misc_feature_requirements.txt b/install/misc_feature_requirements.txt
@@ -1 +1 @@
-globus-compute-sdk==2.12.0
+globus-compute-sdk==2.13.0
diff --git a/install/testing_requirements.txt b/install/testing_requirements.txt
@@ -1,9 +1,9 @@
 flake8==7.0.0
 coverage==7.3.1
-pytest==8.0.0
+pytest==8.0.1
 pytest-cov==4.1.0
 pytest-timeout==2.2.0
 mock==5.1.0
 python-dateutil==2.8.2
-anyio==4.2.0
-matplotlib==3.8.2
+anyio==4.3.0
+matplotlib==3.8.3
diff --git a/libensemble/alloc_funcs/fast_alloc.py b/libensemble/alloc_funcs/fast_alloc.py
@@ -33,9 +33,7 @@ def give_sim_work_first(W, H, sim_specs, gen_specs, alloc_specs, persis_info, li
     gen_in = gen_specs.get("in", [])
 
     for wid in support.avail_worker_ids():
-        # Skip any cancelled points
-        while persis_info["next_to_give"] < len(H) and H[persis_info["next_to_give"]]["cancel_requested"]:
-            persis_info["next_to_give"] += 1
+        persis_info = support.skip_canceled_points(H, persis_info)
 
         # Give sim work if possible
         if persis_info["next_to_give"] < len(H):
diff --git a/libensemble/alloc_funcs/give_pregenerated_work.py b/libensemble/alloc_funcs/give_pregenerated_work.py
@@ -24,9 +24,7 @@ def give_pregenerated_sim_work(W, H, sim_specs, gen_specs, alloc_specs, persis_i
         return Work, persis_info, 1
 
     for i in support.avail_worker_ids():
-        # Skip any cancelled points
-        while persis_info["next_to_give"] < len(H) and H[persis_info["next_to_give"]]["cancel_requested"]:
-            persis_info["next_to_give"] += 1
+        persis_info = support.skip_canceled_points(H, persis_info)
 
         # Give sim work
         try:
diff --git a/libensemble/alloc_funcs/only_one_gen_alloc.py b/libensemble/alloc_funcs/only_one_gen_alloc.py
@@ -22,9 +22,7 @@ def ensure_one_active_gen(W, H, sim_specs, gen_specs, alloc_specs, persis_info,
     gen_in = gen_specs.get("in", [])
 
     for wid in support.avail_worker_ids():
-        # Skip any cancelled points
-        while persis_info["next_to_give"] < len(H) and H[persis_info["next_to_give"]]["cancel_requested"]:
-            persis_info["next_to_give"] += 1
+        persis_info = support.skip_canceled_points(H, persis_info)
 
         if persis_info["next_to_give"] < len(H):
             try:
diff --git a/libensemble/alloc_funcs/persistent_aposmm_alloc.py b/libensemble/alloc_funcs/persistent_aposmm_alloc.py
@@ -54,9 +54,7 @@ def persistent_aposmm_alloc(W, H, sim_specs, gen_specs, alloc_specs, persis_info
                 returned_but_not_given[point_ids] = False
 
     for wid in support.avail_worker_ids(persistent=False):
-        # Skip any cancelled points
-        while persis_info["next_to_give"] < len(H) and H[persis_info["next_to_give"]]["cancel_requested"]:
-            persis_info["next_to_give"] += 1
+        persis_info = support.skip_canceled_points(H, persis_info)
 
         if persis_info["next_to_give"] < len(H):
             # perform sim evaluations (if they exist in History).
diff --git a/libensemble/gen_funcs/aposmm_localopt_support.py b/libensemble/gen_funcs/aposmm_localopt_support.py
@@ -30,8 +30,7 @@ class APOSMMException(Exception):
 optimizers = libensemble.gen_funcs.rc.aposmm_optimizers
 
 if optimizers is not None:
-    if not isinstance(optimizers, list):
-        optimizers = [optimizers]
+    assert isinstance(optimizers, list), "Must have a list"
     unrec = set(optimizers) - set(optimizer_list)
     if unrec:
         raise APOSMMException(f"APOSMM Error: unrecognized optimizers {unrec}")
@@ -47,7 +46,7 @@ class APOSMMException(Exception):
         from ibcdfo import pounders  # noqa: F401
     if "scipy" in optimizers:
         from scipy import optimize as sp_opt  # noqa: F401
-    if "external" in optimizers:
+    if "external_localopt" in optimizers:
         pass
 
 
diff --git a/libensemble/history.py b/libensemble/history.py
@@ -55,27 +55,9 @@ def __init__(
         specs_dtype_list = list(set(libE_fields + sum([k.get("out", []) for k in specs if k], [])))
 
         if len(H0):
-            # a whole lot of work to parse numpy dtypes to python types and 2- or 3-tuples
-            # - dtypes aren't iterable, but you can index into them
-            # - must split out actual numpy type if subdtype refers to sub-array
-            # - then convert that type into a python type in the best way known so far...
-            # - we need to make sure the size of string types is preserved
-            # - if sub-array shape, save as 3-tuple
-
-            H0_fields = []
-            for i in range(len(H0.dtype.names)):
-                dtype = H0.dtype[i]
-                subd = dtype.subdtype[0] if dtype.subdtype else dtype
-                pytype = type(subd.type(0).item())  # kinda redundant innit?
-                size = int(dtype.str.split("<U")[-1]) if "<U" in dtype.str else dtype.shape
-                if size:
-                    H0_fields.append((H0.dtype.names[i], pytype, size))
-                else:
-                    H0_fields.append((H0.dtype.names[i], pytype))
-
             # remove duplicate fields from specs dtype list if those already in H0 (H0 takes precedence)
             pruned_specs_dtype_list = [i for i in specs_dtype_list if i[0] not in H0.dtype.names]
-            H_fields = list(set(pruned_specs_dtype_list + H0_fields))
+            H_fields = list(set(pruned_specs_dtype_list + H0.dtype.descr))
 
             H = np.zeros(L + len(H0), dtype=H_fields)
 
diff --git a/libensemble/resources/gpu_detect.py b/libensemble/resources/gpu_detect.py
@@ -48,30 +48,17 @@ def rocm_smi():
     return gpu_count
 
 
-def zeinfo():
+def zeinfo(string="Number of devices", except_val=None):
     """Detect GPU from zeinfo or return None"""
     try:
         ps = subprocess.Popen(("zeinfo"), stderr=subprocess.PIPE)
-        output = subprocess.check_output(("grep", "Number of devices"), stdin=ps.stderr)
+        output = subprocess.check_output(("grep", string), stdin=ps.stderr)
         gpu_count = int(output.decode().split()[3])
     except Exception:
-        return None
+        return except_val
     return gpu_count
 
 
-def get_num_tiles():
-    """Get number of tiles on GPU if available"""
-
-    # Currently only relevant to Intel GPUs
-    try:
-        ps = subprocess.Popen(("zeinfo"), stderr=subprocess.PIPE)
-        output = subprocess.check_output(("grep", "Number of sub-devices"), stdin=ps.stderr)
-        tile_count = int(output.decode().split()[3])
-    except Exception:
-        return 1
-    return tile_count
-
-
 METHODS = {
     "pynvml": pynvml,
     "nvidia_smi": nvidia_smi,
diff --git a/libensemble/resources/resources.py b/libensemble/resources/resources.py
@@ -9,7 +9,7 @@
 
 from libensemble.resources import node_resources
 from libensemble.resources.env_resources import EnvResources
-from libensemble.resources.gpu_detect import get_num_tiles
+from libensemble.resources.gpu_detect import zeinfo
 from libensemble.resources.mpi_resources import get_MPI_runner
 from libensemble.resources.worker_resources import ResourceManager, WorkerResources
 
@@ -179,7 +179,7 @@ def __init__(self, libE_specs: dict, platform_info: dict = {}, top_level_dir: st
         self.tiles_per_gpu = 1
         if libE_specs.get("use_tiles_as_gpus", False):
             # Not yet detected so only uses tiles if set in platform_info
-            self.tiles_per_gpu = platform_info.get("tiles_per_gpu") or get_num_tiles()
+            self.tiles_per_gpu = platform_info.get("tiles_per_gpu") or zeinfo("Number of sub-devices", except_val=1)
 
         node_file = resource_info.get("node_file", None)
         nodelist_env_slurm = resource_info.get("nodelist_env_slurm", None)
diff --git a/libensemble/sim_funcs/chwirut1.py b/libensemble/sim_funcs/chwirut1.py
@@ -224,17 +224,20 @@
 # fmt: on
 
 
+def func_def(x, i):
+    return y[i] - np.exp(-x[0] * t[i]) / (x[1] + x[2] * t[i])
+
+
 def EvaluateFunction(x, component=np.nan):
     """
     Evaluates the chwirut function
     """
     if np.isnan(component):
         f = np.zeros(NOBSERVATIONS)
         for i in range(NOBSERVATIONS):
-            f[i] = y[i] - np.exp(-x[0] * t[i]) / (x[1] + x[2] * t[i])
+            f[i] = func_def(x, i)
     else:
-        i = component
-        f = y[i] - np.exp(-x[0] * t[i]) / (x[1] + x[2] * t[i])
+        f = func_def(x, component)
 
     return f
 
diff --git a/libensemble/sim_funcs/executor_hworld.py b/libensemble/sim_funcs/executor_hworld.py
@@ -70,7 +70,10 @@ def custom_polling_loop(exctr, task, timeout_sec=5.0, delay=0.3):
 @input_fields(["x"])
 @output_data([("f", float), ("cstat", int)])
 def executor_hworld(H, _, sim_specs, info):
-    """Tests launching and polling task and exiting on task finish"""
+    """
+    Tests launching and polling task and exiting on task finish
+    """
+
     exctr = info["executor"]
     cores = sim_specs["user"]["cores"]
     ELAPSED_TIMEOUT = "elapsed_timeout" in sim_specs["user"]
diff --git a/libensemble/tests/functionality_tests/test_sim_dirs_per_worker.py b/libensemble/tests/functionality_tests/test_sim_dirs_per_worker.py
@@ -13,7 +13,6 @@
 # Do not change these lines - they are parsed by run-tests.sh
 # TESTSUITE_COMMS: mpi local tcp
 # TESTSUITE_NPROCS: 2 4
-# TESTSUITE_EXTRA: true
 
 import os
 
diff --git a/libensemble/tests/regression_tests/test_persistent_aposmm_dfols.py b/libensemble/tests/regression_tests/test_persistent_aposmm_dfols.py
@@ -125,3 +125,12 @@ def combine_component(x):
             # else:
             #     d = np.linalg.solve(np.dot(J.T, J), np.dot(J.T, F))
             #     assert np.linalg.norm(d) <= 1e-5
+
+    if libE_specs["comms"] == "mpi":
+        # Quickly try a different DFO-LS exit condition
+        persis_info = add_unique_random_streams({}, nworkers + 1)
+        gen_specs["user"]["dfols_kwargs"]["rhoend"] = 1e-16
+        H, persis_info, flag = libE(sim_specs, gen_specs, exit_criteria, persis_info, alloc_specs, libE_specs)
+
+        if is_manager:
+            assert flag == 0
diff --git a/libensemble/tests/regression_tests/test_persistent_aposmm_tao_nm.py b/libensemble/tests/regression_tests/test_persistent_aposmm_tao_nm.py
@@ -81,3 +81,10 @@
         print("[Manager]:", H[np.where(H["local_min"])]["x"])
         assert np.sum(~H["local_pt"]) > 100, "Had to do at least 100 sample points"
         assert np.sum(H["local_pt"]) > 100, "Why didn't at least 100 local points occur?"
+
+    if libE_specs["comms"] == "mpi":
+        persis_info = add_unique_random_streams({}, nworkers + 1)
+        gen_specs["user"]["run_max_eval"] = 10 * (n + 1)
+        H, persis_info, flag = libE(sim_specs, gen_specs, exit_criteria, persis_info, alloc_specs, libE_specs)
+        if is_manager:
+            assert flag == 0
diff --git a/libensemble/tests/unit_tests/test_ensemble.py b/libensemble/tests/unit_tests/test_ensemble.py
@@ -3,7 +3,7 @@
 import numpy as np
 
 import libensemble.tests.unit_tests.setup as setup
-from libensemble.utils.misc import pydanticV1, pydanticV2, specs_dump
+from libensemble.utils.misc import pydanticV1, specs_dump
 
 
 def test_ensemble_init():
@@ -132,7 +132,7 @@ def test_flakey_workflow():
     """Test initializing a workflow via Specs and Ensemble.run()"""
     if pydanticV1:
         from pydantic.error_wrappers import ValidationError
-    elif pydanticV2:
+    else:
         from pydantic import ValidationError
 
     from libensemble.ensemble import Ensemble
diff --git a/libensemble/tests/unit_tests/test_models.py b/libensemble/tests/unit_tests/test_models.py
diff --git a/libensemble/tools/alloc_support.py b/libensemble/tools/alloc_support.py
diff --git a/libensemble/utils/misc.py b/libensemble/utils/misc.py
diff --git a/libensemble/utils/output_directory.py b/libensemble/utils/output_directory.py
diff --git a/libensemble/utils/pydantic_bindings.py b/libensemble/utils/pydantic_bindings.py
diff --git a/libensemble/utils/validators.py b/libensemble/utils/validators.py
diff --git a/libensemble/version.py b/libensemble/version.py

Original file line number	Diff line number	Diff line change
`@@ -23,7 +23,7 @@ Can be constructed and passed to libEnsemble as a Python class or a dictionary.`
`23`	`23`	`sim_specs = SimSpecs(`
`24`	`24`	`sim_f=sim_find_sine,`
`25`	`25`	`inputs=["x"],`
`26`		`- out=[("y", float)],`
	`26`	`+ outputs=[("y", float)],`
`27`	`27`	`user={"batch": 1234},`
`28`	`28`	`)`
`29`	`29`	`...`
Original file line number	Diff line number	Diff line change
`@@ -1 +1 @@`
`1`		`-globus-compute-sdk==2.12.0`
	`1`	`+globus-compute-sdk==2.13.0`