Refactor/feb24 (#1249)

* dramatic refactorings courtesy of angel and codeclimate

* refactor Ensemble.copy_back

* add per-worker sim_dir test as non-extra

* this may actually be a pydantic issue that autodoc-pydantic hasn't addressed yet

* fixes model fields not rendering in docs; turns out pydantic.create_model doesn't play nicely with sphinx. other small fixes

* undo pydantic version pin in requirements.txt

* fix GenSpecs code sample

Author: jlnav

docs/data_structures/gen_specs.rst

@@ -23,7 +23,7 @@ Can be constructed and passed to libEnsemble as a Python class or a dictionary.
 
         gen_specs = GenSpecs(
             gen_f=gen_random_sample,
-            out=[("x", float, (1,))],
+            outputs=[("x", float, (1,))],
             user={
                 "lower": np.array([-3]),
                 "upper": np.array([3]),

docs/data_structures/sim_specs.rst

@@ -23,7 +23,7 @@ Can be constructed and passed to libEnsemble as a Python class or a dictionary.
         sim_specs = SimSpecs(
             sim_f=sim_find_sine,
             inputs=["x"],
-            out=[("y", float)],
+            outputs=[("y", float)],
             user={"batch": 1234},
         )
         ...

docs/nitpicky

@@ -13,6 +13,7 @@ py:class numpy.typing.NDArray
 py:class numpy._typing._generic_alias.ScalarType
 py:class numpy._typing._dtype_like._DTypeDict
 py:class numpy._typing._dtype_like._SupportsDType
+py:class numpy._typing._array_like._ScalarType_co
 
 # Pending on python docs links issue #11975
 py:class list
@@ -34,6 +35,7 @@ py:class int
 py:class <class 'dict'>
 py:class <class 'int'>
 py:class +ScalarType
+py:class collections.abc.Sequence
 
 # Internal paths that are verified importable but Sphinx can't find
 py:class libensemble.resources.platforms.Aurora

libensemble/history.py

@@ -55,27 +55,9 @@ def __init__(
         specs_dtype_list = list(set(libE_fields + sum([k.get("out", []) for k in specs if k], [])))
 
         if len(H0):
-            # a whole lot of work to parse numpy dtypes to python types and 2- or 3-tuples
-            # - dtypes aren't iterable, but you can index into them
-            # - must split out actual numpy type if subdtype refers to sub-array
-            # - then convert that type into a python type in the best way known so far...
-            # - we need to make sure the size of string types is preserved
-            # - if sub-array shape, save as 3-tuple
-
-            H0_fields = []
-            for i in range(len(H0.dtype.names)):
-                dtype = H0.dtype[i]
-                subd = dtype.subdtype[0] if dtype.subdtype else dtype
-                pytype = type(subd.type(0).item())  # kinda redundant innit?
-                size = int(dtype.str.split("<U")[-1]) if "<U" in dtype.str else dtype.shape
-                if size:
-                    H0_fields.append((H0.dtype.names[i], pytype, size))
-                else:
-                    H0_fields.append((H0.dtype.names[i], pytype))
-
             # remove duplicate fields from specs dtype list if those already in H0 (H0 takes precedence)
             pruned_specs_dtype_list = [i for i in specs_dtype_list if i[0] not in H0.dtype.names]
-            H_fields = list(set(pruned_specs_dtype_list + H0_fields))
+            H_fields = list(set(pruned_specs_dtype_list + H0.dtype.descr))
 
             H = np.zeros(L + len(H0), dtype=H_fields)
 

libensemble/sim_funcs/executor_hworld.py

@@ -71,6 +71,7 @@ def custom_polling_loop(exctr, task, timeout_sec=5.0, delay=0.3):
 @output_data([("f", float), ("cstat", int)])
 def executor_hworld(H, _, sim_specs, info):
     """Tests launching and polling task and exiting on task finish"""
+
     exctr = info["executor"]
     cores = sim_specs["user"]["cores"]
     ELAPSED_TIMEOUT = "elapsed_timeout" in sim_specs["user"]

libensemble/tests/functionality_tests/test_sim_dirs_per_worker.py

@@ -13,7 +13,6 @@
 # Do not change these lines - they are parsed by run-tests.sh
 # TESTSUITE_COMMS: mpi local tcp
 # TESTSUITE_NPROCS: 2 4
-# TESTSUITE_EXTRA: true
 
 import os
 

libensemble/utils/output_directory.py

@@ -187,35 +187,33 @@ def prep_calc_dir(self, Work: dict, calc_iter: dict, workerID: int, calc_type: i
 
     def copy_back(self) -> None:
         """Copy back all ensemble dir contents to launch location"""
-        if self.ensemble_dir.exists() and self.ensemble_copy_back and self.loc_stack:
-            no_calc_dirs = not self.sim_dirs_make or not self.gen_dirs_make
-
-            for dire in self.loc_stack.dirs.values():
-                dire = Path(dire)
-                dest_path = self.copybackdir / Path(dire.stem)
-                if dire == self.ensemble_dir:  # occurs when no_calc_dirs is True
-                    continue  # otherwise, entire ensemble dir copied into copyback dir
-
+        if not self.ensemble_dir.exists() or not self.ensemble_copy_back or not self.loc_stack:
+            return
+
+        for dire in self.loc_stack.dirs.values():
+            dire = Path(dire)
+            dest_path = self.copybackdir / Path(dire.stem)
+            if dire == self.ensemble_dir:  # occurs when no_calc_dirs is True
+                continue  # otherwise, entire ensemble dir copied into copyback dir
+
+            if self.allow_overwrite:
+                shutil.rmtree(dest_path, ignore_errors=True)
+            shutil.copytree(dire, dest_path, symlinks=True, dirs_exist_ok=True)
+            if dire.stem.startswith("worker"):
+                return  # Worker dir (with all contents) has been copied.
+
+        # If not using calc dirs, likely miscellaneous files to copy back
+        if not self.sim_dirs_make or not self.gen_dirs_make:
+            p = re.compile(r"((^sim)|(^gen))\d")
+            for filep in [i for i in os.listdir(self.ensemble_dir) if not p.match(i)]:  # each noncalc_dir file
+                source_path = self.ensemble_dir / filep
+                dest_path = self.copybackdir / filep
                 if self.allow_overwrite:
                     shutil.rmtree(dest_path, ignore_errors=True)
-                shutil.copytree(dire, dest_path, symlinks=True, dirs_exist_ok=True)
-                if dire.stem.startswith("worker"):
-                    return  # Worker dir (with all contents) has been copied.
-
-            # If not using calc dirs, likely miscellaneous files to copy back
-            if no_calc_dirs:
-                p = re.compile(r"((^sim)|(^gen))\d")
-                for filep in [i for i in os.listdir(self.ensemble_dir) if not p.match(i)]:  # each noncalc_dir file
-                    source_path = os.path.join(self.ensemble_dir, filep)
-                    dest_path = os.path.join(self.copybackdir, filep)
-                    if self.allow_overwrite:
-                        shutil.rmtree(dest_path, ignore_errors=True)
-                    try:
-                        if os.path.isdir(source_path):
-                            shutil.copytree(source_path, dest_path, symlinks=True)
-                        else:
-                            shutil.copy(source_path, dest_path, follow_symlinks=False)
-                    except FileExistsError:
-                        continue
-                    except shutil.SameFileError:  # creating an identical symlink
-                        continue
+                try:
+                    if os.path.isdir(source_path):
+                        shutil.copytree(source_path, dest_path, symlinks=True)
+                    else:
+                        shutil.copy(source_path, dest_path, follow_symlinks=False)
+                except (FileExistsError, shutil.SameFileError):  # creating an identical symlink
+                    continue

libensemble/utils/pydantic_bindings.py

@@ -1,3 +1,5 @@
+import sys
+
 from pydantic import Field, create_model
 
 from libensemble import specs
@@ -62,25 +64,16 @@ class Config:
     specs._EnsembleSpecs.model_config = model_config
     platforms.Platform.model_config = model_config
 
-    specs.SimSpecs.model_fields["inputs"] = FieldInfo.merge_field_infos(
-        specs.SimSpecs.model_fields["inputs"], Field(alias="in")
-    )
+    model = specs.SimSpecs.model_fields
+    model["inputs"] = FieldInfo.merge_field_infos(model["inputs"], Field(alias="in"))
+    model["outputs"] = FieldInfo.merge_field_infos(model["outputs"], Field(alias="out"))
 
-    specs.SimSpecs.model_fields["outputs"] = FieldInfo.merge_field_infos(
-        specs.SimSpecs.model_fields["outputs"], Field(alias="out")
-    )
+    model = specs.GenSpecs.model_fields
+    model["inputs"] = FieldInfo.merge_field_infos(model["inputs"], Field(alias="in"))
+    model["outputs"] = FieldInfo.merge_field_infos(model["outputs"], Field(alias="out"))
 
-    specs.GenSpecs.model_fields["inputs"] = FieldInfo.merge_field_infos(
-        specs.GenSpecs.model_fields["inputs"], Field(alias="in")
-    )
-
-    specs.GenSpecs.model_fields["outputs"] = FieldInfo.merge_field_infos(
-        specs.GenSpecs.model_fields["outputs"], Field(alias="out")
-    )
-
-    specs.AllocSpecs.model_fields["outputs"] = FieldInfo.merge_field_infos(
-        specs.AllocSpecs.model_fields["outputs"], Field(alias="out")
-    )
+    model = specs.AllocSpecs.model_fields
+    model["outputs"] = FieldInfo.merge_field_infos(model["outputs"], Field(alias="out"))
 
     specs.SimSpecs.model_rebuild(force=True)
     specs.GenSpecs.model_rebuild(force=True)
@@ -90,57 +83,60 @@ class Config:
     specs._EnsembleSpecs.model_rebuild(force=True)
     platforms.Platform.model_rebuild(force=True)
 
-specs.SimSpecs = create_model(
-    "SimSpecs",
-    __base__=specs.SimSpecs,
-    __validators__={
-        "check_valid_out": check_valid_out,
-        "check_valid_in": check_valid_in,
-        "simf_set_in_out_from_attrs": simf_set_in_out_from_attrs,
-    },
-)
+# the create_model function removes fields for rendering in docs
+if "sphinx" not in sys.modules:
+
+    specs.SimSpecs = create_model(
+        "SimSpecs",
+        __base__=specs.SimSpecs,
+        __validators__={
+            "check_valid_out": check_valid_out,
+            "check_valid_in": check_valid_in,
+            "simf_set_in_out_from_attrs": simf_set_in_out_from_attrs,
+        },
+    )
 
-specs.GenSpecs = create_model(
-    "GenSpecs",
-    __base__=specs.GenSpecs,
-    __validators__={
-        "check_valid_out": check_valid_out,
-        "check_valid_in": check_valid_in,
-        "genf_set_in_out_from_attrs": genf_set_in_out_from_attrs,
-    },
-)
+    specs.GenSpecs = create_model(
+        "GenSpecs",
+        __base__=specs.GenSpecs,
+        __validators__={
+            "check_valid_out": check_valid_out,
+            "check_valid_in": check_valid_in,
+            "genf_set_in_out_from_attrs": genf_set_in_out_from_attrs,
+        },
+    )
 
-specs.LibeSpecs = create_model(
-    "LibeSpecs",
-    __base__=specs.LibeSpecs,
-    __validators__={
-        "check_valid_comms_type": check_valid_comms_type,
-        "set_platform_specs_to_class": set_platform_specs_to_class,
-        "check_input_dir_exists": check_input_dir_exists,
-        "check_inputs_exist": check_inputs_exist,
-        "check_any_workers_and_disable_rm_if_tcp": check_any_workers_and_disable_rm_if_tcp,
-        "enable_save_H_when_every_K": enable_save_H_when_every_K,
-        "set_workflow_dir": set_workflow_dir,
-    },
-)
+    specs.LibeSpecs = create_model(
+        "LibeSpecs",
+        __base__=specs.LibeSpecs,
+        __validators__={
+            "check_valid_comms_type": check_valid_comms_type,
+            "set_platform_specs_to_class": set_platform_specs_to_class,
+            "check_input_dir_exists": check_input_dir_exists,
+            "check_inputs_exist": check_inputs_exist,
+            "check_any_workers_and_disable_rm_if_tcp": check_any_workers_and_disable_rm_if_tcp,
+            "enable_save_H_when_every_K": enable_save_H_when_every_K,
+            "set_workflow_dir": set_workflow_dir,
+        },
+    )
 
-specs._EnsembleSpecs = create_model(
-    "_EnsembleSpecs",
-    __base__=specs._EnsembleSpecs,
-    __validators__={
-        "check_exit_criteria": check_exit_criteria,
-        "check_output_fields": check_output_fields,
-        "check_H0": check_H0,
-        "check_provided_ufuncs": check_provided_ufuncs,
-    },
-)
+    specs._EnsembleSpecs = create_model(
+        "_EnsembleSpecs",
+        __base__=specs._EnsembleSpecs,
+        __validators__={
+            "check_exit_criteria": check_exit_criteria,
+            "check_output_fields": check_output_fields,
+            "check_H0": check_H0,
+            "check_provided_ufuncs": check_provided_ufuncs,
+        },
+    )
 
-platforms.Platform = create_model(
-    "Platform",
-    __base__=platforms.Platform,
-    __validators__={
-        "check_gpu_setting_type": check_gpu_setting_type,
-        "check_mpi_runner_type": check_mpi_runner_type,
-        "check_logical_cores": check_logical_cores,
-    },
-)
+    platforms.Platform = create_model(
+        "Platform",
+        __base__=platforms.Platform,
+        __validators__={
+            "check_gpu_setting_type": check_gpu_setting_type,
+            "check_mpi_runner_type": check_mpi_runner_type,
+            "check_logical_cores": check_logical_cores,
+        },
+    )