Merge branch 'develop' into refactor/deprecate_from_yaml_toml_json

Author: jlnav

.github/workflows/basic.yml

@@ -17,7 +17,7 @@ jobs:
             matrix:
                 os: [ubuntu-latest]
                 mpi-version: [mpich]
-                python-version: ["py310", "py311", "py312", "py313", "py314"]
+                python-version: ["py311", "py312", "py313", "py314"]
                 comms-type: [m, l]
                 include:
                     - os: macos-latest

.github/workflows/extra.yml

@@ -11,7 +11,7 @@ jobs:
             matrix:
                 os: [ubuntu-latest]
                 mpi-version: [mpich]
-                python-version: ["py310e", "py311e", "py312e", "py313e", "py314e"]
+                python-version: ["py311e", "py312e", "py313e", "py314e"]
                 comms-type: [m, l]
                 include:
                     - os: macos-latest
@@ -70,13 +70,6 @@ jobs:
           run: |
             pixi run -e ${{ matrix.python-version }} pip install gpcam==8.1.13
 
-        - name: Remove xopt tests on old python
-          if: matrix.python-version == 'py310e'
-          run: |
-            rm ./libensemble/tests/regression_tests/test_xopt_EI_xopt_sim.py
-            rm ./libensemble/tests/regression_tests/test_xopt_EI.py
-            rm ./libensemble/tests/regression_tests/test_xopt_nelder_mead.py
-
         - name: Remove test using octave, gpcam, globus-compute on Python 3.13
           if: matrix.python-version == 'py313e' || matrix.python-version == 'py314e'
           run: |

.pre-commit-config.yaml

@@ -32,3 +32,8 @@ repos:
       additional_dependencies: [black==22.12.0]
       files: ^(.*\.py|.*\.rst)$
       args: [--line-length=120]
+
+- repo: https://github.com/pre-commit/mirrors-mypy
+  rev: v1.19.1
+  hooks:
+    - id: mypy

.readthedocs.yml

@@ -3,7 +3,7 @@ version: 2
 build:
   os: "ubuntu-22.04"
   tools:
-    python: "3.10"
+    python: "3.12"
   commands:
     # from https://docs.readthedocs.com/platform/stable/build-customization.html#support-git-lfs-large-file-storage
     # Download and uncompress the binary

AGENTS.md

@@ -0,0 +1,95 @@
+Agent Contributor Guidelines and Information
+============================================
+
+Read the ``README.rst`` for an overview of libEnsemble.
+
+- libEnsemble uses a manager-worker architecture. Points are generated by a generator and sent to a worker, which runs a simulator.
+- The manager determines how and when points get passed to workers via an allocation function.
+- See ``libensemble/tests/regression_tests/test_1d_sampling.py`` for a simple example of the libEnsemble interface.
+
+Repository Layout
+-----------------
+
+- ``libensemble/`` - Source code.
+-   ``/alloc_funcs`` - Allocation functions. Policies for passing work between the manager and workers.
+-   ``/comms`` - Modules and abstractions for communication between the manager and workers.
+-   ``/executors`` - An interface for launching executables, often simulations.
+-   ``/gen_classes`` - Generators that adhere to the `gest-api` standard.
+                       Recommended over entries from ``/gen_funcs`` that perform similar functionality.
+-   ``/gen_funcs`` - Generator functions. Modules for producing points for simulations.
+-   ``/resources`` - Classes and functions for managing compute resources for MPI tasks, libensemble workers.
+-   ``/sim_funcs`` - Simulator functions. Modules for running simulations or performing experiments.
+-   ``/tests`` - Tests.
+    - ``/functionality_tests`` - Primarily tests libEnsemble code only.
+    - ``/regression_tests`` - Tests libEnsemble code with external code. Often more closely resembles actual use-cases.
+    - ``/unit_tests`` - Tests for individual modules.
+-   ``/tools`` - Tools. Misc functions and classes to ease development.
+-   ``/utils`` - Utilities. Misc functions and classes used internally by multiple modules.
+-   ``ensemble.py`` - The primary interface for parameterizing and running libEnsemble.
+-   ``generators.py`` - Base classes for generators that adhere to the `gest-api` standard.
+-   ``history.py`` - Module for recording points that have been generated and simulation results. NumPy array.
+-   ``libE.py`` - libE main file. Previous primary interface for parameterizing and running libEnsemble.
+-   ``logger.py`` - Logging configuration.
+-   ``manager.py`` - Module for maintaining the history array and passing points between the workers.
+-   ``message_numbers.py`` - Constants that represent states of the ensemble.
+-   ``specs.py`` - Dataclasses for parameterizing the ensemble. Most importantly, contains ``LibeSpecs, SimSpecs, GenSpecs``.
+-   ``worker.py`` - Module for running generators and simulators. Communicates with the manager.
+-   ``version.py`` - Version file.
+
+- ``.github/`` - GitHub actions. See ``.github/workflows/`` for the CI.
+- ``docs/`` - Documentation. Check here first for information before reading the source code.
+- ``examples/`` - The ``*_funcs`` and ``calling_scripts`` directories contain symlinks to examples further in the source code.
+-   ``/libE_submission_scripts`` - Example scripts for submitting libEnsemble jobs to HPC systems.
+-   ``/tutorials`` - Tutorials on how to use libEnsemble.
+- ``pyproject.toml`` - Project configuration file. Contains information about the project and its dependencies.
+
+Other files in the root directory should be self-documenting.
+
+Information about Generators
+----------------------------
+
+- Generators are functions or objects that produce points for simulations.
+- The History array is a NumPy structured array that stores points that have been generated and simulation results.
+Its fields match ``sim_specs/gen_specs["out"]`` or ``vocs`` attributes, plus additional reserved fields for metadata.
+- Prior to libEnsemble v1.6.0, generators were plain functions. They often ran in "persistent" mode, meaning they executed in a
+long-running loop, sending and receiving points to and from the manager until the ensemble was complete.
+- A ``gest-api`` or "standardized" generator is a class that at a minimum implements ``suggest`` and ``ingest`` methods, and is parameterized by a ``vocs``.
+- See ``libensemble/generators.py`` for more information about the ``gest-api`` standard.
+- If using a generator that adheres to the ``gest-api`` standard, or a classic persistent generator, use the ``start_only_persistent`` allocation function.
+- Generators are often used for simple sampling, optimization, calibration, uncertainty quantification, and other simulation-based tasks.
+
+General Guidelines
+------------------
+
+- If using classic ``sim_specs`` and ``gen_specs``, then ensure that ``sim_specs["out"]`` and ``gen_specs["in"]`` field names match, and vice-versa.
+- As-of libEnsemble v1.6.0, ``SimSpecs`` and ``GenSpecs`` can also be parameterized by a ``vocs`` object, imported from ``gest_api.vocs`` (NOT xopt.vocs).
+- ``VOCS`` contains variables, objectives, constraints, and other settings that define the problem.
+See ``libensemble/tests/regression_tests/test_xopt_EI.py`` for an example of how to use it.
+- An MPI distribution is not required for libEnsemble to run, but is required to use the ``MPIExecutor``.  ``mpich`` is recommended.
+- New tests are heavily encouraged for new features, bug fixes, or integrations. See ``libensemble/tests/regression_tests`` for examples.
+- Never use destructive git commands unless explicitly requested.
+- Code is in the ``black`` style. This should be enforced by ``pre-commit``.
+- When writing new code, prefer the ``LibeSpecs``, ``SimSpecs``, and ``GenSpecs`` dataclasses over the classic ``sim_specs`` and ``gen_specs`` bare dictionaries.
+- Read ``CONTRIBUTING.md`` for more information.
+- The external ``libE-community-examples`` repository contains past use-cases, generators, and other examples.
+
+Development Environment
+-----------------------
+
+- ``pixi`` is the recommended environment manager for libEnsemble development.  See ``pyproject.toml`` for the list
+of dependencies and the available testing environments.
+- Enter the development environment with ``pixi shell -e dev``. This environment contains the most common dependencies for development and testing.
+- For one-off commands, use ``pixi run -e dev``. This will run a single command in the development environment.
+- If ``pixi`` is not available or not preferred by the user, ``pip install -e .`` can be used instead. Other dependencies may need to be installed manually.
+- If committing, use ``pre-commit`` to ensure that code style and formatting are consistent.  See ``.pre-commit-config.yaml`` for
+the configuration and ``pyproject.toml`` for other configuration.
+
+Testing
+-------
+
+- Run tests with the ``run-tests.py`` script: ``python libensemble/tests/run-tests.py``.  See ``libensemble/tests/run-tests.py`` for usage information.
+- Some tests require third party software to be installed. When developing a feature or fixing a bug, since the entire test suite will be run on Github Actions,
+for local development running individual tests is sufficient.
+- Individual unit tests can be run with ``pixi run -e dev pytest path/to/test_file``.
+- A libEnsemble run typically outputs an ``ensemble.log`` and ``libE_stats.txt`` file in the working directory. Check these files for tracebacks or run statistics.
+- An "ensemble" or "workflow" directory may also be created, often containing per-simulation output directories

docs/advanced_installation.rst

@@ -6,7 +6,7 @@ libEnsemble can be installed from ``pip``, ``uv``, ``Conda``, or ``Spack``.
 libEnsemble requires the following dependencies, which are typically
 automatically installed alongside libEnsemble:
 
-* Python_       ``>= 3.10``
+* Python_       ``>= 3.11``
 * NumPy_        ``>= 1.21``
 * psutil_       ``>= 5.9.4``
 * `pydantic`_   ``>= 2``
@@ -142,7 +142,7 @@ Further recommendations for selected HPC systems are given in the
 
         **Hint**: When combining Spack and Conda, you can access your Conda Python and packages in your
         ``~/.spack/packages.yaml`` while your Conda environment is activated, using ``CONDA_PREFIX``
-        For example, if you have an activated Conda environment with Python 3.10 and SciPy installed:
+        For example, if you have an activated Conda environment with Python 3.11 and SciPy installed:
 
         .. code-block:: yaml
 
@@ -155,12 +155,12 @@ Further recommendations for selected HPC systems are given in the
             py-numpy:
                 externals:
                 - spec: "py-numpy"
-                prefix: $CONDA_PREFIX/lib/python3.10/site-packages/numpy
+                prefix: $CONDA_PREFIX/lib/python3.11/site-packages/numpy
                 buildable: False
             py-scipy:
                 externals:
                 - spec: "py-scipy"
-                prefix: $CONDA_PREFIX/lib/python3.10/site-packages/scipy
+                prefix: $CONDA_PREFIX/lib/python3.11/site-packages/scipy
                 buildable: True
 
         For more information on Spack builds and any particular considerations

docs/platforms/perlmutter.rst

@@ -120,6 +120,7 @@ Change the ``libE_specs`` in **run_libe_forces.py** as follows.
     # Persistent gen does not need resources
     ensemble.libE_specs = LibeSpecs(
         gen_on_manager=True,
+    )
 
 and run with::
 

docs/platforms/summit.rst

@@ -24,7 +24,7 @@ Begin by loading the Python 3 Anaconda module::
 
 You can now create and activate your own custom conda_ environment::
 
-    conda create --name myenv python=3.10
+    conda create --name myenv python=3.11
     export PYTHONNOUSERSITE=1 # Make sure get python from conda env
     . activate myenv
 

docs/tutorials/local_sine_tutorial.rst

@@ -22,7 +22,7 @@ need to write a new allocation function.
 
         .. code-block:: bash
 
-            python --version  # This should be >= 3.10
+            python --version  # This should be >= 3.11
 
         .. _Python: https://www.python.org/
 

libensemble/utils/misc.py

@@ -4,7 +4,6 @@
 
 from itertools import chain, groupby
 from operator import itemgetter
-from typing import List
 
 import numpy as np
 import numpy.typing as npt
@@ -85,6 +84,7 @@ def _get_combinable_multidim_names(first: dict, new_dtype_names: list) -> list:
 
 def _decide_dtype(name: str, entry, size: int) -> tuple:
     """decide dtype of field, and size if needed"""
+    output_type: str | type  # str for numpy string type, type for python type
     if isinstance(entry, str):  # use numpy style for string type
         output_type = "U" + str(len(entry) + 1)
     else:
@@ -115,7 +115,7 @@ def _pack_field(input_dict: dict, field_names: list) -> tuple:
     return tuple(input_dict[name] for name in field_names) if len(field_names) > 1 else input_dict[field_names[0]]
 
 
-def list_dicts_to_np(list_dicts: list, dtype: list = None, mapping: dict = {}) -> npt.NDArray:
+def list_dicts_to_np(list_dicts: list, dtype: list | None = None, mapping: dict = {}) -> npt.NDArray:
     """Convert list of dicts to numpy structured array"""
     if list_dicts is None:
         return None
@@ -231,7 +231,7 @@ def map_numpy_array(array: npt.NDArray, mapping: dict = {}) -> npt.NDArray:
         return array
 
     # Create new dtype with mapped fields
-    new_fields = []
+    new_fields: list[tuple] = []
 
     # Track fields processed by mapping to avoid duplication
     mapped_source_fields = set()
@@ -275,7 +275,7 @@ def map_numpy_array(array: npt.NDArray, mapping: dict = {}) -> npt.NDArray:
     return mapped_array
 
 
-def np_to_list_dicts(array: npt.NDArray, mapping: dict = {}) -> List[dict]:
+def np_to_list_dicts(array: npt.NDArray, mapping: dict = {}) -> list[dict]:
     """Convert numpy structured array to list of dicts"""
     out = []