Organize cleanup

Author: shuds13

docs/platforms/example_scripts.rst

@@ -78,3 +78,10 @@ System Examples
     ..  literalinclude:: ../../examples/libE_submission_scripts/summit_submit_mproc.sh
         :caption: /examples/libE_submission_scripts/summit_submit_mproc.sh
         :language: bash
+
+.. dropdown:: Cobalt - Intermediate node with Multiprocessing
+
+    .. literalinclude:: ../../examples/libE_submission_scripts/cobalt_submit_mproc.sh
+        :caption: /examples/libE_submission_scripts/cobalt_submit_mproc.sh
+        :language: bash
+

docs/platforms/summit.rst

@@ -12,9 +12,56 @@ Users on login nodes submit batch runs to the launch nodes.
 Batch scripts and interactive sessions run on the launch nodes. Only the launch
 nodes can submit MPI runs to the compute nodes via ``jsrun``.
 
-These docs are maintained to guide libEnsemble's usage on three-tier systems similar to Summit.
+These docs are maintained to guide libEnsemble's usage on three-tier systems and/or
+`jsrun` systems similar to Summit.
 
+Configuring Python
+------------------
+
+Begin by loading the Python 3 Anaconda module::
+
+    $ module load python
+
+You can now create and activate your own custom conda_ environment::
+
+    conda create --name myenv python=3.10
+    export PYTHONNOUSERSITE=1 # Make sure get python from conda env
+    . activate myenv
+
+If you are installing any packages with extensions, ensure that the correct compiler
+module is loaded. If using mpi4py_, this must be installed from source,
+referencing the compiler. Currently, mpi4py must be built with gcc::
+
+    module load gcc
+
+With your environment activated, run ::
+
+    CC=mpicc MPICC=mpicc pip install mpi4py --no-binary mpi4py
+
+Installing libEnsemble
+----------------------
+
+Obtaining libEnsemble is now as simple as ``pip install libensemble``.
+Your prompt should be similar to the following line:
+
+.. code-block:: console
+
+    (my_env) user@login5:~$ pip install libensemble
+
+.. note::
+    If you encounter pip errors, run ``python -m pip install --upgrade pip`` first
+
+Or, you can install via ``conda``:
+
+.. code-block:: console
+
+    (my_env) user@login5:~$ conda config --add channels conda-forge
+    (my_env) user@login5:~$ conda install -c conda-forge libensemble
+
+See :doc:`here<../advanced_installation>` for more information on advanced options
+for installing libEnsemble.
 Special note on resource sets and Executor submit options
+
 ---------------------------------------------------------
 
 When using the portable MPI run configuration options (e.g., num_nodes) to the

libensemble/resources/platforms.py

@@ -143,6 +143,7 @@ class Aurora(Platform):
     scheduler_match_slots: bool = True
 
 
+# On SLURM systems, let srun assign free GPUs on the node
 class Frontier(Platform):
     mpi_runner: str = "srun"
     cores_per_node: int = 64
@@ -153,16 +154,6 @@ class Frontier(Platform):
     scheduler_match_slots: bool = False
 
 
-class Summit(Platform):
-    mpi_runner: str = "jsrun"
-    cores_per_node: int = 42
-    logical_cores_per_node: int = 168
-    gpus_per_node: int = 6
-    gpu_setting_type: str = "option_gpus_per_task"
-    gpu_setting_name: str = "-g"
-    scheduler_match_slots: bool = False
-
-
 # Example of a ROCM system
 class GenericROCm(Platform):
     mpi_runner: str = "mpich"
@@ -201,6 +192,16 @@ class Polaris(Platform):
     scheduler_match_slots: bool = True
 
 
+class Summit(Platform):
+    mpi_runner: str = "jsrun"
+    cores_per_node: int = 42
+    logical_cores_per_node: int = 168
+    gpus_per_node: int = 6
+    gpu_setting_type: str = "option_gpus_per_task"
+    gpu_setting_name: str = "-g"
+    scheduler_match_slots: bool = False
+
+
 class Known_platforms(BaseModel):
     """A list of platforms with known configurations.
 

libensemble/tests/functionality_tests/test_mpi_gpu_settings.py

@@ -53,7 +53,7 @@
 
 # Import libEnsemble items for this test
 from libensemble.libE import libE
-from libensemble.resources.platforms import Aurora, Frontier, PerlmutterGPU, Platform, Polaris
+from libensemble.resources.platforms import Aurora, Frontier, PerlmutterGPU, Platform, Polaris, Summit
 from libensemble.sim_funcs import six_hump_camel
 from libensemble.sim_funcs.var_resources import gpu_variable_resources as sim_f
 from libensemble.tools import add_unique_random_streams, parse_args
@@ -203,7 +203,7 @@
     del libE_specs["platform_specs"]
 
     # Fourth set - use platform setting ------------------------------------------------------------
-    for platform in ["perlmutter_g", "polaris", "aurora"]:
+    for platform in ["summit", "frontier", "perlmutter_g", "polaris", "aurora"]:
         print(f"\nRunning GPU setting checks (via known platform) for {platform} ------------------- ")
         libE_specs["platform"] = platform
 
@@ -221,7 +221,7 @@
         del libE_specs["platform"]
 
     # Fifth set - use platform environment setting -----------------------------------------------
-    for platform in ["perlmutter_g", "polaris", "aurora"]:
+    for platform in ["summit", "frontier", "perlmutter_g", "polaris", "aurora"]:
         print(f"\nRunning GPU setting checks (via known platform env. variable) for {platform} ----- ")
         os.environ["LIBE_PLATFORM"] = platform
 
@@ -239,7 +239,7 @@
         del os.environ["LIBE_PLATFORM"]
 
     # Sixth set - use platform_specs with known systems -------------------------------------------
-    for platform in [Frontier, PerlmutterGPU, Polaris, Aurora]:
+    for platform in [Summit, Frontier, PerlmutterGPU, Polaris, Aurora]:
         print(f"\nRunning GPU setting checks (via known platform - platform_specs) for {platform} ------------------- ")
         libE_specs["platform_specs"] = platform()
 

libensemble/tests/scaling_tests/forces/forces_app/build_forces.sh

@@ -45,3 +45,10 @@ mpicc -O3 -o forces.x forces.c -lm
 # Nvidia (nvc) compiler with mpicc and on Cray system with target (Perlmutter)
 # mpicc -DGPU -O3 -fopenmp -mp=gpu -o forces.x forces.c
 # cc -DGPU -Wl,-znoexecstack -O3 -fopenmp -mp=gpu -target-accel=nvidia80 -o forces.x forces.c
+
+# xl (plain and using mpicc on Summit)
+# xlc_r -DGPU -O3 -qsmp=omp -qoffload -o forces.x forces.c
+# mpicc -DGPU -O3 -qsmp=omp -qoffload -o forces.x forces.c
+
+# Summit with gcc (Need up to offload capable gcc: module load gcc/12.1.0) - slower than xlc
+# mpicc -DGPU -Ofast -fopenmp -Wl,-rpath=/sw/summit/gcc/12.1.0-0/lib64 -lm -foffload=nvptx-none forces.c -o forces.x