fixes to examples after system updates

frontier/containers_on_frontier_docs/apptainer_wrappers_lammps/cpu/lammps.def

@@ -1,13 +1,14 @@
 Bootstrap: docker 
-From: savannah.ornl.gov/olcf-container-images/cpe:24.11_gnu_opensuse # this could be the ubuntu, opensuse or rockylinux container
+From: savannah.ornl.gov/olcf-container-images/cpe:25.09_gnu_opensuse # this could be the ubuntu, opensuse or rockylinux container
 
 %post
     zypper install -y cmake
-    wget https://github.com/lammps/lammps/archive/refs/tags/stable_2Aug2023_update3.tar.gz
-    tar xzf stable_2Aug2023_update3.tar.gz
-    cmake -S lammps-stable_2Aug2023_update3/cmake -B lammps_build -D CMAKE_INSTALL_PREFIX=/opt/lammps
+    wget https://github.com/lammps/lammps/releases/download/stable_22Jul2025_update4/lammps-src-22Jul2025_update4.tar.gz
+    tar xzf lammps-src-22Jul2025_update4.tar.gz
+
+    cmake -S lammps-22Jul2025/cmake -B lammps_build -D CMAKE_INSTALL_PREFIX=/opt/lammps
     cmake --build lammps_build --target install -j8
-    rm -r stable_2Aug2023_update3.tar.gz lammps-stable_2Aug2023_update3 lammps_build
+    rm -r lammps-src-22Jul2025_update4.tar.gz lammps-22Jul2025 lammps_build
 
 %environment
     export PATH="/opt/lammps/bin:$PATH"

frontier/containers_on_frontier_docs/apptainer_wrappers_lammps/gpu/lammps.def

@@ -1,14 +1,14 @@
 Bootstrap: docker 
-From: savannah.ornl.gov/olcf-container-images/cpe:24.11_gnu_opensuse # this could be the ubuntu, opensuse or rockylinux container
+From: savannah.ornl.gov/olcf-container-images/cpe:25.09_gnu_opensuse # this could be the ubuntu, opensuse or rockylinux container
 
 %post
     zypper install -y cmake
-    wget https://github.com/lammps/lammps/archive/refs/tags/stable_2Aug2023_update3.tar.gz
-    tar xzf stable_2Aug2023_update3.tar.gz
-    cmake -S lammps-stable_2Aug2023_update3/cmake -B lammps_build -D CMAKE_INSTALL_PREFIX=/opt/lammps -D CMAKE_CXX_COMPILER=hipcc \
-		-D PKG_KOKKOS=yes -D Kokkos_ARCH_HOSTARCH=yes -D Kokkos_ARCH_GPUARCH=yes -D Kokkos_ENABLE_HIP=yes
+    wget https://github.com/lammps/lammps/releases/download/stable_22Jul2025_update4/lammps-src-22Jul2025_update4.tar.gz
+    tar xzf lammps-src-22Jul2025_update4.tar.gz
+    cmake -S lammps-22Jul2025/cmake -B lammps_build -D CMAKE_INSTALL_PREFIX=/opt/lammps -D CMAKE_CXX_COMPILER=hipcc  -D PKG_KOKKOS=yes \
+       -D Kokkos_ARCH_HOSTARCH=yes -D Kokkos_ARCH_GPUARCH=yes -D Kokkos_ENABLE_HIP=yes -DKokkos_ARCH_AMD_GFX90A=ON
     cmake --build lammps_build --target install -j8
-    rm -r stable_2Aug2023_update3.tar.gz lammps-stable_2Aug2023_update3 lammps_build
+    rm -r lammps-src-22Jul2025_update4.tar.gz lammps-22Jul2025 lammps_build
 
 %environment
     export PATH="/opt/lammps/bin:$PATH"

frontier/containers_on_frontier_docs/apptainer_wrappers_lammps/gpu/submit.slurm

@@ -7,6 +7,7 @@
 #SBATCH -t 00:20:00
 
 module reset
+module load rocm/6.4.2
 module load olcf-container-tools
 module load apptainer-enable-mpi apptainer-enable-gpu
 

frontier/containers_on_frontier_docs/debugging/bcastandlammps.def

@@ -1,5 +1,5 @@
 bootstrap: docker
-From: savannah.ornl.gov/olcf-container-images/cpe:24.11_gnu_opensuse # this could be the ubuntu, opensuse or rockylinux container
+From: savannah.ornl.gov/olcf-container-images/cpe:25.09_gnu_opensuse # this could be the ubuntu, opensuse or rockylinux container
 
 %files
 ./mpi_bcast.c /opt/
@@ -10,12 +10,18 @@ mpicc -o mpi_bcast mpi_bcast.c
 zypper install -y gdb gdbserver
 
 zypper install -y cmake
-wget https://github.com/lammps/lammps/archive/refs/tags/stable_2Aug2023_update3.tar.gz
-tar xzf stable_2Aug2023_update3.tar.gz
-cmake -S lammps-stable_2Aug2023_update3/cmake -B lammps_build -D CMAKE_INSTALL_PREFIX=/opt/lammps -D CMAKE_CXX_COMPILER=hipcc \
-	-D PKG_KOKKOS=yes -D Kokkos_ARCH_HOSTARCH=yes -D Kokkos_ARCH_GPUARCH=yes -D Kokkos_ENABLE_HIP=yes -D CMAKE_BUILD_TYPE=Debug
+#wget https://github.com/lammps/lammps/archive/refs/tags/stable_2Aug2023_update3.tar.gz
+#tar xzf stable_2Aug2023_update3.tar.gz
+#cmake -S lammps-stable_2Aug2023_update3/cmake -B lammps_build -D CMAKE_INSTALL_PREFIX=/opt/lammps -D CMAKE_CXX_COMPILER=hipcc \
+#	-D PKG_KOKKOS=yes -D Kokkos_ARCH_HOSTARCH=yes -D Kokkos_ARCH_GPUARCH=yes -D Kokkos_ENABLE_HIP=yes -D CMAKE_BUILD_TYPE=Debug
+#cmake --build lammps_build --target install -j8
+#rm -r stable_2Aug2023_update3.tar.gz 
+wget https://github.com/lammps/lammps/releases/download/stable_22Jul2025_update4/lammps-src-22Jul2025_update4.tar.gz
+tar xzf lammps-src-22Jul2025_update4.tar.gz
+cmake -S lammps-22Jul2025/cmake -B lammps_build -D CMAKE_INSTALL_PREFIX=/opt/lammps -D CMAKE_CXX_COMPILER=hipcc  -D PKG_KOKKOS=yes \
+   -D Kokkos_ARCH_HOSTARCH=yes -D Kokkos_ARCH_GPUARCH=yes -D Kokkos_ENABLE_HIP=yes -DKokkos_ARCH_AMD_GFX90A=ON -D CMAKE_BUILD_TYPE=Debug
 cmake --build lammps_build --target install -j8
-rm -r stable_2Aug2023_update3.tar.gz 
+rm -r lammps-src-22Jul2025_update4.tar.gz 
 
 %startscript
 ./launchapp.sh

frontier/containers_on_frontier_docs/debugging/ij.in

@@ -0,0 +1,30 @@
+# 3d Lennard-Jones melt
+
+variable        x index 36
+variable        y index 36
+variable        z index 36
+
+variable        xx equal 20*$x
+variable        yy equal 20*$y
+variable        zz equal 20*$z
+
+units           lj
+atom_style      atomic
+
+lattice         fcc 0.8442
+region          box block 0 ${xx} 0 ${yy} 0 ${zz}
+create_box      1 box
+create_atoms    1 box
+mass            1 1.0
+
+velocity        all create 1.44 87287 loop geom
+
+pair_style      lj/cut 2.5
+pair_coeff      1 1 1.0 1.0 2.5
+
+neighbor        0.3 bin
+neigh_modify    delay 0 every 20 check no
+
+fix             1 all nve
+
+run             100

frontier/containers_on_frontier_docs/debugging/log.lammps

@@ -0,0 +1,94 @@
+LAMMPS (22 Jul 2025 - Update 4)
+KOKKOS mode with Kokkos version 4.6.2 is enabled
+  will use up to 1 GPU(s) per node
+  using 1 OpenMP thread(s) per MPI task
+package kokkos
+package kokkos gpu/aware on
+# 3d Lennard-Jones melt
+
+variable        x index 36
+variable        y index 36
+variable        z index 36
+
+variable        xx equal 20*$x
+variable        xx equal 20*36
+variable        yy equal 20*$y
+variable        yy equal 20*36
+variable        zz equal 20*$z
+variable        zz equal 20*36
+
+units           lj
+atom_style      atomic
+
+lattice         fcc 0.8442
+Lattice spacing in x,y,z = 1.6795962 1.6795962 1.6795962
+region          box block 0 ${xx} 0 ${yy} 0 ${zz}
+region          box block 0 720 0 ${yy} 0 ${zz}
+region          box block 0 720 0 720 0 ${zz}
+region          box block 0 720 0 720 0 720
+create_box      1 box
+Created orthogonal box = (0 0 0) to (1209.3093 1209.3093 1209.3093)
+  2 by 2 by 4 MPI processor grid
+create_atoms    1 box
+Created 1492992000 atoms
+  using lattice units in orthogonal box = (0 0 0) to (1209.3093 1209.3093 1209.3093)
+  create_atoms CPU = 6.816 seconds
+mass            1 1.0
+
+velocity        all create 1.44 87287 loop geom
+
+pair_style      lj/cut 2.5
+pair_coeff      1 1 1.0 1.0 2.5
+
+neighbor        0.3 bin
+neigh_modify    delay 0 every 20 check no
+
+fix             1 all nve
+
+run             100
+Generated 0 of 0 mixed pair_coeff terms from geometric mixing rule
+Neighbor list info ...
+  update: every = 20 steps, delay = 0 steps, check = no
+  max neighbors/atom: 2000, page size: 100000
+  master list distance cutoff = 2.8
+  ghost atom cutoff = 2.8
+  binsize = 2.8, bins = 432 432 432
+  1 neighbor lists, perpetual/occasional/extra = 1 0 0
+  (1) pair lj/cut/kk, perpetual
+      attributes: full, newton off, kokkos_device
+      pair build: full/bin/kk/device
+      stencil: full/bin/3d
+      bin: kk/device
+Per MPI rank memory allocation (min/avg/max) = 1.271e+04 | 1.271e+04 | 1.271e+04 Mbytes
+   Step          Temp          E_pair         E_mol          TotEng         Press     
+         0   1.44          -6.7733681      0             -4.6133681     -5.0196693    
+       100   0.75941233    -5.7614356      0             -4.6223171      0.19016988   
+Loop time of 71.5051 on 16 procs for 100 steps with 1492992000 atoms
+
+Performance: 604.153 tau/day, 1.399 timesteps/s, 2.088 Gatom-step/s
+101.2% CPU use with 16 MPI tasks x 1 OpenMP threads
+
+MPI task timing breakdown:
+Section |  min time  |  avg time  |  max time  |%varavg| %total
+---------------------------------------------------------------
+Pair    | 0.59484    | 0.62897    | 0.64008    |   1.7 |  0.88
+Neigh   | 8.9862     | 11.225     | 14.466     |  46.7 | 15.70
+Comm    | 6.7342     | 9.5598     | 11.672     |  49.5 | 13.37
+Output  | 0.0028027  | 1.1681     | 1.3759     |  40.6 |  1.63
+Modify  | 47.25      | 48.536     | 49.277     |   9.2 | 67.88
+Other   |            | 0.3873     |            |       |  0.54
+
+Nlocal:     9.3312e+07 ave 9.33132e+07 max 9.33111e+07 min
+Histogram: 4 1 1 0 2 4 1 2 0 1
+Nghost:    3.62905e+06 ave 3.62976e+06 max 3.62784e+06 min
+Histogram: 1 0 0 2 1 5 1 1 1 4
+Neighs:              0 ave           0 max           0 min
+Histogram: 16 0 0 0 0 0 0 0 0 0
+FullNghs:  7.00851e+09 ave 7.00868e+09 max 7.00841e+09 min
+Histogram: 1 5 1 2 2 4 0 0 0 1
+
+Total # of neighbors = 1.1213611e+11
+Ave neighs/atom = 75.10831
+Neighbor list builds = 5
+Dangerous builds not checked
+Total wall time: 0:01:40

frontier/containers_on_frontier_docs/gpu_aware_mpi_example/opensusempich342rocm624.def

@@ -67,7 +67,7 @@ mkdir -p /opt
 # Download
 cd /mpich && wget -O mpich-$MPICH_VERSION.tar.gz $MPICH_URL && tar --no-same-owner -xzf mpich-$MPICH_VERSION.tar.gz
 # Compile and install
-cd /mpich/mpich-$MPICH_VERSION && ./configure  --with-device=ch4:ofi --prefix=$MPICH_DIR && make install
+cd /mpich/mpich-$MPICH_VERSION && ./configure  --with-device=ch4:ofi --prefix=$MPICH_DIR && make -j32 install
 rm -rf /mpich
 
 # build ping_pong_gpu_aware.cpp

frontier/containers_on_frontier_docs/gpu_aware_mpi_example/opensusempich412rocm642.def

@@ -21,36 +21,14 @@ ls
 
 # Installing some prerequisites
 zypper install -y wget sudo gzip gcc-c++ gcc-fortran gcc14-c++ gcc14-fortran tar make autoconf automake binutils cpp glibc-devel m4 makeinfo zlib-devel gcc-info git glibc-info patch pkg-config which
-
+zypper --non-interactive update 
 # installing rocm 6.4.2 (see docs: https://rocm.docs.amd.com/en/latest/deploy/linux/installer/install.html)
-## prereqs for rocm
-zypper --non-interactive --no-gpg-checks addrepo https://download.opensuse.org/repositories/devel:/languages:/perl/15.6/devel:languages:perl.repo 
-zypper --non-interactive --no-gpg-checks addrepo https://download.opensuse.org/repositories/Education/15.6/Education.repo
-zypper --non-interactive --no-gpg-checks addrepo https://download.opensuse.org/repositories/science/SLE_15_SP5/science.repo
-
-ver=6.4.2
-tee /etc/zypp/repos.d/amdgpu.repo <<EOF
-[amdgpu]
-name=amdgpu
-baseurl=https://repo.radeon.com/amdgpu/$ver/sle/15.6/main/x86_64
-enabled=1
-gpgcheck=1
-gpgkey=https://repo.radeon.com/rocm/rocm.gpg.key
-EOF
-zypper --non-interactive  --no-gpg-checks ref
-
-tee --append /etc/zypp/repos.d/rocm.repo <<EOF
-[ROCm-$ver]
-name=ROCm$ver
-name=rocm
-baseurl=https://repo.radeon.com/rocm/zyp/$ver/main
-enabled=1
-gpgcheck=1
-gpgkey=https://repo.radeon.com/rocm/rocm.gpg.key
-EOF
-zypper --non-interactive  --no-gpg-checks ref
-
-zypper --gpg-auto-import-keys install -y rocm
+zypper --non-interactive --no-gpg-checks install https://repo.radeon.com/amdgpu-install/6.4.2/sle/15.6/amdgpu-install-6.4.60402-1.noarch.rpm
+zypper --non-interactive --gpg-auto-import-keys refresh
+zypper --non-interactive addrepo https://download.opensuse.org/repositories/science/SLE_15_SP5/science.repo
+zypper --non-interactive --gpg-auto-import-keys refresh
+zypper --non-interactive install python3-setuptools python3-wheel
+zypper --non-interactive install --force-resolution rocm
 
 # Installing MPICH 4.1.2
 # Information about the version of MPICH to use
@@ -65,7 +43,7 @@ mkdir -p /opt
 # Download
 cd /mpich && wget -O mpich-$MPICH_VERSION.tar.gz $MPICH_URL && tar --no-same-owner -xzf mpich-$MPICH_VERSION.tar.gz
 # Compile and install
-cd /mpich/mpich-$MPICH_VERSION && ./configure  --with-device=ch4:ofi --prefix=$MPICH_DIR && make install
+cd /mpich/mpich-$MPICH_VERSION && ./configure  --with-device=ch4:ofi --prefix=$MPICH_DIR && make -j32 install
 rm -rf /mpich
 
 # build ping_pong_gpu_aware.cpp

frontier/containers_on_frontier_docs/gpu_aware_mpi_example/submit624.sbatch

@@ -39,15 +39,15 @@ export APPTAINER_BINDPATH=/usr/share/libdrm,/var/spool/slurmd,/opt/cray,${PWD}
 # that might've been defined as part of the container build, so make sure to explicitly include 
 # any paths from within the container's file tree that you want available to the application 
 # in the container.
-export APPTAINERENV_LD_LIBRARY_PATH="$CRAY_MPICH_ROOTDIR/gtl/lib:/opt/rocm/lib:/opt/rocm/lib64:$CRAY_LD_LIBRARY_PATH:$LD_LIBRARY_PATH:/opt/cray/pe/lib64" #:/opt/cray/xpmem/2.10.6-1.2_gfaa90a94be64/lib64"
+export APPTAINERENV_LD_LIBRARY_PATH="$CRAY_MPICH_ROOTDIR/gtl/lib:/opt/cray/lib64:/opt/rocm/lib:/opt/rocm/lib64:$CRAY_LD_LIBRARY_PATH:$LD_LIBRARY_PATH:/opt/cray/pe/lib64" 
 
 # APPTAINER_CONTAINLIBS allows you to specify individual libraries from the host that you want
 # visible inside the container and added to the LD_LIBRARY_PATH inside the container.
 # In this GPU aware MPI example, Some specific libraries from the host's /usr/lib64 need to 
 # be visible inside the container for the Cray MPICH libraries (that we are making visible 
 # inside the container with the earlier steps) to work as the Cray MPICH libraries are linked
 # linked to them.
-export APPTAINER_CONTAINLIBS="/usr/lib64/libcxi.so.1,/usr/lib64/libjson-c.so.5,/lib64/libtinfo.so.6,/usr/lib64/libnl-3.so.200,/usr/lib64/libgfortran.so.5,/usr/lib64/libjansson.so.4,/usr/lib64/libxpmem.so.0"
+export APPTAINER_CONTAINLIBS="/usr/lib64/libjansson.so.4,/usr/lib64/libgfortran.so.5,/usr/lib64/libnl-3.so.200,/lib64/libtinfo.so.6,/usr/lib64/libjson-c.so.5,/usr/lib64/libcxi.so.1,/opt/xpmem/lib64/libxpmem.so.0"
 
 # This is required when you have an application that is compiled inside a container that 
 # doesn't have access to the Cray MPICH libraries during the container build process (such as our

frontier/containers_on_frontier_docs/gpu_aware_mpi_example/submit642.sbatch

@@ -39,15 +39,15 @@ export APPTAINER_BINDPATH=/usr/share/libdrm,/var/spool/slurmd,/opt/cray,${PWD}
 # that might've been defined as part of the container build, so make sure to explicitly include 
 # any paths from within the container's file tree that you want available to the application 
 # in the container.
-export APPTAINERENV_LD_LIBRARY_PATH="$CRAY_MPICH_ROOTDIR/gtl/lib:/opt/rocm/lib:/opt/rocm/lib64:$CRAY_LD_LIBRARY_PATH:$LD_LIBRARY_PATH:/opt/cray/pe/lib64" #:/opt/cray/xpmem/2.10.6-1.2_gfaa90a94be64/lib64"
+export APPTAINERENV_LD_LIBRARY_PATH="$CRAY_MPICH_ROOTDIR/gtl/lib:/opt/cray/lib64:/opt/rocm/lib:/opt/rocm/lib64:$CRAY_LD_LIBRARY_PATH:$LD_LIBRARY_PATH:/opt/cray/pe/lib64" 
 
 # APPTAINER_CONTAINLIBS allows you to specify individual libraries from the host that you want
 # visible inside the container and added to the LD_LIBRARY_PATH inside the container.
 # In this GPU aware MPI example, Some specific libraries from the host's /usr/lib64 need to 
 # be visible inside the container for the Cray MPICH libraries (that we are making visible 
 # inside the container with the earlier steps) to work as the Cray MPICH libraries are linked
 # linked to them.
-export APPTAINER_CONTAINLIBS="/usr/lib64/libcxi.so.1,/usr/lib64/libjson-c.so.5,/lib64/libtinfo.so.6,/usr/lib64/libnl-3.so.200,/usr/lib64/libgfortran.so.5,/usr/lib64/libjansson.so.4,/usr/lib64/libxpmem.so.0"
+export APPTAINER_CONTAINLIBS="/usr/lib64/libjansson.so.4,/usr/lib64/libgfortran.so.5,/usr/lib64/libnl-3.so.200,/lib64/libtinfo.so.6,/usr/lib64/libjson-c.so.5,/usr/lib64/libcxi.so.1,/opt/xpmem/lib64/libxpmem.so.0"
 
 # This is required when you have an application that is compiled inside a container that 
 # doesn't have access to the Cray MPICH libraries during the container build process (such as our
@@ -61,5 +61,6 @@ export APPTAINERENV_LD_PRELOAD=$CRAY_MPICH_ROOTDIR/gtl/lib/libmpi_gtl_hsa.so.0:
 # The program passes data allocated on GPU memory of increasing size back and forth between 
 # two MPI processes across two nodes.
 # The --rocm flag is required to support AMD GPUs inside the container.
+srun -N1 -n1 apptainer --silent exec --workdir `pwd` opensusempich412rocm642.sif ldd /ping_pong_gpu_aware.exe
 srun -N2 --tasks-per-node 1 --gpus-per-task=1 apptainer --silent exec --workdir `pwd` --rocm opensusempich412rocm642.sif /ping_pong_gpu_aware.exe
 

frontier/containers_on_frontier_docs/rccltests/README.md

@@ -1,12 +1,4 @@
 If you want to run the RCCL tests with the [aws-ofi-rccl plugin](https://github.com/ROCm/aws-ofi-rccl),
- run the `./awsofircclbuild.sh` script to build aws-ofi-rccl first. 
+Load `module load rccl-net-plugin` along with the other modules.
 
-All aws-ofi-rccl requires to make it usable to RCCL programs 
-is to make sure that its `lib` directory is added to the 
-`LD_LIBRARY_PATH` environment variable. We add it to the `APPTAINERENV_LD_LIBRARY_PATH`
-variable in this example to update the `LD_LIBRARY_PATH` variable inside
-the container.
-
-For more information on the use of the aws-ofi-rccl plugin, see the [slides](https://www.olcf.ornl.gov/wp-content/uploads/OLCF_AI_Training_0417_2024.pdf) 
-and [recording](https://vimeo.com/938233665)
-from the [OLCF AI Training Series event](https://www.olcf.ornl.gov/calendar/pytorch-on-frontier/)
+You will notice a good performance improvement with and withou the rccl-net-plugin.

frontier/containers_on_frontier_docs/rccltests/rcclmpich342rocm624.def → frontier/containers_on_frontier_docs/rccltests/rcclmpich412rocm642.def

@@ -1,5 +1,5 @@
 Bootstrap: localimage
-From: ../gpu_aware_mpi_example/opensusempich342rocm624.sif
+From: ../gpu_aware_mpi_example/opensusempich412rocm642.sif
 
 %environment
     # Point to MPICH binaries, libraries man pages
@@ -14,13 +14,7 @@ From: ../gpu_aware_mpi_example/opensusempich342rocm624.sif
 export PATH=$MPICH_DIR/bin:$PATH
 export LD_LIBRARY_PATH=$MPICH_DIR/lib:$LD_LIBRARY_PATH
 
-echo "Compiling the MPI application..."
-if [[ -d /opt/osu-micro-benchmarks-7.2 ]]
-then
-rm -rf /opt/osu-micro-benchmarks-7.2
-fi
-curl -o osubenchmarks-7.2.tar.gz https://mvapich.cse.ohio-state.edu/download/mvapich/osu-micro-benchmarks-7.2.tar.gz && tar -xzf osubenchmarks-7.2.tar.gz --no-same-owner
-cd osu-micro-benchmarks-7.2 && ./configure CC=mpicc CXX=mpicc --enable-rocm --with-rocm=/opt/rocm && make  && rm ../osubenchmarks-7.2.tar.gz
+zypper install -y libhwloc15
 
 git clone https://github.com/ROCmSoftwarePlatform/rccl-tests.git
 cd rccl-tests && make MPI=1 MPI_HOME=$MPICH_DIR