From e2a66d55dc2a725f71f7eb374b3f16696c86cc24 Mon Sep 17 00:00:00 2001 From: Yona Lapeyre Date: Sun, 7 Jun 2026 14:25:40 +0200 Subject: [PATCH 01/11] baseline --- .../adastra/mi250/acpp-rocm/env_built_acpp.sh | 83 +++++++++++++++++++ .../adastra/mi250/acpp-rocm/exemple_batch.sh | 41 +++++++++ .../adastra/mi250/acpp-rocm/setup-env.py | 53 ++++++++++++ 3 files changed, 177 insertions(+) create mode 100644 env/machine/adastra/mi250/acpp-rocm/env_built_acpp.sh create mode 100755 env/machine/adastra/mi250/acpp-rocm/exemple_batch.sh create mode 100644 env/machine/adastra/mi250/acpp-rocm/setup-env.py diff --git a/env/machine/adastra/mi250/acpp-rocm/env_built_acpp.sh b/env/machine/adastra/mi250/acpp-rocm/env_built_acpp.sh new file mode 100644 index 000000000..b8c95a96d --- /dev/null +++ b/env/machine/adastra/mi250/acpp-rocm/env_built_acpp.sh @@ -0,0 +1,83 @@ +# Everything before this line will be provided by the new-env script + +# ---- Modules ---- +module purge +module load cpe/25.09 +module load craype-accel-amd-gfx90a craype-x86-trento +module load PrgEnv-cray +module load amd-mixed +module load rocm +module load cray-python +module load cmake +module load ninja +module load boost/1.88.0-mpi # CCE-GPU-5.0.0 toolchain + +# ---- AdaptiveCpp config ---- +export ACPP_VERSION=v24.10.0 +export ACPP_TARGETS="hip:gfx90a" +export ACPP_GIT_DIR=$BUILD_DIR/.env/acpp-git +export ACPP_BUILD_DIR=$BUILD_DIR/.env/acpp-builddir +export ACPP_INSTALL_DIR=$BUILD_DIR/.env/acpp-installdir + +# Make ROCm LLVM headers visible to the build +export C_INCLUDE_PATH=$ROCM_PATH/llvm/include +export CPLUS_INCLUDE_PATH=$ROCM_PATH/llvm/include + +export MPICH_GPU_SUPPORT_ENABLED=1 + +# ---- Compiler setup ---- +function setupcompiler { + echo " ---- Running AdaptiveCpp compiler setup ----" + echo " -- Module list" + module list + + clone_acpp || return + + cmake -S ${ACPP_GIT_DIR} -B ${ACPP_BUILD_DIR} \ + -DCMAKE_INSTALL_PREFIX=${ACPP_INSTALL_DIR} \ + -DROCM_PATH=$ROCM_PATH \ + -DCMAKE_C_COMPILER=${ROCM_PATH}/llvm/bin/clang \ + -DCMAKE_CXX_COMPILER=${ROCM_PATH}/llvm/bin/clang++ \ + -DWITH_ACCELERATED_CPU=ON \ + -DWITH_CPU_BACKEND=ON \ + -DWITH_CUDA_BACKEND=OFF \ + -DWITH_ROCM_BACKEND=ON \ + -DWITH_OPENCL_BACKEND=OFF \ + -DWITH_LEVEL_ZERO_BACKEND=OFF \ + -DACPP_TARGETS="gfx90a" \ + -DBoost_NO_BOOST_CMAKE=TRUE \ + -DBoost_NO_SYSTEM_PATHS=TRUE \ + -DWITH_SSCP_COMPILER=OFF \ + -DLLVM_DIR=${ROCM_PATH}/llvm/lib/cmake/llvm/ || return + + (cd ${ACPP_BUILD_DIR} && $MAKE_EXEC "${MAKE_OPT[@]}" && $MAKE_EXEC install) || return +} + +if [ ! -f "$ACPP_INSTALL_DIR/bin/acpp" ]; then + echo " ----- acpp is not configured, compiling it ... -----" + setupcompiler || return + echo " ----- acpp configured ! -----" +fi + +# ---- Shamrock configure ---- +function shamconfigure { + cmake \ + -S $SHAMROCK_DIR \ + -B $BUILD_DIR \ + -DSHAMROCK_ENABLE_BACKEND=SYCL \ + -DSYCL_IMPLEMENTATION=ACPPDirect \ + -DCMAKE_CXX_COMPILER="${ACPP_INSTALL_DIR}/bin/acpp" \ + -DACPP_PATH="${ACPP_INSTALL_DIR}" \ + -DCMAKE_BUILD_TYPE="${SHAMROCK_BUILD_TYPE}" \ + -DCMAKE_CXX_FLAGS="-march=znver3 -isystem ${CRAY_MPICH_PREFIX}/include" \ + -DCMAKE_EXE_LINKER_FLAGS="-lpthread -L\"${CRAY_MPICH_PREFIX}/lib\" -lmpi ${PE_MPICH_GTL_DIR_amd_gfx90a} ${PE_MPICH_GTL_LIBS_amd_gfx90a}" \ + -DBUILD_TEST=Yes \ + -DCXX_FLAG_ARCH_NATIVE=off \ + -DPYTHON_EXECUTABLE=$(python3 -c "import sys; print(sys.executable)") \ + "${CMAKE_OPT[@]}" || return +} + +# ---- Shamrock build ---- +function shammake { + (cd $BUILD_DIR && $MAKE_EXEC "${MAKE_OPT[@]}" "${@}") || return +} diff --git a/env/machine/adastra/mi250/acpp-rocm/exemple_batch.sh b/env/machine/adastra/mi250/acpp-rocm/exemple_batch.sh new file mode 100755 index 000000000..91a431bad --- /dev/null +++ b/env/machine/adastra/mi250/acpp-rocm/exemple_batch.sh @@ -0,0 +1,41 @@ +#!/bin/bash -l +#SBATCH --job-name=examplejob # Job name +#SBATCH --output=examplejob.o%j # Name of stdout output file +#SBATCH --error=examplejob.e%j # Name of stderr error file +#SBATCH --partition=standard-g # partition name +#SBATCH --nodes=1 # Total number of nodes +#SBATCH --ntasks-per-node=8 # 8 MPI ranks per node, 16 total (2x8) +#SBATCH --gpus-per-node=8 # Allocate one gpu per MPI rank +#SBATCH --time=00:10:00 # Run time (d-hh:mm:ss) +#SBATCH --account= # Project for billing +# +echo "The job ${SLURM_JOB_ID} is running on these nodes:" +echo ${SLURM_NODELIST} +echo +# +SHAMROCK_BIN_PATH= +# +RSCRIPT= +# +source activate +# +ldd ./shamrock + +cat << EOF > select_gpu +#!/bin/bash + +export ROCR_VISIBLE_DEVICES=\$SLURM_LOCALID +exec \$* +EOF + +chmod +x ./select_gpu + +CPU_BIND="map_cpu:49,57,17,25,1,9,33,41" + +export MPICH_GPU_SUPPORT_ENABLED=1 +export ACPP_DEBUG_LEVEL=0 + +# +srun --cpu-bind=${CPU_BIND} -- \ + ./select_gpu $SHAMROCK_BIN_PATH/shamrock --force-dgpu-on --sycl-cfg auto:HIP --loglevel 1 --smi \ + --rscript $RSCRIPT diff --git a/env/machine/adastra/mi250/acpp-rocm/setup-env.py b/env/machine/adastra/mi250/acpp-rocm/setup-env.py new file mode 100644 index 000000000..03b1463ad --- /dev/null +++ b/env/machine/adastra/mi250/acpp-rocm/setup-env.py @@ -0,0 +1,53 @@ +import argparse +import os + +import utils.amd_arch +import utils.envscript +import utils.intel_llvm +import utils.sysinfo +from utils.oscmd import * +from utils.setuparg import * + +NAME = "Adastra Intel AdaptiveCpp ROCM/LLVM" +PATH = "machine/adastra/standard-g/acpp-rocm-llvm" + + +def setup(arg: SetupArg, envgen: EnvGen): + argv = arg.argv + builddir = arg.builddir + shamrockdir = arg.shamrockdir + buildtype = arg.buildtype + lib_mode = arg.lib_mode + + # Get current file path + cur_file = os.path.realpath(os.path.expanduser(__file__)) + + parser = argparse.ArgumentParser(prog=PATH, description=NAME + " env for Shamrock") + args = parser.parse_args(argv) + args.gen = "ninja" + + gen, gen_opt, cmake_gen, cmake_build_type = utils.sysinfo.select_generator(args, buildtype) + + ############################## + # Generate env script header + ############################## + + cmake_extra_args = "" + + envgen.export_list = { + "SHAMROCK_DIR": shamrockdir, + "BUILD_DIR": builddir, + "CMAKE_GENERATOR": cmake_gen, + "MAKE_EXEC": gen, + "MAKE_OPT": f"({gen_opt})", + "CMAKE_OPT": f"({cmake_extra_args})", + "SHAMROCK_BUILD_TYPE": f"'{cmake_build_type}'", + } + + envgen.ext_script_list = [ + shamrockdir + "/env/helpers/clone-acpp.sh", + shamrockdir + "/env/helpers/pull_reffiles.sh", + ] + + envgen.copy_env_file("exemple_batch.sh", "exemple_batch.sh") + envgen.gen_env_file("env_built_acpp.sh") From e07d464e20999b77c0f5b02c4c119eb8fb266279 Mon Sep 17 00:00:00 2001 From: Yona Lapeyre Date: Sun, 7 Jun 2026 14:55:01 +0200 Subject: [PATCH 02/11] load prerequisite for boost --- env/machine/adastra/mi250/acpp-rocm/env_built_acpp.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/env/machine/adastra/mi250/acpp-rocm/env_built_acpp.sh b/env/machine/adastra/mi250/acpp-rocm/env_built_acpp.sh index b8c95a96d..76d9e1b08 100644 --- a/env/machine/adastra/mi250/acpp-rocm/env_built_acpp.sh +++ b/env/machine/adastra/mi250/acpp-rocm/env_built_acpp.sh @@ -10,6 +10,7 @@ module load rocm module load cray-python module load cmake module load ninja +module load CCE-GPU-5.0.0 module load boost/1.88.0-mpi # CCE-GPU-5.0.0 toolchain # ---- AdaptiveCpp config ---- From 1899e129ac3673d6e52ec4a9035b8edece01f5cd Mon Sep 17 00:00:00 2001 From: Yona Lapeyre Date: Sun, 7 Jun 2026 15:01:26 +0200 Subject: [PATCH 03/11] add explicit path to boost libraries --- env/machine/adastra/mi250/acpp-rocm/env_built_acpp.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/env/machine/adastra/mi250/acpp-rocm/env_built_acpp.sh b/env/machine/adastra/mi250/acpp-rocm/env_built_acpp.sh index 76d9e1b08..cc5bd6b97 100644 --- a/env/machine/adastra/mi250/acpp-rocm/env_built_acpp.sh +++ b/env/machine/adastra/mi250/acpp-rocm/env_built_acpp.sh @@ -34,6 +34,8 @@ function setupcompiler { clone_acpp || return + local BOOST_ROOT_PATH="${BOOST_ROOT:-/opt/software/gaia/prod/5.0.0/boost-1.88.0-cce-18.0.0-ml3z}" + cmake -S ${ACPP_GIT_DIR} -B ${ACPP_BUILD_DIR} \ -DCMAKE_INSTALL_PREFIX=${ACPP_INSTALL_DIR} \ -DROCM_PATH=$ROCM_PATH \ @@ -46,6 +48,7 @@ function setupcompiler { -DWITH_OPENCL_BACKEND=OFF \ -DWITH_LEVEL_ZERO_BACKEND=OFF \ -DACPP_TARGETS="gfx90a" \ + -DBOOST_ROOT="${BOOST_ROOT_PATH}" \ -DBoost_NO_BOOST_CMAKE=TRUE \ -DBoost_NO_SYSTEM_PATHS=TRUE \ -DWITH_SSCP_COMPILER=OFF \ From da8425d1f2f5081d378e7f052a7cd09cfc5cf277 Mon Sep 17 00:00:00 2001 From: Yona Lapeyre Date: Sun, 7 Jun 2026 15:10:18 +0200 Subject: [PATCH 04/11] switch Cmake config --- env/machine/adastra/mi250/acpp-rocm/env_built_acpp.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/env/machine/adastra/mi250/acpp-rocm/env_built_acpp.sh b/env/machine/adastra/mi250/acpp-rocm/env_built_acpp.sh index cc5bd6b97..d485f5cb7 100644 --- a/env/machine/adastra/mi250/acpp-rocm/env_built_acpp.sh +++ b/env/machine/adastra/mi250/acpp-rocm/env_built_acpp.sh @@ -49,7 +49,8 @@ function setupcompiler { -DWITH_LEVEL_ZERO_BACKEND=OFF \ -DACPP_TARGETS="gfx90a" \ -DBOOST_ROOT="${BOOST_ROOT_PATH}" \ - -DBoost_NO_BOOST_CMAKE=TRUE \ + -DBoost_DIR="${BOOST_ROOT_PATH}/lib/cmake" \ + -DBoost_NO_BOOST_CMAKE=FALSE \ -DBoost_NO_SYSTEM_PATHS=TRUE \ -DWITH_SSCP_COMPILER=OFF \ -DLLVM_DIR=${ROCM_PATH}/llvm/lib/cmake/llvm/ || return From c36caea7d83bdac8c826cdd732cd57738ddddeee Mon Sep 17 00:00:00 2001 From: Yona Lapeyre Date: Sun, 7 Jun 2026 15:22:44 +0200 Subject: [PATCH 05/11] find boost dammit --- env/machine/adastra/mi250/acpp-rocm/env_built_acpp.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/env/machine/adastra/mi250/acpp-rocm/env_built_acpp.sh b/env/machine/adastra/mi250/acpp-rocm/env_built_acpp.sh index d485f5cb7..e7ae9f4a1 100644 --- a/env/machine/adastra/mi250/acpp-rocm/env_built_acpp.sh +++ b/env/machine/adastra/mi250/acpp-rocm/env_built_acpp.sh @@ -26,6 +26,9 @@ export CPLUS_INCLUDE_PATH=$ROCM_PATH/llvm/include export MPICH_GPU_SUPPORT_ENABLED=1 +export BOOST_ROOT_PATH="${BOOST_ROOT:-/opt/software/gaia/prod/5.0.0/boost-1.88.0-cce-18.0.0-ml3z}" +export LD_LIBRARY_PATH="${BOOST_ROOT_PATH}/lib:${LD_LIBRARY_PATH}" + # ---- Compiler setup ---- function setupcompiler { echo " ---- Running AdaptiveCpp compiler setup ----" @@ -75,7 +78,7 @@ function shamconfigure { -DACPP_PATH="${ACPP_INSTALL_DIR}" \ -DCMAKE_BUILD_TYPE="${SHAMROCK_BUILD_TYPE}" \ -DCMAKE_CXX_FLAGS="-march=znver3 -isystem ${CRAY_MPICH_PREFIX}/include" \ - -DCMAKE_EXE_LINKER_FLAGS="-lpthread -L\"${CRAY_MPICH_PREFIX}/lib\" -lmpi ${PE_MPICH_GTL_DIR_amd_gfx90a} ${PE_MPICH_GTL_LIBS_amd_gfx90a}" \ + -DCMAKE_EXE_LINKER_FLAGS="-lpthread -L\"${CRAY_MPICH_PREFIX}/lib\" -lmpi ${PE_MPICH_GTL_DIR_amd_gfx90a} ${PE_MPICH_GTL_LIBS_amd_gfx90a}" -L\"${BOOST_ROOT_PATH}/lib\" -Wl,-rpath,${BOOST_ROOT_PATH}/lib"\ -DBUILD_TEST=Yes \ -DCXX_FLAG_ARCH_NATIVE=off \ -DPYTHON_EXECUTABLE=$(python3 -c "import sys; print(sys.executable)") \ From 12e901068af6be33d6d5570f7bb518ca3a7cd4d0 Mon Sep 17 00:00:00 2001 From: Yona Lapeyre Date: Sun, 7 Jun 2026 15:25:47 +0200 Subject: [PATCH 06/11] formatting --- .../adastra/mi250/acpp-rocm/env_built_acpp.sh | 32 +++++++++---------- 1 file changed, 15 insertions(+), 17 deletions(-) diff --git a/env/machine/adastra/mi250/acpp-rocm/env_built_acpp.sh b/env/machine/adastra/mi250/acpp-rocm/env_built_acpp.sh index e7ae9f4a1..f846874dc 100644 --- a/env/machine/adastra/mi250/acpp-rocm/env_built_acpp.sh +++ b/env/machine/adastra/mi250/acpp-rocm/env_built_acpp.sh @@ -1,5 +1,5 @@ # Everything before this line will be provided by the new-env script - + # ---- Modules ---- module purge module load cpe/25.09 @@ -11,34 +11,31 @@ module load cray-python module load cmake module load ninja module load CCE-GPU-5.0.0 -module load boost/1.88.0-mpi # CCE-GPU-5.0.0 toolchain - +module load boost/1.88.0-mpi + # ---- AdaptiveCpp config ---- export ACPP_VERSION=v24.10.0 export ACPP_TARGETS="hip:gfx90a" export ACPP_GIT_DIR=$BUILD_DIR/.env/acpp-git export ACPP_BUILD_DIR=$BUILD_DIR/.env/acpp-builddir export ACPP_INSTALL_DIR=$BUILD_DIR/.env/acpp-installdir - -# Make ROCm LLVM headers visible to the build + export C_INCLUDE_PATH=$ROCM_PATH/llvm/include export CPLUS_INCLUDE_PATH=$ROCM_PATH/llvm/include - + export MPICH_GPU_SUPPORT_ENABLED=1 - + export BOOST_ROOT_PATH="${BOOST_ROOT:-/opt/software/gaia/prod/5.0.0/boost-1.88.0-cce-18.0.0-ml3z}" export LD_LIBRARY_PATH="${BOOST_ROOT_PATH}/lib:${LD_LIBRARY_PATH}" - + # ---- Compiler setup ---- function setupcompiler { echo " ---- Running AdaptiveCpp compiler setup ----" echo " -- Module list" module list - + clone_acpp || return - - local BOOST_ROOT_PATH="${BOOST_ROOT:-/opt/software/gaia/prod/5.0.0/boost-1.88.0-cce-18.0.0-ml3z}" - + cmake -S ${ACPP_GIT_DIR} -B ${ACPP_BUILD_DIR} \ -DCMAKE_INSTALL_PREFIX=${ACPP_INSTALL_DIR} \ -DROCM_PATH=$ROCM_PATH \ @@ -57,16 +54,16 @@ function setupcompiler { -DBoost_NO_SYSTEM_PATHS=TRUE \ -DWITH_SSCP_COMPILER=OFF \ -DLLVM_DIR=${ROCM_PATH}/llvm/lib/cmake/llvm/ || return - + (cd ${ACPP_BUILD_DIR} && $MAKE_EXEC "${MAKE_OPT[@]}" && $MAKE_EXEC install) || return } - + if [ ! -f "$ACPP_INSTALL_DIR/bin/acpp" ]; then echo " ----- acpp is not configured, compiling it ... -----" setupcompiler || return echo " ----- acpp configured ! -----" fi - + # ---- Shamrock configure ---- function shamconfigure { cmake \ @@ -78,14 +75,15 @@ function shamconfigure { -DACPP_PATH="${ACPP_INSTALL_DIR}" \ -DCMAKE_BUILD_TYPE="${SHAMROCK_BUILD_TYPE}" \ -DCMAKE_CXX_FLAGS="-march=znver3 -isystem ${CRAY_MPICH_PREFIX}/include" \ - -DCMAKE_EXE_LINKER_FLAGS="-lpthread -L\"${CRAY_MPICH_PREFIX}/lib\" -lmpi ${PE_MPICH_GTL_DIR_amd_gfx90a} ${PE_MPICH_GTL_LIBS_amd_gfx90a}" -L\"${BOOST_ROOT_PATH}/lib\" -Wl,-rpath,${BOOST_ROOT_PATH}/lib"\ + -DCMAKE_EXE_LINKER_FLAGS="-lpthread -L\"${CRAY_MPICH_PREFIX}/lib\" -lmpi ${PE_MPICH_GTL_DIR_amd_gfx90a} ${PE_MPICH_GTL_LIBS_amd_gfx90a} -L\"${BOOST_ROOT_PATH}/lib\" -Wl,-rpath,${BOOST_ROOT_PATH}/lib" \ -DBUILD_TEST=Yes \ -DCXX_FLAG_ARCH_NATIVE=off \ -DPYTHON_EXECUTABLE=$(python3 -c "import sys; print(sys.executable)") \ "${CMAKE_OPT[@]}" || return } - + # ---- Shamrock build ---- function shammake { (cd $BUILD_DIR && $MAKE_EXEC "${MAKE_OPT[@]}" "${@}") || return } + \ No newline at end of file From dc99117c145e9a9dcd711f2363cbb7b83605c52d Mon Sep 17 00:00:00 2001 From: Yona Lapeyre Date: Sun, 7 Jun 2026 15:30:31 +0200 Subject: [PATCH 07/11] find boost --- .../adastra/mi250/acpp-rocm/env_built_acpp.sh | 27 ++++++++++++++++--- 1 file changed, 24 insertions(+), 3 deletions(-) diff --git a/env/machine/adastra/mi250/acpp-rocm/env_built_acpp.sh b/env/machine/adastra/mi250/acpp-rocm/env_built_acpp.sh index f846874dc..603db6ab9 100644 --- a/env/machine/adastra/mi250/acpp-rocm/env_built_acpp.sh +++ b/env/machine/adastra/mi250/acpp-rocm/env_built_acpp.sh @@ -24,9 +24,30 @@ export C_INCLUDE_PATH=$ROCM_PATH/llvm/include export CPLUS_INCLUDE_PATH=$ROCM_PATH/llvm/include export MPICH_GPU_SUPPORT_ENABLED=1 - + export BOOST_ROOT_PATH="${BOOST_ROOT:-/opt/software/gaia/prod/5.0.0/boost-1.88.0-cce-18.0.0-ml3z}" -export LD_LIBRARY_PATH="${BOOST_ROOT_PATH}/lib:${LD_LIBRARY_PATH}" + +# The Boost installation on Adastra only ships tagged library names +# (libboost_context-mt-x64.so etc.) but ld.lld expects untagged names +# (libboost_context.so). We create symlinks in a writable directory and +# prepend it to the linker search path. +export BOOST_SYMLINK_DIR=$BUILD_DIR/.env/boost-symlinks + +function setupboost { + mkdir -p "${BOOST_SYMLINK_DIR}" + for tagged in "${BOOST_ROOT_PATH}/lib"/libboost_*-mt-x64.so; do + # e.g. libboost_context-mt-x64.so -> libboost_context.so + base=$(basename "$tagged") + untagged="${base/-mt-x64/}" + if [ ! -e "${BOOST_SYMLINK_DIR}/${untagged}" ]; then + ln -s "$tagged" "${BOOST_SYMLINK_DIR}/${untagged}" + fi + done +} + +setupboost + +export LD_LIBRARY_PATH="${BOOST_SYMLINK_DIR}:${BOOST_ROOT_PATH}/lib:${LD_LIBRARY_PATH}" # ---- Compiler setup ---- function setupcompiler { @@ -75,7 +96,7 @@ function shamconfigure { -DACPP_PATH="${ACPP_INSTALL_DIR}" \ -DCMAKE_BUILD_TYPE="${SHAMROCK_BUILD_TYPE}" \ -DCMAKE_CXX_FLAGS="-march=znver3 -isystem ${CRAY_MPICH_PREFIX}/include" \ - -DCMAKE_EXE_LINKER_FLAGS="-lpthread -L\"${CRAY_MPICH_PREFIX}/lib\" -lmpi ${PE_MPICH_GTL_DIR_amd_gfx90a} ${PE_MPICH_GTL_LIBS_amd_gfx90a} -L\"${BOOST_ROOT_PATH}/lib\" -Wl,-rpath,${BOOST_ROOT_PATH}/lib" \ + -DCMAKE_EXE_LINKER_FLAGS="-lpthread -L\"${CRAY_MPICH_PREFIX}/lib\" -lmpi ${PE_MPICH_GTL_DIR_amd_gfx90a} ${PE_MPICH_GTL_LIBS_amd_gfx90a} -L\"${BOOST_SYMLINK_DIR}\" -Wl,-rpath,${BOOST_SYMLINK_DIR}" \ -DBUILD_TEST=Yes \ -DCXX_FLAG_ARCH_NATIVE=off \ -DPYTHON_EXECUTABLE=$(python3 -c "import sys; print(sys.executable)") \ From 632b8a7c9215f5caa38a6eb652457b6eceaa2992 Mon Sep 17 00:00:00 2001 From: Yona Lapeyre Date: Sun, 7 Jun 2026 15:44:13 +0200 Subject: [PATCH 08/11] linking properly --- env/machine/adastra/mi250/acpp-rocm/env_built_acpp.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/env/machine/adastra/mi250/acpp-rocm/env_built_acpp.sh b/env/machine/adastra/mi250/acpp-rocm/env_built_acpp.sh index 603db6ab9..1879bd05c 100644 --- a/env/machine/adastra/mi250/acpp-rocm/env_built_acpp.sh +++ b/env/machine/adastra/mi250/acpp-rocm/env_built_acpp.sh @@ -96,6 +96,7 @@ function shamconfigure { -DACPP_PATH="${ACPP_INSTALL_DIR}" \ -DCMAKE_BUILD_TYPE="${SHAMROCK_BUILD_TYPE}" \ -DCMAKE_CXX_FLAGS="-march=znver3 -isystem ${CRAY_MPICH_PREFIX}/include" \ + -DCMAKE_SHARED_LINKER_FLAGS="-L\"${BOOST_SYMLINK_DIR}\" -Wl,-rpath,${BOOST_SYMLINK_DIR}" \ -DCMAKE_EXE_LINKER_FLAGS="-lpthread -L\"${CRAY_MPICH_PREFIX}/lib\" -lmpi ${PE_MPICH_GTL_DIR_amd_gfx90a} ${PE_MPICH_GTL_LIBS_amd_gfx90a} -L\"${BOOST_SYMLINK_DIR}\" -Wl,-rpath,${BOOST_SYMLINK_DIR}" \ -DBUILD_TEST=Yes \ -DCXX_FLAG_ARCH_NATIVE=off \ From a51e1ea3db619bc15575636607f5fb2d050fe515 Mon Sep 17 00:00:00 2001 From: Yona Lapeyre Date: Sun, 7 Jun 2026 15:55:15 +0200 Subject: [PATCH 09/11] pass flags to modules --- env/machine/adastra/mi250/acpp-rocm/env_built_acpp.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/env/machine/adastra/mi250/acpp-rocm/env_built_acpp.sh b/env/machine/adastra/mi250/acpp-rocm/env_built_acpp.sh index 1879bd05c..b5182ba75 100644 --- a/env/machine/adastra/mi250/acpp-rocm/env_built_acpp.sh +++ b/env/machine/adastra/mi250/acpp-rocm/env_built_acpp.sh @@ -97,6 +97,7 @@ function shamconfigure { -DCMAKE_BUILD_TYPE="${SHAMROCK_BUILD_TYPE}" \ -DCMAKE_CXX_FLAGS="-march=znver3 -isystem ${CRAY_MPICH_PREFIX}/include" \ -DCMAKE_SHARED_LINKER_FLAGS="-L\"${BOOST_SYMLINK_DIR}\" -Wl,-rpath,${BOOST_SYMLINK_DIR}" \ + -DCMAKE_MODULE_LINKER_FLAGS="-L\"${BOOST_SYMLINK_DIR}\" -Wl,-rpath,${BOOST_SYMLINK_DIR}" \ -DCMAKE_EXE_LINKER_FLAGS="-lpthread -L\"${CRAY_MPICH_PREFIX}/lib\" -lmpi ${PE_MPICH_GTL_DIR_amd_gfx90a} ${PE_MPICH_GTL_LIBS_amd_gfx90a} -L\"${BOOST_SYMLINK_DIR}\" -Wl,-rpath,${BOOST_SYMLINK_DIR}" \ -DBUILD_TEST=Yes \ -DCXX_FLAG_ARCH_NATIVE=off \ From 154ffbe686328a515a72e242c2299b148e3af569 Mon Sep 17 00:00:00 2001 From: Yona Lapeyre Date: Sun, 7 Jun 2026 16:25:52 +0200 Subject: [PATCH 10/11] formatting --- .../adastra/mi250/acpp-rocm/env_built_acpp.sh | 35 ++++++++----------- .../adastra/mi250/acpp-rocm/setup-env.py | 2 +- 2 files changed, 16 insertions(+), 21 deletions(-) diff --git a/env/machine/adastra/mi250/acpp-rocm/env_built_acpp.sh b/env/machine/adastra/mi250/acpp-rocm/env_built_acpp.sh index b5182ba75..bff4898a6 100644 --- a/env/machine/adastra/mi250/acpp-rocm/env_built_acpp.sh +++ b/env/machine/adastra/mi250/acpp-rocm/env_built_acpp.sh @@ -1,5 +1,5 @@ # Everything before this line will be provided by the new-env script - + # ---- Modules ---- module purge module load cpe/25.09 @@ -12,28 +12,24 @@ module load cmake module load ninja module load CCE-GPU-5.0.0 module load boost/1.88.0-mpi - + # ---- AdaptiveCpp config ---- export ACPP_VERSION=v24.10.0 export ACPP_TARGETS="hip:gfx90a" export ACPP_GIT_DIR=$BUILD_DIR/.env/acpp-git export ACPP_BUILD_DIR=$BUILD_DIR/.env/acpp-builddir export ACPP_INSTALL_DIR=$BUILD_DIR/.env/acpp-installdir - + export C_INCLUDE_PATH=$ROCM_PATH/llvm/include export CPLUS_INCLUDE_PATH=$ROCM_PATH/llvm/include - + export MPICH_GPU_SUPPORT_ENABLED=1 export BOOST_ROOT_PATH="${BOOST_ROOT:-/opt/software/gaia/prod/5.0.0/boost-1.88.0-cce-18.0.0-ml3z}" - -# The Boost installation on Adastra only ships tagged library names -# (libboost_context-mt-x64.so etc.) but ld.lld expects untagged names -# (libboost_context.so). We create symlinks in a writable directory and -# prepend it to the linker search path. export BOOST_SYMLINK_DIR=$BUILD_DIR/.env/boost-symlinks - + function setupboost { + # here I lost 2hrs of my life mkdir -p "${BOOST_SYMLINK_DIR}" for tagged in "${BOOST_ROOT_PATH}/lib"/libboost_*-mt-x64.so; do # e.g. libboost_context-mt-x64.so -> libboost_context.so @@ -44,19 +40,19 @@ function setupboost { fi done } - + setupboost - + export LD_LIBRARY_PATH="${BOOST_SYMLINK_DIR}:${BOOST_ROOT_PATH}/lib:${LD_LIBRARY_PATH}" - + # ---- Compiler setup ---- function setupcompiler { echo " ---- Running AdaptiveCpp compiler setup ----" echo " -- Module list" module list - + clone_acpp || return - + cmake -S ${ACPP_GIT_DIR} -B ${ACPP_BUILD_DIR} \ -DCMAKE_INSTALL_PREFIX=${ACPP_INSTALL_DIR} \ -DROCM_PATH=$ROCM_PATH \ @@ -75,16 +71,16 @@ function setupcompiler { -DBoost_NO_SYSTEM_PATHS=TRUE \ -DWITH_SSCP_COMPILER=OFF \ -DLLVM_DIR=${ROCM_PATH}/llvm/lib/cmake/llvm/ || return - + (cd ${ACPP_BUILD_DIR} && $MAKE_EXEC "${MAKE_OPT[@]}" && $MAKE_EXEC install) || return } - + if [ ! -f "$ACPP_INSTALL_DIR/bin/acpp" ]; then echo " ----- acpp is not configured, compiling it ... -----" setupcompiler || return echo " ----- acpp configured ! -----" fi - + # ---- Shamrock configure ---- function shamconfigure { cmake \ @@ -104,9 +100,8 @@ function shamconfigure { -DPYTHON_EXECUTABLE=$(python3 -c "import sys; print(sys.executable)") \ "${CMAKE_OPT[@]}" || return } - + # ---- Shamrock build ---- function shammake { (cd $BUILD_DIR && $MAKE_EXEC "${MAKE_OPT[@]}" "${@}") || return } - \ No newline at end of file diff --git a/env/machine/adastra/mi250/acpp-rocm/setup-env.py b/env/machine/adastra/mi250/acpp-rocm/setup-env.py index 03b1463ad..f87257dc5 100644 --- a/env/machine/adastra/mi250/acpp-rocm/setup-env.py +++ b/env/machine/adastra/mi250/acpp-rocm/setup-env.py @@ -9,7 +9,7 @@ from utils.setuparg import * NAME = "Adastra Intel AdaptiveCpp ROCM/LLVM" -PATH = "machine/adastra/standard-g/acpp-rocm-llvm" +PATH = "machine/adastra/acpp-rocm-llvm" def setup(arg: SetupArg, envgen: EnvGen): From 939d6a365f5a4fba20265b1479ad4a620bbb736f Mon Sep 17 00:00:00 2001 From: Yona Lapeyre Date: Mon, 8 Jun 2026 01:26:25 +0200 Subject: [PATCH 11/11] formatting + fix example batch --- .../adastra/mi250/acpp-rocm/exemple_batch.sh | 63 +++++++++---------- .../adastra/mi250/acpp-rocm/setup-env.py | 2 +- 2 files changed, 30 insertions(+), 35 deletions(-) diff --git a/env/machine/adastra/mi250/acpp-rocm/exemple_batch.sh b/env/machine/adastra/mi250/acpp-rocm/exemple_batch.sh index 91a431bad..7dc1d39ae 100755 --- a/env/machine/adastra/mi250/acpp-rocm/exemple_batch.sh +++ b/env/machine/adastra/mi250/acpp-rocm/exemple_batch.sh @@ -1,41 +1,36 @@ -#!/bin/bash -l -#SBATCH --job-name=examplejob # Job name -#SBATCH --output=examplejob.o%j # Name of stdout output file -#SBATCH --error=examplejob.e%j # Name of stderr error file -#SBATCH --partition=standard-g # partition name -#SBATCH --nodes=1 # Total number of nodes -#SBATCH --ntasks-per-node=8 # 8 MPI ranks per node, 16 total (2x8) -#SBATCH --gpus-per-node=8 # Allocate one gpu per MPI rank -#SBATCH --time=00:10:00 # Run time (d-hh:mm:ss) -#SBATCH --account= # Project for billing -# -echo "The job ${SLURM_JOB_ID} is running on these nodes:" -echo ${SLURM_NODELIST} -echo -# -SHAMROCK_BIN_PATH= -# -RSCRIPT= -# -source activate -# -ldd ./shamrock - -cat << EOF > select_gpu #!/bin/bash +#SBATCH --account= +#SBATCH --job-name= +#SBATCH --constraint=MI250 +#SBATCH --nodes=1 +#SBATCH --exclusive +#SBATCH --time=01:00:00 +#SBATCH --output=job.%j.out +#SBATCH --error=job.%j.out # Same file for both! -export ROCR_VISIBLE_DEVICES=\$SLURM_LOCALID -exec \$* -EOF -chmod +x ./select_gpu +module purge +source /opt/cray/pe/cpe/24.07/restore_lmod_system_defaults.sh +# A CrayPE environment version +module load cpe/24.11 +# An architecture +module load craype-accel-amd-gfx90a craype-x86-trento +# A compiler to target the architecture +module load PrgEnv-cray +# Some architecture related libraries and tools +module load amd-mixed/6.4.3 -CPU_BIND="map_cpu:49,57,17,25,1,9,33,41" +module list export MPICH_GPU_SUPPORT_ENABLED=1 -export ACPP_DEBUG_LEVEL=0 +export SYCL_DEVICE_ALLOCATOR=aligned +# export OMP_ +WORKDIR= -# -srun --cpu-bind=${CPU_BIND} -- \ - ./select_gpu $SHAMROCK_BIN_PATH/shamrock --force-dgpu-on --sycl-cfg auto:HIP --loglevel 1 --smi \ - --rscript $RSCRIPT +. /opt/cray/pe/cpe/24.11/restore_lmod_system_defaults.sh + +export SHAM_MAX_ALLOC_SIZE=4294967296 + +. ./activate + +srun --ntasks-per-node=8 --cpus-per-task=8 --threads-per-core=1 --gpus-per-task=1 --gpu-bind=closest -- ./shamrock --smi-full --sycl-cfg auto:HIP --force-dgpu-off --loglevel 1 --rscript runscript.py diff --git a/env/machine/adastra/mi250/acpp-rocm/setup-env.py b/env/machine/adastra/mi250/acpp-rocm/setup-env.py index f87257dc5..bf5d18eae 100644 --- a/env/machine/adastra/mi250/acpp-rocm/setup-env.py +++ b/env/machine/adastra/mi250/acpp-rocm/setup-env.py @@ -9,7 +9,7 @@ from utils.setuparg import * NAME = "Adastra Intel AdaptiveCpp ROCM/LLVM" -PATH = "machine/adastra/acpp-rocm-llvm" +PATH = "machine/adastra/mi250/acpp-rocm-llvm" def setup(arg: SetupArg, envgen: EnvGen):