[interop] Upgrade CppInterOp to v1.8.0

Author: aaronj0

interpreter/CppInterOp/CMakeLists.txt

@@ -12,6 +12,9 @@ set(CMAKE_CXX_EXTENSIONS NO)
 option(CPPINTEROP_USE_CLING "Use Cling as backend" OFF)
 option(CPPINTEROP_USE_REPL "Use clang-repl as backend" ON)
 option(CPPINTEROP_ENABLE_TESTING "Enable the CppInterOp testing infrastructure." ON)
+if(EMSCRIPTEN)
+  set(CPPINTEROP_EXTRA_WASM_FLAGS "-sSUPPORT_LONGJMP=wasm -fwasm-exceptions" CACHE STRING "Extra flags for wasm")
+endif()
 
 if (CPPINTEROP_USE_CLING AND CPPINTEROP_USE_REPL)
 message(FATAL_ERROR "We can only use Cling (${CPPINTEROP_USE_CLING}=On) or Repl (CPPINTEROP_USE_REPL=On), but not both of them.")
@@ -68,14 +71,14 @@ include(GNUInstallDirs)
   ## Define supported version of clang and llvm
 
   set(CLANG_MIN_SUPPORTED 18.0)
-  set(CLANG_MAX_SUPPORTED "20.1.x")
-  set(CLANG_VERSION_UPPER_BOUND 21.0.0)
+  set(CLANG_MAX_SUPPORTED "21.1.x")
+  set(CLANG_VERSION_UPPER_BOUND 22.0.0)
   set(LLD_MIN_SUPPORTED 18.0)
-  set(LLD_MAX_SUPPORTED "20.1.x")
-  set(LLD_VERSION_UPPER_BOUND 21.0.0)
+  set(LLD_MAX_SUPPORTED "21.1.x")
+  set(LLD_VERSION_UPPER_BOUND 22.0.0)
   set(LLVM_MIN_SUPPORTED 18.0)
-  set(LLVM_MAX_SUPPORTED "20.1.x")
-  set(LLVM_VERSION_UPPER_BOUND 21.0.0)
+  set(LLVM_MAX_SUPPORTED "21.1.x")
+  set(LLVM_VERSION_UPPER_BOUND 22.0.0)
 
   ## Set Cmake packages search order
 
@@ -298,6 +301,18 @@ include_directories(SYSTEM ${LLVM_INCLUDE_DIRS})
 separate_arguments(LLVM_DEFINITIONS_LIST NATIVE_COMMAND ${LLVM_DEFINITIONS})
 add_definitions(${LLVM_DEFINITIONS_LIST})
 
+string(REGEX REPLACE "/lib/cmake/llvm$" "" LLVM_BINARY_LIB_DIR "${LLVM_DIR}")
+add_definitions(-DLLVM_BINARY_LIB_DIR="${LLVM_BINARY_LIB_DIR}")
+
+if(LLVM_BUILT_WITH_OOP_JIT)
+  if((CMAKE_SYSTEM_NAME STREQUAL "Darwin" AND CMAKE_SYSTEM_PROCESSOR MATCHES "arm64") OR
+     (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR MATCHES "x86_64"))
+    add_definitions(-DLLVM_BUILT_WITH_OOP_JIT)
+  else()
+    message(FATAL_ERROR "LLVM_BUILT_WITH_OOP_JIT is only supported on MacOS arm64 or Linux x86_64. Build aborted.")
+  endif()
+endif()
+
 # If the llvm sources are present add them with higher priority.
 if (LLVM_BUILD_MAIN_SRC_DIR)
   # LLVM_INCLUDE_DIRS contains the include paths to both LLVM's source and
@@ -347,6 +362,9 @@ endif()
 if (LLVM_COMPILER_IS_GCC_COMPATIBLE)
   if (APPLE OR EMSCRIPTEN)
     set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fno-common -Woverloaded-virtual -Wcast-qual -fno-strict-aliasing -Wno-long-long -Wall -W -Wno-unused-parameter -Wwrite-strings")
+    if (EMSCRIPTEN)
+      set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${CPPINTEROP_EXTRA_WASM_FLAGS}")
+    endif ()
   else()
     set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fno-common -Woverloaded-virtual -Wcast-qual -fno-strict-aliasing -pedantic -Wno-long-long -Wall -W -Wno-unused-parameter -Wwrite-strings")
   endif ()

interpreter/CppInterOp/Emscripten-build-instructions.md

@@ -19,34 +19,34 @@ cd ./CppInterOp-wasm
 ```
 
 To create a wasm build of CppInterOp we make use of the emsdk toolchain. This can be installed by executing (we only currently  
-support version 3.1.73)
+support version 4.0.9)
 ```bash
 git clone https://github.com/emscripten-core/emsdk.git
-./emsdk/emsdk install  3.1.73
+./emsdk/emsdk install  4.0.9
 ```
 
 and to activate the emsdk environment on Linux and osx execute (we are defining SYSROOT_PATH for use later)
 
 ```bash
-./emsdk/emsdk activate 3.1.73
+./emsdk/emsdk activate 4.0.9
 source ./emsdk/emsdk_env.sh
 export SYSROOT_PATH=$PWD/emsdk/upstream/emscripten/cache/sysroot
 ```
 
 and on Windows execute in Powershell
 
 ```powershell
-.\emsdk\emsdk activate 3.1.73
+.\emsdk\emsdk activate 4.0.9
 .\emsdk\emsdk_env.ps1
 $env:PWD_DIR= $PWD.Path
 $env:SYSROOT_PATH="$env:EMSDK/upstream/emscripten/cache/sysroot"
 ```
 
-Now clone the 20.x release of the LLVM project repository and CppInterOp (the building of the emscripten version of llvm can be
+Now clone the 21.x release of the LLVM project repository and CppInterOp (the building of the emscripten version of llvm can be
 avoided by executing micromamba install llvm -c <https://repo.mamba.pm/emscripten-forge> and setting the LLVM_BUILD_DIR/$env:LLVM_BUILD_DIR appropriately)
 
 ```bash
-git clone --depth=1 --branch release/20.x https://github.com/llvm/llvm-project.git
+git clone --depth=1 --branch release/21.x https://github.com/llvm/llvm-project.git
 git clone --depth=1 https://github.com/compiler-research/CppInterOp.git
 ```
 
@@ -55,16 +55,17 @@ executing
 
 ```bash
 cd ./llvm-project/
-git apply -v ../CppInterOp/patches/llvm/emscripten-clang20-*.patch
+git apply -v ../CppInterOp/patches/llvm/emscripten-clang21-*.patch
 ```
 
 On Windows execute the following
 
 ```powershell
 cd .\llvm-project\
-cp -r ..\patches\llvm\emscripten-clang20*
-git apply -v emscripten-clang20-2-shift-temporary-files-to-tmp-dir.patch
-git apply -v emscripten-clang20-3-enable_exception_handling.patch
+cp -r ..\patches\llvm\emscripten-clang21*
+git apply -v emscripten-clang21-1-shift-temporary-files-to-tmp-dir.patch
+git apply -v emscripten-clang21-2-enable_exception_handling.patch
+git apply -v emscripten-clang21-3-webassembly_target_machine_reordering.patch
 ```
 
 We are now in a position to build an emscripten build of llvm by executing the following on Linux
@@ -101,9 +102,7 @@ emcmake cmake -DCMAKE_BUILD_TYPE=Release \
                         -DCMAKE_CXX_FLAGS_RELEASE="-Oz -g0 -DNDEBUG" \
                         -DLLVM_ENABLE_LTO=Full \
                         ../llvm
-emmake make libclang -j $(nproc --all)
-emmake make clangInterpreter clangStaticAnalyzerCore -j $(nproc --all)
-emmake make lldWasm -j $(nproc --all)
+EMCC_CFLAGS="-sSUPPORT_LONGJMP=wasm -fwasm-exceptions" emmake make libclang clangInterpreter clangStaticAnalyzerCore -j $(nproc --all)
 ```
 
 or executing
@@ -142,7 +141,9 @@ emcmake cmake -DCMAKE_BUILD_TYPE=Release `
                         -DCMAKE_CXX_FLAGS_RELEASE="-Oz -g0 -DNDEBUG" `
                         -DLLVM_ENABLE_LTO=Full `
                         ..\llvm
-emmake ninja libclang clangInterpreter clangStaticAnalyzerCore lldWasm
+$env:EMCC_CFLAGS="-sSUPPORT_LONGJMP=wasm -fwasm-exceptions"
+emmake ninja libclang clangInterpreter clangStaticAnalyzerCore
+$env:EMCC_CFLAGS=""
 ```
 
 on Windows. Once this finishes building we need to take note of where we built our llvm build. This can be done by executing the following on Linux and osx
@@ -163,7 +164,7 @@ by executing (assumes you have micromamba installed and that your shell is initi
 
 ```bash
 cd ../../CppInterOp/
-micromamba create -f environment-wasm.yml --platform=emscripten-wasm32
+micromamba create -f environment-wasm.yml --platform=emscripten-wasm32 -c https://prefix.dev/emscripten-forge-4x -c https://prefix.dev/conda-forge
 micromamba activate CppInterOp-wasm
 ```
 
@@ -178,7 +179,7 @@ export CMAKE_SYSTEM_PREFIX_PATH=$PREFIX
 and
 
 ```powershell
-$env:PREFIX="%CONDA_PREFIX%/envs/CppInterOp-wasm"
+$env:PREFIX="$env:MAMBA_ROOT_PREFIX/envs/CppInterOp-wasm"
 $env:CMAKE_PREFIX_PATH=$env:PREFIX
 $env:CMAKE_SYSTEM_PREFIX_PATH=$env:PREFIX
 ```
@@ -337,13 +338,13 @@ emmake make -j $(nproc --all) install
 ## Xeus-cpp-lite Wasm Build Instructions
 
 A project which makes use of the wasm build of CppInterOp is xeus-cpp. xeus-cpp is a C++ Jupyter kernel. Assuming you are in  
-the CppInterOp build folder, you can build the wasm version of xeus-cpp by executing (replace LLVM_VERSION with the version
+the CppInterOp build folder, you can build the wasm version of xeus-cpp on Linux/MacOS by executing (replace LLVM_VERSION with the version
 of llvm you are building against)
 
 ```bash
 cd ../..
 git clone --depth=1 https://github.com/compiler-research/xeus-cpp.git
-export LLVM_VERSION=20
+export LLVM_VERSION=21
 cd ./xeus-cpp
 mkdir build
 cd build
@@ -356,20 +357,58 @@ emcmake cmake \
           -DXEUS_CPP_RESOURCE_DIR="$LLVM_BUILD_DIR/lib/clang/$LLVM_VERSION" \
           -DSYSROOT_PATH=$SYSROOT_PATH                                   \
           ..
- emmake make -j $(nproc --all) install
+EMCC_CFLAGS="-sSUPPORT_LONGJMP=wasm -fwasm-exceptions" emmake make -j $(nproc --all) install
 ```
 
-To build and test Jupyter Lite with this kernel locally you can execute the following
+and on Windows by executing
+
+```powershell
+cd ..\..
+git clone --depth=1 https://github.com/compiler-research/xeus-cpp.git
+$env:LLVM_VERSION=21
+cd .\xeus-cpp
+mkdir build
+cd build
+emcmake cmake `
+          -DCMAKE_BUILD_TYPE=Release                                     `
+          -DCMAKE_PREFIX_PATH="$env:PREFIX"                                    `
+          -DCMAKE_INSTALL_PREFIX="$env:PREFIX"                                 `
+          -DXEUS_CPP_EMSCRIPTEN_WASM_BUILD=ON                            `
+          -DCMAKE_FIND_ROOT_PATH_MODE_PACKAGE=ON                         `
+          -DXEUS_CPP_RESOURCE_DIR="$env:LLVM_BUILD_DIR/lib/clang/$env:LLVM_VERSION" `
+          -DSYSROOT_PATH="$env:SYSROOT_PATH"                                   `
+          ..
+$env:EMCC_CFLAGS="-sSUPPORT_LONGJMP=wasm -fwasm-exceptions"
+emmake make -j $(nproc --all) install
+$env:EMCC_CFLAGS=""
+```
+
+To build and test Jupyter Lite with this kernel locally on Linux/MacOS you can execute the following
 
 ```bash
 cd ../..
-micromamba create -n xeus-lite-host jupyterlite-core=0.6 jupyterlite-xeus jupyter_server jupyterlab notebook python-libarchive-c -c conda-forge
+micromamba create -n xeus-lite-host jupyterlite-core jupyterlite-xeus jupyter_server jupyterlab notebook python-libarchive-c -c conda-forge
 micromamba activate xeus-lite-host
 jupyter lite serve --XeusAddon.prefix=$PREFIX \
                    --contents xeus-cpp/notebooks/xeus-cpp-lite-demo.ipynb \
-                   --contents xeus-cpp/notebooks/smallpt.ipynb \
+                   --contents xeus-cpp/notebooks/tinyraytracer.ipynb \
                    --contents xeus-cpp/notebooks/images/marie.png \
                    --contents xeus-cpp/notebooks/audio/audio.wav \
                    --XeusAddon.mounts="$PREFIX/share/xeus-cpp/tagfiles:/share/xeus-cpp/tagfiles" \
                    --XeusAddon.mounts="$PREFIX/etc/xeus-cpp/tags.d:/etc/xeus-cpp/tags.d"
 ```
+
+and on Windows execute
+
+```powershell
+cd ..\..
+micromamba create -n xeus-lite-host jupyterlite-core jupyterlite-xeus jupyter_server jupyterlab notebook python-libarchive-c -c conda-forge
+micromamba activate xeus-lite-host
+jupyter lite serve --XeusAddon.prefix="$env:PREFIX" `
+                   --contents xeus-cpp/notebooks/xeus-cpp-lite-demo.ipynb `
+                   --contents xeus-cpp/notebooks/tinyraytracer.ipynb `
+                   --contents xeus-cpp/notebooks/images/marie.png `
+                   --contents xeus-cpp/notebooks/audio/audio.wav `
+                   --XeusAddon.mounts="$env:PREFIX/share/xeus-cpp/tagfiles:/share/xeus-cpp/tagfiles" `
+                   --XeusAddon.mounts="$env:PREFIX/etc/xeus-cpp/tags.d:/etc/xeus-cpp/tags.d"
+```

interpreter/CppInterOp/README.md

@@ -104,13 +104,20 @@ git clone --depth=1 https://github.com/compiler-research/cppyy-backend.git
 
 #### Setup Clang-REPL
 
-Clone the 20.x release of the LLVM project repository.
+Clone the 21.x release of the LLVM project repository.
 
 ```bash
-git clone --depth=1 --branch release/20.x https://github.com/llvm/llvm-project.git
+git clone --depth=1 --branch release/21.x https://github.com/llvm/llvm-project.git
 cd llvm-project
 ```
 
+If you want to have out-of-process JIT execution enabled in CppInterOp, then apply this patch on Linux-x86_64 and MacOS arm64 environment.
+> Note that this patch will not work for Windows because out-of-process JIT execution is currently implemented for Linux-x86_64 and MacOS arm64 only.
+
+```bash
+git apply -v ../CppInterOp/patches/llvm/clang20-1-out-of-process.patch
+```
+
 ##### Build Clang-REPL
 
 Clang-REPL is an interpreter that CppInterOp works alongside. Build Clang (and
@@ -140,6 +147,45 @@ export LLVM_DIR=$PWD
 cd ../
 ```
 
+##### Build Clang-REPL with Out-of-Process JIT Execution
+
+To have ``Out-of-Process JIT Execution`` enabled, run following commands to build clang and clang-repl to support this feature:
+> Only for Linux x86_64 and MacOS arm64
+
+```bash
+mkdir build 
+cd build 
+cmake -DLLVM_ENABLE_PROJECTS="clang;compiler-rt"                   \
+              -DLLVM_TARGETS_TO_BUILD="host;NVPTX"                \
+              -DCMAKE_BUILD_TYPE=Release                          \
+              -DLLVM_ENABLE_ASSERTIONS=ON                         \
+              -DCLANG_ENABLE_STATIC_ANALYZER=OFF                  \
+              -DCLANG_ENABLE_ARCMT=OFF                            \
+              -DCLANG_ENABLE_FORMAT=OFF                           \
+              -DCLANG_ENABLE_BOOTSTRAP=OFF                        \
+              ../llvm
+```
+
+###### For Linux x86_64
+
+```bash
+cmake --build . --target clang clang-repl llvm-jitlink-executor orc_rt-x86_64 --parallel $(nproc --all)
+```
+
+###### For MacOS arm64
+
+```bash
+cmake --build . --target clang clang-repl llvm-jitlink-executor orc_rt_osx --parallel $(sysctl -n hw.ncpu)
+```
+
+Note the 'llvm-project' directory location by executing
+
+```bash
+cd ../
+export LLVM_DIR=$PWD
+cd ../
+```
+
 #### Environment variables
 
 You will need to define the following environment variables for the build of CppInterOp and cppyy (as they clear for a new session, it is recommended that you also add these to your .bashrc in linux, .bash_profile if on MacOS). On Linux and MacOS you define as follows
@@ -163,10 +209,14 @@ Now CppInterOp can be built. This can be done by executing
 ```bash
 mkdir CppInterOp/build/
 cd CppInterOp/build/
-cmake -DBUILD_SHARED_LIBS=ON -DCPPINTEROP_USE_CLING=ON -DCPPINTEROP_USE_REPL=Off -DCling_DIR=$LLVM_DIR/build/tools/cling -DLLVM_DIR=$LLVM_DIR/build/lib/cmake/llvm -DClang_DIR=$LLVM_DIR/build/lib/cmake/clang -DCMAKE_INSTALL_PREFIX=$CPPINTEROP_DIR ..
+cmake -DBUILD_SHARED_LIBS=ON -DLLVM_DIR=$LLVM_DIR/build/lib/cmake/llvm -DClang_DIR=$LLVM_DIR/build/lib/cmake/clang -DCMAKE_INSTALL_PREFIX=$CPPINTEROP_DIR ..
 cmake --build . --target install --parallel $(nproc --all)
 ```
 
+and
+
+> Do make sure to pass ``DLLVM_BUILT_WITH_OOP_JIT=ON``, if you want to have out-of-process JIT execution feature enabled.
+
 #### Testing CppInterOp
 
 To test the built CppInterOp execute the following command in the CppInterOP build folder on Linux and MacOS
@@ -406,10 +456,10 @@ git clone --depth=1 https://github.com/compiler-research/cppyy-backend.git
 
 #### Setup Clang-REPL
 
-Clone the 20.x release of the LLVM project repository.
+Clone the 21.x release of the LLVM project repository.
 
 ```bash
-git clone --depth=1 --branch release/20.x https://github.com/llvm/llvm-project.git
+git clone --depth=1 --branch release/21.x https://github.com/llvm/llvm-project.git
 cd llvm-project
 ```
 

interpreter/CppInterOp/VERSION

@@ -1 +1 @@
-1.8.0;dev
+1.9.0;dev

interpreter/CppInterOp/cmake/modules/GoogleTest.cmake

@@ -20,6 +20,9 @@ endif()
 
 include(ExternalProject)
 if (EMSCRIPTEN)
+  # FIXME: -sSUPPORT_LONGJMP=wasm in the default option causes a warning in the Emscripten build of Googletest
+  # and as we treat warnings as errors in the ci, it causes the ci to fail.
+  string(REPLACE "-sSUPPORT_LONGJMP=wasm" "" GOOGLETEST_CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}")
   set(config_cmd emcmake${EMCC_SUFFIX} cmake)
   if(CMAKE_GENERATOR STREQUAL "Ninja")
     set(build_cmd emmake${EMCC_SUFFIX} ninja)
@@ -34,8 +37,8 @@ endif()
 ExternalProject_Add(
   googletest
   GIT_REPOSITORY https://github.com/google/googletest.git
-  GIT_SHALLOW 1
-  GIT_TAG v1.17.0
+  GIT_SHALLOW FALSE
+  GIT_TAG fa8438ae6b70c57010177de47a9f13d7041a6328
   UPDATE_COMMAND ""
   # # Force separate output paths for debug and release builds to allow easy
   # # identification of correct lib in subsequent TARGET_LINK_LIBRARIES commands
@@ -49,7 +52,7 @@ ExternalProject_Add(
                 -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER}
                 -DCMAKE_C_FLAGS=${CMAKE_C_FLAGS}
                 -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER}
-                -DCMAKE_CXX_FLAGS=${CMAKE_CXX_FLAGS}
+                -DCMAKE_CXX_FLAGS=${GOOGLETEST_CMAKE_CXX_FLAGS}
                 -DCMAKE_AR=${CMAKE_AR}
                 -DCMAKE_INSTALL_PREFIX=${CMAKE_INSTALL_PREFIX}
                 ${EXTRA_GTEST_OPTS}

interpreter/CppInterOp/cppinterop-version.tag

@@ -1 +1 @@
-33a3e7e7629c0818bd92420568671325b016d6b2
+c211948a266d7d42466a3ea5a3ebbf81548486e5