Skip to content

Dynamic Loading IB verbs #253

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
AtlantaPepsi wants to merge 1 commit into
base: candidate
Choose a base branch
from
+143 −11
Draft

Dynamic Loading IB verbs #253

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 9 additions & 0 deletions CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake")
#==================================================================================================
option(BUILD_LOCAL_GPU_TARGET_ONLY "Build only for GPUs detected on this machine" OFF)
option(ENABLE_NIC_EXEC "Enable RDMA NIC Executor in TransferBench" OFF)
option(ENABLE_IBV_DIRECT "Link libibverbs symbols directly (OFF: resolve via dlsym)" ON)
option(ENABLE_MPI_COMM "Enable MPI Communicator support" OFF)
option(ENABLE_DMA_BUF "Enable DMA-BUF support for GPU Direct RDMA" OFF)
option(ENABLE_AMD_SMI "Enable AMD-SMI pod membership queries" OFF)
Expand Down Expand Up @@ -146,6 +147,11 @@ else()
set_target_properties(ibverbs PROPERTIES INTERFACE_INCLUDE_DIRECTORIES "${IBVERBS_INCLUDE_DIR}" IMPORTED_LOCATION "${IBVERBS_LIBRARY}" INTERFACE_SYSTEM_INCLUDE_DIRECTORIES "${IBVERBS_INCLUDE_DIR}")
set(IBVERBS_FOUND 1)
message(STATUS "- Building with NIC executor support. Can set DISABLE_NIC_EXEC=1 to disable")
if(ENABLE_IBV_DIRECT)
message(STATUS "- IBV_DIRECT enabled (direct libibverbs linkage); set -DENABLE_IBV_DIRECT=OFF for dlsym path")
else()
message(STATUS "- IBV_DIRECT disabled: libibverbs symbols resolved via dlsym at runtime")
endif()
else()
if(NOT IBVERBS_LIBRARY)
message(WARNING "- IBVerbs library not found")
Expand Down Expand Up @@ -318,6 +324,9 @@ if(IBVERBS_FOUND)
target_include_directories(TransferBench PRIVATE ${IBVERBS_INCLUDE_DIR})
target_link_libraries(TransferBench PRIVATE ${IBVERBS_LIBRARY})
target_compile_definitions(TransferBench PRIVATE NIC_EXEC_ENABLED)
if(ENABLE_IBV_DIRECT)
Comment on lines 325 to +327
Copy link

Copilot AI Apr 8, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

ENABLE_IBV_DIRECT=OFF claims to resolve symbols via dlsym at runtime, but the target still links against ${IBVERBS_LIBRARY} unconditionally when IBVERBS_FOUND. This keeps libibverbs as a hard dependency and defeats optional loading. If runtime optionality is intended, avoid linking ${IBVERBS_LIBRARY} when ENABLE_IBV_DIRECT is OFF and ensure all ibv_* usage is through the loaded function pointers.

Suggested change
target_link_libraries(TransferBench PRIVATE ${IBVERBS_LIBRARY})
target_compile_definitions(TransferBench PRIVATE NIC_EXEC_ENABLED)
if(ENABLE_IBV_DIRECT)
target_compile_definitions(TransferBench PRIVATE NIC_EXEC_ENABLED)
if(ENABLE_IBV_DIRECT)
target_link_libraries(TransferBench PRIVATE ${IBVERBS_LIBRARY})

Copilot uses AI. Check for mistakes.
target_compile_definitions(TransferBench PRIVATE IBV_DIRECT=1)
endif()
endif()
if(MPI_COMM_FOUND)
if(TARGET MPI::MPI_CXX)
Expand Down
9 changes: 9 additions & 0 deletions Makefile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ MPI_PATH ?= /usr/local/openmpi

# Optional features (set to 0 to disable, 1 to enable)
# DISABLE_NIC_EXEC: Disable RDMA/NIC executor support (default: 0)
# DISABLE_IBV_DIRECT: When NIC support is on, use dlsym for libibverbs instead of direct linkage (default: 0)
# DISABLE_MPI_COMM: Disable MPI communicator support (default: 0)
# DISABLE_DMA_BUF: Disable DMA-BUF support for GPU Direct RDMA (default: 1)
# DISABLE_AMD_SMI: Disable AMD-SMI pod membership checking support (default: 0)
Expand Down Expand Up @@ -85,7 +86,9 @@ ifeq ($(filter clean,$(MAKECMDGOALS)),)
# 1) DISABLE_NIC_EXEC is not set to 1
# 2) IBVerbs is found in the Dynamic Linker cache
# 3) infiniband/verbs.h is found in the default include path
# When enabled, -DIBV_DIRECT=1 is added unless DISABLE_IBV_DIRECT=1 (verbs via direct link + constexpr pfn_*)
DISABLE_NIC_EXEC ?= 0
DISABLE_IBV_DIRECT ?= 0
ifneq ($(DISABLE_NIC_EXEC),1)
$(info Attempting to build with NIC executor support)
ifeq ("$(shell ldconfig -p | grep -c ibverbs)", "0")
Expand All @@ -96,6 +99,9 @@ ifeq ($(filter clean,$(MAKECMDGOALS)),)
COMMON_FLAGS += -DNIC_EXEC_ENABLED
LDFLAGS += -libverbs
NIC_ENABLED = 1
ifneq ($(DISABLE_IBV_DIRECT),1)
COMMON_FLAGS += -DIBV_DIRECT=1
Comment on lines 100 to +103
Copy link

Copilot AI Apr 8, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

When DISABLE_IBV_DIRECT=1 (dlsym path), the build still unconditionally links -libverbs. Any remaining direct ibv_* calls (and even an unused -libverbs without --as-needed) can keep libibverbs as a hard runtime dependency, undermining the goal of optional runtime loading. Consider omitting -libverbs when using the dlsym path (and ensure all ibv_* usage goes through pfn_*), and add -ldl for toolchains/glibc versions where dlopen is not in libc.

Suggested change
LDFLAGS += -libverbs
NIC_ENABLED = 1
ifneq ($(DISABLE_IBV_DIRECT),1)
COMMON_FLAGS += -DIBV_DIRECT=1
NIC_ENABLED = 1
ifneq ($(DISABLE_IBV_DIRECT),1)
COMMON_FLAGS += -DIBV_DIRECT=1
LDFLAGS += -libverbs
else
LDFLAGS += -ldl

Copilot uses AI. Check for mistakes.
endif

# Disable DMA-BUF support by default (set DISABLE_DMA_BUF=0 to enable)
DISABLE_DMA_BUF ?= 1
Expand Down Expand Up @@ -123,6 +129,9 @@ ifeq ($(filter clean,$(MAKECMDGOALS)),)
$(info - To use the TransferBench RDMA executor, check if your system has NICs, the NIC drivers are installed, and libibverbs-dev is installed)
else
$(info - Building with NIC executor support. Can set DISABLE_NIC_EXEC=1 to disable)
ifeq ($(DISABLE_IBV_DIRECT),1)
$(info - IBV_DIRECT disabled: libibverbs via dlsym, DISABLE_IBV_DIRECT=1)
endif
endif
endif

Expand Down
136 changes: 125 additions & 11 deletions src/header/TransferBench.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,12 @@ THE SOFTWARE.

#ifdef NIC_EXEC_ENABLED
#include <infiniband/verbs.h>
#if IBV_DIRECT
#define IBV_FN(name, rettype, arglist) constexpr rettype(*pfn_##name)arglist = name;
#else
#include <dlfcn.h>
#define IBV_FN(name, rettype, arglist) rettype(*pfn_##name)arglist = nullptr;
#endif
#endif

#ifdef MPI_COMM_ENABLED
Expand Down Expand Up @@ -736,11 +742,36 @@ namespace TransferBench
return false; \
} while (0)

namespace {
IBV_FN(ibv_alloc_pd, ibv_pd*, (ibv_context*));
IBV_FN(ibv_close_device, int, (ibv_context*));
IBV_FN(ibv_create_cq, ibv_cq*, (ibv_context*, int, void*, ibv_comp_channel*, int));
IBV_FN(ibv_create_qp, ibv_qp*, (ibv_pd*, ibv_qp_init_attr*));
IBV_FN(ibv_dealloc_pd, int, (ibv_pd*));
IBV_FN(ibv_dereg_mr, int, (ibv_mr*));
IBV_FN(ibv_destroy_cq, int, (ibv_cq*));
IBV_FN(ibv_destroy_qp, int, (ibv_qp*));
IBV_FN(ibv_free_device_list, void, (ibv_device**));
IBV_FN(ibv_get_device_list, ibv_device**, (int*));
IBV_FN(ibv_get_device_name, const char*, (ibv_device*));
IBV_FN(ibv_modify_qp, int, (ibv_qp*, ibv_qp_attr*, int));
IBV_FN(ibv_open_device, ibv_context*, (ibv_device*));
IBV_FN(ibv_poll_cq, int, (ibv_cq*, int, ibv_wc*));
IBV_FN(ibv_post_send, int, (ibv_qp*, ibv_send_wr*, ibv_send_wr**));
IBV_FN(ibv_query_device, int, (ibv_context*, ibv_device_attr*));
IBV_FN(ibv_query_gid, int, (ibv_context*, uint8_t, int, ibv_gid*));
IBV_FN(ibv_query_port, int, (ibv_context*, uint8_t, ibv_port_attr*));
#ifdef HAVE_DMABUF_SUPPORT
IBV_FN(ibv_reg_dmabuf_mr, ibv_mr*, (ibv_pd*, uint64_t, size_t, uint64_t, int, int));
#endif
IBV_FN(ibv_reg_mr, ibv_mr*, (ibv_pd*, void*, size_t, int));
}
Comment on lines +745 to +768
Copy link

Copilot AI Apr 8, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The new IBV_FN() function-pointer declarations are not guarded by NIC_EXEC_ENABLED, but IBV_FN (and ibv_* types) only exist when NIC_EXEC_ENABLED includes <infiniband/verbs.h>. As-is, builds with NIC executor disabled will fail to compile at this block. Wrap the IBV_FN declarations (and any dependent IBV_* call macros) in #ifdef NIC_EXEC_ENABLED, or provide a no-op/alternative definition when NIC support is off.

Copilot uses AI. Check for mistakes.

// Helper macros for calling RDMA functions and reporting errors
#ifdef VERBS_DEBUG
#define IBV_CALL(__func__, ...) \
do { \
int error = __func__(__VA_ARGS__); \
int error = pfn_##__func__(__VA_ARGS__); \
if (error != 0) { \
return {ERR_FATAL, "Encountered IbVerbs error (%d) at line (%d) " \
"and function (%s)", (error), __LINE__, #__func__}; \
Expand All @@ -749,7 +780,7 @@ namespace TransferBench

#define IBV_PTR_CALL(__ptr__, __func__, ...) \
do { \
__ptr__ = __func__(__VA_ARGS__); \
__ptr__ = pfn_##__func__(__VA_ARGS__); \
if (__ptr__ == nullptr) { \
return {ERR_FATAL, "Encountered IbVerbs nullptr error at line (%d) " \
"and function (%s)", __LINE__, #__func__}; \
Expand All @@ -758,7 +789,7 @@ namespace TransferBench
#else
#define IBV_CALL(__func__, ...) \
do { \
int error = __func__(__VA_ARGS__); \
int error = pfn_##__func__(__VA_ARGS__); \
if (error != 0) { \
return {ERR_FATAL, "Encountered IbVerbs error (%d=%s) in func (%s)" \
, error, strerror(errno), #__func__}; \
Expand All @@ -767,7 +798,7 @@ namespace TransferBench

#define IBV_PTR_CALL(__ptr__, __func__, ...) \
do { \
__ptr__ = __func__(__VA_ARGS__); \
__ptr__ = pfn_##__func__(__VA_ARGS__); \
if (__ptr__ == nullptr) { \
return {ERR_FATAL, "Encountered IbVerbs nullptr error (%s) in func (%s) " \
, strerror(errno), #__func__}; \
Expand Down Expand Up @@ -1006,6 +1037,7 @@ namespace {
bool IsSamePod(int targetRank, int sourceRank) const;
std::string GetExecutorName(ExeDevice exeDevice) const;
int NicIsActive(int nicIndex, int targetRank) const;
bool IbvLoaded() const;

#if !defined(__NVCC__)
ErrResult GetHsaAgent(ExeDevice const& exeDevice, hsa_agent_t& agent) const;
Expand All @@ -1032,6 +1064,8 @@ namespace {
bool verbose = false;
bool rankDoesOutput = true;
FILE* dumpCfgFile = nullptr;
bool ibvLoaded = false;
void* ibvLibHandle = nullptr;

#if !defined(__NVCC__)
std::vector<hsa_agent_t> cpuAgents;
Expand Down Expand Up @@ -2341,6 +2375,11 @@ namespace {
case EXE_NIC: case EXE_NIC_NEAREST:
#ifdef NIC_EXEC_ENABLED
{
if (!System::Get().IbvLoaded()) {
errors.push_back({ERR_FATAL, "Transfer %d: NIC executor is requested but IB verbs is not loaded.", i});
hasFatalError = true;
break;
}
// NIC Executors can only execute a copy operation
if (t.srcs.size() != 1 || t.dsts.size() != 1) {
errors.push_back({ERR_FATAL, "Transfer %d: NIC executor requires single SRC and single DST", i});
Expand Down Expand Up @@ -2671,7 +2710,7 @@ namespace {
#ifdef NIC_EXEC_ENABLED
// Function to collect information about IBV devices
//========================================================================================
static bool IsConfiguredGid(union ibv_gid const& gid)
static bool IsConfiguredGid(union ibv_gid const& gid)
{
const struct in6_addr *a = (struct in6_addr *) gid.raw;
int trailer = (a->s6_addr32[1] | a->s6_addr32[2] | a->s6_addr32[3]);
Expand All @@ -2696,7 +2735,8 @@ static bool IsConfiguredGid(union ibv_gid const& gid)
int const& gidIndex,
int& version)
{
char const* deviceName = ibv_get_device_name(context->device);
char const* deviceName;
IBV_PTR_CALL(deviceName, ibv_get_device_name, context->device);
char gidRoceVerStr[16] = {};
char roceTypePath[PATH_MAX] = {};
sprintf(roceTypePath, "/sys/class/infiniband/%s/ports/%d/gid_attrs/types/%d",
Expand Down Expand Up @@ -2778,17 +2818,68 @@ static bool IsConfiguredGid(union ibv_gid const& gid)
return ERR_NONE;
}

// Should only be called with IBV_DIRECT guard
static void* Ibvdl() {
Comment on lines +2821 to +2822
Copy link

Copilot AI Apr 8, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The comment above Ibvdl() says it “should only be called with IBV_DIRECT guard”, but Ibvdl() is the dlsym/dlopen path and should only be used when IBV_DIRECT is disabled. Update the comment (and ideally add a compile-time guard) to avoid confusion about which mode this helper supports.

Copilot uses AI. Check for mistakes.
static void* ibvLibHandle = nullptr;
if (ibvLibHandle) return ibvLibHandle;

void *handle = dlopen("libibverbs.so.1", RTLD_NOW);
if (handle != nullptr) {
Comment on lines +2821 to +2827
Copy link

Copilot AI Apr 8, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Ibvdl() uses dlopen/dlsym/dlclose but is compiled unconditionally under NIC_EXEC_ENABLED. When IBV_DIRECT=1, <dlfcn.h> is not included, so these calls will be undeclared and compilation will fail. Guard Ibvdl() itself (and any dl* usage) with #if !IBV_DIRECT (or include <dlfcn.h> regardless of IBV_DIRECT).

Copilot uses AI. Check for mistakes.
struct Symbol { void **ppfn; char const *name; };
Symbol symbols[] = {
{ (void**)&pfn_ibv_alloc_pd, "ibv_alloc_pd" },
{ (void**)&pfn_ibv_close_device, "ibv_close_device" },
{ (void**)&pfn_ibv_create_cq, "ibv_create_cq" },
{ (void**)&pfn_ibv_create_qp, "ibv_create_qp" },
{ (void**)&pfn_ibv_dealloc_pd, "ibv_dealloc_pd" },
{ (void**)&pfn_ibv_dereg_mr, "ibv_dereg_mr" },
{ (void**)&pfn_ibv_destroy_cq, "ibv_destroy_cq" },
{ (void**)&pfn_ibv_destroy_qp, "ibv_destroy_qp" },
{ (void**)&pfn_ibv_free_device_list, "ibv_free_device_list" },
{ (void**)&pfn_ibv_get_device_list, "ibv_get_device_list" },
{ (void**)&pfn_ibv_get_device_name, "ibv_get_device_name" },
{ (void**)&pfn_ibv_modify_qp, "ibv_modify_qp" },
{ (void**)&pfn_ibv_open_device, "ibv_open_device" },
{ (void**)&pfn_ibv_poll_cq, "ibv_poll_cq" },
{ (void**)&pfn_ibv_post_send, "ibv_post_send" },
{ (void**)&pfn_ibv_query_device, "ibv_query_device" },
{ (void**)&pfn_ibv_query_gid, "ibv_query_gid" },
{ (void**)&pfn_ibv_query_port, "ibv_query_port" },
#ifdef HAVE_DMABUF_SUPPORT
{ (void**)&pfn_ibv_reg_dmabuf_mr, "ibv_reg_dmabuf_mr" },
#endif
{ (void**)&pfn_ibv_reg_mr, "ibv_reg_mr" },
};
for (size_t i = 0; i < sizeof(symbols) / sizeof(symbols[0]); i++) {
*symbols[i].ppfn = dlsym(handle, symbols[i].name);
if (*symbols[i].ppfn == nullptr) {
// Log("[WARN] Failed to load symbol %s", symbols[i].name);
dlclose(handle);
break;
}
}
Comment on lines +2853 to +2860
Copy link

Copilot AI Apr 8, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Ibvdl() currently treats partial symbol resolution as success: on a dlsym failure it dlclose()s the handle but still leaves handle non-null and later assigns it to the cached ibvLibHandle. This can return a closed handle and leave some pfn_* pointers non-null/invalid. Only mark the library loaded if all symbols are resolved; otherwise set handle to nullptr and reset any already-assigned function pointers.

Suggested change
for (size_t i = 0; i < sizeof(symbols) / sizeof(symbols[0]); i++) {
*symbols[i].ppfn = dlsym(handle, symbols[i].name);
if (*symbols[i].ppfn == nullptr) {
// Log("[WARN] Failed to load symbol %s", symbols[i].name);
dlclose(handle);
break;
}
}
auto resetSymbols = [&symbols]() {
for (size_t i = 0; i < sizeof(symbols) / sizeof(symbols[0]); i++) {
*symbols[i].ppfn = nullptr;
}
};
bool loadedAllSymbols = true;
for (size_t i = 0; i < sizeof(symbols) / sizeof(symbols[0]); i++) {
*symbols[i].ppfn = dlsym(handle, symbols[i].name);
if (*symbols[i].ppfn == nullptr) {
// Log("[WARN] Failed to load symbol %s", symbols[i].name);
loadedAllSymbols = false;
break;
}
}
if (!loadedAllSymbols) {
resetSymbols();
dlclose(handle);
handle = nullptr;
}

Copilot uses AI. Check for mistakes.
ibvLibHandle = handle;
}

return ibvLibHandle;
}

static vector<IbvDevice>& GetIbvDeviceList()
{
static bool isInitialized = false;
static vector<IbvDevice> ibvDeviceList = {};

#if !defined(IBV_DIRECT)
if (ibvLibHandle == nullptr) {
Copy link

Copilot AI Apr 8, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

GetIbvDeviceList() checks ibvLibHandle == nullptr, but ibvLibHandle here refers to no visible variable (the System member isn’t accessible from this free function, and the Ibvdl() local static isn’t in scope). This is a compile error. Use a reachable source of truth (e.g., System::Get().IbvLoaded(), store the handle in a shared internal static with external linkage in this anonymous namespace, or call Ibvdl() and cache its result) instead of referencing an out-of-scope identifier.

Suggested change
if (ibvLibHandle == nullptr) {
if (Ibvdl() == nullptr) {

Copilot uses AI. Check for mistakes.
return ibvDeviceList;
}
#endif
// Build list on first use
if (!isInitialized) {

// Query the number of IBV devices
int numIbvDevices = 0;
ibv_device** deviceList = ibv_get_device_list(&numIbvDevices);
ibv_device** deviceList = pfn_ibv_get_device_list(&numIbvDevices);

Comment on lines 2880 to 2883
Copy link

Copilot AI Apr 8, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Within GetIbvDeviceList(), most verbs calls were converted to pfn_* (e.g., pfn_ibv_get_device_list/pfn_ibv_open_device/etc.), but the function still calls ibv_free_device_list(deviceList) directly later in the same function. In the dlsym path this reintroduces a direct libibverbs symbol reference and breaks the “all calls through pfn_*” model. Switch that cleanup call to pfn_ibv_free_device_list as well.

Copilot uses AI. Check for mistakes.
// Check for TB_NIC_FILTER
// By default, accept all NIC names
Expand All @@ -2806,15 +2897,15 @@ static bool IsConfiguredGid(union ibv_gid const& gid)
ibvDevice.name = deviceList[i]->name;
ibvDevice.hasActivePort = false;
{
struct ibv_context *context = ibv_open_device(ibvDevice.devicePtr);
struct ibv_context *context = pfn_ibv_open_device(ibvDevice.devicePtr);
if (context) {
struct ibv_device_attr deviceAttr;
if (!ibv_query_device(context, &deviceAttr)) {
if (!pfn_ibv_query_device(context, &deviceAttr)) {
int activePort;
ibvDevice.gidIndex = -1;
for (int port = 1; port <= deviceAttr.phys_port_cnt; ++port) {
struct ibv_port_attr portAttr;
if (ibv_query_port(context, port, &portAttr)) continue;
if (pfn_ibv_query_port(context, port, &portAttr)) continue;
if (portAttr.state == IBV_PORT_ACTIVE) {
activePort = port;
ibvDevice.hasActivePort = true;
Expand All @@ -2831,7 +2922,7 @@ static bool IsConfiguredGid(union ibv_gid const& gid)
}
}
}
ibv_close_device(context);
pfn_ibv_close_device(context);
}
}
ibvDevice.busId = "";
Expand Down Expand Up @@ -5810,6 +5901,19 @@ static bool IsConfiguredGid(union ibv_gid const& gid)
Log("[INFO] Running in single node mode\n");
}

#ifdef NIC_EXEC_ENABLED
#if IBV_DIRECT
ibvLoaded = true;
#else
ibvLoaded = true;
ibvLibHandle = Ibvdl();
if (ibvLibHandle == nullptr) {
Log("[WARN] Failed to open libibverbs.so.1");
ibvLoaded = false;
}
#endif
#endif

// Collect topology and distribute across all ranks
CollectTopology();
}
Expand Down Expand Up @@ -5842,6 +5946,11 @@ static bool IsConfiguredGid(union ibv_gid const& gid)
fclose(dumpCfgFile);
}

if (ibvLibHandle) {
dlclose(ibvLibHandle);
ibvLibHandle = nullptr;
}
Comment on lines +5949 to +5952
Copy link

Copilot AI Apr 8, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

System::~System() calls dlclose(ibvLibHandle) unconditionally. When IBV_DIRECT=1, <dlfcn.h> is not included, so dlclose will be undeclared and compilation can fail. Also, Ibvdl() keeps its own internal static handle, so closing only the System member can leave Ibvdl() returning a stale (already-closed) handle on future calls. Guard dlclose with #if !IBV_DIRECT and keep a single owner for the handle (either System or Ibvdl, but not both).

Suggested change
if (ibvLibHandle) {
dlclose(ibvLibHandle);
ibvLibHandle = nullptr;
}
#ifdef NIC_EXEC_ENABLED
#if !IBV_DIRECT
// Ibvdl() owns the dynamically loaded libibverbs handle. Avoid closing a
// cached copy here so future Ibvdl() calls cannot observe a stale handle.
ibvLibHandle = nullptr;
#endif
#endif

Copilot uses AI. Check for mistakes.

#ifdef AMD_SMI_ENABLED
amdsmi_shut_down();
#elif defined(__NVCC__) && defined(POD_COMM_ENABLED)
Expand Down Expand Up @@ -7016,6 +7125,11 @@ static bool IsConfiguredGid(union ibv_gid const& gid)
return rankInfo[targetRank].nicIsActive.at(nicIndex);
}

bool System::IbvLoaded() const
{
return ibvLoaded;
}

int GetNumExecutors(ExeType exeType, int targetRank)
{
return System::Get().GetNumExecutors(exeType, targetRank);
Expand Down