diff --git a/unified-runtime/include/unified-runtime/ur_api.h b/unified-runtime/include/unified-runtime/ur_api.h index 798423b6ae55a..0fa0b802f1814 100644 --- a/unified-runtime/include/unified-runtime/ur_api.h +++ b/unified-runtime/include/unified-runtime/ur_api.h @@ -512,6 +512,14 @@ typedef enum ur_function_t { UR_FUNCTION_QUEUE_GET_GRAPH_EXP = 314, /// Enumerator for ::urGraphSetDestructionCallbackExp UR_FUNCTION_GRAPH_SET_DESTRUCTION_CALLBACK_EXP = 315, + /// Enumerator for ::urIPCGetPhysMemHandleExp + UR_FUNCTION_IPC_GET_PHYS_MEM_HANDLE_EXP = 316, + /// Enumerator for ::urIPCPutPhysMemHandleExp + UR_FUNCTION_IPC_PUT_PHYS_MEM_HANDLE_EXP = 317, + /// Enumerator for ::urIPCOpenPhysMemHandleExp + UR_FUNCTION_IPC_OPEN_PHYS_MEM_HANDLE_EXP = 318, + /// Enumerator for ::urIPCClosePhysMemHandleExp + UR_FUNCTION_IPC_CLOSE_PHYS_MEM_HANDLE_EXP = 319, /// @cond UR_FUNCTION_FORCE_UINT32 = 0x7fffffff /// @endcond @@ -2505,6 +2513,9 @@ typedef enum ur_device_info_t { /// [::ur_bool_t] returns true if the device supports inter-process /// communicable memory handles UR_DEVICE_INFO_IPC_MEMORY_SUPPORT_EXP = 0x2023, + /// [::ur_bool_t] returns true if the device supports inter-process + /// communicable physical memory handles + UR_DEVICE_INFO_IPC_PHYSICAL_MEMORY_SUPPORT_EXP = 0x2024, /// [::ur_bool_t] returns true if the device supports enqueueing of /// allocations and frees. UR_DEVICE_INFO_ASYNC_USM_ALLOCATIONS_SUPPORT_EXP = 0x2050, @@ -5336,15 +5347,15 @@ UR_APIEXPORT ur_result_t UR_APICALL urVirtualMemGetInfo( /// @brief Physical memory creation properties. typedef uint32_t ur_physical_mem_flags_t; typedef enum ur_physical_mem_flag_t { - /// reserved for future use. - UR_PHYSICAL_MEM_FLAG_TBD = UR_BIT(0), + /// allocate physical memory that can be shared via IPC handles. + UR_PHYSICAL_MEM_FLAG_ENABLE_IPC = UR_BIT(1), /// @cond UR_PHYSICAL_MEM_FLAG_FORCE_UINT32 = 0x7fffffff /// @endcond } ur_physical_mem_flag_t; /// @brief Bit Mask for validating ur_physical_mem_flags_t -#define UR_PHYSICAL_MEM_FLAGS_MASK 0xfffffffe +#define UR_PHYSICAL_MEM_FLAGS_MASK 0xfffffffd /////////////////////////////////////////////////////////////////////////////// /// @brief Physical memory creation properties. @@ -5438,6 +5449,11 @@ typedef enum ur_physical_mem_info_t { /// It is unsuitable for general use in applications. This feature is /// provided for identifying memory leaks. UR_PHYSICAL_MEM_INFO_REFERENCE_COUNT = 4, + /// [void*] For physical memory objects opened from an IPC handle via + /// ::urIPCOpenPhysMemHandleExp, returns the virtual address established + /// by the driver (i.e. the address returned by zeMemOpenIpcHandle). + /// Returns nullptr for non-IPC physical memory objects. + UR_PHYSICAL_MEM_INFO_IPC_VIRTUAL_ADDRESS = 5, /// @cond UR_PHYSICAL_MEM_INFO_FORCE_UINT32 = 0x7fffffff /// @endcond @@ -5455,7 +5471,7 @@ typedef enum ur_physical_mem_info_t { /// - ::UR_RESULT_ERROR_INVALID_NULL_HANDLE /// + `NULL == hPhysicalMem` /// - ::UR_RESULT_ERROR_INVALID_ENUMERATION -/// + `::UR_PHYSICAL_MEM_INFO_REFERENCE_COUNT < propName` +/// + `::UR_PHYSICAL_MEM_INFO_IPC_VIRTUAL_ADDRESS < propName` UR_APIEXPORT ur_result_t UR_APICALL urPhysicalMemGetInfo( /// [in] handle of the physical memory object to query. ur_physical_mem_handle_t hPhysicalMem, @@ -11242,6 +11258,113 @@ UR_APIEXPORT ur_result_t UR_APICALL urIPCCloseMemHandleExp( /// [in] pointer to device USM memory opened through urIPCOpenMemHandleExp void *pMem); +/////////////////////////////////////////////////////////////////////////////// +/// @brief Gets an inter-process handle for a physical memory object +/// +/// @returns +/// - ::UR_RESULT_SUCCESS +/// - ::UR_RESULT_ERROR_UNINITIALIZED +/// - ::UR_RESULT_ERROR_DEVICE_LOST +/// - ::UR_RESULT_ERROR_ADAPTER_SPECIFIC +/// - ::UR_RESULT_ERROR_INVALID_NULL_HANDLE +/// + `NULL == hContext` +/// + `NULL == hPhysMem` +/// - ::UR_RESULT_ERROR_INVALID_NULL_POINTER +/// + `NULL == ppIPCPhysMemHandleData` +/// + `NULL == pIPCPhysMemHandleDataSizeRet` +/// - ::UR_RESULT_ERROR_INVALID_CONTEXT +/// - ::UR_RESULT_ERROR_INVALID_ARGUMENT +/// + `hPhysMem` was not created with the +/// `::UR_PHYSICAL_MEM_FLAG_ENABLE_IPC` flag +/// - ::UR_RESULT_ERROR_OUT_OF_HOST_MEMORY +/// - ::UR_RESULT_ERROR_OUT_OF_RESOURCES +UR_APIEXPORT ur_result_t UR_APICALL urIPCGetPhysMemHandleExp( + /// [in] handle of the context object + ur_context_handle_t hContext, + /// [in] handle of the physical memory object + ur_physical_mem_handle_t hPhysMem, + /// [out] a pointer to the IPC physical memory handle data + void **ppIPCPhysMemHandleData, + /// [out] size of the resulting IPC physical memory handle data + size_t *pIPCPhysMemHandleDataSizeRet); + +/////////////////////////////////////////////////////////////////////////////// +/// @brief Releases an inter-process physical memory handle +/// +/// @returns +/// - ::UR_RESULT_SUCCESS +/// - ::UR_RESULT_ERROR_UNINITIALIZED +/// - ::UR_RESULT_ERROR_DEVICE_LOST +/// - ::UR_RESULT_ERROR_ADAPTER_SPECIFIC +/// - ::UR_RESULT_ERROR_INVALID_NULL_HANDLE +/// + `NULL == hContext` +/// - ::UR_RESULT_ERROR_INVALID_NULL_POINTER +/// + `NULL == pIPCPhysMemHandleData` +/// - ::UR_RESULT_ERROR_INVALID_CONTEXT +/// - ::UR_RESULT_ERROR_OUT_OF_HOST_MEMORY +/// - ::UR_RESULT_ERROR_OUT_OF_RESOURCES +UR_APIEXPORT ur_result_t UR_APICALL urIPCPutPhysMemHandleExp( + /// [in] handle of the context object + ur_context_handle_t hContext, + /// [in] a pointer to the IPC physical memory handle data obtained with + /// urIPCGetPhysMemHandleExp + const void *pIPCPhysMemHandleData); + +/////////////////////////////////////////////////////////////////////////////// +/// @brief Opens an inter-process physical memory handle to get the +/// corresponding +/// physical memory object +/// +/// @returns +/// - ::UR_RESULT_SUCCESS +/// - ::UR_RESULT_ERROR_UNINITIALIZED +/// - ::UR_RESULT_ERROR_DEVICE_LOST +/// - ::UR_RESULT_ERROR_ADAPTER_SPECIFIC +/// - ::UR_RESULT_ERROR_INVALID_NULL_HANDLE +/// + `NULL == hContext` +/// + `NULL == hDevice` +/// - ::UR_RESULT_ERROR_INVALID_NULL_POINTER +/// + `NULL == phPhysMem` +/// + `NULL == pIPCPhysMemHandleData` +/// - ::UR_RESULT_ERROR_INVALID_CONTEXT +/// - ::UR_RESULT_ERROR_INVALID_VALUE +/// + ipcPhysMemHandleDataSize is not the same as the size of IPC +/// physical memory handle data +/// - ::UR_RESULT_ERROR_OUT_OF_HOST_MEMORY +/// - ::UR_RESULT_ERROR_OUT_OF_RESOURCES +UR_APIEXPORT ur_result_t UR_APICALL urIPCOpenPhysMemHandleExp( + /// [in] handle of the context object + ur_context_handle_t hContext, + /// [in] handle of the device object the physical memory was allocated on + ur_device_handle_t hDevice, + /// [in] the IPC physical memory handle data obtained with + /// urIPCGetPhysMemHandleExp + const void *pIPCPhysMemHandleData, + /// [in] size of the IPC physical memory handle data + size_t ipcPhysMemHandleDataSize, + /// [out] pointer to the physical memory handle + ur_physical_mem_handle_t *phPhysMem); + +/////////////////////////////////////////////////////////////////////////////// +/// @brief Closes an inter-process physical memory handle +/// +/// @returns +/// - ::UR_RESULT_SUCCESS +/// - ::UR_RESULT_ERROR_UNINITIALIZED +/// - ::UR_RESULT_ERROR_DEVICE_LOST +/// - ::UR_RESULT_ERROR_ADAPTER_SPECIFIC +/// - ::UR_RESULT_ERROR_INVALID_NULL_HANDLE +/// + `NULL == hContext` +/// + `NULL == hPhysMem` +/// - ::UR_RESULT_ERROR_INVALID_CONTEXT +/// - ::UR_RESULT_ERROR_OUT_OF_HOST_MEMORY +/// - ::UR_RESULT_ERROR_OUT_OF_RESOURCES +UR_APIEXPORT ur_result_t UR_APICALL urIPCClosePhysMemHandleExp( + /// [in] handle of the context object + ur_context_handle_t hContext, + /// [in] physical memory handle opened through urIPCOpenPhysMemHandleExp + ur_physical_mem_handle_t hPhysMem); + #if !defined(__GNUC__) #pragma endregion #endif @@ -16504,6 +16627,47 @@ typedef struct ur_ipc_close_mem_handle_exp_params_t { void **ppMem; } ur_ipc_close_mem_handle_exp_params_t; +/////////////////////////////////////////////////////////////////////////////// +/// @brief Function parameters for urIPCGetPhysMemHandleExp +/// @details Each entry is a pointer to the parameter passed to the function; +/// allowing the callback the ability to modify the parameter's value +typedef struct ur_ipc_get_phys_mem_handle_exp_params_t { + ur_context_handle_t *phContext; + ur_physical_mem_handle_t *phPhysMem; + void ***pppIPCPhysMemHandleData; + size_t **ppIPCPhysMemHandleDataSizeRet; +} ur_ipc_get_phys_mem_handle_exp_params_t; + +/////////////////////////////////////////////////////////////////////////////// +/// @brief Function parameters for urIPCPutPhysMemHandleExp +/// @details Each entry is a pointer to the parameter passed to the function; +/// allowing the callback the ability to modify the parameter's value +typedef struct ur_ipc_put_phys_mem_handle_exp_params_t { + ur_context_handle_t *phContext; + const void **ppIPCPhysMemHandleData; +} ur_ipc_put_phys_mem_handle_exp_params_t; + +/////////////////////////////////////////////////////////////////////////////// +/// @brief Function parameters for urIPCOpenPhysMemHandleExp +/// @details Each entry is a pointer to the parameter passed to the function; +/// allowing the callback the ability to modify the parameter's value +typedef struct ur_ipc_open_phys_mem_handle_exp_params_t { + ur_context_handle_t *phContext; + ur_device_handle_t *phDevice; + const void **ppIPCPhysMemHandleData; + size_t *pipcPhysMemHandleDataSize; + ur_physical_mem_handle_t **pphPhysMem; +} ur_ipc_open_phys_mem_handle_exp_params_t; + +/////////////////////////////////////////////////////////////////////////////// +/// @brief Function parameters for urIPCClosePhysMemHandleExp +/// @details Each entry is a pointer to the parameter passed to the function; +/// allowing the callback the ability to modify the parameter's value +typedef struct ur_ipc_close_phys_mem_handle_exp_params_t { + ur_context_handle_t *phContext; + ur_physical_mem_handle_t *phPhysMem; +} ur_ipc_close_phys_mem_handle_exp_params_t; + /////////////////////////////////////////////////////////////////////////////// /// @brief Function parameters for urMemoryExportAllocExportableMemoryExp /// @details Each entry is a pointer to the parameter passed to the function; diff --git a/unified-runtime/include/unified-runtime/ur_api_funcs.def b/unified-runtime/include/unified-runtime/ur_api_funcs.def index fc481d947a8c7..f0d68bcf009f0 100644 --- a/unified-runtime/include/unified-runtime/ur_api_funcs.def +++ b/unified-runtime/include/unified-runtime/ur_api_funcs.def @@ -223,6 +223,10 @@ _UR_API(urIPCGetMemHandleExp) _UR_API(urIPCPutMemHandleExp) _UR_API(urIPCOpenMemHandleExp) _UR_API(urIPCCloseMemHandleExp) +_UR_API(urIPCGetPhysMemHandleExp) +_UR_API(urIPCPutPhysMemHandleExp) +_UR_API(urIPCOpenPhysMemHandleExp) +_UR_API(urIPCClosePhysMemHandleExp) _UR_API(urMemoryExportAllocExportableMemoryExp) _UR_API(urMemoryExportFreeExportableMemoryExp) _UR_API(urMemoryExportExportMemoryHandleExp) diff --git a/unified-runtime/include/unified-runtime/ur_ddi.h b/unified-runtime/include/unified-runtime/ur_ddi.h index cc734e6194d92..8f45375c369c1 100644 --- a/unified-runtime/include/unified-runtime/ur_ddi.h +++ b/unified-runtime/include/unified-runtime/ur_ddi.h @@ -1950,6 +1950,27 @@ typedef ur_result_t(UR_APICALL *ur_pfnIPCOpenMemHandleExp_t)( typedef ur_result_t(UR_APICALL *ur_pfnIPCCloseMemHandleExp_t)( ur_context_handle_t, void *); +/////////////////////////////////////////////////////////////////////////////// +/// @brief Function-pointer for urIPCGetPhysMemHandleExp +typedef ur_result_t(UR_APICALL *ur_pfnIPCGetPhysMemHandleExp_t)( + ur_context_handle_t, ur_physical_mem_handle_t, void **, size_t *); + +/////////////////////////////////////////////////////////////////////////////// +/// @brief Function-pointer for urIPCPutPhysMemHandleExp +typedef ur_result_t(UR_APICALL *ur_pfnIPCPutPhysMemHandleExp_t)( + ur_context_handle_t, const void *); + +/////////////////////////////////////////////////////////////////////////////// +/// @brief Function-pointer for urIPCOpenPhysMemHandleExp +typedef ur_result_t(UR_APICALL *ur_pfnIPCOpenPhysMemHandleExp_t)( + ur_context_handle_t, ur_device_handle_t, const void *, size_t, + ur_physical_mem_handle_t *); + +/////////////////////////////////////////////////////////////////////////////// +/// @brief Function-pointer for urIPCClosePhysMemHandleExp +typedef ur_result_t(UR_APICALL *ur_pfnIPCClosePhysMemHandleExp_t)( + ur_context_handle_t, ur_physical_mem_handle_t); + /////////////////////////////////////////////////////////////////////////////// /// @brief Table of IPCExp functions pointers typedef struct ur_ipc_exp_dditable_t { @@ -1957,6 +1978,10 @@ typedef struct ur_ipc_exp_dditable_t { ur_pfnIPCPutMemHandleExp_t pfnPutMemHandleExp; ur_pfnIPCOpenMemHandleExp_t pfnOpenMemHandleExp; ur_pfnIPCCloseMemHandleExp_t pfnCloseMemHandleExp; + ur_pfnIPCGetPhysMemHandleExp_t pfnGetPhysMemHandleExp; + ur_pfnIPCPutPhysMemHandleExp_t pfnPutPhysMemHandleExp; + ur_pfnIPCOpenPhysMemHandleExp_t pfnOpenPhysMemHandleExp; + ur_pfnIPCClosePhysMemHandleExp_t pfnClosePhysMemHandleExp; } ur_ipc_exp_dditable_t; /////////////////////////////////////////////////////////////////////////////// diff --git a/unified-runtime/include/unified-runtime/ur_print.h b/unified-runtime/include/unified-runtime/ur_print.h index d9acae65c7d07..70ff14c95a499 100644 --- a/unified-runtime/include/unified-runtime/ur_print.h +++ b/unified-runtime/include/unified-runtime/ur_print.h @@ -3785,6 +3785,46 @@ UR_APIEXPORT ur_result_t UR_APICALL urPrintIpcCloseMemHandleExpParams( const struct ur_ipc_close_mem_handle_exp_params_t *params, char *buffer, const size_t buff_size, size_t *out_size); +/////////////////////////////////////////////////////////////////////////////// +/// @brief Print ur_ipc_get_phys_mem_handle_exp_params_t struct +/// @returns +/// - ::UR_RESULT_SUCCESS +/// - ::UR_RESULT_ERROR_INVALID_SIZE +/// - `buff_size < out_size` +UR_APIEXPORT ur_result_t UR_APICALL urPrintIpcGetPhysMemHandleExpParams( + const struct ur_ipc_get_phys_mem_handle_exp_params_t *params, char *buffer, + const size_t buff_size, size_t *out_size); + +/////////////////////////////////////////////////////////////////////////////// +/// @brief Print ur_ipc_put_phys_mem_handle_exp_params_t struct +/// @returns +/// - ::UR_RESULT_SUCCESS +/// - ::UR_RESULT_ERROR_INVALID_SIZE +/// - `buff_size < out_size` +UR_APIEXPORT ur_result_t UR_APICALL urPrintIpcPutPhysMemHandleExpParams( + const struct ur_ipc_put_phys_mem_handle_exp_params_t *params, char *buffer, + const size_t buff_size, size_t *out_size); + +/////////////////////////////////////////////////////////////////////////////// +/// @brief Print ur_ipc_open_phys_mem_handle_exp_params_t struct +/// @returns +/// - ::UR_RESULT_SUCCESS +/// - ::UR_RESULT_ERROR_INVALID_SIZE +/// - `buff_size < out_size` +UR_APIEXPORT ur_result_t UR_APICALL urPrintIpcOpenPhysMemHandleExpParams( + const struct ur_ipc_open_phys_mem_handle_exp_params_t *params, char *buffer, + const size_t buff_size, size_t *out_size); + +/////////////////////////////////////////////////////////////////////////////// +/// @brief Print ur_ipc_close_phys_mem_handle_exp_params_t struct +/// @returns +/// - ::UR_RESULT_SUCCESS +/// - ::UR_RESULT_ERROR_INVALID_SIZE +/// - `buff_size < out_size` +UR_APIEXPORT ur_result_t UR_APICALL urPrintIpcClosePhysMemHandleExpParams( + const struct ur_ipc_close_phys_mem_handle_exp_params_t *params, + char *buffer, const size_t buff_size, size_t *out_size); + /////////////////////////////////////////////////////////////////////////////// /// @brief Print ur_memory_export_alloc_exportable_memory_exp_params_t struct /// @returns diff --git a/unified-runtime/include/unified-runtime/ur_print.hpp b/unified-runtime/include/unified-runtime/ur_print.hpp index c29373a7d162a..541b9a6a18e63 100644 --- a/unified-runtime/include/unified-runtime/ur_print.hpp +++ b/unified-runtime/include/unified-runtime/ur_print.hpp @@ -1380,6 +1380,18 @@ inline std::ostream &operator<<(std::ostream &os, enum ur_function_t value) { case UR_FUNCTION_GRAPH_SET_DESTRUCTION_CALLBACK_EXP: os << "UR_FUNCTION_GRAPH_SET_DESTRUCTION_CALLBACK_EXP"; break; + case UR_FUNCTION_IPC_GET_PHYS_MEM_HANDLE_EXP: + os << "UR_FUNCTION_IPC_GET_PHYS_MEM_HANDLE_EXP"; + break; + case UR_FUNCTION_IPC_PUT_PHYS_MEM_HANDLE_EXP: + os << "UR_FUNCTION_IPC_PUT_PHYS_MEM_HANDLE_EXP"; + break; + case UR_FUNCTION_IPC_OPEN_PHYS_MEM_HANDLE_EXP: + os << "UR_FUNCTION_IPC_OPEN_PHYS_MEM_HANDLE_EXP"; + break; + case UR_FUNCTION_IPC_CLOSE_PHYS_MEM_HANDLE_EXP: + os << "UR_FUNCTION_IPC_CLOSE_PHYS_MEM_HANDLE_EXP"; + break; default: os << "unknown enumerator"; break; @@ -3309,6 +3321,9 @@ inline std::ostream &operator<<(std::ostream &os, enum ur_device_info_t value) { case UR_DEVICE_INFO_IPC_MEMORY_SUPPORT_EXP: os << "UR_DEVICE_INFO_IPC_MEMORY_SUPPORT_EXP"; break; + case UR_DEVICE_INFO_IPC_PHYSICAL_MEMORY_SUPPORT_EXP: + os << "UR_DEVICE_INFO_IPC_PHYSICAL_MEMORY_SUPPORT_EXP"; + break; case UR_DEVICE_INFO_ASYNC_USM_ALLOCATIONS_SUPPORT_EXP: os << "UR_DEVICE_INFO_ASYNC_USM_ALLOCATIONS_SUPPORT_EXP"; break; @@ -5594,6 +5609,19 @@ inline ur_result_t printTagged(std::ostream &os, const void *ptr, os << ")"; } break; + case UR_DEVICE_INFO_IPC_PHYSICAL_MEMORY_SUPPORT_EXP: { + const ur_bool_t *tptr = (const ur_bool_t *)ptr; + if (sizeof(ur_bool_t) > size) { + os << "invalid size (is: " << size + << ", expected: >=" << sizeof(ur_bool_t) << ")"; + return UR_RESULT_ERROR_INVALID_SIZE; + } + os << (const void *)(tptr) << " ("; + + os << *tptr; + + os << ")"; + } break; case UR_DEVICE_INFO_ASYNC_USM_ALLOCATIONS_SUPPORT_EXP: { const ur_bool_t *tptr = (const ur_bool_t *)ptr; if (sizeof(ur_bool_t) > size) { @@ -9144,8 +9172,8 @@ inline ur_result_t printTagged(std::ostream &os, const void *ptr, inline std::ostream &operator<<(std::ostream &os, enum ur_physical_mem_flag_t value) { switch (value) { - case UR_PHYSICAL_MEM_FLAG_TBD: - os << "UR_PHYSICAL_MEM_FLAG_TBD"; + case UR_PHYSICAL_MEM_FLAG_ENABLE_IPC: + os << "UR_PHYSICAL_MEM_FLAG_ENABLE_IPC"; break; default: os << "unknown enumerator"; @@ -9163,14 +9191,15 @@ inline ur_result_t printFlag(std::ostream &os, uint32_t val = flag; bool first = true; - if ((val & UR_PHYSICAL_MEM_FLAG_TBD) == (uint32_t)UR_PHYSICAL_MEM_FLAG_TBD) { - val ^= (uint32_t)UR_PHYSICAL_MEM_FLAG_TBD; + if ((val & UR_PHYSICAL_MEM_FLAG_ENABLE_IPC) == + (uint32_t)UR_PHYSICAL_MEM_FLAG_ENABLE_IPC) { + val ^= (uint32_t)UR_PHYSICAL_MEM_FLAG_ENABLE_IPC; if (!first) { os << " | "; } else { first = false; } - os << UR_PHYSICAL_MEM_FLAG_TBD; + os << UR_PHYSICAL_MEM_FLAG_ENABLE_IPC; } if (val != 0) { std::bitset<32> bits(val); @@ -9231,6 +9260,9 @@ inline std::ostream &operator<<(std::ostream &os, case UR_PHYSICAL_MEM_INFO_REFERENCE_COUNT: os << "UR_PHYSICAL_MEM_INFO_REFERENCE_COUNT"; break; + case UR_PHYSICAL_MEM_INFO_IPC_VIRTUAL_ADDRESS: + os << "UR_PHYSICAL_MEM_INFO_IPC_VIRTUAL_ADDRESS"; + break; default: os << "unknown enumerator"; break; @@ -9314,6 +9346,19 @@ inline ur_result_t printTagged(std::ostream &os, const void *ptr, os << ")"; } break; + case UR_PHYSICAL_MEM_INFO_IPC_VIRTUAL_ADDRESS: { + const void *const *tptr = (const void *const *)ptr; + if (sizeof(void *) > size) { + os << "invalid size (is: " << size << ", expected: >=" << sizeof(void *) + << ")"; + return UR_RESULT_ERROR_INVALID_SIZE; + } + os << (const void *)(tptr) << " ("; + + ur::details::printPtr(os, *tptr); + + os << ")"; + } break; default: os << "unknown enumerator"; return UR_RESULT_ERROR_INVALID_ENUMERATION; @@ -21724,6 +21769,115 @@ operator<<(std::ostream &os, return os; } +/////////////////////////////////////////////////////////////////////////////// +/// @brief Print operator for the ur_ipc_get_phys_mem_handle_exp_params_t type +/// @returns +/// std::ostream & +inline std::ostream & +operator<<(std::ostream &os, + [[maybe_unused]] const struct ur_ipc_get_phys_mem_handle_exp_params_t + *params) { + + os << ".hContext = "; + + ur::details::printPtr(os, *(params->phContext)); + + os << ", "; + os << ".hPhysMem = "; + + ur::details::printPtr(os, *(params->phPhysMem)); + + os << ", "; + os << ".ppIPCPhysMemHandleData = "; + + ur::details::printPtr(os, *(params->pppIPCPhysMemHandleData)); + + os << ", "; + os << ".pIPCPhysMemHandleDataSizeRet = "; + + ur::details::printPtr(os, *(params->ppIPCPhysMemHandleDataSizeRet)); + + return os; +} + +/////////////////////////////////////////////////////////////////////////////// +/// @brief Print operator for the ur_ipc_put_phys_mem_handle_exp_params_t type +/// @returns +/// std::ostream & +inline std::ostream & +operator<<(std::ostream &os, + [[maybe_unused]] const struct ur_ipc_put_phys_mem_handle_exp_params_t + *params) { + + os << ".hContext = "; + + ur::details::printPtr(os, *(params->phContext)); + + os << ", "; + os << ".pIPCPhysMemHandleData = "; + + ur::details::printPtr(os, *(params->ppIPCPhysMemHandleData)); + + return os; +} + +/////////////////////////////////////////////////////////////////////////////// +/// @brief Print operator for the ur_ipc_open_phys_mem_handle_exp_params_t type +/// @returns +/// std::ostream & +inline std::ostream &operator<<( + std::ostream &os, + [[maybe_unused]] const struct ur_ipc_open_phys_mem_handle_exp_params_t + *params) { + + os << ".hContext = "; + + ur::details::printPtr(os, *(params->phContext)); + + os << ", "; + os << ".hDevice = "; + + ur::details::printPtr(os, *(params->phDevice)); + + os << ", "; + os << ".pIPCPhysMemHandleData = "; + + os << *(params->ppIPCPhysMemHandleData); + + os << ", "; + os << ".ipcPhysMemHandleDataSize = "; + + os << *(params->pipcPhysMemHandleDataSize); + + os << ", "; + os << ".phPhysMem = "; + + ur::details::printPtr(os, *(params->pphPhysMem)); + + return os; +} + +/////////////////////////////////////////////////////////////////////////////// +/// @brief Print operator for the ur_ipc_close_phys_mem_handle_exp_params_t type +/// @returns +/// std::ostream & +inline std::ostream &operator<<( + std::ostream &os, + [[maybe_unused]] const struct ur_ipc_close_phys_mem_handle_exp_params_t + *params) { + + os << ".hContext = "; + + ur::details::printPtr(os, *(params->phContext)); + + os << ", "; + os << ".hPhysMem = "; + + ur::details::printPtr(os, *(params->phPhysMem)); + + return os; +} + /////////////////////////////////////////////////////////////////////////////// /// @brief Print operator for the /// ur_memory_export_alloc_exportable_memory_exp_params_t type @@ -23280,6 +23434,18 @@ inline ur_result_t UR_APICALL printFunctionParams(std::ostream &os, case UR_FUNCTION_IPC_CLOSE_MEM_HANDLE_EXP: { os << (const struct ur_ipc_close_mem_handle_exp_params_t *)params; } break; + case UR_FUNCTION_IPC_GET_PHYS_MEM_HANDLE_EXP: { + os << (const struct ur_ipc_get_phys_mem_handle_exp_params_t *)params; + } break; + case UR_FUNCTION_IPC_PUT_PHYS_MEM_HANDLE_EXP: { + os << (const struct ur_ipc_put_phys_mem_handle_exp_params_t *)params; + } break; + case UR_FUNCTION_IPC_OPEN_PHYS_MEM_HANDLE_EXP: { + os << (const struct ur_ipc_open_phys_mem_handle_exp_params_t *)params; + } break; + case UR_FUNCTION_IPC_CLOSE_PHYS_MEM_HANDLE_EXP: { + os << (const struct ur_ipc_close_phys_mem_handle_exp_params_t *)params; + } break; case UR_FUNCTION_MEMORY_EXPORT_ALLOC_EXPORTABLE_MEMORY_EXP: { os << (const struct ur_memory_export_alloc_exportable_memory_exp_params_t *) params; diff --git a/unified-runtime/scripts/core/exp-inter-process-communication.yml b/unified-runtime/scripts/core/exp-inter-process-communication.yml index e5019bb40836d..71c15e6b9d71a 100644 --- a/unified-runtime/scripts/core/exp-inter-process-communication.yml +++ b/unified-runtime/scripts/core/exp-inter-process-communication.yml @@ -23,6 +23,16 @@ etors: value: "0x2023" desc: "[$x_bool_t] returns true if the device supports inter-process communicable memory handles" --- #-------------------------------------------------------------------------- +type: enum +extend: true +typed_etors: true +desc: "Extension enums to $x_device_info_t to support inter-process communicable physical memory handles." +name: $x_device_info_t +etors: + - name: IPC_PHYSICAL_MEMORY_SUPPORT_EXP + value: "0x2024" + desc: "[$x_bool_t] returns true if the device supports inter-process communicable physical memory handles" +--- #-------------------------------------------------------------------------- type: function desc: "Gets an inter-process memory handle for a pointer to device USM memory" class: $xIPC @@ -126,3 +136,109 @@ returns: - "`NULL == pMem`" - $X_RESULT_ERROR_OUT_OF_HOST_MEMORY - $X_RESULT_ERROR_OUT_OF_RESOURCES +--- #-------------------------------------------------------------------------- +type: function +desc: "Gets an inter-process handle for a physical memory object" +class: $xIPC +name: GetPhysMemHandleExp +ordinal: "0" +params: + - type: $x_context_handle_t + name: hContext + desc: "[in] handle of the context object" + - type: $x_physical_mem_handle_t + name: hPhysMem + desc: "[in] handle of the physical memory object" + - type: void** + name: ppIPCPhysMemHandleData + desc: "[out] a pointer to the IPC physical memory handle data" + - type: size_t* + name: pIPCPhysMemHandleDataSizeRet + desc: "[out] size of the resulting IPC physical memory handle data" +returns: + - $X_RESULT_ERROR_INVALID_CONTEXT + - $X_RESULT_ERROR_INVALID_NULL_HANDLE: + - "`NULL == hContext`" + - "`NULL == hPhysMem`" + - $X_RESULT_ERROR_INVALID_NULL_POINTER: + - "`NULL == ppIPCPhysMemHandleData`" + - "`NULL == pIPCPhysMemHandleDataSizeRet`" + - $X_RESULT_ERROR_INVALID_ARGUMENT: + - "`hPhysMem` was not created with the `$X_PHYSICAL_MEM_FLAG_ENABLE_IPC` flag" + - $X_RESULT_ERROR_OUT_OF_HOST_MEMORY + - $X_RESULT_ERROR_OUT_OF_RESOURCES +--- #-------------------------------------------------------------------------- +type: function +desc: "Releases an inter-process physical memory handle" +class: $xIPC +name: PutPhysMemHandleExp +ordinal: "0" +params: + - type: $x_context_handle_t + name: hContext + desc: "[in] handle of the context object" + - type: const void* + name: pIPCPhysMemHandleData + desc: "[in] a pointer to the IPC physical memory handle data obtained with urIPCGetPhysMemHandleExp" +returns: + - $X_RESULT_ERROR_INVALID_CONTEXT + - $X_RESULT_ERROR_INVALID_NULL_HANDLE: + - "`NULL == hContext`" + - $X_RESULT_ERROR_INVALID_NULL_POINTER: + - "`NULL == pIPCPhysMemHandleData`" + - $X_RESULT_ERROR_OUT_OF_HOST_MEMORY + - $X_RESULT_ERROR_OUT_OF_RESOURCES +--- #-------------------------------------------------------------------------- +type: function +desc: "Opens an inter-process physical memory handle to get the corresponding physical memory object" +class: $xIPC +name: OpenPhysMemHandleExp +ordinal: "0" +params: + - type: $x_context_handle_t + name: hContext + desc: "[in] handle of the context object" + - type: $x_device_handle_t + name: hDevice + desc: "[in] handle of the device object the physical memory was allocated on" + - type: const void * + name: pIPCPhysMemHandleData + desc: "[in] the IPC physical memory handle data obtained with urIPCGetPhysMemHandleExp" + - type: size_t + name: ipcPhysMemHandleDataSize + desc: "[in] size of the IPC physical memory handle data" + - type: $x_physical_mem_handle_t* + name: phPhysMem + desc: "[out] pointer to the physical memory handle" +returns: + - $X_RESULT_ERROR_INVALID_CONTEXT + - $X_RESULT_ERROR_INVALID_NULL_HANDLE: + - "`NULL == hContext`" + - "`NULL == hDevice`" + - $X_RESULT_ERROR_INVALID_NULL_POINTER: + - "`NULL == pIPCPhysMemHandleData`" + - "`NULL == phPhysMem`" + - $X_RESULT_ERROR_INVALID_VALUE: + - "ipcPhysMemHandleDataSize is not the same as the size of IPC physical memory handle data" + - $X_RESULT_ERROR_OUT_OF_HOST_MEMORY + - $X_RESULT_ERROR_OUT_OF_RESOURCES +--- #-------------------------------------------------------------------------- +type: function +desc: "Closes an inter-process physical memory handle" +class: $xIPC +name: ClosePhysMemHandleExp +ordinal: "0" +params: + - type: $x_context_handle_t + name: hContext + desc: "[in] handle of the context object" + - type: $x_physical_mem_handle_t + name: hPhysMem + desc: "[in] physical memory handle opened through urIPCOpenPhysMemHandleExp" +returns: + - $X_RESULT_ERROR_INVALID_CONTEXT + - $X_RESULT_ERROR_INVALID_NULL_HANDLE: + - "`NULL == hContext`" + - "`NULL == hPhysMem`" + - $X_RESULT_ERROR_OUT_OF_HOST_MEMORY + - $X_RESULT_ERROR_OUT_OF_RESOURCES diff --git a/unified-runtime/scripts/core/registry.yml b/unified-runtime/scripts/core/registry.yml index f7a11b40c6547..3d4f037329ad9 100644 --- a/unified-runtime/scripts/core/registry.yml +++ b/unified-runtime/scripts/core/registry.yml @@ -727,7 +727,19 @@ etors: - name: GRAPH_SET_DESTRUCTION_CALLBACK_EXP desc: Enumerator for $xGraphSetDestructionCallbackExp value: '315' -max_id: '315' +- name: IPC_GET_PHYS_MEM_HANDLE_EXP + desc: Enumerator for $xIPCGetPhysMemHandleExp + value: '316' +- name: IPC_PUT_PHYS_MEM_HANDLE_EXP + desc: Enumerator for $xIPCPutPhysMemHandleExp + value: '317' +- name: IPC_OPEN_PHYS_MEM_HANDLE_EXP + desc: Enumerator for $xIPCOpenPhysMemHandleExp + value: '318' +- name: IPC_CLOSE_PHYS_MEM_HANDLE_EXP + desc: Enumerator for $xIPCClosePhysMemHandleExp + value: '319' +max_id: '319' --- type: enum desc: Defines structure types diff --git a/unified-runtime/scripts/core/virtual_memory.yml b/unified-runtime/scripts/core/virtual_memory.yml index 00d03c54a5726..6bdc6c63c79d7 100644 --- a/unified-runtime/scripts/core/virtual_memory.yml +++ b/unified-runtime/scripts/core/virtual_memory.yml @@ -246,8 +246,9 @@ desc: "Physical memory creation properties." class: $xPhysicalMem name: $x_physical_mem_flags_t etors: - - name: TBD - desc: "reserved for future use." + - name: ENABLE_IPC + value: "$X_BIT(1)" + desc: "allocate physical memory that can be shared via IPC handles." --- #-------------------------------------------------------------------------- type: struct @@ -326,6 +327,12 @@ etors: [uint32_t] Reference count of the physical memory object. The reference count returned should be considered immediately stale. It is unsuitable for general use in applications. This feature is provided for identifying memory leaks. + - name: IPC_VIRTUAL_ADDRESS + desc: | + [void*] For physical memory objects opened from an IPC handle via + $xIPCOpenPhysMemHandleExp, returns the virtual address established + by the driver (i.e. the address returned by zeMemOpenIpcHandle). + Returns nullptr for non-IPC physical memory objects. --- #-------------------------------------------------------------------------- type: function diff --git a/unified-runtime/source/adapters/cuda/device.cpp b/unified-runtime/source/adapters/cuda/device.cpp index d213d51eaeb9b..9ef56d297e330 100644 --- a/unified-runtime/source/adapters/cuda/device.cpp +++ b/unified-runtime/source/adapters/cuda/device.cpp @@ -1173,6 +1173,8 @@ UR_APIEXPORT ur_result_t UR_APICALL urDeviceGetInfo(ur_device_handle_t hDevice, hDevice->get())); return ReturnValue(static_cast(IPCSupported)); } + case UR_DEVICE_INFO_IPC_PHYSICAL_MEMORY_SUPPORT_EXP: + return ReturnValue(false); case UR_DEVICE_INFO_COMMAND_BUFFER_SUPPORT_EXP: case UR_DEVICE_INFO_COMMAND_BUFFER_EVENT_SUPPORT_EXP: return ReturnValue(true); diff --git a/unified-runtime/source/adapters/cuda/memory.cpp b/unified-runtime/source/adapters/cuda/memory.cpp index 81b86d7445381..cf73ac1cbca65 100644 --- a/unified-runtime/source/adapters/cuda/memory.cpp +++ b/unified-runtime/source/adapters/cuda/memory.cpp @@ -639,3 +639,24 @@ UR_APIEXPORT ur_result_t UR_APICALL urIPCCloseMemHandleExp(ur_context_handle_t, void *pMem) { return umf::umf2urResult(umfCloseIPCHandle(pMem)); } + +UR_APIEXPORT ur_result_t UR_APICALL urIPCGetPhysMemHandleExp( + ur_context_handle_t, ur_physical_mem_handle_t, void **, size_t *) { + return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; +} + +UR_APIEXPORT ur_result_t UR_APICALL +urIPCPutPhysMemHandleExp(ur_context_handle_t, const void *) { + return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; +} + +UR_APIEXPORT ur_result_t UR_APICALL +urIPCOpenPhysMemHandleExp(ur_context_handle_t, ur_device_handle_t, const void *, + size_t, ur_physical_mem_handle_t *) { + return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; +} + +UR_APIEXPORT ur_result_t UR_APICALL +urIPCClosePhysMemHandleExp(ur_context_handle_t, ur_physical_mem_handle_t) { + return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; +} diff --git a/unified-runtime/source/adapters/cuda/ur_interface_loader.cpp b/unified-runtime/source/adapters/cuda/ur_interface_loader.cpp index 1f6434b2c140f..e1cce057c6320 100644 --- a/unified-runtime/source/adapters/cuda/ur_interface_loader.cpp +++ b/unified-runtime/source/adapters/cuda/ur_interface_loader.cpp @@ -510,6 +510,10 @@ UR_DLLEXPORT ur_result_t UR_APICALL urGetIPCExpProcAddrTable( pDdiTable->pfnPutMemHandleExp = urIPCPutMemHandleExp; pDdiTable->pfnOpenMemHandleExp = urIPCOpenMemHandleExp; pDdiTable->pfnCloseMemHandleExp = urIPCCloseMemHandleExp; + pDdiTable->pfnGetPhysMemHandleExp = urIPCGetPhysMemHandleExp; + pDdiTable->pfnPutPhysMemHandleExp = urIPCPutPhysMemHandleExp; + pDdiTable->pfnOpenPhysMemHandleExp = urIPCOpenPhysMemHandleExp; + pDdiTable->pfnClosePhysMemHandleExp = urIPCClosePhysMemHandleExp; return UR_RESULT_SUCCESS; } diff --git a/unified-runtime/source/adapters/hip/device.cpp b/unified-runtime/source/adapters/hip/device.cpp index f07a31c03ce4e..e43f7e58164f0 100644 --- a/unified-runtime/source/adapters/hip/device.cpp +++ b/unified-runtime/source/adapters/hip/device.cpp @@ -1021,6 +1021,8 @@ UR_APIEXPORT ur_result_t UR_APICALL urDeviceGetInfo(ur_device_handle_t hDevice, static_cast(0)); case UR_DEVICE_INFO_IPC_MEMORY_SUPPORT_EXP: return ReturnValue(false); + case UR_DEVICE_INFO_IPC_PHYSICAL_MEMORY_SUPPORT_EXP: + return ReturnValue(false); case UR_DEVICE_INFO_COMMAND_BUFFER_SUPPORT_EXP: { int RuntimeVersion = 0; UR_CHECK_ERROR(hipRuntimeGetVersion(&RuntimeVersion)); diff --git a/unified-runtime/source/adapters/hip/memory.cpp b/unified-runtime/source/adapters/hip/memory.cpp index 6360ab0f92352..78c4a7c93ea24 100644 --- a/unified-runtime/source/adapters/hip/memory.cpp +++ b/unified-runtime/source/adapters/hip/memory.cpp @@ -658,3 +658,24 @@ UR_APIEXPORT ur_result_t UR_APICALL urIPCCloseMemHandleExp(ur_context_handle_t, void *) { return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; } + +UR_APIEXPORT ur_result_t UR_APICALL urIPCGetPhysMemHandleExp( + ur_context_handle_t, ur_physical_mem_handle_t, void **, size_t *) { + return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; +} + +UR_APIEXPORT ur_result_t UR_APICALL +urIPCPutPhysMemHandleExp(ur_context_handle_t, const void *) { + return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; +} + +UR_APIEXPORT ur_result_t UR_APICALL +urIPCOpenPhysMemHandleExp(ur_context_handle_t, ur_device_handle_t, const void *, + size_t, ur_physical_mem_handle_t *) { + return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; +} + +UR_APIEXPORT ur_result_t UR_APICALL +urIPCClosePhysMemHandleExp(ur_context_handle_t, ur_physical_mem_handle_t) { + return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; +} diff --git a/unified-runtime/source/adapters/hip/ur_interface_loader.cpp b/unified-runtime/source/adapters/hip/ur_interface_loader.cpp index 1d6fbdf7a9c0c..48c6c1b161546 100644 --- a/unified-runtime/source/adapters/hip/ur_interface_loader.cpp +++ b/unified-runtime/source/adapters/hip/ur_interface_loader.cpp @@ -503,6 +503,10 @@ UR_DLLEXPORT ur_result_t UR_APICALL urGetIPCExpProcAddrTable( pDdiTable->pfnPutMemHandleExp = urIPCPutMemHandleExp; pDdiTable->pfnOpenMemHandleExp = urIPCOpenMemHandleExp; pDdiTable->pfnCloseMemHandleExp = urIPCCloseMemHandleExp; + pDdiTable->pfnGetPhysMemHandleExp = urIPCGetPhysMemHandleExp; + pDdiTable->pfnPutPhysMemHandleExp = urIPCPutPhysMemHandleExp; + pDdiTable->pfnOpenPhysMemHandleExp = urIPCOpenPhysMemHandleExp; + pDdiTable->pfnClosePhysMemHandleExp = urIPCClosePhysMemHandleExp; return UR_RESULT_SUCCESS; } diff --git a/unified-runtime/source/adapters/level_zero/CMakeLists.txt b/unified-runtime/source/adapters/level_zero/CMakeLists.txt index 42e63e952f392..95108f9307601 100644 --- a/unified-runtime/source/adapters/level_zero/CMakeLists.txt +++ b/unified-runtime/source/adapters/level_zero/CMakeLists.txt @@ -25,7 +25,6 @@ if(UR_BUILD_ADAPTER_L0) ${CMAKE_CURRENT_SOURCE_DIR}/usm.hpp ${CMAKE_CURRENT_SOURCE_DIR}/memory.hpp ${CMAKE_CURRENT_SOURCE_DIR}/kernel.hpp - ${CMAKE_CURRENT_SOURCE_DIR}/physical_mem.hpp ${CMAKE_CURRENT_SOURCE_DIR}/platform.hpp ${CMAKE_CURRENT_SOURCE_DIR}/program.hpp ${CMAKE_CURRENT_SOURCE_DIR}/queue.hpp @@ -132,7 +131,6 @@ if(UR_BUILD_ADAPTER_L0_V2) ${CMAKE_CURRENT_SOURCE_DIR}/device.hpp ${CMAKE_CURRENT_SOURCE_DIR}/image_common.hpp ${CMAKE_CURRENT_SOURCE_DIR}/platform.hpp - ${CMAKE_CURRENT_SOURCE_DIR}/physical_mem.hpp ${CMAKE_CURRENT_SOURCE_DIR}/program.hpp ${CMAKE_CURRENT_SOURCE_DIR}/helpers/kernel_helpers.hpp ${CMAKE_CURRENT_SOURCE_DIR}/helpers/memory_helpers.hpp @@ -145,7 +143,6 @@ if(UR_BUILD_ADAPTER_L0_V2) ${CMAKE_CURRENT_SOURCE_DIR}/image_common.cpp ${CMAKE_CURRENT_SOURCE_DIR}/ur_interface_loader.cpp ${CMAKE_CURRENT_SOURCE_DIR}/platform.cpp - ${CMAKE_CURRENT_SOURCE_DIR}/physical_mem.cpp ${CMAKE_CURRENT_SOURCE_DIR}/program.cpp ${CMAKE_CURRENT_SOURCE_DIR}/helpers/kernel_helpers.cpp ${CMAKE_CURRENT_SOURCE_DIR}/helpers/memory_helpers.cpp @@ -170,6 +167,7 @@ if(UR_BUILD_ADAPTER_L0_V2) ${CMAKE_CURRENT_SOURCE_DIR}/v2/kernel.hpp ${CMAKE_CURRENT_SOURCE_DIR}/v2/memory.hpp ${CMAKE_CURRENT_SOURCE_DIR}/v2/lockable.hpp + ${CMAKE_CURRENT_SOURCE_DIR}/v2/physical_mem.hpp ${CMAKE_CURRENT_SOURCE_DIR}/v2/queue_api.hpp ${CMAKE_CURRENT_SOURCE_DIR}/v2/queue_batched.hpp ${CMAKE_CURRENT_SOURCE_DIR}/v2/queue_immediate_in_order.hpp @@ -187,6 +185,7 @@ if(UR_BUILD_ADAPTER_L0_V2) ${CMAKE_CURRENT_SOURCE_DIR}/v2/event.cpp ${CMAKE_CURRENT_SOURCE_DIR}/v2/kernel.cpp ${CMAKE_CURRENT_SOURCE_DIR}/v2/memory.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/v2/physical_mem.cpp ${CMAKE_CURRENT_SOURCE_DIR}/v2/queue_api.cpp ${CMAKE_CURRENT_SOURCE_DIR}/v2/queue_batched.cpp ${CMAKE_CURRENT_SOURCE_DIR}/v2/queue_create.cpp diff --git a/unified-runtime/source/adapters/level_zero/device.cpp b/unified-runtime/source/adapters/level_zero/device.cpp index 03b570bda31d7..e3012bfbda43a 100644 --- a/unified-runtime/source/adapters/level_zero/device.cpp +++ b/unified-runtime/source/adapters/level_zero/device.cpp @@ -1379,6 +1379,12 @@ ur_result_t urDeviceGetInfo( return ReturnValue(false); #else return ReturnValue(true); +#endif + case UR_DEVICE_INFO_IPC_PHYSICAL_MEMORY_SUPPORT_EXP: +#ifdef __linux__ + return ReturnValue(Device->isIntelBMGOrNewer()); +#else + return ReturnValue(false); #endif case UR_DEVICE_INFO_ASYNC_BARRIER: return ReturnValue(false); diff --git a/unified-runtime/source/adapters/level_zero/device.hpp b/unified-runtime/source/adapters/level_zero/device.hpp index 0e184c0c5f01d..7ddba032196e0 100644 --- a/unified-runtime/source/adapters/level_zero/device.hpp +++ b/unified-runtime/source/adapters/level_zero/device.hpp @@ -209,6 +209,11 @@ struct ur_device_handle_t_ : ur_object { ZeDeviceIpVersionExt->ipVersion >= 0x030f0000); } + bool isIntelBMGOrNewer() { + return (ZeDeviceProperties->vendorId == 0x8086 && + ZeDeviceIpVersionExt->ipVersion >= 0x05004000); + } + bool isIntegrated() { return (ZeDeviceProperties->flags & ZE_DEVICE_PROPERTY_FLAG_INTEGRATED); } diff --git a/unified-runtime/source/adapters/level_zero/memory.cpp b/unified-runtime/source/adapters/level_zero/memory.cpp index 97239a8416564..b4a1724de2938 100644 --- a/unified-runtime/source/adapters/level_zero/memory.cpp +++ b/unified-runtime/source/adapters/level_zero/memory.cpp @@ -10,6 +10,10 @@ #include #include #include +#ifdef __linux__ +#include +#include +#endif #include #include "context.hpp" @@ -20,6 +24,7 @@ #include "queue.hpp" #include "ur_interface_loader.hpp" #include "ur_level_zero.hpp" +#include "v2/physical_mem.hpp" // Default to using compute engine for fill operation, but allow to // override this with an environment variable. diff --git a/unified-runtime/source/adapters/level_zero/physical_mem.cpp b/unified-runtime/source/adapters/level_zero/physical_mem.cpp index a0663253f42e3..cc00a99b74ffd 100644 --- a/unified-runtime/source/adapters/level_zero/physical_mem.cpp +++ b/unified-runtime/source/adapters/level_zero/physical_mem.cpp @@ -7,15 +7,10 @@ // //===----------------------------------------------------------------------===// -#include "physical_mem.hpp" +#include "v2/physical_mem.hpp" #include "common.hpp" -#include "device.hpp" - -#ifdef UR_ADAPTER_LEVEL_ZERO_V2 -#include "v2/context.hpp" -#else #include "context.hpp" -#endif +#include "device.hpp" namespace ur::level_zero { @@ -27,14 +22,20 @@ ur_result_t urPhysicalMemCreate( PhysicalMemDesc.flags = 0; PhysicalMemDesc.size = size; + bool EnableIpc = + pProperties && (pProperties->flags & UR_PHYSICAL_MEM_FLAG_ENABLE_IPC); + ze_physical_mem_handle_t ZePhysicalMem; ZE2UR_CALL(zePhysicalMemCreate, (hContext->getZeHandle(), hDevice->ZeDevice, &PhysicalMemDesc, &ZePhysicalMem)); try { - *phPhysicalMem = new ur_physical_mem_handle_t_(ZePhysicalMem, hContext); + *phPhysicalMem = new ur_physical_mem_handle_t_(ZePhysicalMem, hContext, + hDevice, size, EnableIpc); } catch (const std::bad_alloc &) { + zePhysicalMemDestroy(hContext->getZeHandle(), ZePhysicalMem); return UR_RESULT_ERROR_OUT_OF_HOST_MEMORY; } catch (...) { + zePhysicalMemDestroy(hContext->getZeHandle(), ZePhysicalMem); return UR_RESULT_ERROR_UNKNOWN; } return UR_RESULT_SUCCESS; @@ -49,13 +50,25 @@ ur_result_t urPhysicalMemRelease(ur_physical_mem_handle_t hPhysicalMem) { if (!hPhysicalMem->RefCount.release()) return UR_RESULT_SUCCESS; - if (checkL0LoaderTeardown()) { - ZE2UR_CALL(zePhysicalMemDestroy, (hPhysicalMem->Context->getZeHandle(), - hPhysicalMem->ZePhysicalMem)); + ur_result_t Res = UR_RESULT_SUCCESS; + if (hPhysicalMem->IpcVirtualAddress) { + if (checkL0LoaderTeardown()) { + ze_result_t ZeRes = ZE_CALL_NOCHECK(zeMemCloseIpcHandle, + (hPhysicalMem->Context->getZeHandle(), + hPhysicalMem->IpcVirtualAddress)); + Res = ze2urResult(ZeRes); + } + } else if (hPhysicalMem->ZePhysicalMem) { + if (checkL0LoaderTeardown()) { + ze_result_t ZeRes = ZE_CALL_NOCHECK( + zePhysicalMemDestroy, + (hPhysicalMem->Context->getZeHandle(), hPhysicalMem->ZePhysicalMem)); + Res = ze2urResult(ZeRes); + } } delete hPhysicalMem; - return UR_RESULT_SUCCESS; + return Res; } ur_result_t urPhysicalMemGetInfo(ur_physical_mem_handle_t hPhysicalMem, @@ -66,13 +79,51 @@ ur_result_t urPhysicalMemGetInfo(ur_physical_mem_handle_t hPhysicalMem, UrReturnHelper ReturnValue(propSize, pPropValue, pPropSizeRet); switch (propName) { - case UR_PHYSICAL_MEM_INFO_REFERENCE_COUNT: { - return ReturnValue(hPhysicalMem->RefCount.getCount()); + case UR_PHYSICAL_MEM_INFO_CONTEXT: + return ReturnValue(hPhysicalMem->Context); + case UR_PHYSICAL_MEM_INFO_DEVICE: + return ReturnValue(hPhysicalMem->Device); + case UR_PHYSICAL_MEM_INFO_SIZE: + return ReturnValue(hPhysicalMem->Size); + case UR_PHYSICAL_MEM_INFO_PROPERTIES: { + ur_physical_mem_flags_t Flags = static_cast(0); + if (hPhysicalMem->EnableIpc) + Flags = UR_PHYSICAL_MEM_FLAG_ENABLE_IPC; + ur_physical_mem_properties_t Props = { + UR_STRUCTURE_TYPE_PHYSICAL_MEM_PROPERTIES, nullptr, Flags}; + return ReturnValue(Props); } + case UR_PHYSICAL_MEM_INFO_REFERENCE_COUNT: + return ReturnValue(hPhysicalMem->RefCount.getCount()); + case UR_PHYSICAL_MEM_INFO_IPC_VIRTUAL_ADDRESS: + return ReturnValue(hPhysicalMem->IpcVirtualAddress); default: return UR_RESULT_ERROR_UNSUPPORTED_ENUMERATION; } return UR_RESULT_SUCCESS; } +// IPC physical memory functions are only supported on L0v2 (Xe2+/BMG hardware). +// Provide stubs here so the L0v1 adapter links correctly. +ur_result_t urIPCGetPhysMemHandleExp(ur_context_handle_t, + ur_physical_mem_handle_t, void **, + size_t *) { + return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; +} + +ur_result_t urIPCPutPhysMemHandleExp(ur_context_handle_t, const void *) { + return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; +} + +ur_result_t urIPCOpenPhysMemHandleExp(ur_context_handle_t, ur_device_handle_t, + const void *, size_t, + ur_physical_mem_handle_t *) { + return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; +} + +ur_result_t urIPCClosePhysMemHandleExp(ur_context_handle_t, + ur_physical_mem_handle_t) { + return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; +} + } // namespace ur::level_zero diff --git a/unified-runtime/source/adapters/level_zero/physical_mem.hpp b/unified-runtime/source/adapters/level_zero/physical_mem.hpp deleted file mode 100644 index a5db639575e8d..0000000000000 --- a/unified-runtime/source/adapters/level_zero/physical_mem.hpp +++ /dev/null @@ -1,26 +0,0 @@ -//===---------------- physical_mem.hpp - Level Zero Adapter ---------------===// -// -// -// Part of the LLVM Project, under the Apache License v2.0 with LLVM -// Exceptions. See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// -#pragma once - -#include "common.hpp" -#include "common/ur_ref_count.hpp" - -struct ur_physical_mem_handle_t_ : ur_object { - ur_physical_mem_handle_t_(ze_physical_mem_handle_t ZePhysicalMem, - ur_context_handle_t Context) - : ZePhysicalMem{ZePhysicalMem}, Context{Context} {} - - // Level Zero physical memory handle. - ze_physical_mem_handle_t ZePhysicalMem; - - // Keeps the PI context of this memory handle. - ur_context_handle_t Context; - - ur::RefCount RefCount; -}; diff --git a/unified-runtime/source/adapters/level_zero/ur_interface_loader.cpp b/unified-runtime/source/adapters/level_zero/ur_interface_loader.cpp index dc76f7729f937..ce63be24c0a64 100644 --- a/unified-runtime/source/adapters/level_zero/ur_interface_loader.cpp +++ b/unified-runtime/source/adapters/level_zero/ur_interface_loader.cpp @@ -293,6 +293,12 @@ UR_APIEXPORT ur_result_t UR_APICALL urGetIPCExpProcAddrTable( pDdiTable->pfnPutMemHandleExp = ur::level_zero::urIPCPutMemHandleExp; pDdiTable->pfnOpenMemHandleExp = ur::level_zero::urIPCOpenMemHandleExp; pDdiTable->pfnCloseMemHandleExp = ur::level_zero::urIPCCloseMemHandleExp; + pDdiTable->pfnGetPhysMemHandleExp = ur::level_zero::urIPCGetPhysMemHandleExp; + pDdiTable->pfnPutPhysMemHandleExp = ur::level_zero::urIPCPutPhysMemHandleExp; + pDdiTable->pfnOpenPhysMemHandleExp = + ur::level_zero::urIPCOpenPhysMemHandleExp; + pDdiTable->pfnClosePhysMemHandleExp = + ur::level_zero::urIPCClosePhysMemHandleExp; return result; } diff --git a/unified-runtime/source/adapters/level_zero/ur_interface_loader.hpp b/unified-runtime/source/adapters/level_zero/ur_interface_loader.hpp index b56cc5e803c5d..bc4956f6eed2f 100644 --- a/unified-runtime/source/adapters/level_zero/ur_interface_loader.hpp +++ b/unified-runtime/source/adapters/level_zero/ur_interface_loader.hpp @@ -629,6 +629,19 @@ ur_result_t urIPCOpenMemHandleExp(ur_context_handle_t hContext, void *pIPCMemHandleData, size_t ipcMemHandleDataSize, void **ppMem); ur_result_t urIPCCloseMemHandleExp(ur_context_handle_t hContext, void *pMem); +ur_result_t urIPCGetPhysMemHandleExp(ur_context_handle_t hContext, + ur_physical_mem_handle_t hPhysMem, + void **ppIPCPhysMemHandleData, + size_t *pIPCPhysMemHandleDataSizeRet); +ur_result_t urIPCPutPhysMemHandleExp(ur_context_handle_t hContext, + const void *pIPCPhysMemHandleData); +ur_result_t urIPCOpenPhysMemHandleExp(ur_context_handle_t hContext, + ur_device_handle_t hDevice, + const void *pIPCPhysMemHandleData, + size_t ipcPhysMemHandleDataSize, + ur_physical_mem_handle_t *phPhysMem); +ur_result_t urIPCClosePhysMemHandleExp(ur_context_handle_t hContext, + ur_physical_mem_handle_t hPhysMem); ur_result_t urMemoryExportAllocExportableMemoryExp( ur_context_handle_t hContext, ur_device_handle_t hDevice, size_t alignment, size_t size, ur_exp_external_mem_type_t handleTypeToExport, void **ppMem); diff --git a/unified-runtime/source/adapters/level_zero/ur_level_zero.hpp b/unified-runtime/source/adapters/level_zero/ur_level_zero.hpp index 9cd954311283c..4576502df3669 100644 --- a/unified-runtime/source/adapters/level_zero/ur_level_zero.hpp +++ b/unified-runtime/source/adapters/level_zero/ur_level_zero.hpp @@ -29,9 +29,9 @@ #include "event.hpp" #include "kernel.hpp" #include "memory.hpp" -#include "physical_mem.hpp" #include "platform.hpp" #include "program.hpp" #include "queue.hpp" #include "sampler.hpp" #include "usm.hpp" +#include "v2/physical_mem.hpp" diff --git a/unified-runtime/source/adapters/level_zero/v2/memory.cpp b/unified-runtime/source/adapters/level_zero/v2/memory.cpp index b0601ba956af2..ff981e810df7b 100644 --- a/unified-runtime/source/adapters/level_zero/v2/memory.cpp +++ b/unified-runtime/source/adapters/level_zero/v2/memory.cpp @@ -9,11 +9,17 @@ #include "memory.hpp" +#ifdef __linux__ +#include +#include +#endif + #include "../ur_interface_loader.hpp" #include "context.hpp" #include "../helpers/memory_helpers.hpp" #include "../image_common.hpp" +#include "physical_mem.hpp" static bool isAccessCompatible(ur_mem_buffer_t::device_access_mode_t requested, ur_mem_buffer_t::device_access_mode_t actual) { diff --git a/unified-runtime/source/adapters/level_zero/v2/physical_mem.cpp b/unified-runtime/source/adapters/level_zero/v2/physical_mem.cpp new file mode 100644 index 0000000000000..415c374314cd3 --- /dev/null +++ b/unified-runtime/source/adapters/level_zero/v2/physical_mem.cpp @@ -0,0 +1,260 @@ +//===---------------- physical_mem.cpp - Level Zero Adapter ---------------===// +// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM +// Exceptions. See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "physical_mem.hpp" +#include "../device.hpp" +#include "common.hpp" + +#include "context.hpp" + +namespace ur::level_zero { + +ur_result_t urPhysicalMemCreate( + ur_context_handle_t hContext, ur_device_handle_t hDevice, size_t size, + [[maybe_unused]] const ur_physical_mem_properties_t *pProperties, + ur_physical_mem_handle_t *phPhysicalMem) { + ZeStruct PhysicalMemDesc; + PhysicalMemDesc.flags = 0; + PhysicalMemDesc.size = size; + + bool EnableIpc = + pProperties && (pProperties->flags & UR_PHYSICAL_MEM_FLAG_ENABLE_IPC); + + ze_physical_mem_handle_t ZePhysicalMem; + ZE2UR_CALL(zePhysicalMemCreate, (hContext->getZeHandle(), hDevice->ZeDevice, + &PhysicalMemDesc, &ZePhysicalMem)); + try { + *phPhysicalMem = new ur_physical_mem_handle_t_(ZePhysicalMem, hContext, + hDevice, size, EnableIpc); + } catch (const std::bad_alloc &) { + zePhysicalMemDestroy(hContext->getZeHandle(), ZePhysicalMem); + return UR_RESULT_ERROR_OUT_OF_HOST_MEMORY; + } catch (...) { + zePhysicalMemDestroy(hContext->getZeHandle(), ZePhysicalMem); + return UR_RESULT_ERROR_UNKNOWN; + } + return UR_RESULT_SUCCESS; +} + +ur_result_t urPhysicalMemRetain(ur_physical_mem_handle_t hPhysicalMem) { + hPhysicalMem->RefCount.retain(); + return UR_RESULT_SUCCESS; +} + +ur_result_t urPhysicalMemRelease(ur_physical_mem_handle_t hPhysicalMem) { + if (!hPhysicalMem->RefCount.release()) + return UR_RESULT_SUCCESS; + + ur_result_t Res = UR_RESULT_SUCCESS; + if (hPhysicalMem->IpcVirtualAddress) { + // IPC-opened handle: close the IPC virtual mapping instead of destroying + // a physical mem handle (there is no ZePhysicalMem on the consumer side). + if (checkL0LoaderTeardown()) { + ze_result_t ZeRes = ZE_CALL_NOCHECK(zeMemCloseIpcHandle, + (hPhysicalMem->Context->getZeHandle(), + hPhysicalMem->IpcVirtualAddress)); + Res = ze2urResult(ZeRes); + } + } else if (hPhysicalMem->ZePhysicalMem) { + if (checkL0LoaderTeardown()) { + ze_result_t ZeRes = ZE_CALL_NOCHECK( + zePhysicalMemDestroy, + (hPhysicalMem->Context->getZeHandle(), hPhysicalMem->ZePhysicalMem)); + Res = ze2urResult(ZeRes); + } + } + delete hPhysicalMem; + + return Res; +} + +ur_result_t urPhysicalMemGetInfo(ur_physical_mem_handle_t hPhysicalMem, + ur_physical_mem_info_t propName, + size_t propSize, void *pPropValue, + size_t *pPropSizeRet) { + + UrReturnHelper ReturnValue(propSize, pPropValue, pPropSizeRet); + + switch (propName) { + case UR_PHYSICAL_MEM_INFO_CONTEXT: + return ReturnValue(hPhysicalMem->Context); + case UR_PHYSICAL_MEM_INFO_DEVICE: + return ReturnValue(hPhysicalMem->Device); + case UR_PHYSICAL_MEM_INFO_SIZE: + return ReturnValue(hPhysicalMem->Size); + case UR_PHYSICAL_MEM_INFO_PROPERTIES: { + ur_physical_mem_flags_t Flags = static_cast(0); + if (hPhysicalMem->EnableIpc) + Flags = UR_PHYSICAL_MEM_FLAG_ENABLE_IPC; + ur_physical_mem_properties_t Props = { + UR_STRUCTURE_TYPE_PHYSICAL_MEM_PROPERTIES, nullptr, Flags}; + return ReturnValue(Props); + } + case UR_PHYSICAL_MEM_INFO_REFERENCE_COUNT: + return ReturnValue(hPhysicalMem->RefCount.getCount()); + case UR_PHYSICAL_MEM_INFO_IPC_VIRTUAL_ADDRESS: + return ReturnValue(hPhysicalMem->IpcVirtualAddress); + default: + return UR_RESULT_ERROR_UNSUPPORTED_ENUMERATION; + } + return UR_RESULT_SUCCESS; +} + +ur_result_t urIPCGetPhysMemHandleExp(ur_context_handle_t hContext, + ur_physical_mem_handle_t hPhysMem, + void **ppIPCPhysMemHandleData, + size_t *pIPCPhysMemHandleDataSizeRet) { +#ifdef __linux__ + if (!hPhysMem->Device->isIntelBMGOrNewer()) + return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; + + if (!hPhysMem->EnableIpc) + return UR_RESULT_ERROR_INVALID_ARGUMENT; + + // IPC-opened handles (consumer side) have ZePhysicalMem == nullptr; they + // cannot be re-exported. + if (!hPhysMem->ZePhysicalMem) + return UR_RESULT_ERROR_INVALID_ARGUMENT; + + // Pass the physical memory handle directly to zeMemGetIpcHandleWithProperties + // No prior virtual mapping is required or expected; the physical handle + // itself is the key. + ze_ipc_mem_handle_type_ext_desc_t HandleTypeDesc = {}; + HandleTypeDesc.stype = ZE_STRUCTURE_TYPE_IPC_MEM_HANDLE_TYPE_EXT_DESC; + HandleTypeDesc.pNext = nullptr; + HandleTypeDesc.typeFlags = ZE_IPC_MEM_HANDLE_TYPE_FLAG_DEFAULT; + + ze_ipc_mem_handle_t IpcHandle = {}; + ze_result_t ZeRes = + ZE_CALL_NOCHECK(zeMemGetIpcHandleWithProperties, + (hContext->getZeHandle(), + reinterpret_cast(hPhysMem->ZePhysicalMem), + &HandleTypeDesc, &IpcHandle)); + // On drivers that do not support physical-mem IPC the function may return + // ZE_RESULT_ERROR_UNSUPPORTED_FEATURE or ZE_RESULT_ERROR_INVALID_ARGUMENT. + // Map both to UR_RESULT_ERROR_UNSUPPORTED_FEATURE so callers can skip. + if (ZeRes == ZE_RESULT_ERROR_INVALID_ARGUMENT || + ZeRes == ZE_RESULT_ERROR_UNSUPPORTED_FEATURE) + return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; + if (ZeRes != ZE_RESULT_SUCCESS) + return ze2urResult(ZeRes); + + auto *HandleData = new (std::nothrow) ZeIPCPhysMemHandleData; + if (!HandleData) { + zeMemPutIpcHandle(hContext->getZeHandle(), IpcHandle); + return UR_RESULT_ERROR_OUT_OF_HOST_MEMORY; + } + + HandleData->IpcHandle = IpcHandle; + HandleData->Size = hPhysMem->Size; + + *ppIPCPhysMemHandleData = HandleData; + *pIPCPhysMemHandleDataSizeRet = sizeof(ZeIPCPhysMemHandleData); + return UR_RESULT_SUCCESS; +#else + (void)hContext; + (void)hPhysMem; + (void)ppIPCPhysMemHandleData; + (void)pIPCPhysMemHandleDataSizeRet; + return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; +#endif // __linux__ +} + +ur_result_t urIPCPutPhysMemHandleExp(ur_context_handle_t hContext, + const void *pIPCPhysMemHandleData) { +#ifdef __linux__ + auto *HandleData = + static_cast(pIPCPhysMemHandleData); + ze_result_t ZeResult = ZE_CALL_NOCHECK( + zeMemPutIpcHandle, (hContext->getZeHandle(), HandleData->IpcHandle)); + // Always delete the UR wrapper: urIPCPutPhysMemHandleExp takes ownership of + // pIPCPhysMemHandleData and the pointer must not be used by the caller after + // this call, regardless of the return value. If zeMemPutIpcHandle fails, the + // driver-side handle may be in an undefined state, but there is no mechanism + // to retry — the wrapper is freed unconditionally. + delete HandleData; + return ze2urResult(ZeResult); +#else + (void)hContext; + (void)pIPCPhysMemHandleData; + return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; +#endif // __linux__ +} + +ur_result_t urIPCOpenPhysMemHandleExp(ur_context_handle_t hContext, + ur_device_handle_t hDevice, + const void *pIPCPhysMemHandleData, + size_t ipcPhysMemHandleDataSize, + ur_physical_mem_handle_t *phPhysMem) { +#ifdef __linux__ + if (ipcPhysMemHandleDataSize != sizeof(ZeIPCPhysMemHandleData)) + return UR_RESULT_ERROR_INVALID_VALUE; + + if (!hDevice->isIntelBMGOrNewer()) + return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; + + auto *HandleData = + static_cast(pIPCPhysMemHandleData); + + if (HandleData->Size == 0) + return UR_RESULT_ERROR_INVALID_ARGUMENT; + + // Open the IPC handle in this process. zeMemOpenIpcHandle creates a virtual + // mapping backed by the exporter's physical memory and returns a pointer to + // it. No separate zeVirtualMemMap call is needed; the memory is immediately + // accessible at the returned address. + void *VirtualAddress = nullptr; + ZE2UR_CALL(zeMemOpenIpcHandle, (hContext->getZeHandle(), hDevice->ZeDevice, + HandleData->IpcHandle, 0, &VirtualAddress)); + if (!VirtualAddress) + return UR_RESULT_ERROR_UNKNOWN; + + try { + *phPhysMem = new ur_physical_mem_handle_t_( + /*ZePhysicalMem=*/nullptr, hContext, hDevice, HandleData->Size, + /*EnableIpc=*/true, VirtualAddress); + } catch (const std::bad_alloc &) { + zeMemCloseIpcHandle(hContext->getZeHandle(), VirtualAddress); + return UR_RESULT_ERROR_OUT_OF_HOST_MEMORY; + } catch (...) { + zeMemCloseIpcHandle(hContext->getZeHandle(), VirtualAddress); + return UR_RESULT_ERROR_UNKNOWN; + } + + return UR_RESULT_SUCCESS; +#else + (void)hContext; + (void)hDevice; + (void)pIPCPhysMemHandleData; + (void)ipcPhysMemHandleDataSize; + (void)phPhysMem; + return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; +#endif // __linux__ +} + +ur_result_t urIPCClosePhysMemHandleExp(ur_context_handle_t hContext, + ur_physical_mem_handle_t hPhysMem) { +#ifdef __linux__ + (void)hContext; + if (!hPhysMem->Device->isIntelBMGOrNewer()) + return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; + + // Delegate to urPhysicalMemRelease so the refcount is respected. For + // IPC-opened handles (IpcVirtualAddress != nullptr) urPhysicalMemRelease + // calls zeMemCloseIpcHandle; for regular handles it calls + // zePhysicalMemDestroy. + return ur::level_zero::urPhysicalMemRelease(hPhysMem); +#else + (void)hContext; + (void)hPhysMem; + return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; +#endif // __linux__ +} + +} // namespace ur::level_zero diff --git a/unified-runtime/source/adapters/level_zero/v2/physical_mem.hpp b/unified-runtime/source/adapters/level_zero/v2/physical_mem.hpp new file mode 100644 index 0000000000000..bc260c18beb5d --- /dev/null +++ b/unified-runtime/source/adapters/level_zero/v2/physical_mem.hpp @@ -0,0 +1,59 @@ +//===---------------- physical_mem.hpp - Level Zero Adapter ---------------===// +// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM +// Exceptions. See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +#pragma once + +#include "common.hpp" +#include "common/ur_ref_count.hpp" + +// Opaque handle data exchanged between processes for physical memory IPC. +// Contains the IPC handle obtained from zeMemGetIpcHandleWithProperties +// plus the allocation size required to reconstruct the physical_mem on import. +// When the driver correctly implements ZE_IPC_MEM_HANDLE_TYPE_FLAG_DEFAULT for +// physical memory, ze_ipc_mem_handle_t is a self-contained 64-byte opaque blob +// that can be transferred to another process without fd passing. On drivers +// that fall back to an fd-based handle, urIPCGetPhysMemHandleExp detects and +// rejects the fd handle, returning UR_RESULT_ERROR_UNSUPPORTED_FEATURE. +struct ZeIPCPhysMemHandleData { + ze_ipc_mem_handle_t IpcHandle; // opaque IPC handle (64 bytes) + size_t Size; +}; + +struct ur_physical_mem_handle_t_ : ur_object { + ur_physical_mem_handle_t_(ze_physical_mem_handle_t ZePhysicalMem, + ur_context_handle_t Context, + ur_device_handle_t Device, size_t Size, + bool EnableIpc, void *IpcVirtualAddress = nullptr) + : ZePhysicalMem{ZePhysicalMem}, Context{Context}, Device{Device}, + Size{Size}, EnableIpc{EnableIpc}, IpcVirtualAddress{IpcVirtualAddress} { + } + + // Level Zero physical memory handle. Null for IPC-opened handles (consumer + // side), where the memory is accessible via IpcVirtualAddress instead. + ze_physical_mem_handle_t ZePhysicalMem; + + // Keeps the PI context of this memory handle. + ur_context_handle_t Context; + + // Device this physical memory was allocated on. + ur_device_handle_t Device; + + // Size in bytes of this physical memory allocation. + size_t Size; + + // Whether this allocation was created with IPC export enabled. + bool EnableIpc; + + // Virtual address returned by zeMemOpenIpcHandle for IPC-opened handles + // (consumer side). Non-null means the memory is already virtually mapped + // at this address; cleanup uses zeMemCloseIpcHandle rather than + // zePhysicalMemDestroy. + void *IpcVirtualAddress; + + ur::RefCount RefCount; +}; diff --git a/unified-runtime/source/adapters/level_zero/virtual_mem.cpp b/unified-runtime/source/adapters/level_zero/virtual_mem.cpp index 5100c0025fbe5..2870823b5447e 100644 --- a/unified-runtime/source/adapters/level_zero/virtual_mem.cpp +++ b/unified-runtime/source/adapters/level_zero/virtual_mem.cpp @@ -10,7 +10,7 @@ #include "common.hpp" #include "device.hpp" #include "logger/ur_logger.hpp" -#include "physical_mem.hpp" +#include "v2/physical_mem.hpp" #ifdef UR_ADAPTER_LEVEL_ZERO_V2 #include "v2/context.hpp" @@ -80,6 +80,16 @@ ur_result_t urVirtualMemMap(ur_context_handle_t hContext, const void *pStart, size_t size, ur_physical_mem_handle_t hPhysicalMem, size_t offset, ur_virtual_mem_access_flags_t flags) { + // IPC-opened physical memory handles already have a virtual address + // established by zeMemOpenIpcHandle (stored in IpcVirtualAddress). + // The UR virtual memory reservation at pStart is a separate address range + // that cannot be backed by an IPC handle via zeVirtualMemMap — the driver + // requires the physical handle to map, which is null on the consumer side. + // Return INVALID_ARGUMENT so callers know to use + // urPhysicalMemGetInfo(UR_PHYSICAL_MEM_INFO_IPC_VIRTUAL_ADDRESS) instead. + if (hPhysicalMem->IpcVirtualAddress) + return UR_RESULT_ERROR_INVALID_ARGUMENT; + ze_memory_access_attribute_t AccessAttr = ZE_MEMORY_ACCESS_ATTRIBUTE_NONE; if (flags & UR_VIRTUAL_MEM_ACCESS_FLAG_READ_WRITE) AccessAttr = ZE_MEMORY_ACCESS_ATTRIBUTE_READWRITE; diff --git a/unified-runtime/source/adapters/mock/ur_mockddi.cpp b/unified-runtime/source/adapters/mock/ur_mockddi.cpp index 195702990d65c..ddaa42ac60eec 100644 --- a/unified-runtime/source/adapters/mock/ur_mockddi.cpp +++ b/unified-runtime/source/adapters/mock/ur_mockddi.cpp @@ -9661,6 +9661,200 @@ __urdlllocal ur_result_t UR_APICALL urIPCCloseMemHandleExp( return exceptionToResult(std::current_exception()); } +/////////////////////////////////////////////////////////////////////////////// +/// @brief Intercept function for urIPCGetPhysMemHandleExp +__urdlllocal ur_result_t UR_APICALL urIPCGetPhysMemHandleExp( + /// [in] handle of the context object + ur_context_handle_t hContext, + /// [in] handle of the physical memory object + ur_physical_mem_handle_t hPhysMem, + /// [out] a pointer to the IPC physical memory handle data + void **ppIPCPhysMemHandleData, + /// [out] size of the resulting IPC physical memory handle data + size_t *pIPCPhysMemHandleDataSizeRet) try { + ur_result_t result = UR_RESULT_SUCCESS; + + ur_ipc_get_phys_mem_handle_exp_params_t params = { + &hContext, &hPhysMem, &ppIPCPhysMemHandleData, + &pIPCPhysMemHandleDataSizeRet}; + + auto beforeCallback = reinterpret_cast( + mock::getCallbacks().get_before_callback("urIPCGetPhysMemHandleExp")); + if (beforeCallback) { + result = beforeCallback(¶ms); + if (result != UR_RESULT_SUCCESS) { + return result; + } + } + + auto replaceCallback = reinterpret_cast( + mock::getCallbacks().get_replace_callback("urIPCGetPhysMemHandleExp")); + if (replaceCallback) { + result = replaceCallback(¶ms); + } else { + + result = UR_RESULT_SUCCESS; + } + + if (result != UR_RESULT_SUCCESS) { + return result; + } + + auto afterCallback = reinterpret_cast( + mock::getCallbacks().get_after_callback("urIPCGetPhysMemHandleExp")); + if (afterCallback) { + return afterCallback(¶ms); + } + + return result; +} catch (...) { + return exceptionToResult(std::current_exception()); +} + +/////////////////////////////////////////////////////////////////////////////// +/// @brief Intercept function for urIPCPutPhysMemHandleExp +__urdlllocal ur_result_t UR_APICALL urIPCPutPhysMemHandleExp( + /// [in] handle of the context object + ur_context_handle_t hContext, + /// [in] a pointer to the IPC physical memory handle data obtained with + /// urIPCGetPhysMemHandleExp + const void *pIPCPhysMemHandleData) try { + ur_result_t result = UR_RESULT_SUCCESS; + + ur_ipc_put_phys_mem_handle_exp_params_t params = {&hContext, + &pIPCPhysMemHandleData}; + + auto beforeCallback = reinterpret_cast( + mock::getCallbacks().get_before_callback("urIPCPutPhysMemHandleExp")); + if (beforeCallback) { + result = beforeCallback(¶ms); + if (result != UR_RESULT_SUCCESS) { + return result; + } + } + + auto replaceCallback = reinterpret_cast( + mock::getCallbacks().get_replace_callback("urIPCPutPhysMemHandleExp")); + if (replaceCallback) { + result = replaceCallback(¶ms); + } else { + + result = UR_RESULT_SUCCESS; + } + + if (result != UR_RESULT_SUCCESS) { + return result; + } + + auto afterCallback = reinterpret_cast( + mock::getCallbacks().get_after_callback("urIPCPutPhysMemHandleExp")); + if (afterCallback) { + return afterCallback(¶ms); + } + + return result; +} catch (...) { + return exceptionToResult(std::current_exception()); +} + +/////////////////////////////////////////////////////////////////////////////// +/// @brief Intercept function for urIPCOpenPhysMemHandleExp +__urdlllocal ur_result_t UR_APICALL urIPCOpenPhysMemHandleExp( + /// [in] handle of the context object + ur_context_handle_t hContext, + /// [in] handle of the device object the physical memory was allocated on + ur_device_handle_t hDevice, + /// [in] the IPC physical memory handle data obtained with + /// urIPCGetPhysMemHandleExp + const void *pIPCPhysMemHandleData, + /// [in] size of the IPC physical memory handle data + size_t ipcPhysMemHandleDataSize, + /// [out] pointer to the physical memory handle + ur_physical_mem_handle_t *phPhysMem) try { + ur_result_t result = UR_RESULT_SUCCESS; + + ur_ipc_open_phys_mem_handle_exp_params_t params = { + &hContext, &hDevice, &pIPCPhysMemHandleData, &ipcPhysMemHandleDataSize, + &phPhysMem}; + + auto beforeCallback = reinterpret_cast( + mock::getCallbacks().get_before_callback("urIPCOpenPhysMemHandleExp")); + if (beforeCallback) { + result = beforeCallback(¶ms); + if (result != UR_RESULT_SUCCESS) { + return result; + } + } + + auto replaceCallback = reinterpret_cast( + mock::getCallbacks().get_replace_callback("urIPCOpenPhysMemHandleExp")); + if (replaceCallback) { + result = replaceCallback(¶ms); + } else { + + *phPhysMem = mock::createDummyHandle(); + result = UR_RESULT_SUCCESS; + } + + if (result != UR_RESULT_SUCCESS) { + return result; + } + + auto afterCallback = reinterpret_cast( + mock::getCallbacks().get_after_callback("urIPCOpenPhysMemHandleExp")); + if (afterCallback) { + return afterCallback(¶ms); + } + + return result; +} catch (...) { + return exceptionToResult(std::current_exception()); +} + +/////////////////////////////////////////////////////////////////////////////// +/// @brief Intercept function for urIPCClosePhysMemHandleExp +__urdlllocal ur_result_t UR_APICALL urIPCClosePhysMemHandleExp( + /// [in] handle of the context object + ur_context_handle_t hContext, + /// [in] physical memory handle opened through urIPCOpenPhysMemHandleExp + ur_physical_mem_handle_t hPhysMem) try { + ur_result_t result = UR_RESULT_SUCCESS; + + ur_ipc_close_phys_mem_handle_exp_params_t params = {&hContext, &hPhysMem}; + + auto beforeCallback = reinterpret_cast( + mock::getCallbacks().get_before_callback("urIPCClosePhysMemHandleExp")); + if (beforeCallback) { + result = beforeCallback(¶ms); + if (result != UR_RESULT_SUCCESS) { + return result; + } + } + + auto replaceCallback = reinterpret_cast( + mock::getCallbacks().get_replace_callback("urIPCClosePhysMemHandleExp")); + if (replaceCallback) { + result = replaceCallback(¶ms); + } else { + + result = UR_RESULT_SUCCESS; + } + + if (result != UR_RESULT_SUCCESS) { + return result; + } + + auto afterCallback = reinterpret_cast( + mock::getCallbacks().get_after_callback("urIPCClosePhysMemHandleExp")); + if (afterCallback) { + return afterCallback(¶ms); + } + + return result; +} catch (...) { + return exceptionToResult(std::current_exception()); +} + /////////////////////////////////////////////////////////////////////////////// /// @brief Intercept function for urMemoryExportAllocExportableMemoryExp __urdlllocal ur_result_t UR_APICALL urMemoryExportAllocExportableMemoryExp( @@ -13547,6 +13741,14 @@ UR_DLLEXPORT ur_result_t UR_APICALL urGetIPCExpProcAddrTable( pDdiTable->pfnCloseMemHandleExp = driver::urIPCCloseMemHandleExp; + pDdiTable->pfnGetPhysMemHandleExp = driver::urIPCGetPhysMemHandleExp; + + pDdiTable->pfnPutPhysMemHandleExp = driver::urIPCPutPhysMemHandleExp; + + pDdiTable->pfnOpenPhysMemHandleExp = driver::urIPCOpenPhysMemHandleExp; + + pDdiTable->pfnClosePhysMemHandleExp = driver::urIPCClosePhysMemHandleExp; + return result; } catch (...) { return exceptionToResult(std::current_exception()); diff --git a/unified-runtime/source/adapters/native_cpu/device.cpp b/unified-runtime/source/adapters/native_cpu/device.cpp index f08de5694a2f3..e74668f197281 100644 --- a/unified-runtime/source/adapters/native_cpu/device.cpp +++ b/unified-runtime/source/adapters/native_cpu/device.cpp @@ -381,6 +381,8 @@ UR_APIEXPORT ur_result_t UR_APICALL urDeviceGetInfo(ur_device_handle_t hDevice, static_cast(0)); case UR_DEVICE_INFO_IPC_MEMORY_SUPPORT_EXP: return ReturnValue(false); + case UR_DEVICE_INFO_IPC_PHYSICAL_MEMORY_SUPPORT_EXP: + return ReturnValue(false); case UR_DEVICE_INFO_ATOMIC_FENCE_ORDER_CAPABILITIES: { // Currently for Native CPU fences are implemented using OCK // builtins, so we have different capabilities than atomic operations diff --git a/unified-runtime/source/adapters/native_cpu/memory.cpp b/unified-runtime/source/adapters/native_cpu/memory.cpp index 2ac5eccc11cc3..2328dcfc287b8 100644 --- a/unified-runtime/source/adapters/native_cpu/memory.cpp +++ b/unified-runtime/source/adapters/native_cpu/memory.cpp @@ -160,3 +160,24 @@ UR_APIEXPORT ur_result_t UR_APICALL urIPCCloseMemHandleExp(ur_context_handle_t, void *) { return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; } + +UR_APIEXPORT ur_result_t UR_APICALL urIPCGetPhysMemHandleExp( + ur_context_handle_t, ur_physical_mem_handle_t, void **, size_t *) { + return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; +} + +UR_APIEXPORT ur_result_t UR_APICALL +urIPCPutPhysMemHandleExp(ur_context_handle_t, const void *) { + return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; +} + +UR_APIEXPORT ur_result_t UR_APICALL +urIPCOpenPhysMemHandleExp(ur_context_handle_t, ur_device_handle_t, const void *, + size_t, ur_physical_mem_handle_t *) { + return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; +} + +UR_APIEXPORT ur_result_t UR_APICALL +urIPCClosePhysMemHandleExp(ur_context_handle_t, ur_physical_mem_handle_t) { + return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; +} diff --git a/unified-runtime/source/adapters/native_cpu/ur_interface_loader.cpp b/unified-runtime/source/adapters/native_cpu/ur_interface_loader.cpp index c3b121bfb0cdb..5f1228273b006 100644 --- a/unified-runtime/source/adapters/native_cpu/ur_interface_loader.cpp +++ b/unified-runtime/source/adapters/native_cpu/ur_interface_loader.cpp @@ -487,6 +487,10 @@ UR_DLLEXPORT ur_result_t UR_APICALL urGetIPCExpProcAddrTable( pDdiTable->pfnPutMemHandleExp = urIPCPutMemHandleExp; pDdiTable->pfnOpenMemHandleExp = urIPCOpenMemHandleExp; pDdiTable->pfnCloseMemHandleExp = urIPCCloseMemHandleExp; + pDdiTable->pfnGetPhysMemHandleExp = urIPCGetPhysMemHandleExp; + pDdiTable->pfnPutPhysMemHandleExp = urIPCPutPhysMemHandleExp; + pDdiTable->pfnOpenPhysMemHandleExp = urIPCOpenPhysMemHandleExp; + pDdiTable->pfnClosePhysMemHandleExp = urIPCClosePhysMemHandleExp; return UR_RESULT_SUCCESS; } diff --git a/unified-runtime/source/adapters/offload/memory.cpp b/unified-runtime/source/adapters/offload/memory.cpp index c0b7a97fb62ef..7d55d14018481 100644 --- a/unified-runtime/source/adapters/offload/memory.cpp +++ b/unified-runtime/source/adapters/offload/memory.cpp @@ -229,3 +229,24 @@ UR_APIEXPORT ur_result_t UR_APICALL urIPCCloseMemHandleExp(ur_context_handle_t, void *) { return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; } + +UR_APIEXPORT ur_result_t UR_APICALL urIPCGetPhysMemHandleExp( + ur_context_handle_t, ur_physical_mem_handle_t, void **, size_t *) { + return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; +} + +UR_APIEXPORT ur_result_t UR_APICALL +urIPCPutPhysMemHandleExp(ur_context_handle_t, const void *) { + return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; +} + +UR_APIEXPORT ur_result_t UR_APICALL +urIPCOpenPhysMemHandleExp(ur_context_handle_t, ur_device_handle_t, const void *, + size_t, ur_physical_mem_handle_t *) { + return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; +} + +UR_APIEXPORT ur_result_t UR_APICALL +urIPCClosePhysMemHandleExp(ur_context_handle_t, ur_physical_mem_handle_t) { + return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; +} diff --git a/unified-runtime/source/adapters/offload/ur_interface_loader.cpp b/unified-runtime/source/adapters/offload/ur_interface_loader.cpp index ce4ab5351a527..f88af93d6a2b2 100644 --- a/unified-runtime/source/adapters/offload/ur_interface_loader.cpp +++ b/unified-runtime/source/adapters/offload/ur_interface_loader.cpp @@ -437,6 +437,10 @@ UR_DLLEXPORT ur_result_t UR_APICALL urGetIPCExpProcAddrTable( pDdiTable->pfnPutMemHandleExp = urIPCPutMemHandleExp; pDdiTable->pfnOpenMemHandleExp = urIPCOpenMemHandleExp; pDdiTable->pfnCloseMemHandleExp = urIPCCloseMemHandleExp; + pDdiTable->pfnGetPhysMemHandleExp = urIPCGetPhysMemHandleExp; + pDdiTable->pfnPutPhysMemHandleExp = urIPCPutPhysMemHandleExp; + pDdiTable->pfnOpenPhysMemHandleExp = urIPCOpenPhysMemHandleExp; + pDdiTable->pfnClosePhysMemHandleExp = urIPCClosePhysMemHandleExp; return UR_RESULT_SUCCESS; } diff --git a/unified-runtime/source/adapters/opencl/device.cpp b/unified-runtime/source/adapters/opencl/device.cpp index 9654c6b0f764b..f8d11ce26c3ad 100644 --- a/unified-runtime/source/adapters/opencl/device.cpp +++ b/unified-runtime/source/adapters/opencl/device.cpp @@ -1375,6 +1375,8 @@ UR_APIEXPORT ur_result_t UR_APICALL urDeviceGetInfo(ur_device_handle_t hDevice, } case UR_DEVICE_INFO_IPC_MEMORY_SUPPORT_EXP: return ReturnValue(false); + case UR_DEVICE_INFO_IPC_PHYSICAL_MEMORY_SUPPORT_EXP: + return ReturnValue(false); case UR_DEVICE_INFO_BFLOAT16_CONVERSIONS_NATIVE: { bool Supported = false; UR_RETURN_ON_FAILURE(hDevice->checkDeviceExtensions( diff --git a/unified-runtime/source/adapters/opencl/memory.cpp b/unified-runtime/source/adapters/opencl/memory.cpp index ca9e88d4f1ab1..1ba798bc1faa7 100644 --- a/unified-runtime/source/adapters/opencl/memory.cpp +++ b/unified-runtime/source/adapters/opencl/memory.cpp @@ -601,3 +601,24 @@ UR_APIEXPORT ur_result_t UR_APICALL urIPCCloseMemHandleExp(ur_context_handle_t, void *) { return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; } + +UR_APIEXPORT ur_result_t UR_APICALL urIPCGetPhysMemHandleExp( + ur_context_handle_t, ur_physical_mem_handle_t, void **, size_t *) { + return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; +} + +UR_APIEXPORT ur_result_t UR_APICALL +urIPCPutPhysMemHandleExp(ur_context_handle_t, const void *) { + return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; +} + +UR_APIEXPORT ur_result_t UR_APICALL +urIPCOpenPhysMemHandleExp(ur_context_handle_t, ur_device_handle_t, const void *, + size_t, ur_physical_mem_handle_t *) { + return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; +} + +UR_APIEXPORT ur_result_t UR_APICALL +urIPCClosePhysMemHandleExp(ur_context_handle_t, ur_physical_mem_handle_t) { + return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; +} diff --git a/unified-runtime/source/adapters/opencl/ur_interface_loader.cpp b/unified-runtime/source/adapters/opencl/ur_interface_loader.cpp index 3205af2df209e..c9ce08ce199d4 100644 --- a/unified-runtime/source/adapters/opencl/ur_interface_loader.cpp +++ b/unified-runtime/source/adapters/opencl/ur_interface_loader.cpp @@ -458,6 +458,10 @@ UR_DLLEXPORT ur_result_t UR_APICALL urGetIPCExpProcAddrTable( pDdiTable->pfnPutMemHandleExp = urIPCPutMemHandleExp; pDdiTable->pfnOpenMemHandleExp = urIPCOpenMemHandleExp; pDdiTable->pfnCloseMemHandleExp = urIPCCloseMemHandleExp; + pDdiTable->pfnGetPhysMemHandleExp = urIPCGetPhysMemHandleExp; + pDdiTable->pfnPutPhysMemHandleExp = urIPCPutPhysMemHandleExp; + pDdiTable->pfnOpenPhysMemHandleExp = urIPCOpenPhysMemHandleExp; + pDdiTable->pfnClosePhysMemHandleExp = urIPCClosePhysMemHandleExp; return UR_RESULT_SUCCESS; } diff --git a/unified-runtime/source/loader/layers/tracing/ur_trcddi.cpp b/unified-runtime/source/loader/layers/tracing/ur_trcddi.cpp index 5f8bfca375b61..8bd4bf3ab7575 100644 --- a/unified-runtime/source/loader/layers/tracing/ur_trcddi.cpp +++ b/unified-runtime/source/loader/layers/tracing/ur_trcddi.cpp @@ -8159,6 +8159,178 @@ __urdlllocal ur_result_t UR_APICALL urIPCCloseMemHandleExp( return result; } +/////////////////////////////////////////////////////////////////////////////// +/// @brief Intercept function for urIPCGetPhysMemHandleExp +__urdlllocal ur_result_t UR_APICALL urIPCGetPhysMemHandleExp( + /// [in] handle of the context object + ur_context_handle_t hContext, + /// [in] handle of the physical memory object + ur_physical_mem_handle_t hPhysMem, + /// [out] a pointer to the IPC physical memory handle data + void **ppIPCPhysMemHandleData, + /// [out] size of the resulting IPC physical memory handle data + size_t *pIPCPhysMemHandleDataSizeRet) { + auto pfnGetPhysMemHandleExp = + getContext()->urDdiTable.IPCExp.pfnGetPhysMemHandleExp; + + if (nullptr == pfnGetPhysMemHandleExp) + return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; + + ur_ipc_get_phys_mem_handle_exp_params_t params = { + &hContext, &hPhysMem, &ppIPCPhysMemHandleData, + &pIPCPhysMemHandleDataSizeRet}; + uint64_t instance = + getContext()->notify_begin(UR_FUNCTION_IPC_GET_PHYS_MEM_HANDLE_EXP, + "urIPCGetPhysMemHandleExp", ¶ms); + + auto &logger = getContext()->logger; + UR_LOG_L(logger, INFO, " ---> urIPCGetPhysMemHandleExp\n"); + + ur_result_t result = pfnGetPhysMemHandleExp( + hContext, hPhysMem, ppIPCPhysMemHandleData, pIPCPhysMemHandleDataSizeRet); + + getContext()->notify_end(UR_FUNCTION_IPC_GET_PHYS_MEM_HANDLE_EXP, + "urIPCGetPhysMemHandleExp", ¶ms, &result, + instance); + + if (logger.getLevel() <= UR_LOGGER_LEVEL_INFO) { + std::ostringstream args_str; + ur::extras::printFunctionParams( + args_str, UR_FUNCTION_IPC_GET_PHYS_MEM_HANDLE_EXP, ¶ms); + UR_LOG_L(logger, INFO, " <--- urIPCGetPhysMemHandleExp({}) -> {};\n", + args_str.str(), result); + } + + return result; +} + +/////////////////////////////////////////////////////////////////////////////// +/// @brief Intercept function for urIPCPutPhysMemHandleExp +__urdlllocal ur_result_t UR_APICALL urIPCPutPhysMemHandleExp( + /// [in] handle of the context object + ur_context_handle_t hContext, + /// [in] a pointer to the IPC physical memory handle data obtained with + /// urIPCGetPhysMemHandleExp + const void *pIPCPhysMemHandleData) { + auto pfnPutPhysMemHandleExp = + getContext()->urDdiTable.IPCExp.pfnPutPhysMemHandleExp; + + if (nullptr == pfnPutPhysMemHandleExp) + return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; + + ur_ipc_put_phys_mem_handle_exp_params_t params = {&hContext, + &pIPCPhysMemHandleData}; + uint64_t instance = + getContext()->notify_begin(UR_FUNCTION_IPC_PUT_PHYS_MEM_HANDLE_EXP, + "urIPCPutPhysMemHandleExp", ¶ms); + + auto &logger = getContext()->logger; + UR_LOG_L(logger, INFO, " ---> urIPCPutPhysMemHandleExp\n"); + + ur_result_t result = pfnPutPhysMemHandleExp(hContext, pIPCPhysMemHandleData); + + getContext()->notify_end(UR_FUNCTION_IPC_PUT_PHYS_MEM_HANDLE_EXP, + "urIPCPutPhysMemHandleExp", ¶ms, &result, + instance); + + if (logger.getLevel() <= UR_LOGGER_LEVEL_INFO) { + std::ostringstream args_str; + ur::extras::printFunctionParams( + args_str, UR_FUNCTION_IPC_PUT_PHYS_MEM_HANDLE_EXP, ¶ms); + UR_LOG_L(logger, INFO, " <--- urIPCPutPhysMemHandleExp({}) -> {};\n", + args_str.str(), result); + } + + return result; +} + +/////////////////////////////////////////////////////////////////////////////// +/// @brief Intercept function for urIPCOpenPhysMemHandleExp +__urdlllocal ur_result_t UR_APICALL urIPCOpenPhysMemHandleExp( + /// [in] handle of the context object + ur_context_handle_t hContext, + /// [in] handle of the device object the physical memory was allocated on + ur_device_handle_t hDevice, + /// [in] the IPC physical memory handle data obtained with + /// urIPCGetPhysMemHandleExp + const void *pIPCPhysMemHandleData, + /// [in] size of the IPC physical memory handle data + size_t ipcPhysMemHandleDataSize, + /// [out] pointer to the physical memory handle + ur_physical_mem_handle_t *phPhysMem) { + auto pfnOpenPhysMemHandleExp = + getContext()->urDdiTable.IPCExp.pfnOpenPhysMemHandleExp; + + if (nullptr == pfnOpenPhysMemHandleExp) + return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; + + ur_ipc_open_phys_mem_handle_exp_params_t params = { + &hContext, &hDevice, &pIPCPhysMemHandleData, &ipcPhysMemHandleDataSize, + &phPhysMem}; + uint64_t instance = + getContext()->notify_begin(UR_FUNCTION_IPC_OPEN_PHYS_MEM_HANDLE_EXP, + "urIPCOpenPhysMemHandleExp", ¶ms); + + auto &logger = getContext()->logger; + UR_LOG_L(logger, INFO, " ---> urIPCOpenPhysMemHandleExp\n"); + + ur_result_t result = + pfnOpenPhysMemHandleExp(hContext, hDevice, pIPCPhysMemHandleData, + ipcPhysMemHandleDataSize, phPhysMem); + + getContext()->notify_end(UR_FUNCTION_IPC_OPEN_PHYS_MEM_HANDLE_EXP, + "urIPCOpenPhysMemHandleExp", ¶ms, &result, + instance); + + if (logger.getLevel() <= UR_LOGGER_LEVEL_INFO) { + std::ostringstream args_str; + ur::extras::printFunctionParams( + args_str, UR_FUNCTION_IPC_OPEN_PHYS_MEM_HANDLE_EXP, ¶ms); + UR_LOG_L(logger, INFO, " <--- urIPCOpenPhysMemHandleExp({}) -> {};\n", + args_str.str(), result); + } + + return result; +} + +/////////////////////////////////////////////////////////////////////////////// +/// @brief Intercept function for urIPCClosePhysMemHandleExp +__urdlllocal ur_result_t UR_APICALL urIPCClosePhysMemHandleExp( + /// [in] handle of the context object + ur_context_handle_t hContext, + /// [in] physical memory handle opened through urIPCOpenPhysMemHandleExp + ur_physical_mem_handle_t hPhysMem) { + auto pfnClosePhysMemHandleExp = + getContext()->urDdiTable.IPCExp.pfnClosePhysMemHandleExp; + + if (nullptr == pfnClosePhysMemHandleExp) + return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; + + ur_ipc_close_phys_mem_handle_exp_params_t params = {&hContext, &hPhysMem}; + uint64_t instance = + getContext()->notify_begin(UR_FUNCTION_IPC_CLOSE_PHYS_MEM_HANDLE_EXP, + "urIPCClosePhysMemHandleExp", ¶ms); + + auto &logger = getContext()->logger; + UR_LOG_L(logger, INFO, " ---> urIPCClosePhysMemHandleExp\n"); + + ur_result_t result = pfnClosePhysMemHandleExp(hContext, hPhysMem); + + getContext()->notify_end(UR_FUNCTION_IPC_CLOSE_PHYS_MEM_HANDLE_EXP, + "urIPCClosePhysMemHandleExp", ¶ms, &result, + instance); + + if (logger.getLevel() <= UR_LOGGER_LEVEL_INFO) { + std::ostringstream args_str; + ur::extras::printFunctionParams( + args_str, UR_FUNCTION_IPC_CLOSE_PHYS_MEM_HANDLE_EXP, ¶ms); + UR_LOG_L(logger, INFO, " <--- urIPCClosePhysMemHandleExp({}) -> {};\n", + args_str.str(), result); + } + + return result; +} + /////////////////////////////////////////////////////////////////////////////// /// @brief Intercept function for urMemoryExportAllocExportableMemoryExp __urdlllocal ur_result_t UR_APICALL urMemoryExportAllocExportableMemoryExp( @@ -11731,6 +11903,22 @@ __urdlllocal ur_result_t UR_APICALL urGetIPCExpProcAddrTable( dditable.pfnCloseMemHandleExp = pDdiTable->pfnCloseMemHandleExp; pDdiTable->pfnCloseMemHandleExp = ur_tracing_layer::urIPCCloseMemHandleExp; + dditable.pfnGetPhysMemHandleExp = pDdiTable->pfnGetPhysMemHandleExp; + pDdiTable->pfnGetPhysMemHandleExp = + ur_tracing_layer::urIPCGetPhysMemHandleExp; + + dditable.pfnPutPhysMemHandleExp = pDdiTable->pfnPutPhysMemHandleExp; + pDdiTable->pfnPutPhysMemHandleExp = + ur_tracing_layer::urIPCPutPhysMemHandleExp; + + dditable.pfnOpenPhysMemHandleExp = pDdiTable->pfnOpenPhysMemHandleExp; + pDdiTable->pfnOpenPhysMemHandleExp = + ur_tracing_layer::urIPCOpenPhysMemHandleExp; + + dditable.pfnClosePhysMemHandleExp = pDdiTable->pfnClosePhysMemHandleExp; + pDdiTable->pfnClosePhysMemHandleExp = + ur_tracing_layer::urIPCClosePhysMemHandleExp; + return result; } /////////////////////////////////////////////////////////////////////////////// diff --git a/unified-runtime/source/loader/layers/validation/ur_valddi.cpp b/unified-runtime/source/loader/layers/validation/ur_valddi.cpp index 077e6b736846a..a7985d6ea92e2 100644 --- a/unified-runtime/source/loader/layers/validation/ur_valddi.cpp +++ b/unified-runtime/source/loader/layers/validation/ur_valddi.cpp @@ -2615,7 +2615,7 @@ __urdlllocal ur_result_t UR_APICALL urPhysicalMemGetInfo( if (NULL == hPhysicalMem) return UR_RESULT_ERROR_INVALID_NULL_HANDLE; - if (UR_PHYSICAL_MEM_INFO_REFERENCE_COUNT < propName) + if (UR_PHYSICAL_MEM_INFO_IPC_VIRTUAL_ADDRESS < propName) return UR_RESULT_ERROR_INVALID_ENUMERATION; } @@ -9003,6 +9003,176 @@ __urdlllocal ur_result_t UR_APICALL urIPCCloseMemHandleExp( return result; } +/////////////////////////////////////////////////////////////////////////////// +/// @brief Intercept function for urIPCGetPhysMemHandleExp +__urdlllocal ur_result_t UR_APICALL urIPCGetPhysMemHandleExp( + /// [in] handle of the context object + ur_context_handle_t hContext, + /// [in] handle of the physical memory object + ur_physical_mem_handle_t hPhysMem, + /// [out] a pointer to the IPC physical memory handle data + void **ppIPCPhysMemHandleData, + /// [out] size of the resulting IPC physical memory handle data + size_t *pIPCPhysMemHandleDataSizeRet) { + auto pfnGetPhysMemHandleExp = + getContext()->urDdiTable.IPCExp.pfnGetPhysMemHandleExp; + + if (nullptr == pfnGetPhysMemHandleExp) { + return UR_RESULT_ERROR_UNINITIALIZED; + } + + if (getContext()->enableParameterValidation) { + if (NULL == ppIPCPhysMemHandleData) + return UR_RESULT_ERROR_INVALID_NULL_POINTER; + + if (NULL == pIPCPhysMemHandleDataSizeRet) + return UR_RESULT_ERROR_INVALID_NULL_POINTER; + + if (NULL == hContext) + return UR_RESULT_ERROR_INVALID_NULL_HANDLE; + + if (NULL == hPhysMem) + return UR_RESULT_ERROR_INVALID_NULL_HANDLE; + } + + if (getContext()->enableLifetimeValidation && + !getContext()->refCountContext->isReferenceValid(hContext)) { + URLOG_CTX_INVALID_REFERENCE(hContext); + } + + if (getContext()->enableLifetimeValidation && + !getContext()->refCountContext->isReferenceValid(hPhysMem)) { + URLOG_CTX_INVALID_REFERENCE(hPhysMem); + } + + ur_result_t result = pfnGetPhysMemHandleExp( + hContext, hPhysMem, ppIPCPhysMemHandleData, pIPCPhysMemHandleDataSizeRet); + + return result; +} + +/////////////////////////////////////////////////////////////////////////////// +/// @brief Intercept function for urIPCPutPhysMemHandleExp +__urdlllocal ur_result_t UR_APICALL urIPCPutPhysMemHandleExp( + /// [in] handle of the context object + ur_context_handle_t hContext, + /// [in] a pointer to the IPC physical memory handle data obtained with + /// urIPCGetPhysMemHandleExp + const void *pIPCPhysMemHandleData) { + auto pfnPutPhysMemHandleExp = + getContext()->urDdiTable.IPCExp.pfnPutPhysMemHandleExp; + + if (nullptr == pfnPutPhysMemHandleExp) { + return UR_RESULT_ERROR_UNINITIALIZED; + } + + if (getContext()->enableParameterValidation) { + if (NULL == pIPCPhysMemHandleData) + return UR_RESULT_ERROR_INVALID_NULL_POINTER; + + if (NULL == hContext) + return UR_RESULT_ERROR_INVALID_NULL_HANDLE; + } + + if (getContext()->enableLifetimeValidation && + !getContext()->refCountContext->isReferenceValid(hContext)) { + URLOG_CTX_INVALID_REFERENCE(hContext); + } + + ur_result_t result = pfnPutPhysMemHandleExp(hContext, pIPCPhysMemHandleData); + + return result; +} + +/////////////////////////////////////////////////////////////////////////////// +/// @brief Intercept function for urIPCOpenPhysMemHandleExp +__urdlllocal ur_result_t UR_APICALL urIPCOpenPhysMemHandleExp( + /// [in] handle of the context object + ur_context_handle_t hContext, + /// [in] handle of the device object the physical memory was allocated on + ur_device_handle_t hDevice, + /// [in] the IPC physical memory handle data obtained with + /// urIPCGetPhysMemHandleExp + const void *pIPCPhysMemHandleData, + /// [in] size of the IPC physical memory handle data + size_t ipcPhysMemHandleDataSize, + /// [out] pointer to the physical memory handle + ur_physical_mem_handle_t *phPhysMem) { + auto pfnOpenPhysMemHandleExp = + getContext()->urDdiTable.IPCExp.pfnOpenPhysMemHandleExp; + + if (nullptr == pfnOpenPhysMemHandleExp) { + return UR_RESULT_ERROR_UNINITIALIZED; + } + + if (getContext()->enableParameterValidation) { + if (NULL == phPhysMem) + return UR_RESULT_ERROR_INVALID_NULL_POINTER; + + if (NULL == pIPCPhysMemHandleData) + return UR_RESULT_ERROR_INVALID_NULL_POINTER; + + if (NULL == hContext) + return UR_RESULT_ERROR_INVALID_NULL_HANDLE; + + if (NULL == hDevice) + return UR_RESULT_ERROR_INVALID_NULL_HANDLE; + } + + if (getContext()->enableLifetimeValidation && + !getContext()->refCountContext->isReferenceValid(hContext)) { + URLOG_CTX_INVALID_REFERENCE(hContext); + } + + if (getContext()->enableLifetimeValidation && + !getContext()->refCountContext->isReferenceValid(hDevice)) { + URLOG_CTX_INVALID_REFERENCE(hDevice); + } + + ur_result_t result = + pfnOpenPhysMemHandleExp(hContext, hDevice, pIPCPhysMemHandleData, + ipcPhysMemHandleDataSize, phPhysMem); + + return result; +} + +/////////////////////////////////////////////////////////////////////////////// +/// @brief Intercept function for urIPCClosePhysMemHandleExp +__urdlllocal ur_result_t UR_APICALL urIPCClosePhysMemHandleExp( + /// [in] handle of the context object + ur_context_handle_t hContext, + /// [in] physical memory handle opened through urIPCOpenPhysMemHandleExp + ur_physical_mem_handle_t hPhysMem) { + auto pfnClosePhysMemHandleExp = + getContext()->urDdiTable.IPCExp.pfnClosePhysMemHandleExp; + + if (nullptr == pfnClosePhysMemHandleExp) { + return UR_RESULT_ERROR_UNINITIALIZED; + } + + if (getContext()->enableParameterValidation) { + if (NULL == hContext) + return UR_RESULT_ERROR_INVALID_NULL_HANDLE; + + if (NULL == hPhysMem) + return UR_RESULT_ERROR_INVALID_NULL_HANDLE; + } + + if (getContext()->enableLifetimeValidation && + !getContext()->refCountContext->isReferenceValid(hContext)) { + URLOG_CTX_INVALID_REFERENCE(hContext); + } + + if (getContext()->enableLifetimeValidation && + !getContext()->refCountContext->isReferenceValid(hPhysMem)) { + URLOG_CTX_INVALID_REFERENCE(hPhysMem); + } + + ur_result_t result = pfnClosePhysMemHandleExp(hContext, hPhysMem); + + return result; +} + /////////////////////////////////////////////////////////////////////////////// /// @brief Intercept function for urMemoryExportAllocExportableMemoryExp __urdlllocal ur_result_t UR_APICALL urMemoryExportAllocExportableMemoryExp( @@ -12502,6 +12672,22 @@ UR_DLLEXPORT ur_result_t UR_APICALL urGetIPCExpProcAddrTable( dditable.pfnCloseMemHandleExp = pDdiTable->pfnCloseMemHandleExp; pDdiTable->pfnCloseMemHandleExp = ur_validation_layer::urIPCCloseMemHandleExp; + dditable.pfnGetPhysMemHandleExp = pDdiTable->pfnGetPhysMemHandleExp; + pDdiTable->pfnGetPhysMemHandleExp = + ur_validation_layer::urIPCGetPhysMemHandleExp; + + dditable.pfnPutPhysMemHandleExp = pDdiTable->pfnPutPhysMemHandleExp; + pDdiTable->pfnPutPhysMemHandleExp = + ur_validation_layer::urIPCPutPhysMemHandleExp; + + dditable.pfnOpenPhysMemHandleExp = pDdiTable->pfnOpenPhysMemHandleExp; + pDdiTable->pfnOpenPhysMemHandleExp = + ur_validation_layer::urIPCOpenPhysMemHandleExp; + + dditable.pfnClosePhysMemHandleExp = pDdiTable->pfnClosePhysMemHandleExp; + pDdiTable->pfnClosePhysMemHandleExp = + ur_validation_layer::urIPCClosePhysMemHandleExp; + return result; } diff --git a/unified-runtime/source/loader/loader.def.in b/unified-runtime/source/loader/loader.def.in index cc6ec03371edb..4a978ad4584c6 100644 --- a/unified-runtime/source/loader/loader.def.in +++ b/unified-runtime/source/loader/loader.def.in @@ -147,9 +147,13 @@ EXPORTS urGraphIsEmptyExp urGraphSetDestructionCallbackExp urIPCCloseMemHandleExp + urIPCClosePhysMemHandleExp urIPCGetMemHandleExp + urIPCGetPhysMemHandleExp urIPCOpenMemHandleExp + urIPCOpenPhysMemHandleExp urIPCPutMemHandleExp + urIPCPutPhysMemHandleExp urKernelCreate urKernelCreateWithNativeHandle urKernelGetGroupInfo @@ -397,9 +401,13 @@ EXPORTS urPrintImageFormat urPrintImageInfo urPrintIpcCloseMemHandleExpParams + urPrintIpcClosePhysMemHandleExpParams urPrintIpcGetMemHandleExpParams + urPrintIpcGetPhysMemHandleExpParams urPrintIpcOpenMemHandleExpParams + urPrintIpcOpenPhysMemHandleExpParams urPrintIpcPutMemHandleExpParams + urPrintIpcPutPhysMemHandleExpParams urPrintKernelArgLocalProperties urPrintKernelArgMemObjProperties urPrintKernelArgPointerProperties diff --git a/unified-runtime/source/loader/loader.map.in b/unified-runtime/source/loader/loader.map.in index 9ffb00211b6dc..0c06c67d8745a 100644 --- a/unified-runtime/source/loader/loader.map.in +++ b/unified-runtime/source/loader/loader.map.in @@ -147,9 +147,13 @@ urGraphIsEmptyExp; urGraphSetDestructionCallbackExp; urIPCCloseMemHandleExp; + urIPCClosePhysMemHandleExp; urIPCGetMemHandleExp; + urIPCGetPhysMemHandleExp; urIPCOpenMemHandleExp; + urIPCOpenPhysMemHandleExp; urIPCPutMemHandleExp; + urIPCPutPhysMemHandleExp; urKernelCreate; urKernelCreateWithNativeHandle; urKernelGetGroupInfo; @@ -397,9 +401,13 @@ urPrintImageFormat; urPrintImageInfo; urPrintIpcCloseMemHandleExpParams; + urPrintIpcClosePhysMemHandleExpParams; urPrintIpcGetMemHandleExpParams; + urPrintIpcGetPhysMemHandleExpParams; urPrintIpcOpenMemHandleExpParams; + urPrintIpcOpenPhysMemHandleExpParams; urPrintIpcPutMemHandleExpParams; + urPrintIpcPutPhysMemHandleExpParams; urPrintKernelArgLocalProperties; urPrintKernelArgMemObjProperties; urPrintKernelArgPointerProperties; diff --git a/unified-runtime/source/loader/ur_ldrddi.cpp b/unified-runtime/source/loader/ur_ldrddi.cpp index a68a92742892e..d522e58ee53e0 100644 --- a/unified-runtime/source/loader/ur_ldrddi.cpp +++ b/unified-runtime/source/loader/ur_ldrddi.cpp @@ -4643,6 +4643,92 @@ __urdlllocal ur_result_t UR_APICALL urIPCCloseMemHandleExp( return pfnCloseMemHandleExp(hContext, pMem); } +/////////////////////////////////////////////////////////////////////////////// +/// @brief Intercept function for urIPCGetPhysMemHandleExp +__urdlllocal ur_result_t UR_APICALL urIPCGetPhysMemHandleExp( + /// [in] handle of the context object + ur_context_handle_t hContext, + /// [in] handle of the physical memory object + ur_physical_mem_handle_t hPhysMem, + /// [out] a pointer to the IPC physical memory handle data + void **ppIPCPhysMemHandleData, + /// [out] size of the resulting IPC physical memory handle data + size_t *pIPCPhysMemHandleDataSizeRet) { + + auto *dditable = *reinterpret_cast(hContext); + + auto *pfnGetPhysMemHandleExp = dditable->IPCExp.pfnGetPhysMemHandleExp; + if (nullptr == pfnGetPhysMemHandleExp) + return UR_RESULT_ERROR_UNINITIALIZED; + + // forward to device-platform + return pfnGetPhysMemHandleExp(hContext, hPhysMem, ppIPCPhysMemHandleData, + pIPCPhysMemHandleDataSizeRet); +} + +/////////////////////////////////////////////////////////////////////////////// +/// @brief Intercept function for urIPCPutPhysMemHandleExp +__urdlllocal ur_result_t UR_APICALL urIPCPutPhysMemHandleExp( + /// [in] handle of the context object + ur_context_handle_t hContext, + /// [in] a pointer to the IPC physical memory handle data obtained with + /// urIPCGetPhysMemHandleExp + const void *pIPCPhysMemHandleData) { + + auto *dditable = *reinterpret_cast(hContext); + + auto *pfnPutPhysMemHandleExp = dditable->IPCExp.pfnPutPhysMemHandleExp; + if (nullptr == pfnPutPhysMemHandleExp) + return UR_RESULT_ERROR_UNINITIALIZED; + + // forward to device-platform + return pfnPutPhysMemHandleExp(hContext, pIPCPhysMemHandleData); +} + +/////////////////////////////////////////////////////////////////////////////// +/// @brief Intercept function for urIPCOpenPhysMemHandleExp +__urdlllocal ur_result_t UR_APICALL urIPCOpenPhysMemHandleExp( + /// [in] handle of the context object + ur_context_handle_t hContext, + /// [in] handle of the device object the physical memory was allocated on + ur_device_handle_t hDevice, + /// [in] the IPC physical memory handle data obtained with + /// urIPCGetPhysMemHandleExp + const void *pIPCPhysMemHandleData, + /// [in] size of the IPC physical memory handle data + size_t ipcPhysMemHandleDataSize, + /// [out] pointer to the physical memory handle + ur_physical_mem_handle_t *phPhysMem) { + + auto *dditable = *reinterpret_cast(hContext); + + auto *pfnOpenPhysMemHandleExp = dditable->IPCExp.pfnOpenPhysMemHandleExp; + if (nullptr == pfnOpenPhysMemHandleExp) + return UR_RESULT_ERROR_UNINITIALIZED; + + // forward to device-platform + return pfnOpenPhysMemHandleExp(hContext, hDevice, pIPCPhysMemHandleData, + ipcPhysMemHandleDataSize, phPhysMem); +} + +/////////////////////////////////////////////////////////////////////////////// +/// @brief Intercept function for urIPCClosePhysMemHandleExp +__urdlllocal ur_result_t UR_APICALL urIPCClosePhysMemHandleExp( + /// [in] handle of the context object + ur_context_handle_t hContext, + /// [in] physical memory handle opened through urIPCOpenPhysMemHandleExp + ur_physical_mem_handle_t hPhysMem) { + + auto *dditable = *reinterpret_cast(hContext); + + auto *pfnClosePhysMemHandleExp = dditable->IPCExp.pfnClosePhysMemHandleExp; + if (nullptr == pfnClosePhysMemHandleExp) + return UR_RESULT_ERROR_UNINITIALIZED; + + // forward to device-platform + return pfnClosePhysMemHandleExp(hContext, hPhysMem); +} + /////////////////////////////////////////////////////////////////////////////// /// @brief Intercept function for urMemoryExportAllocExportableMemoryExp __urdlllocal ur_result_t UR_APICALL urMemoryExportAllocExportableMemoryExp( @@ -6899,6 +6985,11 @@ UR_DLLEXPORT ur_result_t UR_APICALL urGetIPCExpProcAddrTable( pDdiTable->pfnPutMemHandleExp = ur_loader::urIPCPutMemHandleExp; pDdiTable->pfnOpenMemHandleExp = ur_loader::urIPCOpenMemHandleExp; pDdiTable->pfnCloseMemHandleExp = ur_loader::urIPCCloseMemHandleExp; + pDdiTable->pfnGetPhysMemHandleExp = ur_loader::urIPCGetPhysMemHandleExp; + pDdiTable->pfnPutPhysMemHandleExp = ur_loader::urIPCPutPhysMemHandleExp; + pDdiTable->pfnOpenPhysMemHandleExp = ur_loader::urIPCOpenPhysMemHandleExp; + pDdiTable->pfnClosePhysMemHandleExp = + ur_loader::urIPCClosePhysMemHandleExp; } else { // return pointers directly to platform's DDIs *pDdiTable = ur_loader::getContext()->platforms.front().dditable.IPCExp; diff --git a/unified-runtime/source/loader/ur_libapi.cpp b/unified-runtime/source/loader/ur_libapi.cpp index 1188978674036..2d9f2fa83e21c 100644 --- a/unified-runtime/source/loader/ur_libapi.cpp +++ b/unified-runtime/source/loader/ur_libapi.cpp @@ -3066,7 +3066,7 @@ ur_result_t UR_APICALL urPhysicalMemRelease( /// - ::UR_RESULT_ERROR_INVALID_NULL_HANDLE /// + `NULL == hPhysicalMem` /// - ::UR_RESULT_ERROR_INVALID_ENUMERATION -/// + `::UR_PHYSICAL_MEM_INFO_REFERENCE_COUNT < propName` +/// + `::UR_PHYSICAL_MEM_INFO_IPC_VIRTUAL_ADDRESS < propName` ur_result_t UR_APICALL urPhysicalMemGetInfo( /// [in] handle of the physical memory object to query. ur_physical_mem_handle_t hPhysicalMem, @@ -8640,6 +8640,151 @@ ur_result_t UR_APICALL urIPCCloseMemHandleExp( return exceptionToResult(std::current_exception()); } +/////////////////////////////////////////////////////////////////////////////// +/// @brief Gets an inter-process handle for a physical memory object +/// +/// @returns +/// - ::UR_RESULT_SUCCESS +/// - ::UR_RESULT_ERROR_UNINITIALIZED +/// - ::UR_RESULT_ERROR_DEVICE_LOST +/// - ::UR_RESULT_ERROR_ADAPTER_SPECIFIC +/// - ::UR_RESULT_ERROR_INVALID_NULL_HANDLE +/// + `NULL == hContext` +/// + `NULL == hPhysMem` +/// - ::UR_RESULT_ERROR_INVALID_NULL_POINTER +/// + `NULL == ppIPCPhysMemHandleData` +/// + `NULL == pIPCPhysMemHandleDataSizeRet` +/// - ::UR_RESULT_ERROR_INVALID_CONTEXT +/// - ::UR_RESULT_ERROR_INVALID_ARGUMENT +/// + `hPhysMem` was not created with the +/// `::UR_PHYSICAL_MEM_FLAG_ENABLE_IPC` flag +/// - ::UR_RESULT_ERROR_OUT_OF_HOST_MEMORY +/// - ::UR_RESULT_ERROR_OUT_OF_RESOURCES +ur_result_t UR_APICALL urIPCGetPhysMemHandleExp( + /// [in] handle of the context object + ur_context_handle_t hContext, + /// [in] handle of the physical memory object + ur_physical_mem_handle_t hPhysMem, + /// [out] a pointer to the IPC physical memory handle data + void **ppIPCPhysMemHandleData, + /// [out] size of the resulting IPC physical memory handle data + size_t *pIPCPhysMemHandleDataSizeRet) try { + auto pfnGetPhysMemHandleExp = + ur_lib::getContext()->urDdiTable.IPCExp.pfnGetPhysMemHandleExp; + if (nullptr == pfnGetPhysMemHandleExp) + return UR_RESULT_ERROR_UNINITIALIZED; + + return pfnGetPhysMemHandleExp(hContext, hPhysMem, ppIPCPhysMemHandleData, + pIPCPhysMemHandleDataSizeRet); +} catch (...) { + return exceptionToResult(std::current_exception()); +} + +/////////////////////////////////////////////////////////////////////////////// +/// @brief Releases an inter-process physical memory handle +/// +/// @returns +/// - ::UR_RESULT_SUCCESS +/// - ::UR_RESULT_ERROR_UNINITIALIZED +/// - ::UR_RESULT_ERROR_DEVICE_LOST +/// - ::UR_RESULT_ERROR_ADAPTER_SPECIFIC +/// - ::UR_RESULT_ERROR_INVALID_NULL_HANDLE +/// + `NULL == hContext` +/// - ::UR_RESULT_ERROR_INVALID_NULL_POINTER +/// + `NULL == pIPCPhysMemHandleData` +/// - ::UR_RESULT_ERROR_INVALID_CONTEXT +/// - ::UR_RESULT_ERROR_OUT_OF_HOST_MEMORY +/// - ::UR_RESULT_ERROR_OUT_OF_RESOURCES +ur_result_t UR_APICALL urIPCPutPhysMemHandleExp( + /// [in] handle of the context object + ur_context_handle_t hContext, + /// [in] a pointer to the IPC physical memory handle data obtained with + /// urIPCGetPhysMemHandleExp + const void *pIPCPhysMemHandleData) try { + auto pfnPutPhysMemHandleExp = + ur_lib::getContext()->urDdiTable.IPCExp.pfnPutPhysMemHandleExp; + if (nullptr == pfnPutPhysMemHandleExp) + return UR_RESULT_ERROR_UNINITIALIZED; + + return pfnPutPhysMemHandleExp(hContext, pIPCPhysMemHandleData); +} catch (...) { + return exceptionToResult(std::current_exception()); +} + +/////////////////////////////////////////////////////////////////////////////// +/// @brief Opens an inter-process physical memory handle to get the +/// corresponding +/// physical memory object +/// +/// @returns +/// - ::UR_RESULT_SUCCESS +/// - ::UR_RESULT_ERROR_UNINITIALIZED +/// - ::UR_RESULT_ERROR_DEVICE_LOST +/// - ::UR_RESULT_ERROR_ADAPTER_SPECIFIC +/// - ::UR_RESULT_ERROR_INVALID_NULL_HANDLE +/// + `NULL == hContext` +/// + `NULL == hDevice` +/// - ::UR_RESULT_ERROR_INVALID_NULL_POINTER +/// + `NULL == phPhysMem` +/// + `NULL == pIPCPhysMemHandleData` +/// - ::UR_RESULT_ERROR_INVALID_CONTEXT +/// - ::UR_RESULT_ERROR_INVALID_VALUE +/// + ipcPhysMemHandleDataSize is not the same as the size of IPC +/// physical memory handle data +/// - ::UR_RESULT_ERROR_OUT_OF_HOST_MEMORY +/// - ::UR_RESULT_ERROR_OUT_OF_RESOURCES +ur_result_t UR_APICALL urIPCOpenPhysMemHandleExp( + /// [in] handle of the context object + ur_context_handle_t hContext, + /// [in] handle of the device object the physical memory was allocated on + ur_device_handle_t hDevice, + /// [in] the IPC physical memory handle data obtained with + /// urIPCGetPhysMemHandleExp + const void *pIPCPhysMemHandleData, + /// [in] size of the IPC physical memory handle data + size_t ipcPhysMemHandleDataSize, + /// [out] pointer to the physical memory handle + ur_physical_mem_handle_t *phPhysMem) try { + auto pfnOpenPhysMemHandleExp = + ur_lib::getContext()->urDdiTable.IPCExp.pfnOpenPhysMemHandleExp; + if (nullptr == pfnOpenPhysMemHandleExp) + return UR_RESULT_ERROR_UNINITIALIZED; + + return pfnOpenPhysMemHandleExp(hContext, hDevice, pIPCPhysMemHandleData, + ipcPhysMemHandleDataSize, phPhysMem); +} catch (...) { + return exceptionToResult(std::current_exception()); +} + +/////////////////////////////////////////////////////////////////////////////// +/// @brief Closes an inter-process physical memory handle +/// +/// @returns +/// - ::UR_RESULT_SUCCESS +/// - ::UR_RESULT_ERROR_UNINITIALIZED +/// - ::UR_RESULT_ERROR_DEVICE_LOST +/// - ::UR_RESULT_ERROR_ADAPTER_SPECIFIC +/// - ::UR_RESULT_ERROR_INVALID_NULL_HANDLE +/// + `NULL == hContext` +/// + `NULL == hPhysMem` +/// - ::UR_RESULT_ERROR_INVALID_CONTEXT +/// - ::UR_RESULT_ERROR_OUT_OF_HOST_MEMORY +/// - ::UR_RESULT_ERROR_OUT_OF_RESOURCES +ur_result_t UR_APICALL urIPCClosePhysMemHandleExp( + /// [in] handle of the context object + ur_context_handle_t hContext, + /// [in] physical memory handle opened through urIPCOpenPhysMemHandleExp + ur_physical_mem_handle_t hPhysMem) try { + auto pfnClosePhysMemHandleExp = + ur_lib::getContext()->urDdiTable.IPCExp.pfnClosePhysMemHandleExp; + if (nullptr == pfnClosePhysMemHandleExp) + return UR_RESULT_ERROR_UNINITIALIZED; + + return pfnClosePhysMemHandleExp(hContext, hPhysMem); +} catch (...) { + return exceptionToResult(std::current_exception()); +} + /////////////////////////////////////////////////////////////////////////////// /// @brief Allocate an exportable memory region and return a pointer to that /// allocation. diff --git a/unified-runtime/source/loader/ur_print.cpp b/unified-runtime/source/loader/ur_print.cpp index 8c56e882cde85..1a748a44673f1 100644 --- a/unified-runtime/source/loader/ur_print.cpp +++ b/unified-runtime/source/loader/ur_print.cpp @@ -2203,6 +2203,38 @@ ur_result_t urPrintIpcCloseMemHandleExpParams( return str_copy(&ss, buffer, buff_size, out_size); } +ur_result_t urPrintIpcGetPhysMemHandleExpParams( + const struct ur_ipc_get_phys_mem_handle_exp_params_t *params, char *buffer, + const size_t buff_size, size_t *out_size) { + std::stringstream ss; + ss << params; + return str_copy(&ss, buffer, buff_size, out_size); +} + +ur_result_t urPrintIpcPutPhysMemHandleExpParams( + const struct ur_ipc_put_phys_mem_handle_exp_params_t *params, char *buffer, + const size_t buff_size, size_t *out_size) { + std::stringstream ss; + ss << params; + return str_copy(&ss, buffer, buff_size, out_size); +} + +ur_result_t urPrintIpcOpenPhysMemHandleExpParams( + const struct ur_ipc_open_phys_mem_handle_exp_params_t *params, char *buffer, + const size_t buff_size, size_t *out_size) { + std::stringstream ss; + ss << params; + return str_copy(&ss, buffer, buff_size, out_size); +} + +ur_result_t urPrintIpcClosePhysMemHandleExpParams( + const struct ur_ipc_close_phys_mem_handle_exp_params_t *params, + char *buffer, const size_t buff_size, size_t *out_size) { + std::stringstream ss; + ss << params; + return str_copy(&ss, buffer, buff_size, out_size); +} + ur_result_t urPrintKernelCreateParams(const struct ur_kernel_create_params_t *params, char *buffer, const size_t buff_size, diff --git a/unified-runtime/source/ur_api.cpp b/unified-runtime/source/ur_api.cpp index d6b7be91aa80d..ecb7e4d6ea98f 100644 --- a/unified-runtime/source/ur_api.cpp +++ b/unified-runtime/source/ur_api.cpp @@ -2698,7 +2698,7 @@ ur_result_t UR_APICALL urPhysicalMemRelease( /// - ::UR_RESULT_ERROR_INVALID_NULL_HANDLE /// + `NULL == hPhysicalMem` /// - ::UR_RESULT_ERROR_INVALID_ENUMERATION -/// + `::UR_PHYSICAL_MEM_INFO_REFERENCE_COUNT < propName` +/// + `::UR_PHYSICAL_MEM_INFO_IPC_VIRTUAL_ADDRESS < propName` ur_result_t UR_APICALL urPhysicalMemGetInfo( /// [in] handle of the physical memory object to query. ur_physical_mem_handle_t hPhysicalMem, @@ -7522,6 +7522,125 @@ ur_result_t UR_APICALL urIPCCloseMemHandleExp( return result; } +/////////////////////////////////////////////////////////////////////////////// +/// @brief Gets an inter-process handle for a physical memory object +/// +/// @returns +/// - ::UR_RESULT_SUCCESS +/// - ::UR_RESULT_ERROR_UNINITIALIZED +/// - ::UR_RESULT_ERROR_DEVICE_LOST +/// - ::UR_RESULT_ERROR_ADAPTER_SPECIFIC +/// - ::UR_RESULT_ERROR_INVALID_NULL_HANDLE +/// + `NULL == hContext` +/// + `NULL == hPhysMem` +/// - ::UR_RESULT_ERROR_INVALID_NULL_POINTER +/// + `NULL == ppIPCPhysMemHandleData` +/// + `NULL == pIPCPhysMemHandleDataSizeRet` +/// - ::UR_RESULT_ERROR_INVALID_CONTEXT +/// - ::UR_RESULT_ERROR_INVALID_ARGUMENT +/// + `hPhysMem` was not created with the +/// `::UR_PHYSICAL_MEM_FLAG_ENABLE_IPC` flag +/// - ::UR_RESULT_ERROR_OUT_OF_HOST_MEMORY +/// - ::UR_RESULT_ERROR_OUT_OF_RESOURCES +ur_result_t UR_APICALL urIPCGetPhysMemHandleExp( + /// [in] handle of the context object + ur_context_handle_t hContext, + /// [in] handle of the physical memory object + ur_physical_mem_handle_t hPhysMem, + /// [out] a pointer to the IPC physical memory handle data + void **ppIPCPhysMemHandleData, + /// [out] size of the resulting IPC physical memory handle data + size_t *pIPCPhysMemHandleDataSizeRet) { + ur_result_t result = UR_RESULT_SUCCESS; + return result; +} + +/////////////////////////////////////////////////////////////////////////////// +/// @brief Releases an inter-process physical memory handle +/// +/// @returns +/// - ::UR_RESULT_SUCCESS +/// - ::UR_RESULT_ERROR_UNINITIALIZED +/// - ::UR_RESULT_ERROR_DEVICE_LOST +/// - ::UR_RESULT_ERROR_ADAPTER_SPECIFIC +/// - ::UR_RESULT_ERROR_INVALID_NULL_HANDLE +/// + `NULL == hContext` +/// - ::UR_RESULT_ERROR_INVALID_NULL_POINTER +/// + `NULL == pIPCPhysMemHandleData` +/// - ::UR_RESULT_ERROR_INVALID_CONTEXT +/// - ::UR_RESULT_ERROR_OUT_OF_HOST_MEMORY +/// - ::UR_RESULT_ERROR_OUT_OF_RESOURCES +ur_result_t UR_APICALL urIPCPutPhysMemHandleExp( + /// [in] handle of the context object + ur_context_handle_t hContext, + /// [in] a pointer to the IPC physical memory handle data obtained with + /// urIPCGetPhysMemHandleExp + const void *pIPCPhysMemHandleData) { + ur_result_t result = UR_RESULT_SUCCESS; + return result; +} + +/////////////////////////////////////////////////////////////////////////////// +/// @brief Opens an inter-process physical memory handle to get the +/// corresponding +/// physical memory object +/// +/// @returns +/// - ::UR_RESULT_SUCCESS +/// - ::UR_RESULT_ERROR_UNINITIALIZED +/// - ::UR_RESULT_ERROR_DEVICE_LOST +/// - ::UR_RESULT_ERROR_ADAPTER_SPECIFIC +/// - ::UR_RESULT_ERROR_INVALID_NULL_HANDLE +/// + `NULL == hContext` +/// + `NULL == hDevice` +/// - ::UR_RESULT_ERROR_INVALID_NULL_POINTER +/// + `NULL == phPhysMem` +/// + `NULL == pIPCPhysMemHandleData` +/// - ::UR_RESULT_ERROR_INVALID_CONTEXT +/// - ::UR_RESULT_ERROR_INVALID_VALUE +/// + ipcPhysMemHandleDataSize is not the same as the size of IPC +/// physical memory handle data +/// - ::UR_RESULT_ERROR_OUT_OF_HOST_MEMORY +/// - ::UR_RESULT_ERROR_OUT_OF_RESOURCES +ur_result_t UR_APICALL urIPCOpenPhysMemHandleExp( + /// [in] handle of the context object + ur_context_handle_t hContext, + /// [in] handle of the device object the physical memory was allocated on + ur_device_handle_t hDevice, + /// [in] the IPC physical memory handle data obtained with + /// urIPCGetPhysMemHandleExp + const void *pIPCPhysMemHandleData, + /// [in] size of the IPC physical memory handle data + size_t ipcPhysMemHandleDataSize, + /// [out] pointer to the physical memory handle + ur_physical_mem_handle_t *phPhysMem) { + ur_result_t result = UR_RESULT_SUCCESS; + return result; +} + +/////////////////////////////////////////////////////////////////////////////// +/// @brief Closes an inter-process physical memory handle +/// +/// @returns +/// - ::UR_RESULT_SUCCESS +/// - ::UR_RESULT_ERROR_UNINITIALIZED +/// - ::UR_RESULT_ERROR_DEVICE_LOST +/// - ::UR_RESULT_ERROR_ADAPTER_SPECIFIC +/// - ::UR_RESULT_ERROR_INVALID_NULL_HANDLE +/// + `NULL == hContext` +/// + `NULL == hPhysMem` +/// - ::UR_RESULT_ERROR_INVALID_CONTEXT +/// - ::UR_RESULT_ERROR_OUT_OF_HOST_MEMORY +/// - ::UR_RESULT_ERROR_OUT_OF_RESOURCES +ur_result_t UR_APICALL urIPCClosePhysMemHandleExp( + /// [in] handle of the context object + ur_context_handle_t hContext, + /// [in] physical memory handle opened through urIPCOpenPhysMemHandleExp + ur_physical_mem_handle_t hPhysMem) { + ur_result_t result = UR_RESULT_SUCCESS; + return result; +} + /////////////////////////////////////////////////////////////////////////////// /// @brief Allocate an exportable memory region and return a pointer to that /// allocation. diff --git a/unified-runtime/test/conformance/virtual_memory/CMakeLists.txt b/unified-runtime/test/conformance/virtual_memory/CMakeLists.txt index 65c6c03108f15..dc79f146e5053 100644 --- a/unified-runtime/test/conformance/virtual_memory/CMakeLists.txt +++ b/unified-runtime/test/conformance/virtual_memory/CMakeLists.txt @@ -14,4 +14,8 @@ add_conformance_devices_test(virtual_memory urVirtualMemReserve.cpp urVirtualMemSetAccess.cpp urVirtualMemUnmap.cpp + urIPCGetPhysMemHandleExp.cpp + urIPCPutPhysMemHandleExp.cpp + urIPCOpenPhysMemHandleExp.cpp + urIPCClosePhysMemHandleExp.cpp ) diff --git a/unified-runtime/test/conformance/virtual_memory/urIPCClosePhysMemHandleExp.cpp b/unified-runtime/test/conformance/virtual_memory/urIPCClosePhysMemHandleExp.cpp new file mode 100644 index 0000000000000..4ba5783cdcf08 --- /dev/null +++ b/unified-runtime/test/conformance/virtual_memory/urIPCClosePhysMemHandleExp.cpp @@ -0,0 +1,25 @@ +// Part of the LLVM Project, under the Apache License v2.0 with LLVM +// Exceptions. See https://llvm.org/LICENSE.txt for license information. +// +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +#include "urIPCPhysMemHandleExpFixtures.hpp" + +using urIPCClosePhysMemHandleExpTest = urIPCOpenedPhysMemTest; +UUR_INSTANTIATE_DEVICE_TEST_SUITE(urIPCClosePhysMemHandleExpTest); + +TEST_P(urIPCClosePhysMemHandleExpTest, Success) { + ASSERT_SUCCESS(urIPCClosePhysMemHandleExp(context, opened_physical_mem)); + // Prevent TearDown from attempting a second close. + opened_physical_mem = nullptr; +} + +TEST_P(urIPCClosePhysMemHandleExpTest, InvalidNullHandleContext) { + ASSERT_EQ_RESULT(urIPCClosePhysMemHandleExp(nullptr, opened_physical_mem), + UR_RESULT_ERROR_INVALID_NULL_HANDLE); +} + +TEST_P(urIPCClosePhysMemHandleExpTest, InvalidNullHandlePhysMem) { + ASSERT_EQ_RESULT(urIPCClosePhysMemHandleExp(context, nullptr), + UR_RESULT_ERROR_INVALID_NULL_HANDLE); +} diff --git a/unified-runtime/test/conformance/virtual_memory/urIPCGetPhysMemHandleExp.cpp b/unified-runtime/test/conformance/virtual_memory/urIPCGetPhysMemHandleExp.cpp new file mode 100644 index 0000000000000..22bad3ea36f94 --- /dev/null +++ b/unified-runtime/test/conformance/virtual_memory/urIPCGetPhysMemHandleExp.cpp @@ -0,0 +1,99 @@ +// Part of the LLVM Project, under the Apache License v2.0 with LLVM +// Exceptions. See https://llvm.org/LICENSE.txt for license information. +// +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +#include "urIPCPhysMemHandleExpFixtures.hpp" + +using urIPCGetPhysMemHandleExpTest = urIPCPhysMemTest; +UUR_INSTANTIATE_DEVICE_TEST_SUITE(urIPCGetPhysMemHandleExpTest); + +TEST_P(urIPCGetPhysMemHandleExpTest, Success) { + void *ipc_handle_data = nullptr; + size_t ipc_handle_size = 0; + ur_result_t res = urIPCGetPhysMemHandleExp( + context, physical_mem, &ipc_handle_data, &ipc_handle_size); + if (res == UR_RESULT_ERROR_UNSUPPORTED_FEATURE) { + GTEST_SKIP() << "IPC physical memory handle export is not supported."; + } + ASSERT_SUCCESS(res); + ASSERT_NE(ipc_handle_data, nullptr); + ASSERT_NE(ipc_handle_size, 0U); + ASSERT_SUCCESS(urIPCPutPhysMemHandleExp(context, ipc_handle_data)); +} + +TEST_P(urIPCGetPhysMemHandleExpTest, InvalidNullHandleContext) { + void *ipc_handle_data = nullptr; + size_t ipc_handle_size = 0; + ASSERT_EQ_RESULT(urIPCGetPhysMemHandleExp(nullptr, physical_mem, + &ipc_handle_data, &ipc_handle_size), + UR_RESULT_ERROR_INVALID_NULL_HANDLE); +} + +TEST_P(urIPCGetPhysMemHandleExpTest, InvalidNullHandlePhysMem) { + void *ipc_handle_data = nullptr; + size_t ipc_handle_size = 0; + ASSERT_EQ_RESULT(urIPCGetPhysMemHandleExp(context, nullptr, &ipc_handle_data, + &ipc_handle_size), + UR_RESULT_ERROR_INVALID_NULL_HANDLE); +} + +TEST_P(urIPCGetPhysMemHandleExpTest, InvalidNullPointerIPCHandleData) { + size_t ipc_handle_size = 0; + ASSERT_EQ_RESULT(urIPCGetPhysMemHandleExp(context, physical_mem, nullptr, + &ipc_handle_size), + UR_RESULT_ERROR_INVALID_NULL_POINTER); +} + +TEST_P(urIPCGetPhysMemHandleExpTest, InvalidNullPointerIPCHandleDataSize) { + void *ipc_handle_data = nullptr; + ASSERT_EQ_RESULT(urIPCGetPhysMemHandleExp(context, physical_mem, + &ipc_handle_data, nullptr), + UR_RESULT_ERROR_INVALID_NULL_POINTER); +} + +// Fixture that creates a physical memory object WITHOUT the IPC-export flag. +struct urIPCGetPhysMemHandleExpNoIpcTest : uur::urVirtualMemGranularityTest { + void SetUp() override { + UUR_RETURN_ON_FATAL_FAILURE(urVirtualMemGranularityTest::SetUp()); + size = granularity * 256; + + ur_bool_t ipc_support = false; + ASSERT_SUCCESS( + urDeviceGetInfo(device, UR_DEVICE_INFO_IPC_PHYSICAL_MEMORY_SUPPORT_EXP, + sizeof(ur_bool_t), &ipc_support, nullptr)); + if (!ipc_support) { + GTEST_SKIP() << "IPC physical memory is not supported."; + } + + // Create physical memory without ENABLE_IPC flag. + ur_result_t res = + urPhysicalMemCreate(context, device, size, nullptr, &physical_mem); + if (res == UR_RESULT_ERROR_UNSUPPORTED_FEATURE) { + GTEST_SKIP() << "Physical memory creation is not supported."; + } + ASSERT_SUCCESS(res); + ASSERT_NE(physical_mem, nullptr); + } + + void TearDown() override { + if (physical_mem) { + EXPECT_SUCCESS(urPhysicalMemRelease(physical_mem)); + } + uur::urVirtualMemGranularityTest::TearDown(); + } + + size_t size = 0; + ur_physical_mem_handle_t physical_mem = nullptr; +}; + +UUR_INSTANTIATE_DEVICE_TEST_SUITE(urIPCGetPhysMemHandleExpNoIpcTest); + +TEST_P(urIPCGetPhysMemHandleExpNoIpcTest, InvalidArgumentNoIpcFlag) { + // Exporting a handle for memory not created with ENABLE_IPC must fail. + void *ipc_handle_data = nullptr; + size_t ipc_handle_size = 0; + ASSERT_EQ_RESULT(urIPCGetPhysMemHandleExp(context, physical_mem, + &ipc_handle_data, &ipc_handle_size), + UR_RESULT_ERROR_INVALID_ARGUMENT); +} diff --git a/unified-runtime/test/conformance/virtual_memory/urIPCOpenPhysMemHandleExp.cpp b/unified-runtime/test/conformance/virtual_memory/urIPCOpenPhysMemHandleExp.cpp new file mode 100644 index 0000000000000..0358f69f1390a --- /dev/null +++ b/unified-runtime/test/conformance/virtual_memory/urIPCOpenPhysMemHandleExp.cpp @@ -0,0 +1,57 @@ +// Part of the LLVM Project, under the Apache License v2.0 with LLVM +// Exceptions. See https://llvm.org/LICENSE.txt for license information. +// +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +#include "urIPCPhysMemHandleExpFixtures.hpp" + +using urIPCOpenPhysMemHandleExpTest = urIPCPhysMemHandleTest; +UUR_INSTANTIATE_DEVICE_TEST_SUITE(urIPCOpenPhysMemHandleExpTest); + +TEST_P(urIPCOpenPhysMemHandleExpTest, Success) { + ur_physical_mem_handle_t opened_physical_mem = nullptr; + ASSERT_SUCCESS(urIPCOpenPhysMemHandleExp( + context, device, ipc_handle_data, ipc_handle_size, &opened_physical_mem)); + ASSERT_NE(opened_physical_mem, nullptr); + ASSERT_SUCCESS(urIPCClosePhysMemHandleExp(context, opened_physical_mem)); +} + +TEST_P(urIPCOpenPhysMemHandleExpTest, InvalidNullHandleContext) { + ur_physical_mem_handle_t opened_physical_mem = nullptr; + ASSERT_EQ_RESULT(urIPCOpenPhysMemHandleExp(nullptr, device, ipc_handle_data, + ipc_handle_size, + &opened_physical_mem), + UR_RESULT_ERROR_INVALID_NULL_HANDLE); +} + +TEST_P(urIPCOpenPhysMemHandleExpTest, InvalidNullHandleDevice) { + ur_physical_mem_handle_t opened_physical_mem = nullptr; + ASSERT_EQ_RESULT(urIPCOpenPhysMemHandleExp(context, nullptr, ipc_handle_data, + ipc_handle_size, + &opened_physical_mem), + UR_RESULT_ERROR_INVALID_NULL_HANDLE); +} + +TEST_P(urIPCOpenPhysMemHandleExpTest, InvalidNullPointerIPCHandleData) { + ur_physical_mem_handle_t opened_physical_mem = nullptr; + ASSERT_EQ_RESULT(urIPCOpenPhysMemHandleExp(context, device, nullptr, + ipc_handle_size, + &opened_physical_mem), + UR_RESULT_ERROR_INVALID_NULL_POINTER); +} + +TEST_P(urIPCOpenPhysMemHandleExpTest, InvalidNullPointerPhysMem) { + ASSERT_EQ_RESULT(urIPCOpenPhysMemHandleExp(context, device, ipc_handle_data, + ipc_handle_size, nullptr), + UR_RESULT_ERROR_INVALID_NULL_POINTER); +} + +TEST_P(urIPCOpenPhysMemHandleExpTest, InvalidValue) { + ur_physical_mem_handle_t opened_physical_mem = nullptr; + // Pass a size that differs from the real handle data size to trigger + // UR_RESULT_ERROR_INVALID_VALUE. + ASSERT_EQ_RESULT(urIPCOpenPhysMemHandleExp(context, device, ipc_handle_data, + ipc_handle_size + 1, + &opened_physical_mem), + UR_RESULT_ERROR_INVALID_VALUE); +} diff --git a/unified-runtime/test/conformance/virtual_memory/urIPCPhysMemHandleExpFixtures.hpp b/unified-runtime/test/conformance/virtual_memory/urIPCPhysMemHandleExpFixtures.hpp new file mode 100644 index 0000000000000..93430e0dacc8e --- /dev/null +++ b/unified-runtime/test/conformance/virtual_memory/urIPCPhysMemHandleExpFixtures.hpp @@ -0,0 +1,92 @@ +// Part of the LLVM Project, under the Apache License v2.0 with LLVM +// Exceptions. See https://llvm.org/LICENSE.txt for license information. +// +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +#pragma once + +#include + +// Fixture that creates a physical memory object with the IPC-export flag. +struct urIPCPhysMemTest : uur::urVirtualMemGranularityTest { + void SetUp() override { + UUR_RETURN_ON_FATAL_FAILURE(urVirtualMemGranularityTest::SetUp()); + size = granularity * 256; + + ur_bool_t ipc_support = false; + ASSERT_SUCCESS( + urDeviceGetInfo(device, UR_DEVICE_INFO_IPC_PHYSICAL_MEMORY_SUPPORT_EXP, + sizeof(ur_bool_t), &ipc_support, nullptr)); + if (!ipc_support) { + GTEST_SKIP() << "IPC physical memory is not supported."; + } + + ur_physical_mem_properties_t properties{ + UR_STRUCTURE_TYPE_PHYSICAL_MEM_PROPERTIES, nullptr, + UR_PHYSICAL_MEM_FLAG_ENABLE_IPC}; + ur_result_t res = + urPhysicalMemCreate(context, device, size, &properties, &physical_mem); + if (res == UR_RESULT_ERROR_UNSUPPORTED_FEATURE) { + GTEST_SKIP() << "IPC physical memory is not supported."; + } + ASSERT_SUCCESS(res); + ASSERT_NE(physical_mem, nullptr); + } + + void TearDown() override { + if (physical_mem) { + EXPECT_SUCCESS(urPhysicalMemRelease(physical_mem)); + } + uur::urVirtualMemGranularityTest::TearDown(); + } + + size_t size = 0; + ur_physical_mem_handle_t physical_mem = nullptr; +}; + +// Fixture that also acquires an IPC handle for the physical memory object. +struct urIPCPhysMemHandleTest : urIPCPhysMemTest { + void SetUp() override { + UUR_RETURN_ON_FATAL_FAILURE(urIPCPhysMemTest::SetUp()); + ur_result_t res = urIPCGetPhysMemHandleExp( + context, physical_mem, &ipc_handle_data, &ipc_handle_size); + if (res == UR_RESULT_ERROR_UNSUPPORTED_FEATURE) { + GTEST_SKIP() << "IPC physical memory handle export is not supported."; + } + ASSERT_SUCCESS(res); + ASSERT_NE(ipc_handle_data, nullptr); + ASSERT_NE(ipc_handle_size, 0U); + } + + void TearDown() override { + if (ipc_handle_data) { + EXPECT_SUCCESS(urIPCPutPhysMemHandleExp(context, ipc_handle_data)); + ipc_handle_data = nullptr; + } + urIPCPhysMemTest::TearDown(); + } + + void *ipc_handle_data = nullptr; + size_t ipc_handle_size = 0; +}; + +// Fixture that also opens the IPC handle to produce a second physical_mem. +struct urIPCOpenedPhysMemTest : urIPCPhysMemHandleTest { + void SetUp() override { + UUR_RETURN_ON_FATAL_FAILURE(urIPCPhysMemHandleTest::SetUp()); + ASSERT_SUCCESS(urIPCOpenPhysMemHandleExp(context, device, ipc_handle_data, + ipc_handle_size, + &opened_physical_mem)); + ASSERT_NE(opened_physical_mem, nullptr); + } + + void TearDown() override { + if (opened_physical_mem) { + EXPECT_SUCCESS(urIPCClosePhysMemHandleExp(context, opened_physical_mem)); + opened_physical_mem = nullptr; + } + urIPCPhysMemHandleTest::TearDown(); + } + + ur_physical_mem_handle_t opened_physical_mem = nullptr; +}; diff --git a/unified-runtime/test/conformance/virtual_memory/urIPCPutPhysMemHandleExp.cpp b/unified-runtime/test/conformance/virtual_memory/urIPCPutPhysMemHandleExp.cpp new file mode 100644 index 0000000000000..7a824b7ab6efb --- /dev/null +++ b/unified-runtime/test/conformance/virtual_memory/urIPCPutPhysMemHandleExp.cpp @@ -0,0 +1,25 @@ +// Part of the LLVM Project, under the Apache License v2.0 with LLVM +// Exceptions. See https://llvm.org/LICENSE.txt for license information. +// +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +#include "urIPCPhysMemHandleExpFixtures.hpp" + +using urIPCPutPhysMemHandleExpTest = urIPCPhysMemHandleTest; +UUR_INSTANTIATE_DEVICE_TEST_SUITE(urIPCPutPhysMemHandleExpTest); + +TEST_P(urIPCPutPhysMemHandleExpTest, Success) { + ASSERT_SUCCESS(urIPCPutPhysMemHandleExp(context, ipc_handle_data)); + // Prevent TearDown from attempting a second put. + ipc_handle_data = nullptr; +} + +TEST_P(urIPCPutPhysMemHandleExpTest, InvalidNullHandleContext) { + ASSERT_EQ_RESULT(urIPCPutPhysMemHandleExp(nullptr, ipc_handle_data), + UR_RESULT_ERROR_INVALID_NULL_HANDLE); +} + +TEST_P(urIPCPutPhysMemHandleExpTest, InvalidNullPointerIPCHandleData) { + ASSERT_EQ_RESULT(urIPCPutPhysMemHandleExp(context, nullptr), + UR_RESULT_ERROR_INVALID_NULL_POINTER); +} diff --git a/unified-runtime/test/conformance/virtual_memory/urPhysicalMemCreate.cpp b/unified-runtime/test/conformance/virtual_memory/urPhysicalMemCreate.cpp index f7c9538d79294..77ef56462f14c 100644 --- a/unified-runtime/test/conformance/virtual_memory/urPhysicalMemCreate.cpp +++ b/unified-runtime/test/conformance/virtual_memory/urPhysicalMemCreate.cpp @@ -55,7 +55,7 @@ using urPhysicalMemCreateWithFlagsParamTest = uur::urPhysicalMemTestWithParam; UUR_DEVICE_TEST_SUITE_WITH_PARAM( urPhysicalMemCreateWithFlagsParamTest, - ::testing::Values(UR_PHYSICAL_MEM_FLAG_TBD), + ::testing::Values(UR_PHYSICAL_MEM_FLAG_ENABLE_IPC), uur::deviceTestWithParamPrinter); TEST_P(urPhysicalMemCreateWithFlagsParamTest, Success) { diff --git a/unified-runtime/tools/urinfo/urinfo.hpp b/unified-runtime/tools/urinfo/urinfo.hpp index d5b15e5aaa00b..fe99cf3ef5f00 100644 --- a/unified-runtime/tools/urinfo/urinfo.hpp +++ b/unified-runtime/tools/urinfo/urinfo.hpp @@ -466,6 +466,9 @@ inline void printDeviceInfos(ur_device_handle_t hDevice, std::cout << prefix; printDeviceInfo(hDevice, UR_DEVICE_INFO_IPC_MEMORY_SUPPORT_EXP); std::cout << prefix; + printDeviceInfo(hDevice, + UR_DEVICE_INFO_IPC_PHYSICAL_MEMORY_SUPPORT_EXP); + std::cout << prefix; printDeviceInfo(hDevice, UR_DEVICE_INFO_ASYNC_USM_ALLOCATIONS_SUPPORT_EXP); std::cout << prefix;