From d98f4991e9118b244633e08e0f5aced256aa4859 Mon Sep 17 00:00:00 2001 From: Lukasz Dorau Date: Wed, 27 May 2026 11:32:08 +0000 Subject: [PATCH 1/5] [UR] Add IPC spec for physical_mem objects Add four new function specs to exp-inter-process-communication.yml and update registry.yml to support IPC operations on physical memory objects: - urIPCGetPhysMemHandleExp: export an IPC handle for a physical_mem - urIPCPutPhysMemHandleExp: release the exported IPC handle - urIPCOpenPhysMemHandleExp: import an IPC handle into a physical_mem - urIPCClosePhysMemHandleExp: close the imported physical_mem Also add ENABLE_IPC flag to ur_physical_mem_flags_t in virtual_memory.yml to allow allocating physical memory that can be shared via IPC handles. Add a new enumerator to ur_physical_mem_info_t that allows querying the virtual address established by zeMemOpenIpcHandle when a physical memory handle is opened via urIPCOpenPhysMemHandleExp. Returns nullptr for non-IPC handles. Signed-off-by: Lukasz Dorau --- .../core/exp-inter-process-communication.yml | 116 ++++++++++++++++++ unified-runtime/scripts/core/registry.yml | 14 ++- .../scripts/core/virtual_memory.yml | 11 +- 3 files changed, 138 insertions(+), 3 deletions(-) diff --git a/unified-runtime/scripts/core/exp-inter-process-communication.yml b/unified-runtime/scripts/core/exp-inter-process-communication.yml index e5019bb40836d..c492228b015e6 100644 --- a/unified-runtime/scripts/core/exp-inter-process-communication.yml +++ b/unified-runtime/scripts/core/exp-inter-process-communication.yml @@ -23,6 +23,16 @@ etors: value: "0x2023" desc: "[$x_bool_t] returns true if the device supports inter-process communicable memory handles" --- #-------------------------------------------------------------------------- +type: enum +extend: true +typed_etors: true +desc: "Extension enums to $x_device_info_t to support inter-process communicable physical memory handles." +name: $x_device_info_t +etors: + - name: IPC_PHYSICAL_MEMORY_SUPPORT_EXP + value: "0x2024" + desc: "[$x_bool_t] returns true if the device supports inter-process communicable physical memory handles" +--- #-------------------------------------------------------------------------- type: function desc: "Gets an inter-process memory handle for a pointer to device USM memory" class: $xIPC @@ -126,3 +136,109 @@ returns: - "`NULL == pMem`" - $X_RESULT_ERROR_OUT_OF_HOST_MEMORY - $X_RESULT_ERROR_OUT_OF_RESOURCES +--- #-------------------------------------------------------------------------- +type: function +desc: "Gets an inter-process handle for a physical memory object" +class: $xIPC +name: GetPhysMemHandleExp +ordinal: "0" +params: + - type: $x_context_handle_t + name: hContext + desc: "[in] handle of the context object" + - type: $x_physical_mem_handle_t + name: hPhysMem + desc: "[in] handle of the physical memory object" + - type: void** + name: ppIPCPhysMemHandleData + desc: "[out] a pointer to the IPC physical memory handle data" + - type: size_t* + name: pIPCPhysMemHandleDataSizeRet + desc: "[out] size of the resulting IPC physical memory handle data" +returns: + - $X_RESULT_ERROR_INVALID_CONTEXT + - $X_RESULT_ERROR_INVALID_NULL_HANDLE: + - "`NULL == hContext`" + - "`NULL == hPhysMem`" + - $X_RESULT_ERROR_INVALID_NULL_POINTER: + - "`NULL == ppIPCPhysMemHandleData`" + - "`NULL == pIPCPhysMemHandleDataSizeRet`" + - $X_RESULT_ERROR_INVALID_ARGUMENT: + - "`hPhysMem` was not created with the `$X_PHYSICAL_MEM_FLAG_ENABLE_IPC` flag" + - $X_RESULT_ERROR_OUT_OF_HOST_MEMORY + - $X_RESULT_ERROR_OUT_OF_RESOURCES +--- #-------------------------------------------------------------------------- +type: function +desc: "Releases an inter-process physical memory handle" +class: $xIPC +name: PutPhysMemHandleExp +ordinal: "0" +params: + - type: $x_context_handle_t + name: hContext + desc: "[in] handle of the context object" + - type: void* + name: pIPCPhysMemHandleData + desc: "[in] a pointer to the IPC physical memory handle data" +returns: + - $X_RESULT_ERROR_INVALID_CONTEXT + - $X_RESULT_ERROR_INVALID_NULL_HANDLE: + - "`NULL == hContext`" + - $X_RESULT_ERROR_INVALID_NULL_POINTER: + - "`NULL == pIPCPhysMemHandleData`" + - $X_RESULT_ERROR_OUT_OF_HOST_MEMORY + - $X_RESULT_ERROR_OUT_OF_RESOURCES +--- #-------------------------------------------------------------------------- +type: function +desc: "Opens an inter-process physical memory handle to get the corresponding physical memory object" +class: $xIPC +name: OpenPhysMemHandleExp +ordinal: "0" +params: + - type: $x_context_handle_t + name: hContext + desc: "[in] handle of the context object" + - type: $x_device_handle_t + name: hDevice + desc: "[in] handle of the device object the physical memory was allocated on" + - type: void * + name: pIPCPhysMemHandleData + desc: "[in] the IPC physical memory handle data" + - type: size_t + name: ipcPhysMemHandleDataSize + desc: "[in] size of the IPC physical memory handle data" + - type: $x_physical_mem_handle_t* + name: phPhysMem + desc: "[out] pointer to the physical memory handle" +returns: + - $X_RESULT_ERROR_INVALID_CONTEXT + - $X_RESULT_ERROR_INVALID_NULL_HANDLE: + - "`NULL == hContext`" + - "`NULL == hDevice`" + - $X_RESULT_ERROR_INVALID_NULL_POINTER: + - "`NULL == pIPCPhysMemHandleData`" + - "`NULL == phPhysMem`" + - $X_RESULT_ERROR_INVALID_VALUE: + - "ipcPhysMemHandleDataSize is not the same as the size of IPC physical memory handle data" + - $X_RESULT_ERROR_OUT_OF_HOST_MEMORY + - $X_RESULT_ERROR_OUT_OF_RESOURCES +--- #-------------------------------------------------------------------------- +type: function +desc: "Closes an inter-process physical memory handle" +class: $xIPC +name: ClosePhysMemHandleExp +ordinal: "0" +params: + - type: $x_context_handle_t + name: hContext + desc: "[in] handle of the context object" + - type: $x_physical_mem_handle_t + name: hPhysMem + desc: "[in] physical memory handle opened through urIPCOpenPhysMemHandleExp" +returns: + - $X_RESULT_ERROR_INVALID_CONTEXT + - $X_RESULT_ERROR_INVALID_NULL_HANDLE: + - "`NULL == hContext`" + - "`NULL == hPhysMem`" + - $X_RESULT_ERROR_OUT_OF_HOST_MEMORY + - $X_RESULT_ERROR_OUT_OF_RESOURCES diff --git a/unified-runtime/scripts/core/registry.yml b/unified-runtime/scripts/core/registry.yml index f7a11b40c6547..3d4f037329ad9 100644 --- a/unified-runtime/scripts/core/registry.yml +++ b/unified-runtime/scripts/core/registry.yml @@ -727,7 +727,19 @@ etors: - name: GRAPH_SET_DESTRUCTION_CALLBACK_EXP desc: Enumerator for $xGraphSetDestructionCallbackExp value: '315' -max_id: '315' +- name: IPC_GET_PHYS_MEM_HANDLE_EXP + desc: Enumerator for $xIPCGetPhysMemHandleExp + value: '316' +- name: IPC_PUT_PHYS_MEM_HANDLE_EXP + desc: Enumerator for $xIPCPutPhysMemHandleExp + value: '317' +- name: IPC_OPEN_PHYS_MEM_HANDLE_EXP + desc: Enumerator for $xIPCOpenPhysMemHandleExp + value: '318' +- name: IPC_CLOSE_PHYS_MEM_HANDLE_EXP + desc: Enumerator for $xIPCClosePhysMemHandleExp + value: '319' +max_id: '319' --- type: enum desc: Defines structure types diff --git a/unified-runtime/scripts/core/virtual_memory.yml b/unified-runtime/scripts/core/virtual_memory.yml index 00d03c54a5726..6bdc6c63c79d7 100644 --- a/unified-runtime/scripts/core/virtual_memory.yml +++ b/unified-runtime/scripts/core/virtual_memory.yml @@ -246,8 +246,9 @@ desc: "Physical memory creation properties." class: $xPhysicalMem name: $x_physical_mem_flags_t etors: - - name: TBD - desc: "reserved for future use." + - name: ENABLE_IPC + value: "$X_BIT(1)" + desc: "allocate physical memory that can be shared via IPC handles." --- #-------------------------------------------------------------------------- type: struct @@ -326,6 +327,12 @@ etors: [uint32_t] Reference count of the physical memory object. The reference count returned should be considered immediately stale. It is unsuitable for general use in applications. This feature is provided for identifying memory leaks. + - name: IPC_VIRTUAL_ADDRESS + desc: | + [void*] For physical memory objects opened from an IPC handle via + $xIPCOpenPhysMemHandleExp, returns the virtual address established + by the driver (i.e. the address returned by zeMemOpenIpcHandle). + Returns nullptr for non-IPC physical memory objects. --- #-------------------------------------------------------------------------- type: function From 33b8d68c890fea45dcb30e079b2faea0944e7a38 Mon Sep 17 00:00:00 2001 From: Lukasz Dorau Date: Wed, 27 May 2026 11:59:11 +0000 Subject: [PATCH 2/5] [UR] Add generated files for IPC physical_mem API Add auto-generated files based on the IPC physical_mem spec changes (urIPCGetPhysMemHandleExp, urIPCPutPhysMemHandleExp, urIPCOpenPhysMemHandleExp, urIPCClosePhysMemHandleExp, UR_PHYSICAL_MEM_FLAG_ENABLE_IPC, UR_DEVICE_INFO_IPC_PHYSICAL_MEMORY_SUPPORT_EXP): - ur_api.h, ur_api_funcs.def, ur_ddi.h: function declarations - ur_print.h/hpp, ur_print.cpp: printing support - ur_api.cpp: API dispatch stubs - ur_ldrddi.cpp, ur_libapi.cpp: loader implementations - ur_trcddi.cpp, ur_valddi.cpp: tracing and validation layers - ur_mockddi.cpp: mock adapter - loader.def.in, loader.map.in: loader symbol exports - level_zero ur_interface_loader.cpp/.hpp: Level-Zero DDI table entries - urinfo.hpp: updated urinfo tool Signed-off-by: Lukasz Dorau --- .../include/unified-runtime/ur_api.h | 170 ++++++++++++++- .../include/unified-runtime/ur_api_funcs.def | 4 + .../include/unified-runtime/ur_ddi.h | 25 +++ .../include/unified-runtime/ur_print.h | 40 ++++ .../include/unified-runtime/ur_print.hpp | 176 ++++++++++++++- .../level_zero/ur_interface_loader.cpp | 6 + .../level_zero/ur_interface_loader.hpp | 13 ++ .../source/adapters/mock/ur_mockddi.cpp | 200 ++++++++++++++++++ .../loader/layers/tracing/ur_trcddi.cpp | 186 ++++++++++++++++ .../loader/layers/validation/ur_valddi.cpp | 186 +++++++++++++++- unified-runtime/source/loader/loader.def.in | 8 + unified-runtime/source/loader/loader.map.in | 8 + unified-runtime/source/loader/ur_ldrddi.cpp | 89 ++++++++ unified-runtime/source/loader/ur_libapi.cpp | 145 ++++++++++++- unified-runtime/source/loader/ur_print.cpp | 32 +++ unified-runtime/source/ur_api.cpp | 119 ++++++++++- unified-runtime/tools/urinfo/urinfo.hpp | 3 + 17 files changed, 1398 insertions(+), 12 deletions(-) diff --git a/unified-runtime/include/unified-runtime/ur_api.h b/unified-runtime/include/unified-runtime/ur_api.h index 798423b6ae55a..f4915c9d5822f 100644 --- a/unified-runtime/include/unified-runtime/ur_api.h +++ b/unified-runtime/include/unified-runtime/ur_api.h @@ -512,6 +512,14 @@ typedef enum ur_function_t { UR_FUNCTION_QUEUE_GET_GRAPH_EXP = 314, /// Enumerator for ::urGraphSetDestructionCallbackExp UR_FUNCTION_GRAPH_SET_DESTRUCTION_CALLBACK_EXP = 315, + /// Enumerator for ::urIPCGetPhysMemHandleExp + UR_FUNCTION_IPC_GET_PHYS_MEM_HANDLE_EXP = 316, + /// Enumerator for ::urIPCPutPhysMemHandleExp + UR_FUNCTION_IPC_PUT_PHYS_MEM_HANDLE_EXP = 317, + /// Enumerator for ::urIPCOpenPhysMemHandleExp + UR_FUNCTION_IPC_OPEN_PHYS_MEM_HANDLE_EXP = 318, + /// Enumerator for ::urIPCClosePhysMemHandleExp + UR_FUNCTION_IPC_CLOSE_PHYS_MEM_HANDLE_EXP = 319, /// @cond UR_FUNCTION_FORCE_UINT32 = 0x7fffffff /// @endcond @@ -2505,6 +2513,9 @@ typedef enum ur_device_info_t { /// [::ur_bool_t] returns true if the device supports inter-process /// communicable memory handles UR_DEVICE_INFO_IPC_MEMORY_SUPPORT_EXP = 0x2023, + /// [::ur_bool_t] returns true if the device supports inter-process + /// communicable physical memory handles + UR_DEVICE_INFO_IPC_PHYSICAL_MEMORY_SUPPORT_EXP = 0x2024, /// [::ur_bool_t] returns true if the device supports enqueueing of /// allocations and frees. UR_DEVICE_INFO_ASYNC_USM_ALLOCATIONS_SUPPORT_EXP = 0x2050, @@ -5336,15 +5347,15 @@ UR_APIEXPORT ur_result_t UR_APICALL urVirtualMemGetInfo( /// @brief Physical memory creation properties. typedef uint32_t ur_physical_mem_flags_t; typedef enum ur_physical_mem_flag_t { - /// reserved for future use. - UR_PHYSICAL_MEM_FLAG_TBD = UR_BIT(0), + /// allocate physical memory that can be shared via IPC handles. + UR_PHYSICAL_MEM_FLAG_ENABLE_IPC = UR_BIT(1), /// @cond UR_PHYSICAL_MEM_FLAG_FORCE_UINT32 = 0x7fffffff /// @endcond } ur_physical_mem_flag_t; /// @brief Bit Mask for validating ur_physical_mem_flags_t -#define UR_PHYSICAL_MEM_FLAGS_MASK 0xfffffffe +#define UR_PHYSICAL_MEM_FLAGS_MASK 0xfffffffd /////////////////////////////////////////////////////////////////////////////// /// @brief Physical memory creation properties. @@ -5438,6 +5449,11 @@ typedef enum ur_physical_mem_info_t { /// It is unsuitable for general use in applications. This feature is /// provided for identifying memory leaks. UR_PHYSICAL_MEM_INFO_REFERENCE_COUNT = 4, + /// [void*] For physical memory objects opened from an IPC handle via + /// ::urIPCOpenPhysMemHandleExp, returns the virtual address established + /// by the driver (i.e. the address returned by zeMemOpenIpcHandle). + /// Returns nullptr for non-IPC physical memory objects. + UR_PHYSICAL_MEM_INFO_IPC_VIRTUAL_ADDRESS = 5, /// @cond UR_PHYSICAL_MEM_INFO_FORCE_UINT32 = 0x7fffffff /// @endcond @@ -5455,7 +5471,7 @@ typedef enum ur_physical_mem_info_t { /// - ::UR_RESULT_ERROR_INVALID_NULL_HANDLE /// + `NULL == hPhysicalMem` /// - ::UR_RESULT_ERROR_INVALID_ENUMERATION -/// + `::UR_PHYSICAL_MEM_INFO_REFERENCE_COUNT < propName` +/// + `::UR_PHYSICAL_MEM_INFO_IPC_VIRTUAL_ADDRESS < propName` UR_APIEXPORT ur_result_t UR_APICALL urPhysicalMemGetInfo( /// [in] handle of the physical memory object to query. ur_physical_mem_handle_t hPhysicalMem, @@ -11242,6 +11258,111 @@ UR_APIEXPORT ur_result_t UR_APICALL urIPCCloseMemHandleExp( /// [in] pointer to device USM memory opened through urIPCOpenMemHandleExp void *pMem); +/////////////////////////////////////////////////////////////////////////////// +/// @brief Gets an inter-process handle for a physical memory object +/// +/// @returns +/// - ::UR_RESULT_SUCCESS +/// - ::UR_RESULT_ERROR_UNINITIALIZED +/// - ::UR_RESULT_ERROR_DEVICE_LOST +/// - ::UR_RESULT_ERROR_ADAPTER_SPECIFIC +/// - ::UR_RESULT_ERROR_INVALID_NULL_HANDLE +/// + `NULL == hContext` +/// + `NULL == hPhysMem` +/// - ::UR_RESULT_ERROR_INVALID_NULL_POINTER +/// + `NULL == ppIPCPhysMemHandleData` +/// + `NULL == pIPCPhysMemHandleDataSizeRet` +/// - ::UR_RESULT_ERROR_INVALID_CONTEXT +/// - ::UR_RESULT_ERROR_INVALID_ARGUMENT +/// + `hPhysMem` was not created with the +/// `::UR_PHYSICAL_MEM_FLAG_ENABLE_IPC` flag +/// - ::UR_RESULT_ERROR_OUT_OF_HOST_MEMORY +/// - ::UR_RESULT_ERROR_OUT_OF_RESOURCES +UR_APIEXPORT ur_result_t UR_APICALL urIPCGetPhysMemHandleExp( + /// [in] handle of the context object + ur_context_handle_t hContext, + /// [in] handle of the physical memory object + ur_physical_mem_handle_t hPhysMem, + /// [out] a pointer to the IPC physical memory handle data + void **ppIPCPhysMemHandleData, + /// [out] size of the resulting IPC physical memory handle data + size_t *pIPCPhysMemHandleDataSizeRet); + +/////////////////////////////////////////////////////////////////////////////// +/// @brief Releases an inter-process physical memory handle +/// +/// @returns +/// - ::UR_RESULT_SUCCESS +/// - ::UR_RESULT_ERROR_UNINITIALIZED +/// - ::UR_RESULT_ERROR_DEVICE_LOST +/// - ::UR_RESULT_ERROR_ADAPTER_SPECIFIC +/// - ::UR_RESULT_ERROR_INVALID_NULL_HANDLE +/// + `NULL == hContext` +/// - ::UR_RESULT_ERROR_INVALID_NULL_POINTER +/// + `NULL == pIPCPhysMemHandleData` +/// - ::UR_RESULT_ERROR_INVALID_CONTEXT +/// - ::UR_RESULT_ERROR_OUT_OF_HOST_MEMORY +/// - ::UR_RESULT_ERROR_OUT_OF_RESOURCES +UR_APIEXPORT ur_result_t UR_APICALL urIPCPutPhysMemHandleExp( + /// [in] handle of the context object + ur_context_handle_t hContext, + /// [in] a pointer to the IPC physical memory handle data + void *pIPCPhysMemHandleData); + +/////////////////////////////////////////////////////////////////////////////// +/// @brief Opens an inter-process physical memory handle to get the +/// corresponding +/// physical memory object +/// +/// @returns +/// - ::UR_RESULT_SUCCESS +/// - ::UR_RESULT_ERROR_UNINITIALIZED +/// - ::UR_RESULT_ERROR_DEVICE_LOST +/// - ::UR_RESULT_ERROR_ADAPTER_SPECIFIC +/// - ::UR_RESULT_ERROR_INVALID_NULL_HANDLE +/// + `NULL == hContext` +/// + `NULL == hDevice` +/// - ::UR_RESULT_ERROR_INVALID_NULL_POINTER +/// + `NULL == phPhysMem` +/// + `NULL == pIPCPhysMemHandleData` +/// - ::UR_RESULT_ERROR_INVALID_CONTEXT +/// - ::UR_RESULT_ERROR_INVALID_VALUE +/// + ipcPhysMemHandleDataSize is not the same as the size of IPC +/// physical memory handle data +/// - ::UR_RESULT_ERROR_OUT_OF_HOST_MEMORY +/// - ::UR_RESULT_ERROR_OUT_OF_RESOURCES +UR_APIEXPORT ur_result_t UR_APICALL urIPCOpenPhysMemHandleExp( + /// [in] handle of the context object + ur_context_handle_t hContext, + /// [in] handle of the device object the physical memory was allocated on + ur_device_handle_t hDevice, + /// [in] the IPC physical memory handle data + void *pIPCPhysMemHandleData, + /// [in] size of the IPC physical memory handle data + size_t ipcPhysMemHandleDataSize, + /// [out] pointer to the physical memory handle + ur_physical_mem_handle_t *phPhysMem); + +/////////////////////////////////////////////////////////////////////////////// +/// @brief Closes an inter-process physical memory handle +/// +/// @returns +/// - ::UR_RESULT_SUCCESS +/// - ::UR_RESULT_ERROR_UNINITIALIZED +/// - ::UR_RESULT_ERROR_DEVICE_LOST +/// - ::UR_RESULT_ERROR_ADAPTER_SPECIFIC +/// - ::UR_RESULT_ERROR_INVALID_NULL_HANDLE +/// + `NULL == hContext` +/// + `NULL == hPhysMem` +/// - ::UR_RESULT_ERROR_INVALID_CONTEXT +/// - ::UR_RESULT_ERROR_OUT_OF_HOST_MEMORY +/// - ::UR_RESULT_ERROR_OUT_OF_RESOURCES +UR_APIEXPORT ur_result_t UR_APICALL urIPCClosePhysMemHandleExp( + /// [in] handle of the context object + ur_context_handle_t hContext, + /// [in] physical memory handle opened through urIPCOpenPhysMemHandleExp + ur_physical_mem_handle_t hPhysMem); + #if !defined(__GNUC__) #pragma endregion #endif @@ -16504,6 +16625,47 @@ typedef struct ur_ipc_close_mem_handle_exp_params_t { void **ppMem; } ur_ipc_close_mem_handle_exp_params_t; +/////////////////////////////////////////////////////////////////////////////// +/// @brief Function parameters for urIPCGetPhysMemHandleExp +/// @details Each entry is a pointer to the parameter passed to the function; +/// allowing the callback the ability to modify the parameter's value +typedef struct ur_ipc_get_phys_mem_handle_exp_params_t { + ur_context_handle_t *phContext; + ur_physical_mem_handle_t *phPhysMem; + void ***pppIPCPhysMemHandleData; + size_t **ppIPCPhysMemHandleDataSizeRet; +} ur_ipc_get_phys_mem_handle_exp_params_t; + +/////////////////////////////////////////////////////////////////////////////// +/// @brief Function parameters for urIPCPutPhysMemHandleExp +/// @details Each entry is a pointer to the parameter passed to the function; +/// allowing the callback the ability to modify the parameter's value +typedef struct ur_ipc_put_phys_mem_handle_exp_params_t { + ur_context_handle_t *phContext; + void **ppIPCPhysMemHandleData; +} ur_ipc_put_phys_mem_handle_exp_params_t; + +/////////////////////////////////////////////////////////////////////////////// +/// @brief Function parameters for urIPCOpenPhysMemHandleExp +/// @details Each entry is a pointer to the parameter passed to the function; +/// allowing the callback the ability to modify the parameter's value +typedef struct ur_ipc_open_phys_mem_handle_exp_params_t { + ur_context_handle_t *phContext; + ur_device_handle_t *phDevice; + void **ppIPCPhysMemHandleData; + size_t *pipcPhysMemHandleDataSize; + ur_physical_mem_handle_t **pphPhysMem; +} ur_ipc_open_phys_mem_handle_exp_params_t; + +/////////////////////////////////////////////////////////////////////////////// +/// @brief Function parameters for urIPCClosePhysMemHandleExp +/// @details Each entry is a pointer to the parameter passed to the function; +/// allowing the callback the ability to modify the parameter's value +typedef struct ur_ipc_close_phys_mem_handle_exp_params_t { + ur_context_handle_t *phContext; + ur_physical_mem_handle_t *phPhysMem; +} ur_ipc_close_phys_mem_handle_exp_params_t; + /////////////////////////////////////////////////////////////////////////////// /// @brief Function parameters for urMemoryExportAllocExportableMemoryExp /// @details Each entry is a pointer to the parameter passed to the function; diff --git a/unified-runtime/include/unified-runtime/ur_api_funcs.def b/unified-runtime/include/unified-runtime/ur_api_funcs.def index fc481d947a8c7..f0d68bcf009f0 100644 --- a/unified-runtime/include/unified-runtime/ur_api_funcs.def +++ b/unified-runtime/include/unified-runtime/ur_api_funcs.def @@ -223,6 +223,10 @@ _UR_API(urIPCGetMemHandleExp) _UR_API(urIPCPutMemHandleExp) _UR_API(urIPCOpenMemHandleExp) _UR_API(urIPCCloseMemHandleExp) +_UR_API(urIPCGetPhysMemHandleExp) +_UR_API(urIPCPutPhysMemHandleExp) +_UR_API(urIPCOpenPhysMemHandleExp) +_UR_API(urIPCClosePhysMemHandleExp) _UR_API(urMemoryExportAllocExportableMemoryExp) _UR_API(urMemoryExportFreeExportableMemoryExp) _UR_API(urMemoryExportExportMemoryHandleExp) diff --git a/unified-runtime/include/unified-runtime/ur_ddi.h b/unified-runtime/include/unified-runtime/ur_ddi.h index cc734e6194d92..6731cd2c88053 100644 --- a/unified-runtime/include/unified-runtime/ur_ddi.h +++ b/unified-runtime/include/unified-runtime/ur_ddi.h @@ -1950,6 +1950,27 @@ typedef ur_result_t(UR_APICALL *ur_pfnIPCOpenMemHandleExp_t)( typedef ur_result_t(UR_APICALL *ur_pfnIPCCloseMemHandleExp_t)( ur_context_handle_t, void *); +/////////////////////////////////////////////////////////////////////////////// +/// @brief Function-pointer for urIPCGetPhysMemHandleExp +typedef ur_result_t(UR_APICALL *ur_pfnIPCGetPhysMemHandleExp_t)( + ur_context_handle_t, ur_physical_mem_handle_t, void **, size_t *); + +/////////////////////////////////////////////////////////////////////////////// +/// @brief Function-pointer for urIPCPutPhysMemHandleExp +typedef ur_result_t(UR_APICALL *ur_pfnIPCPutPhysMemHandleExp_t)( + ur_context_handle_t, void *); + +/////////////////////////////////////////////////////////////////////////////// +/// @brief Function-pointer for urIPCOpenPhysMemHandleExp +typedef ur_result_t(UR_APICALL *ur_pfnIPCOpenPhysMemHandleExp_t)( + ur_context_handle_t, ur_device_handle_t, void *, size_t, + ur_physical_mem_handle_t *); + +/////////////////////////////////////////////////////////////////////////////// +/// @brief Function-pointer for urIPCClosePhysMemHandleExp +typedef ur_result_t(UR_APICALL *ur_pfnIPCClosePhysMemHandleExp_t)( + ur_context_handle_t, ur_physical_mem_handle_t); + /////////////////////////////////////////////////////////////////////////////// /// @brief Table of IPCExp functions pointers typedef struct ur_ipc_exp_dditable_t { @@ -1957,6 +1978,10 @@ typedef struct ur_ipc_exp_dditable_t { ur_pfnIPCPutMemHandleExp_t pfnPutMemHandleExp; ur_pfnIPCOpenMemHandleExp_t pfnOpenMemHandleExp; ur_pfnIPCCloseMemHandleExp_t pfnCloseMemHandleExp; + ur_pfnIPCGetPhysMemHandleExp_t pfnGetPhysMemHandleExp; + ur_pfnIPCPutPhysMemHandleExp_t pfnPutPhysMemHandleExp; + ur_pfnIPCOpenPhysMemHandleExp_t pfnOpenPhysMemHandleExp; + ur_pfnIPCClosePhysMemHandleExp_t pfnClosePhysMemHandleExp; } ur_ipc_exp_dditable_t; /////////////////////////////////////////////////////////////////////////////// diff --git a/unified-runtime/include/unified-runtime/ur_print.h b/unified-runtime/include/unified-runtime/ur_print.h index d9acae65c7d07..70ff14c95a499 100644 --- a/unified-runtime/include/unified-runtime/ur_print.h +++ b/unified-runtime/include/unified-runtime/ur_print.h @@ -3785,6 +3785,46 @@ UR_APIEXPORT ur_result_t UR_APICALL urPrintIpcCloseMemHandleExpParams( const struct ur_ipc_close_mem_handle_exp_params_t *params, char *buffer, const size_t buff_size, size_t *out_size); +/////////////////////////////////////////////////////////////////////////////// +/// @brief Print ur_ipc_get_phys_mem_handle_exp_params_t struct +/// @returns +/// - ::UR_RESULT_SUCCESS +/// - ::UR_RESULT_ERROR_INVALID_SIZE +/// - `buff_size < out_size` +UR_APIEXPORT ur_result_t UR_APICALL urPrintIpcGetPhysMemHandleExpParams( + const struct ur_ipc_get_phys_mem_handle_exp_params_t *params, char *buffer, + const size_t buff_size, size_t *out_size); + +/////////////////////////////////////////////////////////////////////////////// +/// @brief Print ur_ipc_put_phys_mem_handle_exp_params_t struct +/// @returns +/// - ::UR_RESULT_SUCCESS +/// - ::UR_RESULT_ERROR_INVALID_SIZE +/// - `buff_size < out_size` +UR_APIEXPORT ur_result_t UR_APICALL urPrintIpcPutPhysMemHandleExpParams( + const struct ur_ipc_put_phys_mem_handle_exp_params_t *params, char *buffer, + const size_t buff_size, size_t *out_size); + +/////////////////////////////////////////////////////////////////////////////// +/// @brief Print ur_ipc_open_phys_mem_handle_exp_params_t struct +/// @returns +/// - ::UR_RESULT_SUCCESS +/// - ::UR_RESULT_ERROR_INVALID_SIZE +/// - `buff_size < out_size` +UR_APIEXPORT ur_result_t UR_APICALL urPrintIpcOpenPhysMemHandleExpParams( + const struct ur_ipc_open_phys_mem_handle_exp_params_t *params, char *buffer, + const size_t buff_size, size_t *out_size); + +/////////////////////////////////////////////////////////////////////////////// +/// @brief Print ur_ipc_close_phys_mem_handle_exp_params_t struct +/// @returns +/// - ::UR_RESULT_SUCCESS +/// - ::UR_RESULT_ERROR_INVALID_SIZE +/// - `buff_size < out_size` +UR_APIEXPORT ur_result_t UR_APICALL urPrintIpcClosePhysMemHandleExpParams( + const struct ur_ipc_close_phys_mem_handle_exp_params_t *params, + char *buffer, const size_t buff_size, size_t *out_size); + /////////////////////////////////////////////////////////////////////////////// /// @brief Print ur_memory_export_alloc_exportable_memory_exp_params_t struct /// @returns diff --git a/unified-runtime/include/unified-runtime/ur_print.hpp b/unified-runtime/include/unified-runtime/ur_print.hpp index c29373a7d162a..541b9a6a18e63 100644 --- a/unified-runtime/include/unified-runtime/ur_print.hpp +++ b/unified-runtime/include/unified-runtime/ur_print.hpp @@ -1380,6 +1380,18 @@ inline std::ostream &operator<<(std::ostream &os, enum ur_function_t value) { case UR_FUNCTION_GRAPH_SET_DESTRUCTION_CALLBACK_EXP: os << "UR_FUNCTION_GRAPH_SET_DESTRUCTION_CALLBACK_EXP"; break; + case UR_FUNCTION_IPC_GET_PHYS_MEM_HANDLE_EXP: + os << "UR_FUNCTION_IPC_GET_PHYS_MEM_HANDLE_EXP"; + break; + case UR_FUNCTION_IPC_PUT_PHYS_MEM_HANDLE_EXP: + os << "UR_FUNCTION_IPC_PUT_PHYS_MEM_HANDLE_EXP"; + break; + case UR_FUNCTION_IPC_OPEN_PHYS_MEM_HANDLE_EXP: + os << "UR_FUNCTION_IPC_OPEN_PHYS_MEM_HANDLE_EXP"; + break; + case UR_FUNCTION_IPC_CLOSE_PHYS_MEM_HANDLE_EXP: + os << "UR_FUNCTION_IPC_CLOSE_PHYS_MEM_HANDLE_EXP"; + break; default: os << "unknown enumerator"; break; @@ -3309,6 +3321,9 @@ inline std::ostream &operator<<(std::ostream &os, enum ur_device_info_t value) { case UR_DEVICE_INFO_IPC_MEMORY_SUPPORT_EXP: os << "UR_DEVICE_INFO_IPC_MEMORY_SUPPORT_EXP"; break; + case UR_DEVICE_INFO_IPC_PHYSICAL_MEMORY_SUPPORT_EXP: + os << "UR_DEVICE_INFO_IPC_PHYSICAL_MEMORY_SUPPORT_EXP"; + break; case UR_DEVICE_INFO_ASYNC_USM_ALLOCATIONS_SUPPORT_EXP: os << "UR_DEVICE_INFO_ASYNC_USM_ALLOCATIONS_SUPPORT_EXP"; break; @@ -5594,6 +5609,19 @@ inline ur_result_t printTagged(std::ostream &os, const void *ptr, os << ")"; } break; + case UR_DEVICE_INFO_IPC_PHYSICAL_MEMORY_SUPPORT_EXP: { + const ur_bool_t *tptr = (const ur_bool_t *)ptr; + if (sizeof(ur_bool_t) > size) { + os << "invalid size (is: " << size + << ", expected: >=" << sizeof(ur_bool_t) << ")"; + return UR_RESULT_ERROR_INVALID_SIZE; + } + os << (const void *)(tptr) << " ("; + + os << *tptr; + + os << ")"; + } break; case UR_DEVICE_INFO_ASYNC_USM_ALLOCATIONS_SUPPORT_EXP: { const ur_bool_t *tptr = (const ur_bool_t *)ptr; if (sizeof(ur_bool_t) > size) { @@ -9144,8 +9172,8 @@ inline ur_result_t printTagged(std::ostream &os, const void *ptr, inline std::ostream &operator<<(std::ostream &os, enum ur_physical_mem_flag_t value) { switch (value) { - case UR_PHYSICAL_MEM_FLAG_TBD: - os << "UR_PHYSICAL_MEM_FLAG_TBD"; + case UR_PHYSICAL_MEM_FLAG_ENABLE_IPC: + os << "UR_PHYSICAL_MEM_FLAG_ENABLE_IPC"; break; default: os << "unknown enumerator"; @@ -9163,14 +9191,15 @@ inline ur_result_t printFlag(std::ostream &os, uint32_t val = flag; bool first = true; - if ((val & UR_PHYSICAL_MEM_FLAG_TBD) == (uint32_t)UR_PHYSICAL_MEM_FLAG_TBD) { - val ^= (uint32_t)UR_PHYSICAL_MEM_FLAG_TBD; + if ((val & UR_PHYSICAL_MEM_FLAG_ENABLE_IPC) == + (uint32_t)UR_PHYSICAL_MEM_FLAG_ENABLE_IPC) { + val ^= (uint32_t)UR_PHYSICAL_MEM_FLAG_ENABLE_IPC; if (!first) { os << " | "; } else { first = false; } - os << UR_PHYSICAL_MEM_FLAG_TBD; + os << UR_PHYSICAL_MEM_FLAG_ENABLE_IPC; } if (val != 0) { std::bitset<32> bits(val); @@ -9231,6 +9260,9 @@ inline std::ostream &operator<<(std::ostream &os, case UR_PHYSICAL_MEM_INFO_REFERENCE_COUNT: os << "UR_PHYSICAL_MEM_INFO_REFERENCE_COUNT"; break; + case UR_PHYSICAL_MEM_INFO_IPC_VIRTUAL_ADDRESS: + os << "UR_PHYSICAL_MEM_INFO_IPC_VIRTUAL_ADDRESS"; + break; default: os << "unknown enumerator"; break; @@ -9314,6 +9346,19 @@ inline ur_result_t printTagged(std::ostream &os, const void *ptr, os << ")"; } break; + case UR_PHYSICAL_MEM_INFO_IPC_VIRTUAL_ADDRESS: { + const void *const *tptr = (const void *const *)ptr; + if (sizeof(void *) > size) { + os << "invalid size (is: " << size << ", expected: >=" << sizeof(void *) + << ")"; + return UR_RESULT_ERROR_INVALID_SIZE; + } + os << (const void *)(tptr) << " ("; + + ur::details::printPtr(os, *tptr); + + os << ")"; + } break; default: os << "unknown enumerator"; return UR_RESULT_ERROR_INVALID_ENUMERATION; @@ -21724,6 +21769,115 @@ operator<<(std::ostream &os, return os; } +/////////////////////////////////////////////////////////////////////////////// +/// @brief Print operator for the ur_ipc_get_phys_mem_handle_exp_params_t type +/// @returns +/// std::ostream & +inline std::ostream & +operator<<(std::ostream &os, + [[maybe_unused]] const struct ur_ipc_get_phys_mem_handle_exp_params_t + *params) { + + os << ".hContext = "; + + ur::details::printPtr(os, *(params->phContext)); + + os << ", "; + os << ".hPhysMem = "; + + ur::details::printPtr(os, *(params->phPhysMem)); + + os << ", "; + os << ".ppIPCPhysMemHandleData = "; + + ur::details::printPtr(os, *(params->pppIPCPhysMemHandleData)); + + os << ", "; + os << ".pIPCPhysMemHandleDataSizeRet = "; + + ur::details::printPtr(os, *(params->ppIPCPhysMemHandleDataSizeRet)); + + return os; +} + +/////////////////////////////////////////////////////////////////////////////// +/// @brief Print operator for the ur_ipc_put_phys_mem_handle_exp_params_t type +/// @returns +/// std::ostream & +inline std::ostream & +operator<<(std::ostream &os, + [[maybe_unused]] const struct ur_ipc_put_phys_mem_handle_exp_params_t + *params) { + + os << ".hContext = "; + + ur::details::printPtr(os, *(params->phContext)); + + os << ", "; + os << ".pIPCPhysMemHandleData = "; + + ur::details::printPtr(os, *(params->ppIPCPhysMemHandleData)); + + return os; +} + +/////////////////////////////////////////////////////////////////////////////// +/// @brief Print operator for the ur_ipc_open_phys_mem_handle_exp_params_t type +/// @returns +/// std::ostream & +inline std::ostream &operator<<( + std::ostream &os, + [[maybe_unused]] const struct ur_ipc_open_phys_mem_handle_exp_params_t + *params) { + + os << ".hContext = "; + + ur::details::printPtr(os, *(params->phContext)); + + os << ", "; + os << ".hDevice = "; + + ur::details::printPtr(os, *(params->phDevice)); + + os << ", "; + os << ".pIPCPhysMemHandleData = "; + + os << *(params->ppIPCPhysMemHandleData); + + os << ", "; + os << ".ipcPhysMemHandleDataSize = "; + + os << *(params->pipcPhysMemHandleDataSize); + + os << ", "; + os << ".phPhysMem = "; + + ur::details::printPtr(os, *(params->pphPhysMem)); + + return os; +} + +/////////////////////////////////////////////////////////////////////////////// +/// @brief Print operator for the ur_ipc_close_phys_mem_handle_exp_params_t type +/// @returns +/// std::ostream & +inline std::ostream &operator<<( + std::ostream &os, + [[maybe_unused]] const struct ur_ipc_close_phys_mem_handle_exp_params_t + *params) { + + os << ".hContext = "; + + ur::details::printPtr(os, *(params->phContext)); + + os << ", "; + os << ".hPhysMem = "; + + ur::details::printPtr(os, *(params->phPhysMem)); + + return os; +} + /////////////////////////////////////////////////////////////////////////////// /// @brief Print operator for the /// ur_memory_export_alloc_exportable_memory_exp_params_t type @@ -23280,6 +23434,18 @@ inline ur_result_t UR_APICALL printFunctionParams(std::ostream &os, case UR_FUNCTION_IPC_CLOSE_MEM_HANDLE_EXP: { os << (const struct ur_ipc_close_mem_handle_exp_params_t *)params; } break; + case UR_FUNCTION_IPC_GET_PHYS_MEM_HANDLE_EXP: { + os << (const struct ur_ipc_get_phys_mem_handle_exp_params_t *)params; + } break; + case UR_FUNCTION_IPC_PUT_PHYS_MEM_HANDLE_EXP: { + os << (const struct ur_ipc_put_phys_mem_handle_exp_params_t *)params; + } break; + case UR_FUNCTION_IPC_OPEN_PHYS_MEM_HANDLE_EXP: { + os << (const struct ur_ipc_open_phys_mem_handle_exp_params_t *)params; + } break; + case UR_FUNCTION_IPC_CLOSE_PHYS_MEM_HANDLE_EXP: { + os << (const struct ur_ipc_close_phys_mem_handle_exp_params_t *)params; + } break; case UR_FUNCTION_MEMORY_EXPORT_ALLOC_EXPORTABLE_MEMORY_EXP: { os << (const struct ur_memory_export_alloc_exportable_memory_exp_params_t *) params; diff --git a/unified-runtime/source/adapters/level_zero/ur_interface_loader.cpp b/unified-runtime/source/adapters/level_zero/ur_interface_loader.cpp index dc76f7729f937..ce63be24c0a64 100644 --- a/unified-runtime/source/adapters/level_zero/ur_interface_loader.cpp +++ b/unified-runtime/source/adapters/level_zero/ur_interface_loader.cpp @@ -293,6 +293,12 @@ UR_APIEXPORT ur_result_t UR_APICALL urGetIPCExpProcAddrTable( pDdiTable->pfnPutMemHandleExp = ur::level_zero::urIPCPutMemHandleExp; pDdiTable->pfnOpenMemHandleExp = ur::level_zero::urIPCOpenMemHandleExp; pDdiTable->pfnCloseMemHandleExp = ur::level_zero::urIPCCloseMemHandleExp; + pDdiTable->pfnGetPhysMemHandleExp = ur::level_zero::urIPCGetPhysMemHandleExp; + pDdiTable->pfnPutPhysMemHandleExp = ur::level_zero::urIPCPutPhysMemHandleExp; + pDdiTable->pfnOpenPhysMemHandleExp = + ur::level_zero::urIPCOpenPhysMemHandleExp; + pDdiTable->pfnClosePhysMemHandleExp = + ur::level_zero::urIPCClosePhysMemHandleExp; return result; } diff --git a/unified-runtime/source/adapters/level_zero/ur_interface_loader.hpp b/unified-runtime/source/adapters/level_zero/ur_interface_loader.hpp index b56cc5e803c5d..7a0efa06d9595 100644 --- a/unified-runtime/source/adapters/level_zero/ur_interface_loader.hpp +++ b/unified-runtime/source/adapters/level_zero/ur_interface_loader.hpp @@ -629,6 +629,19 @@ ur_result_t urIPCOpenMemHandleExp(ur_context_handle_t hContext, void *pIPCMemHandleData, size_t ipcMemHandleDataSize, void **ppMem); ur_result_t urIPCCloseMemHandleExp(ur_context_handle_t hContext, void *pMem); +ur_result_t urIPCGetPhysMemHandleExp(ur_context_handle_t hContext, + ur_physical_mem_handle_t hPhysMem, + void **ppIPCPhysMemHandleData, + size_t *pIPCPhysMemHandleDataSizeRet); +ur_result_t urIPCPutPhysMemHandleExp(ur_context_handle_t hContext, + void *pIPCPhysMemHandleData); +ur_result_t urIPCOpenPhysMemHandleExp(ur_context_handle_t hContext, + ur_device_handle_t hDevice, + void *pIPCPhysMemHandleData, + size_t ipcPhysMemHandleDataSize, + ur_physical_mem_handle_t *phPhysMem); +ur_result_t urIPCClosePhysMemHandleExp(ur_context_handle_t hContext, + ur_physical_mem_handle_t hPhysMem); ur_result_t urMemoryExportAllocExportableMemoryExp( ur_context_handle_t hContext, ur_device_handle_t hDevice, size_t alignment, size_t size, ur_exp_external_mem_type_t handleTypeToExport, void **ppMem); diff --git a/unified-runtime/source/adapters/mock/ur_mockddi.cpp b/unified-runtime/source/adapters/mock/ur_mockddi.cpp index 195702990d65c..cc77f6eccf203 100644 --- a/unified-runtime/source/adapters/mock/ur_mockddi.cpp +++ b/unified-runtime/source/adapters/mock/ur_mockddi.cpp @@ -9661,6 +9661,198 @@ __urdlllocal ur_result_t UR_APICALL urIPCCloseMemHandleExp( return exceptionToResult(std::current_exception()); } +/////////////////////////////////////////////////////////////////////////////// +/// @brief Intercept function for urIPCGetPhysMemHandleExp +__urdlllocal ur_result_t UR_APICALL urIPCGetPhysMemHandleExp( + /// [in] handle of the context object + ur_context_handle_t hContext, + /// [in] handle of the physical memory object + ur_physical_mem_handle_t hPhysMem, + /// [out] a pointer to the IPC physical memory handle data + void **ppIPCPhysMemHandleData, + /// [out] size of the resulting IPC physical memory handle data + size_t *pIPCPhysMemHandleDataSizeRet) try { + ur_result_t result = UR_RESULT_SUCCESS; + + ur_ipc_get_phys_mem_handle_exp_params_t params = { + &hContext, &hPhysMem, &ppIPCPhysMemHandleData, + &pIPCPhysMemHandleDataSizeRet}; + + auto beforeCallback = reinterpret_cast( + mock::getCallbacks().get_before_callback("urIPCGetPhysMemHandleExp")); + if (beforeCallback) { + result = beforeCallback(¶ms); + if (result != UR_RESULT_SUCCESS) { + return result; + } + } + + auto replaceCallback = reinterpret_cast( + mock::getCallbacks().get_replace_callback("urIPCGetPhysMemHandleExp")); + if (replaceCallback) { + result = replaceCallback(¶ms); + } else { + + result = UR_RESULT_SUCCESS; + } + + if (result != UR_RESULT_SUCCESS) { + return result; + } + + auto afterCallback = reinterpret_cast( + mock::getCallbacks().get_after_callback("urIPCGetPhysMemHandleExp")); + if (afterCallback) { + return afterCallback(¶ms); + } + + return result; +} catch (...) { + return exceptionToResult(std::current_exception()); +} + +/////////////////////////////////////////////////////////////////////////////// +/// @brief Intercept function for urIPCPutPhysMemHandleExp +__urdlllocal ur_result_t UR_APICALL urIPCPutPhysMemHandleExp( + /// [in] handle of the context object + ur_context_handle_t hContext, + /// [in] a pointer to the IPC physical memory handle data + void *pIPCPhysMemHandleData) try { + ur_result_t result = UR_RESULT_SUCCESS; + + ur_ipc_put_phys_mem_handle_exp_params_t params = {&hContext, + &pIPCPhysMemHandleData}; + + auto beforeCallback = reinterpret_cast( + mock::getCallbacks().get_before_callback("urIPCPutPhysMemHandleExp")); + if (beforeCallback) { + result = beforeCallback(¶ms); + if (result != UR_RESULT_SUCCESS) { + return result; + } + } + + auto replaceCallback = reinterpret_cast( + mock::getCallbacks().get_replace_callback("urIPCPutPhysMemHandleExp")); + if (replaceCallback) { + result = replaceCallback(¶ms); + } else { + + result = UR_RESULT_SUCCESS; + } + + if (result != UR_RESULT_SUCCESS) { + return result; + } + + auto afterCallback = reinterpret_cast( + mock::getCallbacks().get_after_callback("urIPCPutPhysMemHandleExp")); + if (afterCallback) { + return afterCallback(¶ms); + } + + return result; +} catch (...) { + return exceptionToResult(std::current_exception()); +} + +/////////////////////////////////////////////////////////////////////////////// +/// @brief Intercept function for urIPCOpenPhysMemHandleExp +__urdlllocal ur_result_t UR_APICALL urIPCOpenPhysMemHandleExp( + /// [in] handle of the context object + ur_context_handle_t hContext, + /// [in] handle of the device object the physical memory was allocated on + ur_device_handle_t hDevice, + /// [in] the IPC physical memory handle data + void *pIPCPhysMemHandleData, + /// [in] size of the IPC physical memory handle data + size_t ipcPhysMemHandleDataSize, + /// [out] pointer to the physical memory handle + ur_physical_mem_handle_t *phPhysMem) try { + ur_result_t result = UR_RESULT_SUCCESS; + + ur_ipc_open_phys_mem_handle_exp_params_t params = { + &hContext, &hDevice, &pIPCPhysMemHandleData, &ipcPhysMemHandleDataSize, + &phPhysMem}; + + auto beforeCallback = reinterpret_cast( + mock::getCallbacks().get_before_callback("urIPCOpenPhysMemHandleExp")); + if (beforeCallback) { + result = beforeCallback(¶ms); + if (result != UR_RESULT_SUCCESS) { + return result; + } + } + + auto replaceCallback = reinterpret_cast( + mock::getCallbacks().get_replace_callback("urIPCOpenPhysMemHandleExp")); + if (replaceCallback) { + result = replaceCallback(¶ms); + } else { + + *phPhysMem = mock::createDummyHandle(); + result = UR_RESULT_SUCCESS; + } + + if (result != UR_RESULT_SUCCESS) { + return result; + } + + auto afterCallback = reinterpret_cast( + mock::getCallbacks().get_after_callback("urIPCOpenPhysMemHandleExp")); + if (afterCallback) { + return afterCallback(¶ms); + } + + return result; +} catch (...) { + return exceptionToResult(std::current_exception()); +} + +/////////////////////////////////////////////////////////////////////////////// +/// @brief Intercept function for urIPCClosePhysMemHandleExp +__urdlllocal ur_result_t UR_APICALL urIPCClosePhysMemHandleExp( + /// [in] handle of the context object + ur_context_handle_t hContext, + /// [in] physical memory handle opened through urIPCOpenPhysMemHandleExp + ur_physical_mem_handle_t hPhysMem) try { + ur_result_t result = UR_RESULT_SUCCESS; + + ur_ipc_close_phys_mem_handle_exp_params_t params = {&hContext, &hPhysMem}; + + auto beforeCallback = reinterpret_cast( + mock::getCallbacks().get_before_callback("urIPCClosePhysMemHandleExp")); + if (beforeCallback) { + result = beforeCallback(¶ms); + if (result != UR_RESULT_SUCCESS) { + return result; + } + } + + auto replaceCallback = reinterpret_cast( + mock::getCallbacks().get_replace_callback("urIPCClosePhysMemHandleExp")); + if (replaceCallback) { + result = replaceCallback(¶ms); + } else { + + result = UR_RESULT_SUCCESS; + } + + if (result != UR_RESULT_SUCCESS) { + return result; + } + + auto afterCallback = reinterpret_cast( + mock::getCallbacks().get_after_callback("urIPCClosePhysMemHandleExp")); + if (afterCallback) { + return afterCallback(¶ms); + } + + return result; +} catch (...) { + return exceptionToResult(std::current_exception()); +} + /////////////////////////////////////////////////////////////////////////////// /// @brief Intercept function for urMemoryExportAllocExportableMemoryExp __urdlllocal ur_result_t UR_APICALL urMemoryExportAllocExportableMemoryExp( @@ -13547,6 +13739,14 @@ UR_DLLEXPORT ur_result_t UR_APICALL urGetIPCExpProcAddrTable( pDdiTable->pfnCloseMemHandleExp = driver::urIPCCloseMemHandleExp; + pDdiTable->pfnGetPhysMemHandleExp = driver::urIPCGetPhysMemHandleExp; + + pDdiTable->pfnPutPhysMemHandleExp = driver::urIPCPutPhysMemHandleExp; + + pDdiTable->pfnOpenPhysMemHandleExp = driver::urIPCOpenPhysMemHandleExp; + + pDdiTable->pfnClosePhysMemHandleExp = driver::urIPCClosePhysMemHandleExp; + return result; } catch (...) { return exceptionToResult(std::current_exception()); diff --git a/unified-runtime/source/loader/layers/tracing/ur_trcddi.cpp b/unified-runtime/source/loader/layers/tracing/ur_trcddi.cpp index 5f8bfca375b61..fe9154b5d882c 100644 --- a/unified-runtime/source/loader/layers/tracing/ur_trcddi.cpp +++ b/unified-runtime/source/loader/layers/tracing/ur_trcddi.cpp @@ -8159,6 +8159,176 @@ __urdlllocal ur_result_t UR_APICALL urIPCCloseMemHandleExp( return result; } +/////////////////////////////////////////////////////////////////////////////// +/// @brief Intercept function for urIPCGetPhysMemHandleExp +__urdlllocal ur_result_t UR_APICALL urIPCGetPhysMemHandleExp( + /// [in] handle of the context object + ur_context_handle_t hContext, + /// [in] handle of the physical memory object + ur_physical_mem_handle_t hPhysMem, + /// [out] a pointer to the IPC physical memory handle data + void **ppIPCPhysMemHandleData, + /// [out] size of the resulting IPC physical memory handle data + size_t *pIPCPhysMemHandleDataSizeRet) { + auto pfnGetPhysMemHandleExp = + getContext()->urDdiTable.IPCExp.pfnGetPhysMemHandleExp; + + if (nullptr == pfnGetPhysMemHandleExp) + return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; + + ur_ipc_get_phys_mem_handle_exp_params_t params = { + &hContext, &hPhysMem, &ppIPCPhysMemHandleData, + &pIPCPhysMemHandleDataSizeRet}; + uint64_t instance = + getContext()->notify_begin(UR_FUNCTION_IPC_GET_PHYS_MEM_HANDLE_EXP, + "urIPCGetPhysMemHandleExp", ¶ms); + + auto &logger = getContext()->logger; + UR_LOG_L(logger, INFO, " ---> urIPCGetPhysMemHandleExp\n"); + + ur_result_t result = pfnGetPhysMemHandleExp( + hContext, hPhysMem, ppIPCPhysMemHandleData, pIPCPhysMemHandleDataSizeRet); + + getContext()->notify_end(UR_FUNCTION_IPC_GET_PHYS_MEM_HANDLE_EXP, + "urIPCGetPhysMemHandleExp", ¶ms, &result, + instance); + + if (logger.getLevel() <= UR_LOGGER_LEVEL_INFO) { + std::ostringstream args_str; + ur::extras::printFunctionParams( + args_str, UR_FUNCTION_IPC_GET_PHYS_MEM_HANDLE_EXP, ¶ms); + UR_LOG_L(logger, INFO, " <--- urIPCGetPhysMemHandleExp({}) -> {};\n", + args_str.str(), result); + } + + return result; +} + +/////////////////////////////////////////////////////////////////////////////// +/// @brief Intercept function for urIPCPutPhysMemHandleExp +__urdlllocal ur_result_t UR_APICALL urIPCPutPhysMemHandleExp( + /// [in] handle of the context object + ur_context_handle_t hContext, + /// [in] a pointer to the IPC physical memory handle data + void *pIPCPhysMemHandleData) { + auto pfnPutPhysMemHandleExp = + getContext()->urDdiTable.IPCExp.pfnPutPhysMemHandleExp; + + if (nullptr == pfnPutPhysMemHandleExp) + return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; + + ur_ipc_put_phys_mem_handle_exp_params_t params = {&hContext, + &pIPCPhysMemHandleData}; + uint64_t instance = + getContext()->notify_begin(UR_FUNCTION_IPC_PUT_PHYS_MEM_HANDLE_EXP, + "urIPCPutPhysMemHandleExp", ¶ms); + + auto &logger = getContext()->logger; + UR_LOG_L(logger, INFO, " ---> urIPCPutPhysMemHandleExp\n"); + + ur_result_t result = pfnPutPhysMemHandleExp(hContext, pIPCPhysMemHandleData); + + getContext()->notify_end(UR_FUNCTION_IPC_PUT_PHYS_MEM_HANDLE_EXP, + "urIPCPutPhysMemHandleExp", ¶ms, &result, + instance); + + if (logger.getLevel() <= UR_LOGGER_LEVEL_INFO) { + std::ostringstream args_str; + ur::extras::printFunctionParams( + args_str, UR_FUNCTION_IPC_PUT_PHYS_MEM_HANDLE_EXP, ¶ms); + UR_LOG_L(logger, INFO, " <--- urIPCPutPhysMemHandleExp({}) -> {};\n", + args_str.str(), result); + } + + return result; +} + +/////////////////////////////////////////////////////////////////////////////// +/// @brief Intercept function for urIPCOpenPhysMemHandleExp +__urdlllocal ur_result_t UR_APICALL urIPCOpenPhysMemHandleExp( + /// [in] handle of the context object + ur_context_handle_t hContext, + /// [in] handle of the device object the physical memory was allocated on + ur_device_handle_t hDevice, + /// [in] the IPC physical memory handle data + void *pIPCPhysMemHandleData, + /// [in] size of the IPC physical memory handle data + size_t ipcPhysMemHandleDataSize, + /// [out] pointer to the physical memory handle + ur_physical_mem_handle_t *phPhysMem) { + auto pfnOpenPhysMemHandleExp = + getContext()->urDdiTable.IPCExp.pfnOpenPhysMemHandleExp; + + if (nullptr == pfnOpenPhysMemHandleExp) + return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; + + ur_ipc_open_phys_mem_handle_exp_params_t params = { + &hContext, &hDevice, &pIPCPhysMemHandleData, &ipcPhysMemHandleDataSize, + &phPhysMem}; + uint64_t instance = + getContext()->notify_begin(UR_FUNCTION_IPC_OPEN_PHYS_MEM_HANDLE_EXP, + "urIPCOpenPhysMemHandleExp", ¶ms); + + auto &logger = getContext()->logger; + UR_LOG_L(logger, INFO, " ---> urIPCOpenPhysMemHandleExp\n"); + + ur_result_t result = + pfnOpenPhysMemHandleExp(hContext, hDevice, pIPCPhysMemHandleData, + ipcPhysMemHandleDataSize, phPhysMem); + + getContext()->notify_end(UR_FUNCTION_IPC_OPEN_PHYS_MEM_HANDLE_EXP, + "urIPCOpenPhysMemHandleExp", ¶ms, &result, + instance); + + if (logger.getLevel() <= UR_LOGGER_LEVEL_INFO) { + std::ostringstream args_str; + ur::extras::printFunctionParams( + args_str, UR_FUNCTION_IPC_OPEN_PHYS_MEM_HANDLE_EXP, ¶ms); + UR_LOG_L(logger, INFO, " <--- urIPCOpenPhysMemHandleExp({}) -> {};\n", + args_str.str(), result); + } + + return result; +} + +/////////////////////////////////////////////////////////////////////////////// +/// @brief Intercept function for urIPCClosePhysMemHandleExp +__urdlllocal ur_result_t UR_APICALL urIPCClosePhysMemHandleExp( + /// [in] handle of the context object + ur_context_handle_t hContext, + /// [in] physical memory handle opened through urIPCOpenPhysMemHandleExp + ur_physical_mem_handle_t hPhysMem) { + auto pfnClosePhysMemHandleExp = + getContext()->urDdiTable.IPCExp.pfnClosePhysMemHandleExp; + + if (nullptr == pfnClosePhysMemHandleExp) + return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; + + ur_ipc_close_phys_mem_handle_exp_params_t params = {&hContext, &hPhysMem}; + uint64_t instance = + getContext()->notify_begin(UR_FUNCTION_IPC_CLOSE_PHYS_MEM_HANDLE_EXP, + "urIPCClosePhysMemHandleExp", ¶ms); + + auto &logger = getContext()->logger; + UR_LOG_L(logger, INFO, " ---> urIPCClosePhysMemHandleExp\n"); + + ur_result_t result = pfnClosePhysMemHandleExp(hContext, hPhysMem); + + getContext()->notify_end(UR_FUNCTION_IPC_CLOSE_PHYS_MEM_HANDLE_EXP, + "urIPCClosePhysMemHandleExp", ¶ms, &result, + instance); + + if (logger.getLevel() <= UR_LOGGER_LEVEL_INFO) { + std::ostringstream args_str; + ur::extras::printFunctionParams( + args_str, UR_FUNCTION_IPC_CLOSE_PHYS_MEM_HANDLE_EXP, ¶ms); + UR_LOG_L(logger, INFO, " <--- urIPCClosePhysMemHandleExp({}) -> {};\n", + args_str.str(), result); + } + + return result; +} + /////////////////////////////////////////////////////////////////////////////// /// @brief Intercept function for urMemoryExportAllocExportableMemoryExp __urdlllocal ur_result_t UR_APICALL urMemoryExportAllocExportableMemoryExp( @@ -11731,6 +11901,22 @@ __urdlllocal ur_result_t UR_APICALL urGetIPCExpProcAddrTable( dditable.pfnCloseMemHandleExp = pDdiTable->pfnCloseMemHandleExp; pDdiTable->pfnCloseMemHandleExp = ur_tracing_layer::urIPCCloseMemHandleExp; + dditable.pfnGetPhysMemHandleExp = pDdiTable->pfnGetPhysMemHandleExp; + pDdiTable->pfnGetPhysMemHandleExp = + ur_tracing_layer::urIPCGetPhysMemHandleExp; + + dditable.pfnPutPhysMemHandleExp = pDdiTable->pfnPutPhysMemHandleExp; + pDdiTable->pfnPutPhysMemHandleExp = + ur_tracing_layer::urIPCPutPhysMemHandleExp; + + dditable.pfnOpenPhysMemHandleExp = pDdiTable->pfnOpenPhysMemHandleExp; + pDdiTable->pfnOpenPhysMemHandleExp = + ur_tracing_layer::urIPCOpenPhysMemHandleExp; + + dditable.pfnClosePhysMemHandleExp = pDdiTable->pfnClosePhysMemHandleExp; + pDdiTable->pfnClosePhysMemHandleExp = + ur_tracing_layer::urIPCClosePhysMemHandleExp; + return result; } /////////////////////////////////////////////////////////////////////////////// diff --git a/unified-runtime/source/loader/layers/validation/ur_valddi.cpp b/unified-runtime/source/loader/layers/validation/ur_valddi.cpp index 077e6b736846a..d1a8ab623988b 100644 --- a/unified-runtime/source/loader/layers/validation/ur_valddi.cpp +++ b/unified-runtime/source/loader/layers/validation/ur_valddi.cpp @@ -2615,7 +2615,7 @@ __urdlllocal ur_result_t UR_APICALL urPhysicalMemGetInfo( if (NULL == hPhysicalMem) return UR_RESULT_ERROR_INVALID_NULL_HANDLE; - if (UR_PHYSICAL_MEM_INFO_REFERENCE_COUNT < propName) + if (UR_PHYSICAL_MEM_INFO_IPC_VIRTUAL_ADDRESS < propName) return UR_RESULT_ERROR_INVALID_ENUMERATION; } @@ -9003,6 +9003,174 @@ __urdlllocal ur_result_t UR_APICALL urIPCCloseMemHandleExp( return result; } +/////////////////////////////////////////////////////////////////////////////// +/// @brief Intercept function for urIPCGetPhysMemHandleExp +__urdlllocal ur_result_t UR_APICALL urIPCGetPhysMemHandleExp( + /// [in] handle of the context object + ur_context_handle_t hContext, + /// [in] handle of the physical memory object + ur_physical_mem_handle_t hPhysMem, + /// [out] a pointer to the IPC physical memory handle data + void **ppIPCPhysMemHandleData, + /// [out] size of the resulting IPC physical memory handle data + size_t *pIPCPhysMemHandleDataSizeRet) { + auto pfnGetPhysMemHandleExp = + getContext()->urDdiTable.IPCExp.pfnGetPhysMemHandleExp; + + if (nullptr == pfnGetPhysMemHandleExp) { + return UR_RESULT_ERROR_UNINITIALIZED; + } + + if (getContext()->enableParameterValidation) { + if (NULL == ppIPCPhysMemHandleData) + return UR_RESULT_ERROR_INVALID_NULL_POINTER; + + if (NULL == pIPCPhysMemHandleDataSizeRet) + return UR_RESULT_ERROR_INVALID_NULL_POINTER; + + if (NULL == hContext) + return UR_RESULT_ERROR_INVALID_NULL_HANDLE; + + if (NULL == hPhysMem) + return UR_RESULT_ERROR_INVALID_NULL_HANDLE; + } + + if (getContext()->enableLifetimeValidation && + !getContext()->refCountContext->isReferenceValid(hContext)) { + URLOG_CTX_INVALID_REFERENCE(hContext); + } + + if (getContext()->enableLifetimeValidation && + !getContext()->refCountContext->isReferenceValid(hPhysMem)) { + URLOG_CTX_INVALID_REFERENCE(hPhysMem); + } + + ur_result_t result = pfnGetPhysMemHandleExp( + hContext, hPhysMem, ppIPCPhysMemHandleData, pIPCPhysMemHandleDataSizeRet); + + return result; +} + +/////////////////////////////////////////////////////////////////////////////// +/// @brief Intercept function for urIPCPutPhysMemHandleExp +__urdlllocal ur_result_t UR_APICALL urIPCPutPhysMemHandleExp( + /// [in] handle of the context object + ur_context_handle_t hContext, + /// [in] a pointer to the IPC physical memory handle data + void *pIPCPhysMemHandleData) { + auto pfnPutPhysMemHandleExp = + getContext()->urDdiTable.IPCExp.pfnPutPhysMemHandleExp; + + if (nullptr == pfnPutPhysMemHandleExp) { + return UR_RESULT_ERROR_UNINITIALIZED; + } + + if (getContext()->enableParameterValidation) { + if (NULL == pIPCPhysMemHandleData) + return UR_RESULT_ERROR_INVALID_NULL_POINTER; + + if (NULL == hContext) + return UR_RESULT_ERROR_INVALID_NULL_HANDLE; + } + + if (getContext()->enableLifetimeValidation && + !getContext()->refCountContext->isReferenceValid(hContext)) { + URLOG_CTX_INVALID_REFERENCE(hContext); + } + + ur_result_t result = pfnPutPhysMemHandleExp(hContext, pIPCPhysMemHandleData); + + return result; +} + +/////////////////////////////////////////////////////////////////////////////// +/// @brief Intercept function for urIPCOpenPhysMemHandleExp +__urdlllocal ur_result_t UR_APICALL urIPCOpenPhysMemHandleExp( + /// [in] handle of the context object + ur_context_handle_t hContext, + /// [in] handle of the device object the physical memory was allocated on + ur_device_handle_t hDevice, + /// [in] the IPC physical memory handle data + void *pIPCPhysMemHandleData, + /// [in] size of the IPC physical memory handle data + size_t ipcPhysMemHandleDataSize, + /// [out] pointer to the physical memory handle + ur_physical_mem_handle_t *phPhysMem) { + auto pfnOpenPhysMemHandleExp = + getContext()->urDdiTable.IPCExp.pfnOpenPhysMemHandleExp; + + if (nullptr == pfnOpenPhysMemHandleExp) { + return UR_RESULT_ERROR_UNINITIALIZED; + } + + if (getContext()->enableParameterValidation) { + if (NULL == phPhysMem) + return UR_RESULT_ERROR_INVALID_NULL_POINTER; + + if (NULL == pIPCPhysMemHandleData) + return UR_RESULT_ERROR_INVALID_NULL_POINTER; + + if (NULL == hContext) + return UR_RESULT_ERROR_INVALID_NULL_HANDLE; + + if (NULL == hDevice) + return UR_RESULT_ERROR_INVALID_NULL_HANDLE; + } + + if (getContext()->enableLifetimeValidation && + !getContext()->refCountContext->isReferenceValid(hContext)) { + URLOG_CTX_INVALID_REFERENCE(hContext); + } + + if (getContext()->enableLifetimeValidation && + !getContext()->refCountContext->isReferenceValid(hDevice)) { + URLOG_CTX_INVALID_REFERENCE(hDevice); + } + + ur_result_t result = + pfnOpenPhysMemHandleExp(hContext, hDevice, pIPCPhysMemHandleData, + ipcPhysMemHandleDataSize, phPhysMem); + + return result; +} + +/////////////////////////////////////////////////////////////////////////////// +/// @brief Intercept function for urIPCClosePhysMemHandleExp +__urdlllocal ur_result_t UR_APICALL urIPCClosePhysMemHandleExp( + /// [in] handle of the context object + ur_context_handle_t hContext, + /// [in] physical memory handle opened through urIPCOpenPhysMemHandleExp + ur_physical_mem_handle_t hPhysMem) { + auto pfnClosePhysMemHandleExp = + getContext()->urDdiTable.IPCExp.pfnClosePhysMemHandleExp; + + if (nullptr == pfnClosePhysMemHandleExp) { + return UR_RESULT_ERROR_UNINITIALIZED; + } + + if (getContext()->enableParameterValidation) { + if (NULL == hContext) + return UR_RESULT_ERROR_INVALID_NULL_HANDLE; + + if (NULL == hPhysMem) + return UR_RESULT_ERROR_INVALID_NULL_HANDLE; + } + + if (getContext()->enableLifetimeValidation && + !getContext()->refCountContext->isReferenceValid(hContext)) { + URLOG_CTX_INVALID_REFERENCE(hContext); + } + + if (getContext()->enableLifetimeValidation && + !getContext()->refCountContext->isReferenceValid(hPhysMem)) { + URLOG_CTX_INVALID_REFERENCE(hPhysMem); + } + + ur_result_t result = pfnClosePhysMemHandleExp(hContext, hPhysMem); + + return result; +} + /////////////////////////////////////////////////////////////////////////////// /// @brief Intercept function for urMemoryExportAllocExportableMemoryExp __urdlllocal ur_result_t UR_APICALL urMemoryExportAllocExportableMemoryExp( @@ -12502,6 +12670,22 @@ UR_DLLEXPORT ur_result_t UR_APICALL urGetIPCExpProcAddrTable( dditable.pfnCloseMemHandleExp = pDdiTable->pfnCloseMemHandleExp; pDdiTable->pfnCloseMemHandleExp = ur_validation_layer::urIPCCloseMemHandleExp; + dditable.pfnGetPhysMemHandleExp = pDdiTable->pfnGetPhysMemHandleExp; + pDdiTable->pfnGetPhysMemHandleExp = + ur_validation_layer::urIPCGetPhysMemHandleExp; + + dditable.pfnPutPhysMemHandleExp = pDdiTable->pfnPutPhysMemHandleExp; + pDdiTable->pfnPutPhysMemHandleExp = + ur_validation_layer::urIPCPutPhysMemHandleExp; + + dditable.pfnOpenPhysMemHandleExp = pDdiTable->pfnOpenPhysMemHandleExp; + pDdiTable->pfnOpenPhysMemHandleExp = + ur_validation_layer::urIPCOpenPhysMemHandleExp; + + dditable.pfnClosePhysMemHandleExp = pDdiTable->pfnClosePhysMemHandleExp; + pDdiTable->pfnClosePhysMemHandleExp = + ur_validation_layer::urIPCClosePhysMemHandleExp; + return result; } diff --git a/unified-runtime/source/loader/loader.def.in b/unified-runtime/source/loader/loader.def.in index cc6ec03371edb..4a978ad4584c6 100644 --- a/unified-runtime/source/loader/loader.def.in +++ b/unified-runtime/source/loader/loader.def.in @@ -147,9 +147,13 @@ EXPORTS urGraphIsEmptyExp urGraphSetDestructionCallbackExp urIPCCloseMemHandleExp + urIPCClosePhysMemHandleExp urIPCGetMemHandleExp + urIPCGetPhysMemHandleExp urIPCOpenMemHandleExp + urIPCOpenPhysMemHandleExp urIPCPutMemHandleExp + urIPCPutPhysMemHandleExp urKernelCreate urKernelCreateWithNativeHandle urKernelGetGroupInfo @@ -397,9 +401,13 @@ EXPORTS urPrintImageFormat urPrintImageInfo urPrintIpcCloseMemHandleExpParams + urPrintIpcClosePhysMemHandleExpParams urPrintIpcGetMemHandleExpParams + urPrintIpcGetPhysMemHandleExpParams urPrintIpcOpenMemHandleExpParams + urPrintIpcOpenPhysMemHandleExpParams urPrintIpcPutMemHandleExpParams + urPrintIpcPutPhysMemHandleExpParams urPrintKernelArgLocalProperties urPrintKernelArgMemObjProperties urPrintKernelArgPointerProperties diff --git a/unified-runtime/source/loader/loader.map.in b/unified-runtime/source/loader/loader.map.in index 9ffb00211b6dc..0c06c67d8745a 100644 --- a/unified-runtime/source/loader/loader.map.in +++ b/unified-runtime/source/loader/loader.map.in @@ -147,9 +147,13 @@ urGraphIsEmptyExp; urGraphSetDestructionCallbackExp; urIPCCloseMemHandleExp; + urIPCClosePhysMemHandleExp; urIPCGetMemHandleExp; + urIPCGetPhysMemHandleExp; urIPCOpenMemHandleExp; + urIPCOpenPhysMemHandleExp; urIPCPutMemHandleExp; + urIPCPutPhysMemHandleExp; urKernelCreate; urKernelCreateWithNativeHandle; urKernelGetGroupInfo; @@ -397,9 +401,13 @@ urPrintImageFormat; urPrintImageInfo; urPrintIpcCloseMemHandleExpParams; + urPrintIpcClosePhysMemHandleExpParams; urPrintIpcGetMemHandleExpParams; + urPrintIpcGetPhysMemHandleExpParams; urPrintIpcOpenMemHandleExpParams; + urPrintIpcOpenPhysMemHandleExpParams; urPrintIpcPutMemHandleExpParams; + urPrintIpcPutPhysMemHandleExpParams; urPrintKernelArgLocalProperties; urPrintKernelArgMemObjProperties; urPrintKernelArgPointerProperties; diff --git a/unified-runtime/source/loader/ur_ldrddi.cpp b/unified-runtime/source/loader/ur_ldrddi.cpp index a68a92742892e..0c31622dff65b 100644 --- a/unified-runtime/source/loader/ur_ldrddi.cpp +++ b/unified-runtime/source/loader/ur_ldrddi.cpp @@ -4643,6 +4643,90 @@ __urdlllocal ur_result_t UR_APICALL urIPCCloseMemHandleExp( return pfnCloseMemHandleExp(hContext, pMem); } +/////////////////////////////////////////////////////////////////////////////// +/// @brief Intercept function for urIPCGetPhysMemHandleExp +__urdlllocal ur_result_t UR_APICALL urIPCGetPhysMemHandleExp( + /// [in] handle of the context object + ur_context_handle_t hContext, + /// [in] handle of the physical memory object + ur_physical_mem_handle_t hPhysMem, + /// [out] a pointer to the IPC physical memory handle data + void **ppIPCPhysMemHandleData, + /// [out] size of the resulting IPC physical memory handle data + size_t *pIPCPhysMemHandleDataSizeRet) { + + auto *dditable = *reinterpret_cast(hContext); + + auto *pfnGetPhysMemHandleExp = dditable->IPCExp.pfnGetPhysMemHandleExp; + if (nullptr == pfnGetPhysMemHandleExp) + return UR_RESULT_ERROR_UNINITIALIZED; + + // forward to device-platform + return pfnGetPhysMemHandleExp(hContext, hPhysMem, ppIPCPhysMemHandleData, + pIPCPhysMemHandleDataSizeRet); +} + +/////////////////////////////////////////////////////////////////////////////// +/// @brief Intercept function for urIPCPutPhysMemHandleExp +__urdlllocal ur_result_t UR_APICALL urIPCPutPhysMemHandleExp( + /// [in] handle of the context object + ur_context_handle_t hContext, + /// [in] a pointer to the IPC physical memory handle data + void *pIPCPhysMemHandleData) { + + auto *dditable = *reinterpret_cast(hContext); + + auto *pfnPutPhysMemHandleExp = dditable->IPCExp.pfnPutPhysMemHandleExp; + if (nullptr == pfnPutPhysMemHandleExp) + return UR_RESULT_ERROR_UNINITIALIZED; + + // forward to device-platform + return pfnPutPhysMemHandleExp(hContext, pIPCPhysMemHandleData); +} + +/////////////////////////////////////////////////////////////////////////////// +/// @brief Intercept function for urIPCOpenPhysMemHandleExp +__urdlllocal ur_result_t UR_APICALL urIPCOpenPhysMemHandleExp( + /// [in] handle of the context object + ur_context_handle_t hContext, + /// [in] handle of the device object the physical memory was allocated on + ur_device_handle_t hDevice, + /// [in] the IPC physical memory handle data + void *pIPCPhysMemHandleData, + /// [in] size of the IPC physical memory handle data + size_t ipcPhysMemHandleDataSize, + /// [out] pointer to the physical memory handle + ur_physical_mem_handle_t *phPhysMem) { + + auto *dditable = *reinterpret_cast(hContext); + + auto *pfnOpenPhysMemHandleExp = dditable->IPCExp.pfnOpenPhysMemHandleExp; + if (nullptr == pfnOpenPhysMemHandleExp) + return UR_RESULT_ERROR_UNINITIALIZED; + + // forward to device-platform + return pfnOpenPhysMemHandleExp(hContext, hDevice, pIPCPhysMemHandleData, + ipcPhysMemHandleDataSize, phPhysMem); +} + +/////////////////////////////////////////////////////////////////////////////// +/// @brief Intercept function for urIPCClosePhysMemHandleExp +__urdlllocal ur_result_t UR_APICALL urIPCClosePhysMemHandleExp( + /// [in] handle of the context object + ur_context_handle_t hContext, + /// [in] physical memory handle opened through urIPCOpenPhysMemHandleExp + ur_physical_mem_handle_t hPhysMem) { + + auto *dditable = *reinterpret_cast(hContext); + + auto *pfnClosePhysMemHandleExp = dditable->IPCExp.pfnClosePhysMemHandleExp; + if (nullptr == pfnClosePhysMemHandleExp) + return UR_RESULT_ERROR_UNINITIALIZED; + + // forward to device-platform + return pfnClosePhysMemHandleExp(hContext, hPhysMem); +} + /////////////////////////////////////////////////////////////////////////////// /// @brief Intercept function for urMemoryExportAllocExportableMemoryExp __urdlllocal ur_result_t UR_APICALL urMemoryExportAllocExportableMemoryExp( @@ -6899,6 +6983,11 @@ UR_DLLEXPORT ur_result_t UR_APICALL urGetIPCExpProcAddrTable( pDdiTable->pfnPutMemHandleExp = ur_loader::urIPCPutMemHandleExp; pDdiTable->pfnOpenMemHandleExp = ur_loader::urIPCOpenMemHandleExp; pDdiTable->pfnCloseMemHandleExp = ur_loader::urIPCCloseMemHandleExp; + pDdiTable->pfnGetPhysMemHandleExp = ur_loader::urIPCGetPhysMemHandleExp; + pDdiTable->pfnPutPhysMemHandleExp = ur_loader::urIPCPutPhysMemHandleExp; + pDdiTable->pfnOpenPhysMemHandleExp = ur_loader::urIPCOpenPhysMemHandleExp; + pDdiTable->pfnClosePhysMemHandleExp = + ur_loader::urIPCClosePhysMemHandleExp; } else { // return pointers directly to platform's DDIs *pDdiTable = ur_loader::getContext()->platforms.front().dditable.IPCExp; diff --git a/unified-runtime/source/loader/ur_libapi.cpp b/unified-runtime/source/loader/ur_libapi.cpp index 1188978674036..76220540f5a34 100644 --- a/unified-runtime/source/loader/ur_libapi.cpp +++ b/unified-runtime/source/loader/ur_libapi.cpp @@ -3066,7 +3066,7 @@ ur_result_t UR_APICALL urPhysicalMemRelease( /// - ::UR_RESULT_ERROR_INVALID_NULL_HANDLE /// + `NULL == hPhysicalMem` /// - ::UR_RESULT_ERROR_INVALID_ENUMERATION -/// + `::UR_PHYSICAL_MEM_INFO_REFERENCE_COUNT < propName` +/// + `::UR_PHYSICAL_MEM_INFO_IPC_VIRTUAL_ADDRESS < propName` ur_result_t UR_APICALL urPhysicalMemGetInfo( /// [in] handle of the physical memory object to query. ur_physical_mem_handle_t hPhysicalMem, @@ -8640,6 +8640,149 @@ ur_result_t UR_APICALL urIPCCloseMemHandleExp( return exceptionToResult(std::current_exception()); } +/////////////////////////////////////////////////////////////////////////////// +/// @brief Gets an inter-process handle for a physical memory object +/// +/// @returns +/// - ::UR_RESULT_SUCCESS +/// - ::UR_RESULT_ERROR_UNINITIALIZED +/// - ::UR_RESULT_ERROR_DEVICE_LOST +/// - ::UR_RESULT_ERROR_ADAPTER_SPECIFIC +/// - ::UR_RESULT_ERROR_INVALID_NULL_HANDLE +/// + `NULL == hContext` +/// + `NULL == hPhysMem` +/// - ::UR_RESULT_ERROR_INVALID_NULL_POINTER +/// + `NULL == ppIPCPhysMemHandleData` +/// + `NULL == pIPCPhysMemHandleDataSizeRet` +/// - ::UR_RESULT_ERROR_INVALID_CONTEXT +/// - ::UR_RESULT_ERROR_INVALID_ARGUMENT +/// + `hPhysMem` was not created with the +/// `::UR_PHYSICAL_MEM_FLAG_ENABLE_IPC` flag +/// - ::UR_RESULT_ERROR_OUT_OF_HOST_MEMORY +/// - ::UR_RESULT_ERROR_OUT_OF_RESOURCES +ur_result_t UR_APICALL urIPCGetPhysMemHandleExp( + /// [in] handle of the context object + ur_context_handle_t hContext, + /// [in] handle of the physical memory object + ur_physical_mem_handle_t hPhysMem, + /// [out] a pointer to the IPC physical memory handle data + void **ppIPCPhysMemHandleData, + /// [out] size of the resulting IPC physical memory handle data + size_t *pIPCPhysMemHandleDataSizeRet) try { + auto pfnGetPhysMemHandleExp = + ur_lib::getContext()->urDdiTable.IPCExp.pfnGetPhysMemHandleExp; + if (nullptr == pfnGetPhysMemHandleExp) + return UR_RESULT_ERROR_UNINITIALIZED; + + return pfnGetPhysMemHandleExp(hContext, hPhysMem, ppIPCPhysMemHandleData, + pIPCPhysMemHandleDataSizeRet); +} catch (...) { + return exceptionToResult(std::current_exception()); +} + +/////////////////////////////////////////////////////////////////////////////// +/// @brief Releases an inter-process physical memory handle +/// +/// @returns +/// - ::UR_RESULT_SUCCESS +/// - ::UR_RESULT_ERROR_UNINITIALIZED +/// - ::UR_RESULT_ERROR_DEVICE_LOST +/// - ::UR_RESULT_ERROR_ADAPTER_SPECIFIC +/// - ::UR_RESULT_ERROR_INVALID_NULL_HANDLE +/// + `NULL == hContext` +/// - ::UR_RESULT_ERROR_INVALID_NULL_POINTER +/// + `NULL == pIPCPhysMemHandleData` +/// - ::UR_RESULT_ERROR_INVALID_CONTEXT +/// - ::UR_RESULT_ERROR_OUT_OF_HOST_MEMORY +/// - ::UR_RESULT_ERROR_OUT_OF_RESOURCES +ur_result_t UR_APICALL urIPCPutPhysMemHandleExp( + /// [in] handle of the context object + ur_context_handle_t hContext, + /// [in] a pointer to the IPC physical memory handle data + void *pIPCPhysMemHandleData) try { + auto pfnPutPhysMemHandleExp = + ur_lib::getContext()->urDdiTable.IPCExp.pfnPutPhysMemHandleExp; + if (nullptr == pfnPutPhysMemHandleExp) + return UR_RESULT_ERROR_UNINITIALIZED; + + return pfnPutPhysMemHandleExp(hContext, pIPCPhysMemHandleData); +} catch (...) { + return exceptionToResult(std::current_exception()); +} + +/////////////////////////////////////////////////////////////////////////////// +/// @brief Opens an inter-process physical memory handle to get the +/// corresponding +/// physical memory object +/// +/// @returns +/// - ::UR_RESULT_SUCCESS +/// - ::UR_RESULT_ERROR_UNINITIALIZED +/// - ::UR_RESULT_ERROR_DEVICE_LOST +/// - ::UR_RESULT_ERROR_ADAPTER_SPECIFIC +/// - ::UR_RESULT_ERROR_INVALID_NULL_HANDLE +/// + `NULL == hContext` +/// + `NULL == hDevice` +/// - ::UR_RESULT_ERROR_INVALID_NULL_POINTER +/// + `NULL == phPhysMem` +/// + `NULL == pIPCPhysMemHandleData` +/// - ::UR_RESULT_ERROR_INVALID_CONTEXT +/// - ::UR_RESULT_ERROR_INVALID_VALUE +/// + ipcPhysMemHandleDataSize is not the same as the size of IPC +/// physical memory handle data +/// - ::UR_RESULT_ERROR_OUT_OF_HOST_MEMORY +/// - ::UR_RESULT_ERROR_OUT_OF_RESOURCES +ur_result_t UR_APICALL urIPCOpenPhysMemHandleExp( + /// [in] handle of the context object + ur_context_handle_t hContext, + /// [in] handle of the device object the physical memory was allocated on + ur_device_handle_t hDevice, + /// [in] the IPC physical memory handle data + void *pIPCPhysMemHandleData, + /// [in] size of the IPC physical memory handle data + size_t ipcPhysMemHandleDataSize, + /// [out] pointer to the physical memory handle + ur_physical_mem_handle_t *phPhysMem) try { + auto pfnOpenPhysMemHandleExp = + ur_lib::getContext()->urDdiTable.IPCExp.pfnOpenPhysMemHandleExp; + if (nullptr == pfnOpenPhysMemHandleExp) + return UR_RESULT_ERROR_UNINITIALIZED; + + return pfnOpenPhysMemHandleExp(hContext, hDevice, pIPCPhysMemHandleData, + ipcPhysMemHandleDataSize, phPhysMem); +} catch (...) { + return exceptionToResult(std::current_exception()); +} + +/////////////////////////////////////////////////////////////////////////////// +/// @brief Closes an inter-process physical memory handle +/// +/// @returns +/// - ::UR_RESULT_SUCCESS +/// - ::UR_RESULT_ERROR_UNINITIALIZED +/// - ::UR_RESULT_ERROR_DEVICE_LOST +/// - ::UR_RESULT_ERROR_ADAPTER_SPECIFIC +/// - ::UR_RESULT_ERROR_INVALID_NULL_HANDLE +/// + `NULL == hContext` +/// + `NULL == hPhysMem` +/// - ::UR_RESULT_ERROR_INVALID_CONTEXT +/// - ::UR_RESULT_ERROR_OUT_OF_HOST_MEMORY +/// - ::UR_RESULT_ERROR_OUT_OF_RESOURCES +ur_result_t UR_APICALL urIPCClosePhysMemHandleExp( + /// [in] handle of the context object + ur_context_handle_t hContext, + /// [in] physical memory handle opened through urIPCOpenPhysMemHandleExp + ur_physical_mem_handle_t hPhysMem) try { + auto pfnClosePhysMemHandleExp = + ur_lib::getContext()->urDdiTable.IPCExp.pfnClosePhysMemHandleExp; + if (nullptr == pfnClosePhysMemHandleExp) + return UR_RESULT_ERROR_UNINITIALIZED; + + return pfnClosePhysMemHandleExp(hContext, hPhysMem); +} catch (...) { + return exceptionToResult(std::current_exception()); +} + /////////////////////////////////////////////////////////////////////////////// /// @brief Allocate an exportable memory region and return a pointer to that /// allocation. diff --git a/unified-runtime/source/loader/ur_print.cpp b/unified-runtime/source/loader/ur_print.cpp index 8c56e882cde85..1a748a44673f1 100644 --- a/unified-runtime/source/loader/ur_print.cpp +++ b/unified-runtime/source/loader/ur_print.cpp @@ -2203,6 +2203,38 @@ ur_result_t urPrintIpcCloseMemHandleExpParams( return str_copy(&ss, buffer, buff_size, out_size); } +ur_result_t urPrintIpcGetPhysMemHandleExpParams( + const struct ur_ipc_get_phys_mem_handle_exp_params_t *params, char *buffer, + const size_t buff_size, size_t *out_size) { + std::stringstream ss; + ss << params; + return str_copy(&ss, buffer, buff_size, out_size); +} + +ur_result_t urPrintIpcPutPhysMemHandleExpParams( + const struct ur_ipc_put_phys_mem_handle_exp_params_t *params, char *buffer, + const size_t buff_size, size_t *out_size) { + std::stringstream ss; + ss << params; + return str_copy(&ss, buffer, buff_size, out_size); +} + +ur_result_t urPrintIpcOpenPhysMemHandleExpParams( + const struct ur_ipc_open_phys_mem_handle_exp_params_t *params, char *buffer, + const size_t buff_size, size_t *out_size) { + std::stringstream ss; + ss << params; + return str_copy(&ss, buffer, buff_size, out_size); +} + +ur_result_t urPrintIpcClosePhysMemHandleExpParams( + const struct ur_ipc_close_phys_mem_handle_exp_params_t *params, + char *buffer, const size_t buff_size, size_t *out_size) { + std::stringstream ss; + ss << params; + return str_copy(&ss, buffer, buff_size, out_size); +} + ur_result_t urPrintKernelCreateParams(const struct ur_kernel_create_params_t *params, char *buffer, const size_t buff_size, diff --git a/unified-runtime/source/ur_api.cpp b/unified-runtime/source/ur_api.cpp index d6b7be91aa80d..8d752589ec194 100644 --- a/unified-runtime/source/ur_api.cpp +++ b/unified-runtime/source/ur_api.cpp @@ -2698,7 +2698,7 @@ ur_result_t UR_APICALL urPhysicalMemRelease( /// - ::UR_RESULT_ERROR_INVALID_NULL_HANDLE /// + `NULL == hPhysicalMem` /// - ::UR_RESULT_ERROR_INVALID_ENUMERATION -/// + `::UR_PHYSICAL_MEM_INFO_REFERENCE_COUNT < propName` +/// + `::UR_PHYSICAL_MEM_INFO_IPC_VIRTUAL_ADDRESS < propName` ur_result_t UR_APICALL urPhysicalMemGetInfo( /// [in] handle of the physical memory object to query. ur_physical_mem_handle_t hPhysicalMem, @@ -7522,6 +7522,123 @@ ur_result_t UR_APICALL urIPCCloseMemHandleExp( return result; } +/////////////////////////////////////////////////////////////////////////////// +/// @brief Gets an inter-process handle for a physical memory object +/// +/// @returns +/// - ::UR_RESULT_SUCCESS +/// - ::UR_RESULT_ERROR_UNINITIALIZED +/// - ::UR_RESULT_ERROR_DEVICE_LOST +/// - ::UR_RESULT_ERROR_ADAPTER_SPECIFIC +/// - ::UR_RESULT_ERROR_INVALID_NULL_HANDLE +/// + `NULL == hContext` +/// + `NULL == hPhysMem` +/// - ::UR_RESULT_ERROR_INVALID_NULL_POINTER +/// + `NULL == ppIPCPhysMemHandleData` +/// + `NULL == pIPCPhysMemHandleDataSizeRet` +/// - ::UR_RESULT_ERROR_INVALID_CONTEXT +/// - ::UR_RESULT_ERROR_INVALID_ARGUMENT +/// + `hPhysMem` was not created with the +/// `::UR_PHYSICAL_MEM_FLAG_ENABLE_IPC` flag +/// - ::UR_RESULT_ERROR_OUT_OF_HOST_MEMORY +/// - ::UR_RESULT_ERROR_OUT_OF_RESOURCES +ur_result_t UR_APICALL urIPCGetPhysMemHandleExp( + /// [in] handle of the context object + ur_context_handle_t hContext, + /// [in] handle of the physical memory object + ur_physical_mem_handle_t hPhysMem, + /// [out] a pointer to the IPC physical memory handle data + void **ppIPCPhysMemHandleData, + /// [out] size of the resulting IPC physical memory handle data + size_t *pIPCPhysMemHandleDataSizeRet) { + ur_result_t result = UR_RESULT_SUCCESS; + return result; +} + +/////////////////////////////////////////////////////////////////////////////// +/// @brief Releases an inter-process physical memory handle +/// +/// @returns +/// - ::UR_RESULT_SUCCESS +/// - ::UR_RESULT_ERROR_UNINITIALIZED +/// - ::UR_RESULT_ERROR_DEVICE_LOST +/// - ::UR_RESULT_ERROR_ADAPTER_SPECIFIC +/// - ::UR_RESULT_ERROR_INVALID_NULL_HANDLE +/// + `NULL == hContext` +/// - ::UR_RESULT_ERROR_INVALID_NULL_POINTER +/// + `NULL == pIPCPhysMemHandleData` +/// - ::UR_RESULT_ERROR_INVALID_CONTEXT +/// - ::UR_RESULT_ERROR_OUT_OF_HOST_MEMORY +/// - ::UR_RESULT_ERROR_OUT_OF_RESOURCES +ur_result_t UR_APICALL urIPCPutPhysMemHandleExp( + /// [in] handle of the context object + ur_context_handle_t hContext, + /// [in] a pointer to the IPC physical memory handle data + void *pIPCPhysMemHandleData) { + ur_result_t result = UR_RESULT_SUCCESS; + return result; +} + +/////////////////////////////////////////////////////////////////////////////// +/// @brief Opens an inter-process physical memory handle to get the +/// corresponding +/// physical memory object +/// +/// @returns +/// - ::UR_RESULT_SUCCESS +/// - ::UR_RESULT_ERROR_UNINITIALIZED +/// - ::UR_RESULT_ERROR_DEVICE_LOST +/// - ::UR_RESULT_ERROR_ADAPTER_SPECIFIC +/// - ::UR_RESULT_ERROR_INVALID_NULL_HANDLE +/// + `NULL == hContext` +/// + `NULL == hDevice` +/// - ::UR_RESULT_ERROR_INVALID_NULL_POINTER +/// + `NULL == phPhysMem` +/// + `NULL == pIPCPhysMemHandleData` +/// - ::UR_RESULT_ERROR_INVALID_CONTEXT +/// - ::UR_RESULT_ERROR_INVALID_VALUE +/// + ipcPhysMemHandleDataSize is not the same as the size of IPC +/// physical memory handle data +/// - ::UR_RESULT_ERROR_OUT_OF_HOST_MEMORY +/// - ::UR_RESULT_ERROR_OUT_OF_RESOURCES +ur_result_t UR_APICALL urIPCOpenPhysMemHandleExp( + /// [in] handle of the context object + ur_context_handle_t hContext, + /// [in] handle of the device object the physical memory was allocated on + ur_device_handle_t hDevice, + /// [in] the IPC physical memory handle data + void *pIPCPhysMemHandleData, + /// [in] size of the IPC physical memory handle data + size_t ipcPhysMemHandleDataSize, + /// [out] pointer to the physical memory handle + ur_physical_mem_handle_t *phPhysMem) { + ur_result_t result = UR_RESULT_SUCCESS; + return result; +} + +/////////////////////////////////////////////////////////////////////////////// +/// @brief Closes an inter-process physical memory handle +/// +/// @returns +/// - ::UR_RESULT_SUCCESS +/// - ::UR_RESULT_ERROR_UNINITIALIZED +/// - ::UR_RESULT_ERROR_DEVICE_LOST +/// - ::UR_RESULT_ERROR_ADAPTER_SPECIFIC +/// - ::UR_RESULT_ERROR_INVALID_NULL_HANDLE +/// + `NULL == hContext` +/// + `NULL == hPhysMem` +/// - ::UR_RESULT_ERROR_INVALID_CONTEXT +/// - ::UR_RESULT_ERROR_OUT_OF_HOST_MEMORY +/// - ::UR_RESULT_ERROR_OUT_OF_RESOURCES +ur_result_t UR_APICALL urIPCClosePhysMemHandleExp( + /// [in] handle of the context object + ur_context_handle_t hContext, + /// [in] physical memory handle opened through urIPCOpenPhysMemHandleExp + ur_physical_mem_handle_t hPhysMem) { + ur_result_t result = UR_RESULT_SUCCESS; + return result; +} + /////////////////////////////////////////////////////////////////////////////// /// @brief Allocate an exportable memory region and return a pointer to that /// allocation. diff --git a/unified-runtime/tools/urinfo/urinfo.hpp b/unified-runtime/tools/urinfo/urinfo.hpp index d5b15e5aaa00b..fe99cf3ef5f00 100644 --- a/unified-runtime/tools/urinfo/urinfo.hpp +++ b/unified-runtime/tools/urinfo/urinfo.hpp @@ -466,6 +466,9 @@ inline void printDeviceInfos(ur_device_handle_t hDevice, std::cout << prefix; printDeviceInfo(hDevice, UR_DEVICE_INFO_IPC_MEMORY_SUPPORT_EXP); std::cout << prefix; + printDeviceInfo(hDevice, + UR_DEVICE_INFO_IPC_PHYSICAL_MEMORY_SUPPORT_EXP); + std::cout << prefix; printDeviceInfo(hDevice, UR_DEVICE_INFO_ASYNC_USM_ALLOCATIONS_SUPPORT_EXP); std::cout << prefix; From fe60cba40e4186bed6bbc8945cbad25c1ec4f7ae Mon Sep 17 00:00:00 2001 From: Lukasz Dorau Date: Mon, 1 Jun 2026 20:19:23 +0000 Subject: [PATCH 3/5] [UR] Implement IPC physical_mem API Implement urIPCGetPhysMemHandleExp, urIPCPutPhysMemHandleExp, urIPCOpenPhysMemHandleExp, and urIPCClosePhysMemHandleExp. Level Zero v2 adapter (Linux, BMG/Xe2+ only): - physical_mem.hpp and physical_mem.cpp moved to v2/ subdirectory - urIPCGetPhysMemHandleExp: calls zeMemGetIpcHandleWithProperties passing ze_physical_mem_handle_t cast to (const void *) directly, using ze_ipc_mem_handle_type_ext_desc_t with ZE_IPC_MEM_HANDLE_TYPE_FLAG_DEFAULT; detects fd-based handles via fcntl(F_GETFD) and returns UR_RESULT_ERROR_UNSUPPORTED_FEATURE - urIPCOpenPhysMemHandleExp: calls zeMemOpenIpcHandle with flags=0 - urIPCPutPhysMemHandleExp: calls zeMemPutIpcHandle - urIPCClosePhysMemHandleExp: delegates to urPhysicalMemRelease - virtual_mem.cpp: urVirtualMemMap returns UR_RESULT_ERROR_INVALID_ARGUMENT for IPC-opened handles; the virtual address is already established by zeMemOpenIpcHandle and is accessed via urPhysicalMemGetInfo(UR_PHYSICAL_MEM_INFO_IPC_VIRTUAL_ADDRESS) - device.cpp: report UR_DEVICE_INFO_IPC_PHYSICAL_MEMORY_SUPPORT_EXP as isIntelBMGOrNewer() on Linux, false on Windows - Add isIntelBMGOrNewer() helper (IP version threshold 0x05004000, first Xe2 stepping BMG_G21_A0); IPC entry points return UR_RESULT_ERROR_UNSUPPORTED_FEATURE on pre-Xe2 devices Level Zero v1 adapter: - physical_mem.cpp provides only the four non-IPC functions (urPhysicalMemCreate/Retain/Release/GetInfo) - Add stub definitions for the four IPC functions returning UR_RESULT_ERROR_UNSUPPORTED_FEATURE so the L0v1 adapter links correctly Other adapters (CUDA, HIP, OpenCL, Native CPU, Offload): - Return UR_RESULT_ERROR_UNSUPPORTED_FEATURE from all four functions - Report UR_DEVICE_INFO_IPC_PHYSICAL_MEMORY_SUPPORT_EXP as false Conformance tests: - Add urIPCGetPhysMemHandleExp.cpp, urIPCPutPhysMemHandleExp.cpp, urIPCOpenPhysMemHandleExp.cpp, urIPCClosePhysMemHandleExp.cpp - Add urIPCPhysMemHandleExpFixtures.hpp with urIPCPhysMemTest, urIPCPhysMemHandleTest, urIPCOpenedPhysMemTest fixtures Signed-off-by: Lukasz Dorau Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../include/unified-runtime/ur_api.h | 14 +- .../include/unified-runtime/ur_ddi.h | 4 +- .../core/exp-inter-process-communication.yml | 8 +- .../source/adapters/cuda/device.cpp | 2 + .../source/adapters/cuda/memory.cpp | 21 ++ .../adapters/cuda/ur_interface_loader.cpp | 4 + .../source/adapters/hip/device.cpp | 2 + .../source/adapters/hip/memory.cpp | 21 ++ .../adapters/hip/ur_interface_loader.cpp | 4 + .../source/adapters/level_zero/CMakeLists.txt | 5 +- .../source/adapters/level_zero/device.cpp | 6 + .../source/adapters/level_zero/device.hpp | 5 + .../source/adapters/level_zero/memory.cpp | 5 + .../adapters/level_zero/physical_mem.cpp | 79 ++++- .../adapters/level_zero/physical_mem.hpp | 26 -- .../level_zero/ur_interface_loader.hpp | 4 +- .../adapters/level_zero/ur_level_zero.hpp | 2 +- .../source/adapters/level_zero/v2/memory.cpp | 6 + .../adapters/level_zero/v2/physical_mem.cpp | 282 ++++++++++++++++++ .../adapters/level_zero/v2/physical_mem.hpp | 59 ++++ .../adapters/level_zero/virtual_mem.cpp | 12 +- .../source/adapters/mock/ur_mockddi.cpp | 10 +- .../source/adapters/native_cpu/device.cpp | 2 + .../source/adapters/native_cpu/memory.cpp | 21 ++ .../native_cpu/ur_interface_loader.cpp | 4 + .../source/adapters/offload/memory.cpp | 21 ++ .../adapters/offload/ur_interface_loader.cpp | 4 + .../source/adapters/opencl/device.cpp | 2 + .../source/adapters/opencl/memory.cpp | 21 ++ .../adapters/opencl/ur_interface_loader.cpp | 4 + .../loader/layers/tracing/ur_trcddi.cpp | 10 +- .../loader/layers/validation/ur_valddi.cpp | 10 +- unified-runtime/source/loader/ur_ldrddi.cpp | 10 +- unified-runtime/source/loader/ur_libapi.cpp | 10 +- unified-runtime/source/ur_api.cpp | 10 +- .../conformance/virtual_memory/CMakeLists.txt | 4 + .../urIPCClosePhysMemHandleExp.cpp | 25 ++ .../urIPCGetPhysMemHandleExp.cpp | 99 ++++++ .../urIPCOpenPhysMemHandleExp.cpp | 57 ++++ .../urIPCPhysMemHandleExpFixtures.hpp | 92 ++++++ .../urIPCPutPhysMemHandleExp.cpp | 25 ++ .../virtual_memory/urPhysicalMemCreate.cpp | 2 +- 42 files changed, 930 insertions(+), 84 deletions(-) delete mode 100644 unified-runtime/source/adapters/level_zero/physical_mem.hpp create mode 100644 unified-runtime/source/adapters/level_zero/v2/physical_mem.cpp create mode 100644 unified-runtime/source/adapters/level_zero/v2/physical_mem.hpp create mode 100644 unified-runtime/test/conformance/virtual_memory/urIPCClosePhysMemHandleExp.cpp create mode 100644 unified-runtime/test/conformance/virtual_memory/urIPCGetPhysMemHandleExp.cpp create mode 100644 unified-runtime/test/conformance/virtual_memory/urIPCOpenPhysMemHandleExp.cpp create mode 100644 unified-runtime/test/conformance/virtual_memory/urIPCPhysMemHandleExpFixtures.hpp create mode 100644 unified-runtime/test/conformance/virtual_memory/urIPCPutPhysMemHandleExp.cpp diff --git a/unified-runtime/include/unified-runtime/ur_api.h b/unified-runtime/include/unified-runtime/ur_api.h index f4915c9d5822f..0fa0b802f1814 100644 --- a/unified-runtime/include/unified-runtime/ur_api.h +++ b/unified-runtime/include/unified-runtime/ur_api.h @@ -11306,8 +11306,9 @@ UR_APIEXPORT ur_result_t UR_APICALL urIPCGetPhysMemHandleExp( UR_APIEXPORT ur_result_t UR_APICALL urIPCPutPhysMemHandleExp( /// [in] handle of the context object ur_context_handle_t hContext, - /// [in] a pointer to the IPC physical memory handle data - void *pIPCPhysMemHandleData); + /// [in] a pointer to the IPC physical memory handle data obtained with + /// urIPCGetPhysMemHandleExp + const void *pIPCPhysMemHandleData); /////////////////////////////////////////////////////////////////////////////// /// @brief Opens an inter-process physical memory handle to get the @@ -11336,8 +11337,9 @@ UR_APIEXPORT ur_result_t UR_APICALL urIPCOpenPhysMemHandleExp( ur_context_handle_t hContext, /// [in] handle of the device object the physical memory was allocated on ur_device_handle_t hDevice, - /// [in] the IPC physical memory handle data - void *pIPCPhysMemHandleData, + /// [in] the IPC physical memory handle data obtained with + /// urIPCGetPhysMemHandleExp + const void *pIPCPhysMemHandleData, /// [in] size of the IPC physical memory handle data size_t ipcPhysMemHandleDataSize, /// [out] pointer to the physical memory handle @@ -16642,7 +16644,7 @@ typedef struct ur_ipc_get_phys_mem_handle_exp_params_t { /// allowing the callback the ability to modify the parameter's value typedef struct ur_ipc_put_phys_mem_handle_exp_params_t { ur_context_handle_t *phContext; - void **ppIPCPhysMemHandleData; + const void **ppIPCPhysMemHandleData; } ur_ipc_put_phys_mem_handle_exp_params_t; /////////////////////////////////////////////////////////////////////////////// @@ -16652,7 +16654,7 @@ typedef struct ur_ipc_put_phys_mem_handle_exp_params_t { typedef struct ur_ipc_open_phys_mem_handle_exp_params_t { ur_context_handle_t *phContext; ur_device_handle_t *phDevice; - void **ppIPCPhysMemHandleData; + const void **ppIPCPhysMemHandleData; size_t *pipcPhysMemHandleDataSize; ur_physical_mem_handle_t **pphPhysMem; } ur_ipc_open_phys_mem_handle_exp_params_t; diff --git a/unified-runtime/include/unified-runtime/ur_ddi.h b/unified-runtime/include/unified-runtime/ur_ddi.h index 6731cd2c88053..8f45375c369c1 100644 --- a/unified-runtime/include/unified-runtime/ur_ddi.h +++ b/unified-runtime/include/unified-runtime/ur_ddi.h @@ -1958,12 +1958,12 @@ typedef ur_result_t(UR_APICALL *ur_pfnIPCGetPhysMemHandleExp_t)( /////////////////////////////////////////////////////////////////////////////// /// @brief Function-pointer for urIPCPutPhysMemHandleExp typedef ur_result_t(UR_APICALL *ur_pfnIPCPutPhysMemHandleExp_t)( - ur_context_handle_t, void *); + ur_context_handle_t, const void *); /////////////////////////////////////////////////////////////////////////////// /// @brief Function-pointer for urIPCOpenPhysMemHandleExp typedef ur_result_t(UR_APICALL *ur_pfnIPCOpenPhysMemHandleExp_t)( - ur_context_handle_t, ur_device_handle_t, void *, size_t, + ur_context_handle_t, ur_device_handle_t, const void *, size_t, ur_physical_mem_handle_t *); /////////////////////////////////////////////////////////////////////////////// diff --git a/unified-runtime/scripts/core/exp-inter-process-communication.yml b/unified-runtime/scripts/core/exp-inter-process-communication.yml index c492228b015e6..71c15e6b9d71a 100644 --- a/unified-runtime/scripts/core/exp-inter-process-communication.yml +++ b/unified-runtime/scripts/core/exp-inter-process-communication.yml @@ -177,9 +177,9 @@ params: - type: $x_context_handle_t name: hContext desc: "[in] handle of the context object" - - type: void* + - type: const void* name: pIPCPhysMemHandleData - desc: "[in] a pointer to the IPC physical memory handle data" + desc: "[in] a pointer to the IPC physical memory handle data obtained with urIPCGetPhysMemHandleExp" returns: - $X_RESULT_ERROR_INVALID_CONTEXT - $X_RESULT_ERROR_INVALID_NULL_HANDLE: @@ -201,9 +201,9 @@ params: - type: $x_device_handle_t name: hDevice desc: "[in] handle of the device object the physical memory was allocated on" - - type: void * + - type: const void * name: pIPCPhysMemHandleData - desc: "[in] the IPC physical memory handle data" + desc: "[in] the IPC physical memory handle data obtained with urIPCGetPhysMemHandleExp" - type: size_t name: ipcPhysMemHandleDataSize desc: "[in] size of the IPC physical memory handle data" diff --git a/unified-runtime/source/adapters/cuda/device.cpp b/unified-runtime/source/adapters/cuda/device.cpp index d213d51eaeb9b..9ef56d297e330 100644 --- a/unified-runtime/source/adapters/cuda/device.cpp +++ b/unified-runtime/source/adapters/cuda/device.cpp @@ -1173,6 +1173,8 @@ UR_APIEXPORT ur_result_t UR_APICALL urDeviceGetInfo(ur_device_handle_t hDevice, hDevice->get())); return ReturnValue(static_cast(IPCSupported)); } + case UR_DEVICE_INFO_IPC_PHYSICAL_MEMORY_SUPPORT_EXP: + return ReturnValue(false); case UR_DEVICE_INFO_COMMAND_BUFFER_SUPPORT_EXP: case UR_DEVICE_INFO_COMMAND_BUFFER_EVENT_SUPPORT_EXP: return ReturnValue(true); diff --git a/unified-runtime/source/adapters/cuda/memory.cpp b/unified-runtime/source/adapters/cuda/memory.cpp index 81b86d7445381..cf73ac1cbca65 100644 --- a/unified-runtime/source/adapters/cuda/memory.cpp +++ b/unified-runtime/source/adapters/cuda/memory.cpp @@ -639,3 +639,24 @@ UR_APIEXPORT ur_result_t UR_APICALL urIPCCloseMemHandleExp(ur_context_handle_t, void *pMem) { return umf::umf2urResult(umfCloseIPCHandle(pMem)); } + +UR_APIEXPORT ur_result_t UR_APICALL urIPCGetPhysMemHandleExp( + ur_context_handle_t, ur_physical_mem_handle_t, void **, size_t *) { + return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; +} + +UR_APIEXPORT ur_result_t UR_APICALL +urIPCPutPhysMemHandleExp(ur_context_handle_t, const void *) { + return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; +} + +UR_APIEXPORT ur_result_t UR_APICALL +urIPCOpenPhysMemHandleExp(ur_context_handle_t, ur_device_handle_t, const void *, + size_t, ur_physical_mem_handle_t *) { + return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; +} + +UR_APIEXPORT ur_result_t UR_APICALL +urIPCClosePhysMemHandleExp(ur_context_handle_t, ur_physical_mem_handle_t) { + return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; +} diff --git a/unified-runtime/source/adapters/cuda/ur_interface_loader.cpp b/unified-runtime/source/adapters/cuda/ur_interface_loader.cpp index 1f6434b2c140f..e1cce057c6320 100644 --- a/unified-runtime/source/adapters/cuda/ur_interface_loader.cpp +++ b/unified-runtime/source/adapters/cuda/ur_interface_loader.cpp @@ -510,6 +510,10 @@ UR_DLLEXPORT ur_result_t UR_APICALL urGetIPCExpProcAddrTable( pDdiTable->pfnPutMemHandleExp = urIPCPutMemHandleExp; pDdiTable->pfnOpenMemHandleExp = urIPCOpenMemHandleExp; pDdiTable->pfnCloseMemHandleExp = urIPCCloseMemHandleExp; + pDdiTable->pfnGetPhysMemHandleExp = urIPCGetPhysMemHandleExp; + pDdiTable->pfnPutPhysMemHandleExp = urIPCPutPhysMemHandleExp; + pDdiTable->pfnOpenPhysMemHandleExp = urIPCOpenPhysMemHandleExp; + pDdiTable->pfnClosePhysMemHandleExp = urIPCClosePhysMemHandleExp; return UR_RESULT_SUCCESS; } diff --git a/unified-runtime/source/adapters/hip/device.cpp b/unified-runtime/source/adapters/hip/device.cpp index f07a31c03ce4e..e43f7e58164f0 100644 --- a/unified-runtime/source/adapters/hip/device.cpp +++ b/unified-runtime/source/adapters/hip/device.cpp @@ -1021,6 +1021,8 @@ UR_APIEXPORT ur_result_t UR_APICALL urDeviceGetInfo(ur_device_handle_t hDevice, static_cast(0)); case UR_DEVICE_INFO_IPC_MEMORY_SUPPORT_EXP: return ReturnValue(false); + case UR_DEVICE_INFO_IPC_PHYSICAL_MEMORY_SUPPORT_EXP: + return ReturnValue(false); case UR_DEVICE_INFO_COMMAND_BUFFER_SUPPORT_EXP: { int RuntimeVersion = 0; UR_CHECK_ERROR(hipRuntimeGetVersion(&RuntimeVersion)); diff --git a/unified-runtime/source/adapters/hip/memory.cpp b/unified-runtime/source/adapters/hip/memory.cpp index 6360ab0f92352..78c4a7c93ea24 100644 --- a/unified-runtime/source/adapters/hip/memory.cpp +++ b/unified-runtime/source/adapters/hip/memory.cpp @@ -658,3 +658,24 @@ UR_APIEXPORT ur_result_t UR_APICALL urIPCCloseMemHandleExp(ur_context_handle_t, void *) { return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; } + +UR_APIEXPORT ur_result_t UR_APICALL urIPCGetPhysMemHandleExp( + ur_context_handle_t, ur_physical_mem_handle_t, void **, size_t *) { + return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; +} + +UR_APIEXPORT ur_result_t UR_APICALL +urIPCPutPhysMemHandleExp(ur_context_handle_t, const void *) { + return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; +} + +UR_APIEXPORT ur_result_t UR_APICALL +urIPCOpenPhysMemHandleExp(ur_context_handle_t, ur_device_handle_t, const void *, + size_t, ur_physical_mem_handle_t *) { + return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; +} + +UR_APIEXPORT ur_result_t UR_APICALL +urIPCClosePhysMemHandleExp(ur_context_handle_t, ur_physical_mem_handle_t) { + return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; +} diff --git a/unified-runtime/source/adapters/hip/ur_interface_loader.cpp b/unified-runtime/source/adapters/hip/ur_interface_loader.cpp index 1d6fbdf7a9c0c..48c6c1b161546 100644 --- a/unified-runtime/source/adapters/hip/ur_interface_loader.cpp +++ b/unified-runtime/source/adapters/hip/ur_interface_loader.cpp @@ -503,6 +503,10 @@ UR_DLLEXPORT ur_result_t UR_APICALL urGetIPCExpProcAddrTable( pDdiTable->pfnPutMemHandleExp = urIPCPutMemHandleExp; pDdiTable->pfnOpenMemHandleExp = urIPCOpenMemHandleExp; pDdiTable->pfnCloseMemHandleExp = urIPCCloseMemHandleExp; + pDdiTable->pfnGetPhysMemHandleExp = urIPCGetPhysMemHandleExp; + pDdiTable->pfnPutPhysMemHandleExp = urIPCPutPhysMemHandleExp; + pDdiTable->pfnOpenPhysMemHandleExp = urIPCOpenPhysMemHandleExp; + pDdiTable->pfnClosePhysMemHandleExp = urIPCClosePhysMemHandleExp; return UR_RESULT_SUCCESS; } diff --git a/unified-runtime/source/adapters/level_zero/CMakeLists.txt b/unified-runtime/source/adapters/level_zero/CMakeLists.txt index 42e63e952f392..95108f9307601 100644 --- a/unified-runtime/source/adapters/level_zero/CMakeLists.txt +++ b/unified-runtime/source/adapters/level_zero/CMakeLists.txt @@ -25,7 +25,6 @@ if(UR_BUILD_ADAPTER_L0) ${CMAKE_CURRENT_SOURCE_DIR}/usm.hpp ${CMAKE_CURRENT_SOURCE_DIR}/memory.hpp ${CMAKE_CURRENT_SOURCE_DIR}/kernel.hpp - ${CMAKE_CURRENT_SOURCE_DIR}/physical_mem.hpp ${CMAKE_CURRENT_SOURCE_DIR}/platform.hpp ${CMAKE_CURRENT_SOURCE_DIR}/program.hpp ${CMAKE_CURRENT_SOURCE_DIR}/queue.hpp @@ -132,7 +131,6 @@ if(UR_BUILD_ADAPTER_L0_V2) ${CMAKE_CURRENT_SOURCE_DIR}/device.hpp ${CMAKE_CURRENT_SOURCE_DIR}/image_common.hpp ${CMAKE_CURRENT_SOURCE_DIR}/platform.hpp - ${CMAKE_CURRENT_SOURCE_DIR}/physical_mem.hpp ${CMAKE_CURRENT_SOURCE_DIR}/program.hpp ${CMAKE_CURRENT_SOURCE_DIR}/helpers/kernel_helpers.hpp ${CMAKE_CURRENT_SOURCE_DIR}/helpers/memory_helpers.hpp @@ -145,7 +143,6 @@ if(UR_BUILD_ADAPTER_L0_V2) ${CMAKE_CURRENT_SOURCE_DIR}/image_common.cpp ${CMAKE_CURRENT_SOURCE_DIR}/ur_interface_loader.cpp ${CMAKE_CURRENT_SOURCE_DIR}/platform.cpp - ${CMAKE_CURRENT_SOURCE_DIR}/physical_mem.cpp ${CMAKE_CURRENT_SOURCE_DIR}/program.cpp ${CMAKE_CURRENT_SOURCE_DIR}/helpers/kernel_helpers.cpp ${CMAKE_CURRENT_SOURCE_DIR}/helpers/memory_helpers.cpp @@ -170,6 +167,7 @@ if(UR_BUILD_ADAPTER_L0_V2) ${CMAKE_CURRENT_SOURCE_DIR}/v2/kernel.hpp ${CMAKE_CURRENT_SOURCE_DIR}/v2/memory.hpp ${CMAKE_CURRENT_SOURCE_DIR}/v2/lockable.hpp + ${CMAKE_CURRENT_SOURCE_DIR}/v2/physical_mem.hpp ${CMAKE_CURRENT_SOURCE_DIR}/v2/queue_api.hpp ${CMAKE_CURRENT_SOURCE_DIR}/v2/queue_batched.hpp ${CMAKE_CURRENT_SOURCE_DIR}/v2/queue_immediate_in_order.hpp @@ -187,6 +185,7 @@ if(UR_BUILD_ADAPTER_L0_V2) ${CMAKE_CURRENT_SOURCE_DIR}/v2/event.cpp ${CMAKE_CURRENT_SOURCE_DIR}/v2/kernel.cpp ${CMAKE_CURRENT_SOURCE_DIR}/v2/memory.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/v2/physical_mem.cpp ${CMAKE_CURRENT_SOURCE_DIR}/v2/queue_api.cpp ${CMAKE_CURRENT_SOURCE_DIR}/v2/queue_batched.cpp ${CMAKE_CURRENT_SOURCE_DIR}/v2/queue_create.cpp diff --git a/unified-runtime/source/adapters/level_zero/device.cpp b/unified-runtime/source/adapters/level_zero/device.cpp index 03b570bda31d7..e3012bfbda43a 100644 --- a/unified-runtime/source/adapters/level_zero/device.cpp +++ b/unified-runtime/source/adapters/level_zero/device.cpp @@ -1379,6 +1379,12 @@ ur_result_t urDeviceGetInfo( return ReturnValue(false); #else return ReturnValue(true); +#endif + case UR_DEVICE_INFO_IPC_PHYSICAL_MEMORY_SUPPORT_EXP: +#ifdef __linux__ + return ReturnValue(Device->isIntelBMGOrNewer()); +#else + return ReturnValue(false); #endif case UR_DEVICE_INFO_ASYNC_BARRIER: return ReturnValue(false); diff --git a/unified-runtime/source/adapters/level_zero/device.hpp b/unified-runtime/source/adapters/level_zero/device.hpp index 0e184c0c5f01d..7ddba032196e0 100644 --- a/unified-runtime/source/adapters/level_zero/device.hpp +++ b/unified-runtime/source/adapters/level_zero/device.hpp @@ -209,6 +209,11 @@ struct ur_device_handle_t_ : ur_object { ZeDeviceIpVersionExt->ipVersion >= 0x030f0000); } + bool isIntelBMGOrNewer() { + return (ZeDeviceProperties->vendorId == 0x8086 && + ZeDeviceIpVersionExt->ipVersion >= 0x05004000); + } + bool isIntegrated() { return (ZeDeviceProperties->flags & ZE_DEVICE_PROPERTY_FLAG_INTEGRATED); } diff --git a/unified-runtime/source/adapters/level_zero/memory.cpp b/unified-runtime/source/adapters/level_zero/memory.cpp index 97239a8416564..b4a1724de2938 100644 --- a/unified-runtime/source/adapters/level_zero/memory.cpp +++ b/unified-runtime/source/adapters/level_zero/memory.cpp @@ -10,6 +10,10 @@ #include #include #include +#ifdef __linux__ +#include +#include +#endif #include #include "context.hpp" @@ -20,6 +24,7 @@ #include "queue.hpp" #include "ur_interface_loader.hpp" #include "ur_level_zero.hpp" +#include "v2/physical_mem.hpp" // Default to using compute engine for fill operation, but allow to // override this with an environment variable. diff --git a/unified-runtime/source/adapters/level_zero/physical_mem.cpp b/unified-runtime/source/adapters/level_zero/physical_mem.cpp index a0663253f42e3..cc00a99b74ffd 100644 --- a/unified-runtime/source/adapters/level_zero/physical_mem.cpp +++ b/unified-runtime/source/adapters/level_zero/physical_mem.cpp @@ -7,15 +7,10 @@ // //===----------------------------------------------------------------------===// -#include "physical_mem.hpp" +#include "v2/physical_mem.hpp" #include "common.hpp" -#include "device.hpp" - -#ifdef UR_ADAPTER_LEVEL_ZERO_V2 -#include "v2/context.hpp" -#else #include "context.hpp" -#endif +#include "device.hpp" namespace ur::level_zero { @@ -27,14 +22,20 @@ ur_result_t urPhysicalMemCreate( PhysicalMemDesc.flags = 0; PhysicalMemDesc.size = size; + bool EnableIpc = + pProperties && (pProperties->flags & UR_PHYSICAL_MEM_FLAG_ENABLE_IPC); + ze_physical_mem_handle_t ZePhysicalMem; ZE2UR_CALL(zePhysicalMemCreate, (hContext->getZeHandle(), hDevice->ZeDevice, &PhysicalMemDesc, &ZePhysicalMem)); try { - *phPhysicalMem = new ur_physical_mem_handle_t_(ZePhysicalMem, hContext); + *phPhysicalMem = new ur_physical_mem_handle_t_(ZePhysicalMem, hContext, + hDevice, size, EnableIpc); } catch (const std::bad_alloc &) { + zePhysicalMemDestroy(hContext->getZeHandle(), ZePhysicalMem); return UR_RESULT_ERROR_OUT_OF_HOST_MEMORY; } catch (...) { + zePhysicalMemDestroy(hContext->getZeHandle(), ZePhysicalMem); return UR_RESULT_ERROR_UNKNOWN; } return UR_RESULT_SUCCESS; @@ -49,13 +50,25 @@ ur_result_t urPhysicalMemRelease(ur_physical_mem_handle_t hPhysicalMem) { if (!hPhysicalMem->RefCount.release()) return UR_RESULT_SUCCESS; - if (checkL0LoaderTeardown()) { - ZE2UR_CALL(zePhysicalMemDestroy, (hPhysicalMem->Context->getZeHandle(), - hPhysicalMem->ZePhysicalMem)); + ur_result_t Res = UR_RESULT_SUCCESS; + if (hPhysicalMem->IpcVirtualAddress) { + if (checkL0LoaderTeardown()) { + ze_result_t ZeRes = ZE_CALL_NOCHECK(zeMemCloseIpcHandle, + (hPhysicalMem->Context->getZeHandle(), + hPhysicalMem->IpcVirtualAddress)); + Res = ze2urResult(ZeRes); + } + } else if (hPhysicalMem->ZePhysicalMem) { + if (checkL0LoaderTeardown()) { + ze_result_t ZeRes = ZE_CALL_NOCHECK( + zePhysicalMemDestroy, + (hPhysicalMem->Context->getZeHandle(), hPhysicalMem->ZePhysicalMem)); + Res = ze2urResult(ZeRes); + } } delete hPhysicalMem; - return UR_RESULT_SUCCESS; + return Res; } ur_result_t urPhysicalMemGetInfo(ur_physical_mem_handle_t hPhysicalMem, @@ -66,13 +79,51 @@ ur_result_t urPhysicalMemGetInfo(ur_physical_mem_handle_t hPhysicalMem, UrReturnHelper ReturnValue(propSize, pPropValue, pPropSizeRet); switch (propName) { - case UR_PHYSICAL_MEM_INFO_REFERENCE_COUNT: { - return ReturnValue(hPhysicalMem->RefCount.getCount()); + case UR_PHYSICAL_MEM_INFO_CONTEXT: + return ReturnValue(hPhysicalMem->Context); + case UR_PHYSICAL_MEM_INFO_DEVICE: + return ReturnValue(hPhysicalMem->Device); + case UR_PHYSICAL_MEM_INFO_SIZE: + return ReturnValue(hPhysicalMem->Size); + case UR_PHYSICAL_MEM_INFO_PROPERTIES: { + ur_physical_mem_flags_t Flags = static_cast(0); + if (hPhysicalMem->EnableIpc) + Flags = UR_PHYSICAL_MEM_FLAG_ENABLE_IPC; + ur_physical_mem_properties_t Props = { + UR_STRUCTURE_TYPE_PHYSICAL_MEM_PROPERTIES, nullptr, Flags}; + return ReturnValue(Props); } + case UR_PHYSICAL_MEM_INFO_REFERENCE_COUNT: + return ReturnValue(hPhysicalMem->RefCount.getCount()); + case UR_PHYSICAL_MEM_INFO_IPC_VIRTUAL_ADDRESS: + return ReturnValue(hPhysicalMem->IpcVirtualAddress); default: return UR_RESULT_ERROR_UNSUPPORTED_ENUMERATION; } return UR_RESULT_SUCCESS; } +// IPC physical memory functions are only supported on L0v2 (Xe2+/BMG hardware). +// Provide stubs here so the L0v1 adapter links correctly. +ur_result_t urIPCGetPhysMemHandleExp(ur_context_handle_t, + ur_physical_mem_handle_t, void **, + size_t *) { + return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; +} + +ur_result_t urIPCPutPhysMemHandleExp(ur_context_handle_t, const void *) { + return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; +} + +ur_result_t urIPCOpenPhysMemHandleExp(ur_context_handle_t, ur_device_handle_t, + const void *, size_t, + ur_physical_mem_handle_t *) { + return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; +} + +ur_result_t urIPCClosePhysMemHandleExp(ur_context_handle_t, + ur_physical_mem_handle_t) { + return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; +} + } // namespace ur::level_zero diff --git a/unified-runtime/source/adapters/level_zero/physical_mem.hpp b/unified-runtime/source/adapters/level_zero/physical_mem.hpp deleted file mode 100644 index a5db639575e8d..0000000000000 --- a/unified-runtime/source/adapters/level_zero/physical_mem.hpp +++ /dev/null @@ -1,26 +0,0 @@ -//===---------------- physical_mem.hpp - Level Zero Adapter ---------------===// -// -// -// Part of the LLVM Project, under the Apache License v2.0 with LLVM -// Exceptions. See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// -#pragma once - -#include "common.hpp" -#include "common/ur_ref_count.hpp" - -struct ur_physical_mem_handle_t_ : ur_object { - ur_physical_mem_handle_t_(ze_physical_mem_handle_t ZePhysicalMem, - ur_context_handle_t Context) - : ZePhysicalMem{ZePhysicalMem}, Context{Context} {} - - // Level Zero physical memory handle. - ze_physical_mem_handle_t ZePhysicalMem; - - // Keeps the PI context of this memory handle. - ur_context_handle_t Context; - - ur::RefCount RefCount; -}; diff --git a/unified-runtime/source/adapters/level_zero/ur_interface_loader.hpp b/unified-runtime/source/adapters/level_zero/ur_interface_loader.hpp index 7a0efa06d9595..bc4956f6eed2f 100644 --- a/unified-runtime/source/adapters/level_zero/ur_interface_loader.hpp +++ b/unified-runtime/source/adapters/level_zero/ur_interface_loader.hpp @@ -634,10 +634,10 @@ ur_result_t urIPCGetPhysMemHandleExp(ur_context_handle_t hContext, void **ppIPCPhysMemHandleData, size_t *pIPCPhysMemHandleDataSizeRet); ur_result_t urIPCPutPhysMemHandleExp(ur_context_handle_t hContext, - void *pIPCPhysMemHandleData); + const void *pIPCPhysMemHandleData); ur_result_t urIPCOpenPhysMemHandleExp(ur_context_handle_t hContext, ur_device_handle_t hDevice, - void *pIPCPhysMemHandleData, + const void *pIPCPhysMemHandleData, size_t ipcPhysMemHandleDataSize, ur_physical_mem_handle_t *phPhysMem); ur_result_t urIPCClosePhysMemHandleExp(ur_context_handle_t hContext, diff --git a/unified-runtime/source/adapters/level_zero/ur_level_zero.hpp b/unified-runtime/source/adapters/level_zero/ur_level_zero.hpp index 9cd954311283c..4576502df3669 100644 --- a/unified-runtime/source/adapters/level_zero/ur_level_zero.hpp +++ b/unified-runtime/source/adapters/level_zero/ur_level_zero.hpp @@ -29,9 +29,9 @@ #include "event.hpp" #include "kernel.hpp" #include "memory.hpp" -#include "physical_mem.hpp" #include "platform.hpp" #include "program.hpp" #include "queue.hpp" #include "sampler.hpp" #include "usm.hpp" +#include "v2/physical_mem.hpp" diff --git a/unified-runtime/source/adapters/level_zero/v2/memory.cpp b/unified-runtime/source/adapters/level_zero/v2/memory.cpp index b0601ba956af2..ff981e810df7b 100644 --- a/unified-runtime/source/adapters/level_zero/v2/memory.cpp +++ b/unified-runtime/source/adapters/level_zero/v2/memory.cpp @@ -9,11 +9,17 @@ #include "memory.hpp" +#ifdef __linux__ +#include +#include +#endif + #include "../ur_interface_loader.hpp" #include "context.hpp" #include "../helpers/memory_helpers.hpp" #include "../image_common.hpp" +#include "physical_mem.hpp" static bool isAccessCompatible(ur_mem_buffer_t::device_access_mode_t requested, ur_mem_buffer_t::device_access_mode_t actual) { diff --git a/unified-runtime/source/adapters/level_zero/v2/physical_mem.cpp b/unified-runtime/source/adapters/level_zero/v2/physical_mem.cpp new file mode 100644 index 0000000000000..7d055e15d3456 --- /dev/null +++ b/unified-runtime/source/adapters/level_zero/v2/physical_mem.cpp @@ -0,0 +1,282 @@ +//===---------------- physical_mem.cpp - Level Zero Adapter ---------------===// +// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM +// Exceptions. See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "physical_mem.hpp" +#include "../device.hpp" +#include "common.hpp" + +#ifdef __linux__ +#include +#endif + +#include "context.hpp" + +namespace ur::level_zero { + +ur_result_t urPhysicalMemCreate( + ur_context_handle_t hContext, ur_device_handle_t hDevice, size_t size, + [[maybe_unused]] const ur_physical_mem_properties_t *pProperties, + ur_physical_mem_handle_t *phPhysicalMem) { + ZeStruct PhysicalMemDesc; + PhysicalMemDesc.flags = 0; + PhysicalMemDesc.size = size; + + bool EnableIpc = + pProperties && (pProperties->flags & UR_PHYSICAL_MEM_FLAG_ENABLE_IPC); + + ze_physical_mem_handle_t ZePhysicalMem; + ZE2UR_CALL(zePhysicalMemCreate, (hContext->getZeHandle(), hDevice->ZeDevice, + &PhysicalMemDesc, &ZePhysicalMem)); + try { + *phPhysicalMem = new ur_physical_mem_handle_t_(ZePhysicalMem, hContext, + hDevice, size, EnableIpc); + } catch (const std::bad_alloc &) { + zePhysicalMemDestroy(hContext->getZeHandle(), ZePhysicalMem); + return UR_RESULT_ERROR_OUT_OF_HOST_MEMORY; + } catch (...) { + zePhysicalMemDestroy(hContext->getZeHandle(), ZePhysicalMem); + return UR_RESULT_ERROR_UNKNOWN; + } + return UR_RESULT_SUCCESS; +} + +ur_result_t urPhysicalMemRetain(ur_physical_mem_handle_t hPhysicalMem) { + hPhysicalMem->RefCount.retain(); + return UR_RESULT_SUCCESS; +} + +ur_result_t urPhysicalMemRelease(ur_physical_mem_handle_t hPhysicalMem) { + if (!hPhysicalMem->RefCount.release()) + return UR_RESULT_SUCCESS; + + ur_result_t Res = UR_RESULT_SUCCESS; + if (hPhysicalMem->IpcVirtualAddress) { + // IPC-opened handle: close the IPC virtual mapping instead of destroying + // a physical mem handle (there is no ZePhysicalMem on the consumer side). + if (checkL0LoaderTeardown()) { + ze_result_t ZeRes = ZE_CALL_NOCHECK(zeMemCloseIpcHandle, + (hPhysicalMem->Context->getZeHandle(), + hPhysicalMem->IpcVirtualAddress)); + Res = ze2urResult(ZeRes); + } + } else if (hPhysicalMem->ZePhysicalMem) { + if (checkL0LoaderTeardown()) { + ze_result_t ZeRes = ZE_CALL_NOCHECK( + zePhysicalMemDestroy, + (hPhysicalMem->Context->getZeHandle(), hPhysicalMem->ZePhysicalMem)); + Res = ze2urResult(ZeRes); + } + } + delete hPhysicalMem; + + return Res; +} + +ur_result_t urPhysicalMemGetInfo(ur_physical_mem_handle_t hPhysicalMem, + ur_physical_mem_info_t propName, + size_t propSize, void *pPropValue, + size_t *pPropSizeRet) { + + UrReturnHelper ReturnValue(propSize, pPropValue, pPropSizeRet); + + switch (propName) { + case UR_PHYSICAL_MEM_INFO_CONTEXT: + return ReturnValue(hPhysicalMem->Context); + case UR_PHYSICAL_MEM_INFO_DEVICE: + return ReturnValue(hPhysicalMem->Device); + case UR_PHYSICAL_MEM_INFO_SIZE: + return ReturnValue(hPhysicalMem->Size); + case UR_PHYSICAL_MEM_INFO_PROPERTIES: { + ur_physical_mem_flags_t Flags = static_cast(0); + if (hPhysicalMem->EnableIpc) + Flags = UR_PHYSICAL_MEM_FLAG_ENABLE_IPC; + ur_physical_mem_properties_t Props = { + UR_STRUCTURE_TYPE_PHYSICAL_MEM_PROPERTIES, nullptr, Flags}; + return ReturnValue(Props); + } + case UR_PHYSICAL_MEM_INFO_REFERENCE_COUNT: + return ReturnValue(hPhysicalMem->RefCount.getCount()); + case UR_PHYSICAL_MEM_INFO_IPC_VIRTUAL_ADDRESS: + return ReturnValue(hPhysicalMem->IpcVirtualAddress); + default: + return UR_RESULT_ERROR_UNSUPPORTED_ENUMERATION; + } + return UR_RESULT_SUCCESS; +} + +ur_result_t urIPCGetPhysMemHandleExp(ur_context_handle_t hContext, + ur_physical_mem_handle_t hPhysMem, + void **ppIPCPhysMemHandleData, + size_t *pIPCPhysMemHandleDataSizeRet) { +#ifdef __linux__ + if (!hPhysMem->Device->isIntelBMGOrNewer()) + return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; + + if (!hPhysMem->EnableIpc) + return UR_RESULT_ERROR_INVALID_ARGUMENT; + + // IPC-opened handles (consumer side) have ZePhysicalMem == nullptr; they + // cannot be re-exported. + if (!hPhysMem->ZePhysicalMem) + return UR_RESULT_ERROR_INVALID_ARGUMENT; + + // Pass the physical memory handle directly to zeMemGetIpcHandleWithProperties + // No prior virtual mapping is required or expected; the physical handle + // itself is the key. + ze_ipc_mem_handle_type_ext_desc_t HandleTypeDesc = {}; + HandleTypeDesc.stype = ZE_STRUCTURE_TYPE_IPC_MEM_HANDLE_TYPE_EXT_DESC; + HandleTypeDesc.pNext = nullptr; + HandleTypeDesc.typeFlags = ZE_IPC_MEM_HANDLE_TYPE_FLAG_DEFAULT; + + ze_ipc_mem_handle_t IpcHandle = {}; + ze_result_t ZeRes = + ZE_CALL_NOCHECK(zeMemGetIpcHandleWithProperties, + (hContext->getZeHandle(), + reinterpret_cast(hPhysMem->ZePhysicalMem), + &HandleTypeDesc, &IpcHandle)); + // On drivers that do not support physical-mem IPC the function may return + // ZE_RESULT_ERROR_UNSUPPORTED_FEATURE or ZE_RESULT_ERROR_INVALID_ARGUMENT. + // Map both to UR_RESULT_ERROR_UNSUPPORTED_FEATURE so callers can skip. + if (ZeRes == ZE_RESULT_ERROR_INVALID_ARGUMENT || + ZeRes == ZE_RESULT_ERROR_UNSUPPORTED_FEATURE) + return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; + if (ZeRes != ZE_RESULT_SUCCESS) + return ze2urResult(ZeRes); + + // Some driver versions accept this call but return an fd-based handle even + // when ZE_IPC_MEM_HANDLE_TYPE_FLAG_DEFAULT is requested, because they do not + // yet support passing a ze_physical_mem_handle_t directly. An fd-based + // handle cannot be serialized to a plain byte buffer for cross-process + // transfer without SCM_RIGHTS socket transfer. Detect this case: if the + // first bytes of the handle form a valid open file descriptor in this + // process, the handle is fd-based. Release it and return UNSUPPORTED_FEATURE + // so callers can skip gracefully. + { + int FdVal = 0; + static_assert(sizeof(IpcHandle.data) >= sizeof(FdVal)); + memcpy(&FdVal, IpcHandle.data, sizeof(FdVal)); + if (FdVal >= 0 && ::fcntl(FdVal, F_GETFD) >= 0) { + zeMemPutIpcHandle(hContext->getZeHandle(), IpcHandle); + return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; + } + } + + auto *HandleData = new (std::nothrow) ZeIPCPhysMemHandleData; + if (!HandleData) { + zeMemPutIpcHandle(hContext->getZeHandle(), IpcHandle); + return UR_RESULT_ERROR_OUT_OF_HOST_MEMORY; + } + + HandleData->IpcHandle = IpcHandle; + HandleData->Size = hPhysMem->Size; + + *ppIPCPhysMemHandleData = HandleData; + *pIPCPhysMemHandleDataSizeRet = sizeof(ZeIPCPhysMemHandleData); + return UR_RESULT_SUCCESS; +#else + (void)hContext; + (void)hPhysMem; + (void)ppIPCPhysMemHandleData; + (void)pIPCPhysMemHandleDataSizeRet; + return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; +#endif // __linux__ +} + +ur_result_t urIPCPutPhysMemHandleExp(ur_context_handle_t hContext, + const void *pIPCPhysMemHandleData) { +#ifdef __linux__ + auto *HandleData = + static_cast(pIPCPhysMemHandleData); + ze_result_t ZeResult = ZE_CALL_NOCHECK( + zeMemPutIpcHandle, (hContext->getZeHandle(), HandleData->IpcHandle)); + // Always delete the UR wrapper: urIPCPutPhysMemHandleExp takes ownership of + // pIPCPhysMemHandleData and the pointer must not be used by the caller after + // this call, regardless of the return value. If zeMemPutIpcHandle fails, the + // driver-side handle may be in an undefined state, but there is no mechanism + // to retry — the wrapper is freed unconditionally. + delete HandleData; + return ze2urResult(ZeResult); +#else + (void)hContext; + (void)pIPCPhysMemHandleData; + return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; +#endif // __linux__ +} + +ur_result_t urIPCOpenPhysMemHandleExp(ur_context_handle_t hContext, + ur_device_handle_t hDevice, + const void *pIPCPhysMemHandleData, + size_t ipcPhysMemHandleDataSize, + ur_physical_mem_handle_t *phPhysMem) { +#ifdef __linux__ + if (ipcPhysMemHandleDataSize != sizeof(ZeIPCPhysMemHandleData)) + return UR_RESULT_ERROR_INVALID_VALUE; + + if (!hDevice->isIntelBMGOrNewer()) + return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; + + auto *HandleData = + static_cast(pIPCPhysMemHandleData); + + if (HandleData->Size == 0) + return UR_RESULT_ERROR_INVALID_VALUE; + + // Open the IPC handle in this process. zeMemOpenIpcHandle creates a virtual + // mapping backed by the exporter's physical memory and returns a pointer to + // it. No separate zeVirtualMemMap call is needed; the memory is immediately + // accessible at the returned address. + void *VirtualAddress = nullptr; + ZE2UR_CALL(zeMemOpenIpcHandle, (hContext->getZeHandle(), hDevice->ZeDevice, + HandleData->IpcHandle, 0, &VirtualAddress)); + if (!VirtualAddress) + return UR_RESULT_ERROR_UNKNOWN; + + try { + *phPhysMem = new ur_physical_mem_handle_t_( + /*ZePhysicalMem=*/nullptr, hContext, hDevice, HandleData->Size, + /*EnableIpc=*/true, VirtualAddress); + } catch (const std::bad_alloc &) { + zeMemCloseIpcHandle(hContext->getZeHandle(), VirtualAddress); + return UR_RESULT_ERROR_OUT_OF_HOST_MEMORY; + } catch (...) { + zeMemCloseIpcHandle(hContext->getZeHandle(), VirtualAddress); + return UR_RESULT_ERROR_UNKNOWN; + } + + return UR_RESULT_SUCCESS; +#else + (void)hContext; + (void)hDevice; + (void)pIPCPhysMemHandleData; + (void)ipcPhysMemHandleDataSize; + (void)phPhysMem; + return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; +#endif // __linux__ +} + +ur_result_t urIPCClosePhysMemHandleExp(ur_context_handle_t hContext, + ur_physical_mem_handle_t hPhysMem) { +#ifdef __linux__ + (void)hContext; + if (!hPhysMem->Device->isIntelBMGOrNewer()) + return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; + + // Delegate to urPhysicalMemRelease so the refcount is respected. For + // IPC-opened handles (IpcVirtualAddress != nullptr) urPhysicalMemRelease + // calls zeMemCloseIpcHandle; for regular handles it calls + // zePhysicalMemDestroy. + return ur::level_zero::urPhysicalMemRelease(hPhysMem); +#else + (void)hContext; + (void)hPhysMem; + return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; +#endif // __linux__ +} + +} // namespace ur::level_zero diff --git a/unified-runtime/source/adapters/level_zero/v2/physical_mem.hpp b/unified-runtime/source/adapters/level_zero/v2/physical_mem.hpp new file mode 100644 index 0000000000000..bc260c18beb5d --- /dev/null +++ b/unified-runtime/source/adapters/level_zero/v2/physical_mem.hpp @@ -0,0 +1,59 @@ +//===---------------- physical_mem.hpp - Level Zero Adapter ---------------===// +// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM +// Exceptions. See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +#pragma once + +#include "common.hpp" +#include "common/ur_ref_count.hpp" + +// Opaque handle data exchanged between processes for physical memory IPC. +// Contains the IPC handle obtained from zeMemGetIpcHandleWithProperties +// plus the allocation size required to reconstruct the physical_mem on import. +// When the driver correctly implements ZE_IPC_MEM_HANDLE_TYPE_FLAG_DEFAULT for +// physical memory, ze_ipc_mem_handle_t is a self-contained 64-byte opaque blob +// that can be transferred to another process without fd passing. On drivers +// that fall back to an fd-based handle, urIPCGetPhysMemHandleExp detects and +// rejects the fd handle, returning UR_RESULT_ERROR_UNSUPPORTED_FEATURE. +struct ZeIPCPhysMemHandleData { + ze_ipc_mem_handle_t IpcHandle; // opaque IPC handle (64 bytes) + size_t Size; +}; + +struct ur_physical_mem_handle_t_ : ur_object { + ur_physical_mem_handle_t_(ze_physical_mem_handle_t ZePhysicalMem, + ur_context_handle_t Context, + ur_device_handle_t Device, size_t Size, + bool EnableIpc, void *IpcVirtualAddress = nullptr) + : ZePhysicalMem{ZePhysicalMem}, Context{Context}, Device{Device}, + Size{Size}, EnableIpc{EnableIpc}, IpcVirtualAddress{IpcVirtualAddress} { + } + + // Level Zero physical memory handle. Null for IPC-opened handles (consumer + // side), where the memory is accessible via IpcVirtualAddress instead. + ze_physical_mem_handle_t ZePhysicalMem; + + // Keeps the PI context of this memory handle. + ur_context_handle_t Context; + + // Device this physical memory was allocated on. + ur_device_handle_t Device; + + // Size in bytes of this physical memory allocation. + size_t Size; + + // Whether this allocation was created with IPC export enabled. + bool EnableIpc; + + // Virtual address returned by zeMemOpenIpcHandle for IPC-opened handles + // (consumer side). Non-null means the memory is already virtually mapped + // at this address; cleanup uses zeMemCloseIpcHandle rather than + // zePhysicalMemDestroy. + void *IpcVirtualAddress; + + ur::RefCount RefCount; +}; diff --git a/unified-runtime/source/adapters/level_zero/virtual_mem.cpp b/unified-runtime/source/adapters/level_zero/virtual_mem.cpp index 5100c0025fbe5..2870823b5447e 100644 --- a/unified-runtime/source/adapters/level_zero/virtual_mem.cpp +++ b/unified-runtime/source/adapters/level_zero/virtual_mem.cpp @@ -10,7 +10,7 @@ #include "common.hpp" #include "device.hpp" #include "logger/ur_logger.hpp" -#include "physical_mem.hpp" +#include "v2/physical_mem.hpp" #ifdef UR_ADAPTER_LEVEL_ZERO_V2 #include "v2/context.hpp" @@ -80,6 +80,16 @@ ur_result_t urVirtualMemMap(ur_context_handle_t hContext, const void *pStart, size_t size, ur_physical_mem_handle_t hPhysicalMem, size_t offset, ur_virtual_mem_access_flags_t flags) { + // IPC-opened physical memory handles already have a virtual address + // established by zeMemOpenIpcHandle (stored in IpcVirtualAddress). + // The UR virtual memory reservation at pStart is a separate address range + // that cannot be backed by an IPC handle via zeVirtualMemMap — the driver + // requires the physical handle to map, which is null on the consumer side. + // Return INVALID_ARGUMENT so callers know to use + // urPhysicalMemGetInfo(UR_PHYSICAL_MEM_INFO_IPC_VIRTUAL_ADDRESS) instead. + if (hPhysicalMem->IpcVirtualAddress) + return UR_RESULT_ERROR_INVALID_ARGUMENT; + ze_memory_access_attribute_t AccessAttr = ZE_MEMORY_ACCESS_ATTRIBUTE_NONE; if (flags & UR_VIRTUAL_MEM_ACCESS_FLAG_READ_WRITE) AccessAttr = ZE_MEMORY_ACCESS_ATTRIBUTE_READWRITE; diff --git a/unified-runtime/source/adapters/mock/ur_mockddi.cpp b/unified-runtime/source/adapters/mock/ur_mockddi.cpp index cc77f6eccf203..ddaa42ac60eec 100644 --- a/unified-runtime/source/adapters/mock/ur_mockddi.cpp +++ b/unified-runtime/source/adapters/mock/ur_mockddi.cpp @@ -9716,8 +9716,9 @@ __urdlllocal ur_result_t UR_APICALL urIPCGetPhysMemHandleExp( __urdlllocal ur_result_t UR_APICALL urIPCPutPhysMemHandleExp( /// [in] handle of the context object ur_context_handle_t hContext, - /// [in] a pointer to the IPC physical memory handle data - void *pIPCPhysMemHandleData) try { + /// [in] a pointer to the IPC physical memory handle data obtained with + /// urIPCGetPhysMemHandleExp + const void *pIPCPhysMemHandleData) try { ur_result_t result = UR_RESULT_SUCCESS; ur_ipc_put_phys_mem_handle_exp_params_t params = {&hContext, @@ -9763,8 +9764,9 @@ __urdlllocal ur_result_t UR_APICALL urIPCOpenPhysMemHandleExp( ur_context_handle_t hContext, /// [in] handle of the device object the physical memory was allocated on ur_device_handle_t hDevice, - /// [in] the IPC physical memory handle data - void *pIPCPhysMemHandleData, + /// [in] the IPC physical memory handle data obtained with + /// urIPCGetPhysMemHandleExp + const void *pIPCPhysMemHandleData, /// [in] size of the IPC physical memory handle data size_t ipcPhysMemHandleDataSize, /// [out] pointer to the physical memory handle diff --git a/unified-runtime/source/adapters/native_cpu/device.cpp b/unified-runtime/source/adapters/native_cpu/device.cpp index f08de5694a2f3..e74668f197281 100644 --- a/unified-runtime/source/adapters/native_cpu/device.cpp +++ b/unified-runtime/source/adapters/native_cpu/device.cpp @@ -381,6 +381,8 @@ UR_APIEXPORT ur_result_t UR_APICALL urDeviceGetInfo(ur_device_handle_t hDevice, static_cast(0)); case UR_DEVICE_INFO_IPC_MEMORY_SUPPORT_EXP: return ReturnValue(false); + case UR_DEVICE_INFO_IPC_PHYSICAL_MEMORY_SUPPORT_EXP: + return ReturnValue(false); case UR_DEVICE_INFO_ATOMIC_FENCE_ORDER_CAPABILITIES: { // Currently for Native CPU fences are implemented using OCK // builtins, so we have different capabilities than atomic operations diff --git a/unified-runtime/source/adapters/native_cpu/memory.cpp b/unified-runtime/source/adapters/native_cpu/memory.cpp index 2ac5eccc11cc3..2328dcfc287b8 100644 --- a/unified-runtime/source/adapters/native_cpu/memory.cpp +++ b/unified-runtime/source/adapters/native_cpu/memory.cpp @@ -160,3 +160,24 @@ UR_APIEXPORT ur_result_t UR_APICALL urIPCCloseMemHandleExp(ur_context_handle_t, void *) { return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; } + +UR_APIEXPORT ur_result_t UR_APICALL urIPCGetPhysMemHandleExp( + ur_context_handle_t, ur_physical_mem_handle_t, void **, size_t *) { + return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; +} + +UR_APIEXPORT ur_result_t UR_APICALL +urIPCPutPhysMemHandleExp(ur_context_handle_t, const void *) { + return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; +} + +UR_APIEXPORT ur_result_t UR_APICALL +urIPCOpenPhysMemHandleExp(ur_context_handle_t, ur_device_handle_t, const void *, + size_t, ur_physical_mem_handle_t *) { + return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; +} + +UR_APIEXPORT ur_result_t UR_APICALL +urIPCClosePhysMemHandleExp(ur_context_handle_t, ur_physical_mem_handle_t) { + return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; +} diff --git a/unified-runtime/source/adapters/native_cpu/ur_interface_loader.cpp b/unified-runtime/source/adapters/native_cpu/ur_interface_loader.cpp index c3b121bfb0cdb..5f1228273b006 100644 --- a/unified-runtime/source/adapters/native_cpu/ur_interface_loader.cpp +++ b/unified-runtime/source/adapters/native_cpu/ur_interface_loader.cpp @@ -487,6 +487,10 @@ UR_DLLEXPORT ur_result_t UR_APICALL urGetIPCExpProcAddrTable( pDdiTable->pfnPutMemHandleExp = urIPCPutMemHandleExp; pDdiTable->pfnOpenMemHandleExp = urIPCOpenMemHandleExp; pDdiTable->pfnCloseMemHandleExp = urIPCCloseMemHandleExp; + pDdiTable->pfnGetPhysMemHandleExp = urIPCGetPhysMemHandleExp; + pDdiTable->pfnPutPhysMemHandleExp = urIPCPutPhysMemHandleExp; + pDdiTable->pfnOpenPhysMemHandleExp = urIPCOpenPhysMemHandleExp; + pDdiTable->pfnClosePhysMemHandleExp = urIPCClosePhysMemHandleExp; return UR_RESULT_SUCCESS; } diff --git a/unified-runtime/source/adapters/offload/memory.cpp b/unified-runtime/source/adapters/offload/memory.cpp index c0b7a97fb62ef..7d55d14018481 100644 --- a/unified-runtime/source/adapters/offload/memory.cpp +++ b/unified-runtime/source/adapters/offload/memory.cpp @@ -229,3 +229,24 @@ UR_APIEXPORT ur_result_t UR_APICALL urIPCCloseMemHandleExp(ur_context_handle_t, void *) { return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; } + +UR_APIEXPORT ur_result_t UR_APICALL urIPCGetPhysMemHandleExp( + ur_context_handle_t, ur_physical_mem_handle_t, void **, size_t *) { + return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; +} + +UR_APIEXPORT ur_result_t UR_APICALL +urIPCPutPhysMemHandleExp(ur_context_handle_t, const void *) { + return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; +} + +UR_APIEXPORT ur_result_t UR_APICALL +urIPCOpenPhysMemHandleExp(ur_context_handle_t, ur_device_handle_t, const void *, + size_t, ur_physical_mem_handle_t *) { + return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; +} + +UR_APIEXPORT ur_result_t UR_APICALL +urIPCClosePhysMemHandleExp(ur_context_handle_t, ur_physical_mem_handle_t) { + return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; +} diff --git a/unified-runtime/source/adapters/offload/ur_interface_loader.cpp b/unified-runtime/source/adapters/offload/ur_interface_loader.cpp index ce4ab5351a527..f88af93d6a2b2 100644 --- a/unified-runtime/source/adapters/offload/ur_interface_loader.cpp +++ b/unified-runtime/source/adapters/offload/ur_interface_loader.cpp @@ -437,6 +437,10 @@ UR_DLLEXPORT ur_result_t UR_APICALL urGetIPCExpProcAddrTable( pDdiTable->pfnPutMemHandleExp = urIPCPutMemHandleExp; pDdiTable->pfnOpenMemHandleExp = urIPCOpenMemHandleExp; pDdiTable->pfnCloseMemHandleExp = urIPCCloseMemHandleExp; + pDdiTable->pfnGetPhysMemHandleExp = urIPCGetPhysMemHandleExp; + pDdiTable->pfnPutPhysMemHandleExp = urIPCPutPhysMemHandleExp; + pDdiTable->pfnOpenPhysMemHandleExp = urIPCOpenPhysMemHandleExp; + pDdiTable->pfnClosePhysMemHandleExp = urIPCClosePhysMemHandleExp; return UR_RESULT_SUCCESS; } diff --git a/unified-runtime/source/adapters/opencl/device.cpp b/unified-runtime/source/adapters/opencl/device.cpp index 9654c6b0f764b..f8d11ce26c3ad 100644 --- a/unified-runtime/source/adapters/opencl/device.cpp +++ b/unified-runtime/source/adapters/opencl/device.cpp @@ -1375,6 +1375,8 @@ UR_APIEXPORT ur_result_t UR_APICALL urDeviceGetInfo(ur_device_handle_t hDevice, } case UR_DEVICE_INFO_IPC_MEMORY_SUPPORT_EXP: return ReturnValue(false); + case UR_DEVICE_INFO_IPC_PHYSICAL_MEMORY_SUPPORT_EXP: + return ReturnValue(false); case UR_DEVICE_INFO_BFLOAT16_CONVERSIONS_NATIVE: { bool Supported = false; UR_RETURN_ON_FAILURE(hDevice->checkDeviceExtensions( diff --git a/unified-runtime/source/adapters/opencl/memory.cpp b/unified-runtime/source/adapters/opencl/memory.cpp index ca9e88d4f1ab1..1ba798bc1faa7 100644 --- a/unified-runtime/source/adapters/opencl/memory.cpp +++ b/unified-runtime/source/adapters/opencl/memory.cpp @@ -601,3 +601,24 @@ UR_APIEXPORT ur_result_t UR_APICALL urIPCCloseMemHandleExp(ur_context_handle_t, void *) { return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; } + +UR_APIEXPORT ur_result_t UR_APICALL urIPCGetPhysMemHandleExp( + ur_context_handle_t, ur_physical_mem_handle_t, void **, size_t *) { + return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; +} + +UR_APIEXPORT ur_result_t UR_APICALL +urIPCPutPhysMemHandleExp(ur_context_handle_t, const void *) { + return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; +} + +UR_APIEXPORT ur_result_t UR_APICALL +urIPCOpenPhysMemHandleExp(ur_context_handle_t, ur_device_handle_t, const void *, + size_t, ur_physical_mem_handle_t *) { + return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; +} + +UR_APIEXPORT ur_result_t UR_APICALL +urIPCClosePhysMemHandleExp(ur_context_handle_t, ur_physical_mem_handle_t) { + return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; +} diff --git a/unified-runtime/source/adapters/opencl/ur_interface_loader.cpp b/unified-runtime/source/adapters/opencl/ur_interface_loader.cpp index 3205af2df209e..c9ce08ce199d4 100644 --- a/unified-runtime/source/adapters/opencl/ur_interface_loader.cpp +++ b/unified-runtime/source/adapters/opencl/ur_interface_loader.cpp @@ -458,6 +458,10 @@ UR_DLLEXPORT ur_result_t UR_APICALL urGetIPCExpProcAddrTable( pDdiTable->pfnPutMemHandleExp = urIPCPutMemHandleExp; pDdiTable->pfnOpenMemHandleExp = urIPCOpenMemHandleExp; pDdiTable->pfnCloseMemHandleExp = urIPCCloseMemHandleExp; + pDdiTable->pfnGetPhysMemHandleExp = urIPCGetPhysMemHandleExp; + pDdiTable->pfnPutPhysMemHandleExp = urIPCPutPhysMemHandleExp; + pDdiTable->pfnOpenPhysMemHandleExp = urIPCOpenPhysMemHandleExp; + pDdiTable->pfnClosePhysMemHandleExp = urIPCClosePhysMemHandleExp; return UR_RESULT_SUCCESS; } diff --git a/unified-runtime/source/loader/layers/tracing/ur_trcddi.cpp b/unified-runtime/source/loader/layers/tracing/ur_trcddi.cpp index fe9154b5d882c..8bd4bf3ab7575 100644 --- a/unified-runtime/source/loader/layers/tracing/ur_trcddi.cpp +++ b/unified-runtime/source/loader/layers/tracing/ur_trcddi.cpp @@ -8209,8 +8209,9 @@ __urdlllocal ur_result_t UR_APICALL urIPCGetPhysMemHandleExp( __urdlllocal ur_result_t UR_APICALL urIPCPutPhysMemHandleExp( /// [in] handle of the context object ur_context_handle_t hContext, - /// [in] a pointer to the IPC physical memory handle data - void *pIPCPhysMemHandleData) { + /// [in] a pointer to the IPC physical memory handle data obtained with + /// urIPCGetPhysMemHandleExp + const void *pIPCPhysMemHandleData) { auto pfnPutPhysMemHandleExp = getContext()->urDdiTable.IPCExp.pfnPutPhysMemHandleExp; @@ -8250,8 +8251,9 @@ __urdlllocal ur_result_t UR_APICALL urIPCOpenPhysMemHandleExp( ur_context_handle_t hContext, /// [in] handle of the device object the physical memory was allocated on ur_device_handle_t hDevice, - /// [in] the IPC physical memory handle data - void *pIPCPhysMemHandleData, + /// [in] the IPC physical memory handle data obtained with + /// urIPCGetPhysMemHandleExp + const void *pIPCPhysMemHandleData, /// [in] size of the IPC physical memory handle data size_t ipcPhysMemHandleDataSize, /// [out] pointer to the physical memory handle diff --git a/unified-runtime/source/loader/layers/validation/ur_valddi.cpp b/unified-runtime/source/loader/layers/validation/ur_valddi.cpp index d1a8ab623988b..a7985d6ea92e2 100644 --- a/unified-runtime/source/loader/layers/validation/ur_valddi.cpp +++ b/unified-runtime/source/loader/layers/validation/ur_valddi.cpp @@ -9056,8 +9056,9 @@ __urdlllocal ur_result_t UR_APICALL urIPCGetPhysMemHandleExp( __urdlllocal ur_result_t UR_APICALL urIPCPutPhysMemHandleExp( /// [in] handle of the context object ur_context_handle_t hContext, - /// [in] a pointer to the IPC physical memory handle data - void *pIPCPhysMemHandleData) { + /// [in] a pointer to the IPC physical memory handle data obtained with + /// urIPCGetPhysMemHandleExp + const void *pIPCPhysMemHandleData) { auto pfnPutPhysMemHandleExp = getContext()->urDdiTable.IPCExp.pfnPutPhysMemHandleExp; @@ -9090,8 +9091,9 @@ __urdlllocal ur_result_t UR_APICALL urIPCOpenPhysMemHandleExp( ur_context_handle_t hContext, /// [in] handle of the device object the physical memory was allocated on ur_device_handle_t hDevice, - /// [in] the IPC physical memory handle data - void *pIPCPhysMemHandleData, + /// [in] the IPC physical memory handle data obtained with + /// urIPCGetPhysMemHandleExp + const void *pIPCPhysMemHandleData, /// [in] size of the IPC physical memory handle data size_t ipcPhysMemHandleDataSize, /// [out] pointer to the physical memory handle diff --git a/unified-runtime/source/loader/ur_ldrddi.cpp b/unified-runtime/source/loader/ur_ldrddi.cpp index 0c31622dff65b..d522e58ee53e0 100644 --- a/unified-runtime/source/loader/ur_ldrddi.cpp +++ b/unified-runtime/source/loader/ur_ldrddi.cpp @@ -4671,8 +4671,9 @@ __urdlllocal ur_result_t UR_APICALL urIPCGetPhysMemHandleExp( __urdlllocal ur_result_t UR_APICALL urIPCPutPhysMemHandleExp( /// [in] handle of the context object ur_context_handle_t hContext, - /// [in] a pointer to the IPC physical memory handle data - void *pIPCPhysMemHandleData) { + /// [in] a pointer to the IPC physical memory handle data obtained with + /// urIPCGetPhysMemHandleExp + const void *pIPCPhysMemHandleData) { auto *dditable = *reinterpret_cast(hContext); @@ -4691,8 +4692,9 @@ __urdlllocal ur_result_t UR_APICALL urIPCOpenPhysMemHandleExp( ur_context_handle_t hContext, /// [in] handle of the device object the physical memory was allocated on ur_device_handle_t hDevice, - /// [in] the IPC physical memory handle data - void *pIPCPhysMemHandleData, + /// [in] the IPC physical memory handle data obtained with + /// urIPCGetPhysMemHandleExp + const void *pIPCPhysMemHandleData, /// [in] size of the IPC physical memory handle data size_t ipcPhysMemHandleDataSize, /// [out] pointer to the physical memory handle diff --git a/unified-runtime/source/loader/ur_libapi.cpp b/unified-runtime/source/loader/ur_libapi.cpp index 76220540f5a34..2d9f2fa83e21c 100644 --- a/unified-runtime/source/loader/ur_libapi.cpp +++ b/unified-runtime/source/loader/ur_libapi.cpp @@ -8698,8 +8698,9 @@ ur_result_t UR_APICALL urIPCGetPhysMemHandleExp( ur_result_t UR_APICALL urIPCPutPhysMemHandleExp( /// [in] handle of the context object ur_context_handle_t hContext, - /// [in] a pointer to the IPC physical memory handle data - void *pIPCPhysMemHandleData) try { + /// [in] a pointer to the IPC physical memory handle data obtained with + /// urIPCGetPhysMemHandleExp + const void *pIPCPhysMemHandleData) try { auto pfnPutPhysMemHandleExp = ur_lib::getContext()->urDdiTable.IPCExp.pfnPutPhysMemHandleExp; if (nullptr == pfnPutPhysMemHandleExp) @@ -8737,8 +8738,9 @@ ur_result_t UR_APICALL urIPCOpenPhysMemHandleExp( ur_context_handle_t hContext, /// [in] handle of the device object the physical memory was allocated on ur_device_handle_t hDevice, - /// [in] the IPC physical memory handle data - void *pIPCPhysMemHandleData, + /// [in] the IPC physical memory handle data obtained with + /// urIPCGetPhysMemHandleExp + const void *pIPCPhysMemHandleData, /// [in] size of the IPC physical memory handle data size_t ipcPhysMemHandleDataSize, /// [out] pointer to the physical memory handle diff --git a/unified-runtime/source/ur_api.cpp b/unified-runtime/source/ur_api.cpp index 8d752589ec194..ecb7e4d6ea98f 100644 --- a/unified-runtime/source/ur_api.cpp +++ b/unified-runtime/source/ur_api.cpp @@ -7573,8 +7573,9 @@ ur_result_t UR_APICALL urIPCGetPhysMemHandleExp( ur_result_t UR_APICALL urIPCPutPhysMemHandleExp( /// [in] handle of the context object ur_context_handle_t hContext, - /// [in] a pointer to the IPC physical memory handle data - void *pIPCPhysMemHandleData) { + /// [in] a pointer to the IPC physical memory handle data obtained with + /// urIPCGetPhysMemHandleExp + const void *pIPCPhysMemHandleData) { ur_result_t result = UR_RESULT_SUCCESS; return result; } @@ -7606,8 +7607,9 @@ ur_result_t UR_APICALL urIPCOpenPhysMemHandleExp( ur_context_handle_t hContext, /// [in] handle of the device object the physical memory was allocated on ur_device_handle_t hDevice, - /// [in] the IPC physical memory handle data - void *pIPCPhysMemHandleData, + /// [in] the IPC physical memory handle data obtained with + /// urIPCGetPhysMemHandleExp + const void *pIPCPhysMemHandleData, /// [in] size of the IPC physical memory handle data size_t ipcPhysMemHandleDataSize, /// [out] pointer to the physical memory handle diff --git a/unified-runtime/test/conformance/virtual_memory/CMakeLists.txt b/unified-runtime/test/conformance/virtual_memory/CMakeLists.txt index 65c6c03108f15..dc79f146e5053 100644 --- a/unified-runtime/test/conformance/virtual_memory/CMakeLists.txt +++ b/unified-runtime/test/conformance/virtual_memory/CMakeLists.txt @@ -14,4 +14,8 @@ add_conformance_devices_test(virtual_memory urVirtualMemReserve.cpp urVirtualMemSetAccess.cpp urVirtualMemUnmap.cpp + urIPCGetPhysMemHandleExp.cpp + urIPCPutPhysMemHandleExp.cpp + urIPCOpenPhysMemHandleExp.cpp + urIPCClosePhysMemHandleExp.cpp ) diff --git a/unified-runtime/test/conformance/virtual_memory/urIPCClosePhysMemHandleExp.cpp b/unified-runtime/test/conformance/virtual_memory/urIPCClosePhysMemHandleExp.cpp new file mode 100644 index 0000000000000..4ba5783cdcf08 --- /dev/null +++ b/unified-runtime/test/conformance/virtual_memory/urIPCClosePhysMemHandleExp.cpp @@ -0,0 +1,25 @@ +// Part of the LLVM Project, under the Apache License v2.0 with LLVM +// Exceptions. See https://llvm.org/LICENSE.txt for license information. +// +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +#include "urIPCPhysMemHandleExpFixtures.hpp" + +using urIPCClosePhysMemHandleExpTest = urIPCOpenedPhysMemTest; +UUR_INSTANTIATE_DEVICE_TEST_SUITE(urIPCClosePhysMemHandleExpTest); + +TEST_P(urIPCClosePhysMemHandleExpTest, Success) { + ASSERT_SUCCESS(urIPCClosePhysMemHandleExp(context, opened_physical_mem)); + // Prevent TearDown from attempting a second close. + opened_physical_mem = nullptr; +} + +TEST_P(urIPCClosePhysMemHandleExpTest, InvalidNullHandleContext) { + ASSERT_EQ_RESULT(urIPCClosePhysMemHandleExp(nullptr, opened_physical_mem), + UR_RESULT_ERROR_INVALID_NULL_HANDLE); +} + +TEST_P(urIPCClosePhysMemHandleExpTest, InvalidNullHandlePhysMem) { + ASSERT_EQ_RESULT(urIPCClosePhysMemHandleExp(context, nullptr), + UR_RESULT_ERROR_INVALID_NULL_HANDLE); +} diff --git a/unified-runtime/test/conformance/virtual_memory/urIPCGetPhysMemHandleExp.cpp b/unified-runtime/test/conformance/virtual_memory/urIPCGetPhysMemHandleExp.cpp new file mode 100644 index 0000000000000..22bad3ea36f94 --- /dev/null +++ b/unified-runtime/test/conformance/virtual_memory/urIPCGetPhysMemHandleExp.cpp @@ -0,0 +1,99 @@ +// Part of the LLVM Project, under the Apache License v2.0 with LLVM +// Exceptions. See https://llvm.org/LICENSE.txt for license information. +// +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +#include "urIPCPhysMemHandleExpFixtures.hpp" + +using urIPCGetPhysMemHandleExpTest = urIPCPhysMemTest; +UUR_INSTANTIATE_DEVICE_TEST_SUITE(urIPCGetPhysMemHandleExpTest); + +TEST_P(urIPCGetPhysMemHandleExpTest, Success) { + void *ipc_handle_data = nullptr; + size_t ipc_handle_size = 0; + ur_result_t res = urIPCGetPhysMemHandleExp( + context, physical_mem, &ipc_handle_data, &ipc_handle_size); + if (res == UR_RESULT_ERROR_UNSUPPORTED_FEATURE) { + GTEST_SKIP() << "IPC physical memory handle export is not supported."; + } + ASSERT_SUCCESS(res); + ASSERT_NE(ipc_handle_data, nullptr); + ASSERT_NE(ipc_handle_size, 0U); + ASSERT_SUCCESS(urIPCPutPhysMemHandleExp(context, ipc_handle_data)); +} + +TEST_P(urIPCGetPhysMemHandleExpTest, InvalidNullHandleContext) { + void *ipc_handle_data = nullptr; + size_t ipc_handle_size = 0; + ASSERT_EQ_RESULT(urIPCGetPhysMemHandleExp(nullptr, physical_mem, + &ipc_handle_data, &ipc_handle_size), + UR_RESULT_ERROR_INVALID_NULL_HANDLE); +} + +TEST_P(urIPCGetPhysMemHandleExpTest, InvalidNullHandlePhysMem) { + void *ipc_handle_data = nullptr; + size_t ipc_handle_size = 0; + ASSERT_EQ_RESULT(urIPCGetPhysMemHandleExp(context, nullptr, &ipc_handle_data, + &ipc_handle_size), + UR_RESULT_ERROR_INVALID_NULL_HANDLE); +} + +TEST_P(urIPCGetPhysMemHandleExpTest, InvalidNullPointerIPCHandleData) { + size_t ipc_handle_size = 0; + ASSERT_EQ_RESULT(urIPCGetPhysMemHandleExp(context, physical_mem, nullptr, + &ipc_handle_size), + UR_RESULT_ERROR_INVALID_NULL_POINTER); +} + +TEST_P(urIPCGetPhysMemHandleExpTest, InvalidNullPointerIPCHandleDataSize) { + void *ipc_handle_data = nullptr; + ASSERT_EQ_RESULT(urIPCGetPhysMemHandleExp(context, physical_mem, + &ipc_handle_data, nullptr), + UR_RESULT_ERROR_INVALID_NULL_POINTER); +} + +// Fixture that creates a physical memory object WITHOUT the IPC-export flag. +struct urIPCGetPhysMemHandleExpNoIpcTest : uur::urVirtualMemGranularityTest { + void SetUp() override { + UUR_RETURN_ON_FATAL_FAILURE(urVirtualMemGranularityTest::SetUp()); + size = granularity * 256; + + ur_bool_t ipc_support = false; + ASSERT_SUCCESS( + urDeviceGetInfo(device, UR_DEVICE_INFO_IPC_PHYSICAL_MEMORY_SUPPORT_EXP, + sizeof(ur_bool_t), &ipc_support, nullptr)); + if (!ipc_support) { + GTEST_SKIP() << "IPC physical memory is not supported."; + } + + // Create physical memory without ENABLE_IPC flag. + ur_result_t res = + urPhysicalMemCreate(context, device, size, nullptr, &physical_mem); + if (res == UR_RESULT_ERROR_UNSUPPORTED_FEATURE) { + GTEST_SKIP() << "Physical memory creation is not supported."; + } + ASSERT_SUCCESS(res); + ASSERT_NE(physical_mem, nullptr); + } + + void TearDown() override { + if (physical_mem) { + EXPECT_SUCCESS(urPhysicalMemRelease(physical_mem)); + } + uur::urVirtualMemGranularityTest::TearDown(); + } + + size_t size = 0; + ur_physical_mem_handle_t physical_mem = nullptr; +}; + +UUR_INSTANTIATE_DEVICE_TEST_SUITE(urIPCGetPhysMemHandleExpNoIpcTest); + +TEST_P(urIPCGetPhysMemHandleExpNoIpcTest, InvalidArgumentNoIpcFlag) { + // Exporting a handle for memory not created with ENABLE_IPC must fail. + void *ipc_handle_data = nullptr; + size_t ipc_handle_size = 0; + ASSERT_EQ_RESULT(urIPCGetPhysMemHandleExp(context, physical_mem, + &ipc_handle_data, &ipc_handle_size), + UR_RESULT_ERROR_INVALID_ARGUMENT); +} diff --git a/unified-runtime/test/conformance/virtual_memory/urIPCOpenPhysMemHandleExp.cpp b/unified-runtime/test/conformance/virtual_memory/urIPCOpenPhysMemHandleExp.cpp new file mode 100644 index 0000000000000..0358f69f1390a --- /dev/null +++ b/unified-runtime/test/conformance/virtual_memory/urIPCOpenPhysMemHandleExp.cpp @@ -0,0 +1,57 @@ +// Part of the LLVM Project, under the Apache License v2.0 with LLVM +// Exceptions. See https://llvm.org/LICENSE.txt for license information. +// +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +#include "urIPCPhysMemHandleExpFixtures.hpp" + +using urIPCOpenPhysMemHandleExpTest = urIPCPhysMemHandleTest; +UUR_INSTANTIATE_DEVICE_TEST_SUITE(urIPCOpenPhysMemHandleExpTest); + +TEST_P(urIPCOpenPhysMemHandleExpTest, Success) { + ur_physical_mem_handle_t opened_physical_mem = nullptr; + ASSERT_SUCCESS(urIPCOpenPhysMemHandleExp( + context, device, ipc_handle_data, ipc_handle_size, &opened_physical_mem)); + ASSERT_NE(opened_physical_mem, nullptr); + ASSERT_SUCCESS(urIPCClosePhysMemHandleExp(context, opened_physical_mem)); +} + +TEST_P(urIPCOpenPhysMemHandleExpTest, InvalidNullHandleContext) { + ur_physical_mem_handle_t opened_physical_mem = nullptr; + ASSERT_EQ_RESULT(urIPCOpenPhysMemHandleExp(nullptr, device, ipc_handle_data, + ipc_handle_size, + &opened_physical_mem), + UR_RESULT_ERROR_INVALID_NULL_HANDLE); +} + +TEST_P(urIPCOpenPhysMemHandleExpTest, InvalidNullHandleDevice) { + ur_physical_mem_handle_t opened_physical_mem = nullptr; + ASSERT_EQ_RESULT(urIPCOpenPhysMemHandleExp(context, nullptr, ipc_handle_data, + ipc_handle_size, + &opened_physical_mem), + UR_RESULT_ERROR_INVALID_NULL_HANDLE); +} + +TEST_P(urIPCOpenPhysMemHandleExpTest, InvalidNullPointerIPCHandleData) { + ur_physical_mem_handle_t opened_physical_mem = nullptr; + ASSERT_EQ_RESULT(urIPCOpenPhysMemHandleExp(context, device, nullptr, + ipc_handle_size, + &opened_physical_mem), + UR_RESULT_ERROR_INVALID_NULL_POINTER); +} + +TEST_P(urIPCOpenPhysMemHandleExpTest, InvalidNullPointerPhysMem) { + ASSERT_EQ_RESULT(urIPCOpenPhysMemHandleExp(context, device, ipc_handle_data, + ipc_handle_size, nullptr), + UR_RESULT_ERROR_INVALID_NULL_POINTER); +} + +TEST_P(urIPCOpenPhysMemHandleExpTest, InvalidValue) { + ur_physical_mem_handle_t opened_physical_mem = nullptr; + // Pass a size that differs from the real handle data size to trigger + // UR_RESULT_ERROR_INVALID_VALUE. + ASSERT_EQ_RESULT(urIPCOpenPhysMemHandleExp(context, device, ipc_handle_data, + ipc_handle_size + 1, + &opened_physical_mem), + UR_RESULT_ERROR_INVALID_VALUE); +} diff --git a/unified-runtime/test/conformance/virtual_memory/urIPCPhysMemHandleExpFixtures.hpp b/unified-runtime/test/conformance/virtual_memory/urIPCPhysMemHandleExpFixtures.hpp new file mode 100644 index 0000000000000..93430e0dacc8e --- /dev/null +++ b/unified-runtime/test/conformance/virtual_memory/urIPCPhysMemHandleExpFixtures.hpp @@ -0,0 +1,92 @@ +// Part of the LLVM Project, under the Apache License v2.0 with LLVM +// Exceptions. See https://llvm.org/LICENSE.txt for license information. +// +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +#pragma once + +#include + +// Fixture that creates a physical memory object with the IPC-export flag. +struct urIPCPhysMemTest : uur::urVirtualMemGranularityTest { + void SetUp() override { + UUR_RETURN_ON_FATAL_FAILURE(urVirtualMemGranularityTest::SetUp()); + size = granularity * 256; + + ur_bool_t ipc_support = false; + ASSERT_SUCCESS( + urDeviceGetInfo(device, UR_DEVICE_INFO_IPC_PHYSICAL_MEMORY_SUPPORT_EXP, + sizeof(ur_bool_t), &ipc_support, nullptr)); + if (!ipc_support) { + GTEST_SKIP() << "IPC physical memory is not supported."; + } + + ur_physical_mem_properties_t properties{ + UR_STRUCTURE_TYPE_PHYSICAL_MEM_PROPERTIES, nullptr, + UR_PHYSICAL_MEM_FLAG_ENABLE_IPC}; + ur_result_t res = + urPhysicalMemCreate(context, device, size, &properties, &physical_mem); + if (res == UR_RESULT_ERROR_UNSUPPORTED_FEATURE) { + GTEST_SKIP() << "IPC physical memory is not supported."; + } + ASSERT_SUCCESS(res); + ASSERT_NE(physical_mem, nullptr); + } + + void TearDown() override { + if (physical_mem) { + EXPECT_SUCCESS(urPhysicalMemRelease(physical_mem)); + } + uur::urVirtualMemGranularityTest::TearDown(); + } + + size_t size = 0; + ur_physical_mem_handle_t physical_mem = nullptr; +}; + +// Fixture that also acquires an IPC handle for the physical memory object. +struct urIPCPhysMemHandleTest : urIPCPhysMemTest { + void SetUp() override { + UUR_RETURN_ON_FATAL_FAILURE(urIPCPhysMemTest::SetUp()); + ur_result_t res = urIPCGetPhysMemHandleExp( + context, physical_mem, &ipc_handle_data, &ipc_handle_size); + if (res == UR_RESULT_ERROR_UNSUPPORTED_FEATURE) { + GTEST_SKIP() << "IPC physical memory handle export is not supported."; + } + ASSERT_SUCCESS(res); + ASSERT_NE(ipc_handle_data, nullptr); + ASSERT_NE(ipc_handle_size, 0U); + } + + void TearDown() override { + if (ipc_handle_data) { + EXPECT_SUCCESS(urIPCPutPhysMemHandleExp(context, ipc_handle_data)); + ipc_handle_data = nullptr; + } + urIPCPhysMemTest::TearDown(); + } + + void *ipc_handle_data = nullptr; + size_t ipc_handle_size = 0; +}; + +// Fixture that also opens the IPC handle to produce a second physical_mem. +struct urIPCOpenedPhysMemTest : urIPCPhysMemHandleTest { + void SetUp() override { + UUR_RETURN_ON_FATAL_FAILURE(urIPCPhysMemHandleTest::SetUp()); + ASSERT_SUCCESS(urIPCOpenPhysMemHandleExp(context, device, ipc_handle_data, + ipc_handle_size, + &opened_physical_mem)); + ASSERT_NE(opened_physical_mem, nullptr); + } + + void TearDown() override { + if (opened_physical_mem) { + EXPECT_SUCCESS(urIPCClosePhysMemHandleExp(context, opened_physical_mem)); + opened_physical_mem = nullptr; + } + urIPCPhysMemHandleTest::TearDown(); + } + + ur_physical_mem_handle_t opened_physical_mem = nullptr; +}; diff --git a/unified-runtime/test/conformance/virtual_memory/urIPCPutPhysMemHandleExp.cpp b/unified-runtime/test/conformance/virtual_memory/urIPCPutPhysMemHandleExp.cpp new file mode 100644 index 0000000000000..7a824b7ab6efb --- /dev/null +++ b/unified-runtime/test/conformance/virtual_memory/urIPCPutPhysMemHandleExp.cpp @@ -0,0 +1,25 @@ +// Part of the LLVM Project, under the Apache License v2.0 with LLVM +// Exceptions. See https://llvm.org/LICENSE.txt for license information. +// +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +#include "urIPCPhysMemHandleExpFixtures.hpp" + +using urIPCPutPhysMemHandleExpTest = urIPCPhysMemHandleTest; +UUR_INSTANTIATE_DEVICE_TEST_SUITE(urIPCPutPhysMemHandleExpTest); + +TEST_P(urIPCPutPhysMemHandleExpTest, Success) { + ASSERT_SUCCESS(urIPCPutPhysMemHandleExp(context, ipc_handle_data)); + // Prevent TearDown from attempting a second put. + ipc_handle_data = nullptr; +} + +TEST_P(urIPCPutPhysMemHandleExpTest, InvalidNullHandleContext) { + ASSERT_EQ_RESULT(urIPCPutPhysMemHandleExp(nullptr, ipc_handle_data), + UR_RESULT_ERROR_INVALID_NULL_HANDLE); +} + +TEST_P(urIPCPutPhysMemHandleExpTest, InvalidNullPointerIPCHandleData) { + ASSERT_EQ_RESULT(urIPCPutPhysMemHandleExp(context, nullptr), + UR_RESULT_ERROR_INVALID_NULL_POINTER); +} diff --git a/unified-runtime/test/conformance/virtual_memory/urPhysicalMemCreate.cpp b/unified-runtime/test/conformance/virtual_memory/urPhysicalMemCreate.cpp index f7c9538d79294..77ef56462f14c 100644 --- a/unified-runtime/test/conformance/virtual_memory/urPhysicalMemCreate.cpp +++ b/unified-runtime/test/conformance/virtual_memory/urPhysicalMemCreate.cpp @@ -55,7 +55,7 @@ using urPhysicalMemCreateWithFlagsParamTest = uur::urPhysicalMemTestWithParam; UUR_DEVICE_TEST_SUITE_WITH_PARAM( urPhysicalMemCreateWithFlagsParamTest, - ::testing::Values(UR_PHYSICAL_MEM_FLAG_TBD), + ::testing::Values(UR_PHYSICAL_MEM_FLAG_ENABLE_IPC), uur::deviceTestWithParamPrinter); TEST_P(urPhysicalMemCreateWithFlagsParamTest, Success) { From 81927dfa29a1cf6cd7b593fc1225af717c8cf53b Mon Sep 17 00:00:00 2001 From: Lukasz Dorau Date: Mon, 1 Jun 2026 20:47:38 +0000 Subject: [PATCH 4/5] [UR][L0v2] Fix error code for zero-size IPC phys mem handle urIPCOpenPhysMemHandleExp returned UR_RESULT_ERROR_INVALID_VALUE when HandleData->Size == 0. The SYCL layer catches INVALID_VALUE and maps it to a "HandleData data size does not correspond..." error message, which is misleading for the zero-size case (the handle data size is correct; the allocation size embedded in the handle is zero). Return UR_RESULT_ERROR_INVALID_ARGUMENT instead, which correctly signals that the handle data contents are invalid, and avoids the misleading SYCL error message. Signed-off-by: Lukasz Dorau --- unified-runtime/source/adapters/level_zero/v2/physical_mem.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/unified-runtime/source/adapters/level_zero/v2/physical_mem.cpp b/unified-runtime/source/adapters/level_zero/v2/physical_mem.cpp index 7d055e15d3456..b1f6b60f2c2e8 100644 --- a/unified-runtime/source/adapters/level_zero/v2/physical_mem.cpp +++ b/unified-runtime/source/adapters/level_zero/v2/physical_mem.cpp @@ -225,7 +225,7 @@ ur_result_t urIPCOpenPhysMemHandleExp(ur_context_handle_t hContext, static_cast(pIPCPhysMemHandleData); if (HandleData->Size == 0) - return UR_RESULT_ERROR_INVALID_VALUE; + return UR_RESULT_ERROR_INVALID_ARGUMENT; // Open the IPC handle in this process. zeMemOpenIpcHandle creates a virtual // mapping backed by the exporter's physical memory and returns a pointer to From 3acc55acd08f81523962ec1e7fc070cff9a43fed Mon Sep 17 00:00:00 2001 From: "Dorau, Lukasz" Date: Tue, 2 Jun 2026 09:47:45 +0000 Subject: [PATCH 5/5] [UR][L0v2] Remove incorrect fd-detection guard in urIPCGetPhysMemHandleExp The fd-detection guard in urIPCGetPhysMemHandleExp was rejecting IPC handles whose first bytes formed a valid open file descriptor, on the assumption that such fd-based handles cannot be transferred cross-process without SCM_RIGHTS socket passing. That assumption is wrong for the current NEO driver. The driver intentionally returns fd-based (DMA-BUF) IPC handles and stores the exporter's process ID alongside the fd in the 64-byte handle blob. On the consumer side it uses pidfd_open + pidfd_getfd to copy the fd from the exporting process, so writing the handle bytes to a plain file and reading them in another process works correctly without any fd passing via sockets. Remove the guard and its now-unused include so that the ipc_physical_memory test no longer skips on BMG. --- .../adapters/level_zero/v2/physical_mem.cpp | 22 ------------------- 1 file changed, 22 deletions(-) diff --git a/unified-runtime/source/adapters/level_zero/v2/physical_mem.cpp b/unified-runtime/source/adapters/level_zero/v2/physical_mem.cpp index b1f6b60f2c2e8..415c374314cd3 100644 --- a/unified-runtime/source/adapters/level_zero/v2/physical_mem.cpp +++ b/unified-runtime/source/adapters/level_zero/v2/physical_mem.cpp @@ -11,10 +11,6 @@ #include "../device.hpp" #include "common.hpp" -#ifdef __linux__ -#include -#endif - #include "context.hpp" namespace ur::level_zero { @@ -149,24 +145,6 @@ ur_result_t urIPCGetPhysMemHandleExp(ur_context_handle_t hContext, if (ZeRes != ZE_RESULT_SUCCESS) return ze2urResult(ZeRes); - // Some driver versions accept this call but return an fd-based handle even - // when ZE_IPC_MEM_HANDLE_TYPE_FLAG_DEFAULT is requested, because they do not - // yet support passing a ze_physical_mem_handle_t directly. An fd-based - // handle cannot be serialized to a plain byte buffer for cross-process - // transfer without SCM_RIGHTS socket transfer. Detect this case: if the - // first bytes of the handle form a valid open file descriptor in this - // process, the handle is fd-based. Release it and return UNSUPPORTED_FEATURE - // so callers can skip gracefully. - { - int FdVal = 0; - static_assert(sizeof(IpcHandle.data) >= sizeof(FdVal)); - memcpy(&FdVal, IpcHandle.data, sizeof(FdVal)); - if (FdVal >= 0 && ::fcntl(FdVal, F_GETFD) >= 0) { - zeMemPutIpcHandle(hContext->getZeHandle(), IpcHandle); - return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; - } - } - auto *HandleData = new (std::nothrow) ZeIPCPhysMemHandleData; if (!HandleData) { zeMemPutIpcHandle(hContext->getZeHandle(), IpcHandle);