[rocm-libraries] ROCm/rocm-libraries#6462 (commit d41d078)

[hiptensor] Several fixes and improvements for tests on
 Windows (#6462)

## Motivation

Improve reliability of tests and fix Windows specific errors.

## Technical Details

- Fix cleanup on yaml_test
- Add extra datatype for plan cache tests to perform test on all ASICs
- Fix error when providing stride data on reduction_test
- Copy relevant amd*.dll to hiptensor/bin path so they have priority
over the system DLLs
- Force HIP runtime initialization before any test to get a clean error
in case cause of issues

## Test Plan

Run full set of tests on Windows and on Linux.

## Test Result

- Tests are passing.

## Submission Checklist

- [x] Look over the contributing guidelines at
https://github.com/ROCm/ROCm/blob/develop/CONTRIBUTING.md#pull-requests.

Author: evedovelli

test/00_unit/yaml_test.cpp

@@ -26,6 +26,7 @@
 
 #include <algorithm>
 #include <cstdio>
+#include <cstdlib>
 #include <fstream>
 #include <iterator>
 #include <numeric>
@@ -138,10 +139,19 @@ int main(int argc, char* argv[])
     hiptensor::YamlConfigLoader<hiptensor::ContractionTestParams>::storeToFile(tmpFile, yee);
     auto yee1
         = hiptensor::YamlConfigLoader<hiptensor::ContractionTestParams>::loadFromFile(tmpFile);
-    if(!yee1)
-    {
-        return -1;
-    }
 
-    return 0;
+    int result = yee1 ? 0 : -1;
+
+    // Remove temp file explicitly before quick_exit() bypasses the destructor.
+    ::remove(tmpFile.c_str());
+
+    // Flush LLVM's buffered streams and destroy ManagedStatic objects while all
+    // libraries are still loaded, before post-main cleanup runs.
+    hiptensor::llvmShutdown();
+
+    // Use quick_exit() to bypass C++ static destructors and atexit() handlers.
+    // Post-main cleanup in linked DLLs (hiptensor, HIP runtime) crashes under
+    // ctest due to DLL unload order differences vs a direct run. Our cleanup is
+    // already complete via llvmShutdown() above.
+    std::quick_exit(result);
 }

test/01_contraction/configs/plan_cache_test_params.yaml

@@ -2,6 +2,7 @@
 Log Level:       [ HIPTENSOR_LOG_LEVEL_ERROR, HIPTENSOR_LOG_LEVEL_PERF_TRACE ]
 Tensor Data Types:
   - [ HIPTENSOR_R_32F, HIPTENSOR_R_32F, HIPTENSOR_R_32F, HIPTENSOR_R_32F, HIPTENSOR_R_32F ]
+  - [ HIPTENSOR_R_16F, HIPTENSOR_R_16F, HIPTENSOR_R_16F, HIPTENSOR_R_16F, HIPTENSOR_R_16F ]
 Algorithm Types:
   - HIPTENSOR_ALGO_DEFAULT
 Operators:

test/03_reduction/reduction_test.cpp

@@ -404,7 +404,7 @@ namespace hiptensor
                                                 &descC,
                                                 nmodeC,
                                                 extentCD.data(),
-                                                stridesCD.data() ? nullptr : stridesCD.data(),
+                                                stridesCD.empty() ? nullptr : stridesCD.data(),
                                                 acDataType,
                                                 0));
 
@@ -414,7 +414,7 @@ namespace hiptensor
                                                 &descD,
                                                 nmodeD,
                                                 extentCD.data(),
-                                                stridesCD.data() ? nullptr : stridesCD.data(),
+                                                stridesCD.empty() ? nullptr : stridesCD.data(),
                                                 acDataType,
                                                 0));
 

test/CMakeLists.txt

@@ -63,6 +63,44 @@ file(WRITE "${INSTALL_TEST_FILE}"
 # Target that will trigger build of all tests
 add_custom_target(hiptensor_tests)
 
+# On Windows, DLL search order is: application dir -> System32 -> PATH.
+# A system-installed ROCm places amdhip64_7.dll and amd_comgr_3.dll in
+# System32, which always wins over PATH. If the test binary is compiled with
+# a newer toolchain but an older runtime is loaded from System32, kernel launches
+# may crash (SEH 0xC0000005) if the code-object format is incompatible.
+# Staging the correct runtime DLLs into CMAKE_RUNTIME_OUTPUT_DIRECTORY (the application
+# directory) ensures Windows finds them before System32.
+# This runs at CMake configure time so it takes effect on every cmake invocation.
+if(WIN32)
+    # hip_DIR is set by find_package(HIP) to <HIP_ROOT>/lib/cmake/hip.
+    # Walk up three directory levels to reach <HIP_ROOT>, then append /bin.
+    get_filename_component(_hip_cmake_dir  "${hip_DIR}"          DIRECTORY) # .../lib/cmake
+    get_filename_component(_hip_lib_dir    "${_hip_cmake_dir}"   DIRECTORY) # .../lib
+    get_filename_component(_hip_root       "${_hip_lib_dir}"     DIRECTORY) # <HIP_ROOT>
+    set(_hip_bin_dir "${_hip_root}/bin")
+
+    message(STATUS "HIP runtime DLL source: ${_hip_bin_dir}")
+    message(STATUS "HIP runtime DLL destination: ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}")
+
+    if(EXISTS "${_hip_bin_dir}")
+        file(MAKE_DIRECTORY "${CMAKE_RUNTIME_OUTPUT_DIRECTORY}")
+        file(GLOB _hip_dlls "${_hip_bin_dir}/amdhip64_7.dll")
+        if(_hip_dlls)
+            foreach(_dll ${_hip_dlls})
+                get_filename_component(_dll_name "${_dll}" NAME)
+                file(COPY "${_dll}" DESTINATION "${CMAKE_RUNTIME_OUTPUT_DIRECTORY}")
+            endforeach()
+            list(LENGTH _hip_dlls _hip_dll_count)
+            message(STATUS "Staged ${_hip_dll_count} HIP runtime DLL(s) into ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}")
+        else()
+            message(WARNING "No DLLs found in ${_hip_bin_dir} - tests may load wrong HIP runtime from System32")
+        endif()
+    else()
+        message(WARNING "HIP DLL source directory not found: ${_hip_bin_dir} - "
+                        "tests may load wrong HIP runtime from System32")
+    endif()
+endif()
+
 set(HIPTENSOR_COMMON_TEST_SOURCES ${CMAKE_CURRENT_SOURCE_DIR}/common.cpp
                                   ${CMAKE_CURRENT_SOURCE_DIR}/hip_resource.cpp
                                   ${CMAKE_CURRENT_SOURCE_DIR}/hiptensor_gtest_main.cpp)
@@ -141,6 +179,14 @@ function(add_hiptensor_test BINARY_NAME YAML_CONFIG_FILE TEST_SOURCES)
     # Set working directory to bin/ for proper DLL/resource loading on Windows
     set_property(TEST ${BINARY_NAME} PROPERTY WORKING_DIRECTORY ${CMAKE_RUNTIME_OUTPUT_DIRECTORY})
 
+    # On Windows, ensure the ROCm/HIP DLLs (amdhip64_7.dll, etc.) are discoverable by ctest.
+    # ctest may not inherit the user's PATH that includes the ROCm bin directory, causing
+    # HIP runtime DLL load failures (SEH 0xC0000005) on the first HIP call in a test fixture.
+    if(WIN32)
+        set_property(TEST ${BINARY_NAME} PROPERTY ENVIRONMENT
+            "PATH=${CMAKE_INSTALL_PREFIX}/bin$<SEMICOLON>${CMAKE_RUNTIME_OUTPUT_DIRECTORY}$<SEMICOLON>$ENV{PATH}")
+    endif()
+
     # Install with rocm pkg
     rocm_install_targets(
     TARGETS ${BINARY_NAME}
@@ -203,6 +249,14 @@ function(add_hiptensor_unit_test)
     # Set working directory to bin/ for proper DLL/resource loading on Windows
     set_property(TEST ${BINARY_NAME} PROPERTY WORKING_DIRECTORY ${CMAKE_RUNTIME_OUTPUT_DIRECTORY})
 
+    # On Windows, ensure the ROCm/HIP DLLs (amdhip64_7.dll, etc.) are discoverable by ctest.
+    # ctest may not inherit the user's PATH that includes the ROCm bin directory, causing
+    # HIP runtime DLL load failures (SEH 0xC0000005) on the first HIP call in a test fixture.
+    if(WIN32)
+        set_property(TEST ${BINARY_NAME} PROPERTY ENVIRONMENT
+            "PATH=${CMAKE_INSTALL_PREFIX}/bin$<SEMICOLON>${CMAKE_RUNTIME_OUTPUT_DIRECTORY}$<SEMICOLON>$ENV{PATH}")
+    endif()
+
     # Install with rocm pkg
     rocm_install_targets(
     TARGETS ${BINARY_NAME}

test/hiptensor_gtest_main.cpp

@@ -26,7 +26,9 @@
 #include "hiptensor_options.hpp"
 #include "utils.hpp"
 
+#include <cstdio>
 #include <gtest/gtest.h>
+#include <hip/hip_runtime_api.h>
 
 #include "llvm/command_line_parser.hpp"
 
@@ -35,11 +37,26 @@ int main(int argc, char** argv)
     // Parse hiptensor test options
     hiptensor::parseOptions(argc, argv);
 
+    // Force HIP runtime initialization before any test fixture runs.
+    // If HIP fails here we get a clean error rather than an SEH crash inside a
+    // fixture, which would leave the C++ static-initialization guard locked and
+    // cause all subsequent tests to deadlock ("resource deadlock would occur").
+    int        deviceCount = 0;
+    hipError_t hipErr      = hipGetDeviceCount(&deviceCount);
+    if((hipErr != hipSuccess) || (deviceCount <= 0))
+    {
+        fprintf(
+            stderr,
+            "hipGetDeviceCount failed (%d: %s) — Device count: %d — aborting before tests run.\n",
+            static_cast<int>(hipErr),
+            hipGetErrorString(hipErr),
+            deviceCount);
+        return 1;
+    }
+
     // Initialize Google Tests
     testing::InitGoogleTest(&argc, argv);
 
     // Run the tests
-    int status = RUN_ALL_TESTS();
-
-    return status;
+    return RUN_ALL_TESTS();
 }

test/llvm/yaml_parser.hpp

@@ -39,4 +39,10 @@ namespace hiptensor
         static std::optional<ConfigT> loadFromString(std::string const& yaml = "");
         static void storeToFile(std::string const& filePath, ConfigT const& config);
     };
+
+    // Flush LLVM's buffered output streams and shut down LLVM's ManagedStatic
+    // objects. Must be called before returning from main() in any binary that
+    // uses LLVM YAML I/O, to prevent use-after-free at process exit when the
+    // DLL unload order differs from a direct run (e.g. under ctest on Windows).
+    void llvmShutdown();
 }

test/llvm/yaml_parser_config.cpp

@@ -27,6 +27,8 @@
 #include <vector>
 
 #include <llvm/ObjectYAML/YAML.h>
+#include <llvm/Support/ManagedStatic.h>
+#include <llvm/Support/raw_ostream.h>
 
 #include <hiptensor/hiptensor.h>
 
@@ -543,4 +545,11 @@ namespace hiptensor
     template struct YamlConfigLoader<ContractionTestParams>;
     template struct YamlConfigLoader<PermutationTestParams>;
     template struct YamlConfigLoader<ReductionTestParams>;
+
+    void llvmShutdown()
+    {
+        llvm::outs().flush();
+        llvm::errs().flush();
+        llvm::llvm_shutdown();
+    }
 }