diff --git a/.ci/scripts/test-cuda-build.sh b/.ci/scripts/test-cuda-build.sh index e717718be66..5e9f008beb1 100755 --- a/.ci/scripts/test-cuda-build.sh +++ b/.ci/scripts/test-cuda-build.sh @@ -80,6 +80,68 @@ except Exception as e: exit(1) " + # The CUDA delegate ships as its own shared library. Nothing else here would + # notice if it were built into more than one place, and a process with two + # copies of the delegate has two copies of its state, so check that the + # installed tree defines it exactly once. + python -c " +import shutil +import subprocess +import sys +from pathlib import Path + +if shutil.which('nm') is None: + print('INFO: nm unavailable, skipping the delegate duplication check') + sys.exit(0) + +import executorch + +# A namespace package has no __file__, so derive the directory from the loader's +# search path instead. +locations = list(getattr(executorch, '__path__', []) or []) +if not locations: + print('INFO: cannot locate the installed package, skipping the check') + sys.exit(0) +package = Path(locations[0]) +symbol = 'executorch::backends::cuda::clearCurrentCUDAStream' +libraries = [p for p in package.rglob('*.so*') if p.is_file() and not p.is_symlink()] +definers = [] +for library in libraries: + result = subprocess.run( + ['nm', '-DC', str(library)], capture_output=True, text=True, check=False + ) + if result.returncode != 0: + continue + for line in result.stdout.splitlines(): + parts = line.split(maxsplit=2) + if len(parts) == 3 and parts[1] in 'TtWVu' and parts[2].startswith(symbol): + definers.append(str(library.relative_to(package))) + break + +# The symbol above belongs to the shim layer, so it stays resolvable even if the +# delegate library itself stops being packaged. Check for the delegate file too, +# otherwise losing it entirely would go unnoticed here. +delegates = [ + p for p in package.rglob('libexecutorch_cuda_backend.so*') + if p.is_file() and not p.is_symlink() +] +if len(delegates) != 1: + print(f'ERROR: expected one shipped CUDA delegate library, found {delegates}') + sys.exit(1) +print(f'SUCCESS: one CUDA delegate library at {delegates[0].relative_to(package)}') + +if not definers: + # This runs inside a job that just built with CUDA enabled, so a missing + # delegate means the build or the packaging stopped producing it. Treating + # that as nothing to check would let the regression through. + print('ERROR: CUDA was enabled but no shipped library defines the delegate') + sys.exit(1) +if len(definers) != 1: + print(f'ERROR: expected one library to define the CUDA delegate, found {definers}') + sys.exit(1) +print(f'SUCCESS: exactly one CUDA delegate across {len(libraries)} shipped libraries') +" || exit $? + echo "SUCCESS: ExecuTorch CUDA ${cuda_version} build and verification completed successfully" } diff --git a/.ci/scripts/wheel/test_cpp_sdk.py b/.ci/scripts/wheel/test_cpp_sdk.py index d800fc648ce..f20ce9ae59b 100644 --- a/.ci/scripts/wheel/test_cpp_sdk.py +++ b/.ci/scripts/wheel/test_cpp_sdk.py @@ -27,6 +27,7 @@ import shutil import subprocess import sys +import tempfile from pathlib import Path # Registry entry points. A second definer of any of these means a second @@ -60,6 +61,10 @@ "executorch::backends::xnnpack::XnnpackBackendOptions::workspace_manager", ) +# A representative symbol from the CUDA delegate's shim layer. The delegate's own +# methods are weak symbols, so this checks a strong one instead. +_CUDA_SYMBOLS = ("executorch::backends::cuda::clearCurrentCUDAStream",) + # `nm -DC` prints " " for a definition and # " U " for an undefined reference. _DEFINED = re.compile(r"^[0-9a-fA-F]+\s+(?P[A-Za-z])\s+(?P.+)$") @@ -68,6 +73,7 @@ _OWNING_KINDS = frozenset("TtBbDdGgSsRrWV") _CONSUMER_SOURCE = """\ +#include #include #include @@ -75,9 +81,16 @@ int main() { executorch::runtime::runtime_init(); + // Printed rather than asserted on purpose. This consumer links only the + // runtime, exactly as the documented two-line example does, and the runtime + // alone registers no backend. Requiring a nonzero count here would be + // asserting that the runtime does something it is not supposed to do. std::printf( "registered backends: %zu\\n", (size_t)executorch::runtime::get_num_registered_backends()); + // Compile against the Module header too. It is shipped and advertised as the + // way to load a program, so a consumer must be able to include it. + (void)sizeof(executorch::extension::Module); return 0; } """ @@ -130,16 +143,243 @@ def _defines_symbol(library: Path, symbol: str) -> bool: return False -def _assert_single_definer(symbols, what: str) -> None: - """Exactly one shipped library may define each of `symbols`.""" +def report_wheel_composition() -> None: + """Print what the wheel ships and what each library needs. + + Not an assertion. A size jump or an unexpected external dependency is the + first visible sign that a component got statically duplicated again, so the + numbers are worth having in the log of every run. + """ + package_dir = _installed_package_dir() + libraries = _shipped_shared_objects(package_dir) + + print("shipped libraries:") + total = 0 + for library in sorted(libraries, key=lambda path: path.name): + size = library.stat().st_size + total += size + print(f" {size / 1024:9.1f} KiB {library.relative_to(package_dir)}") + print(f" {total / 1024:9.1f} KiB total") + + if shutil.which("readelf") is None: + return + # Anything the libraries need that the wheel does not itself ship has to be + # present on the user's machine, so it belongs in the report. Compare against + # the shipped file names rather than guessing from name prefixes. + shipped = {library.name for library in libraries} + external = set() + for library in libraries: + dynamic = subprocess.run( + ["readelf", "-d", str(library)], + capture_output=True, + text=True, + check=False, + ).stdout + for line in dynamic.splitlines(): + if "(NEEDED)" not in line or "[" not in line: + continue + name = line.split("[", 1)[1].rstrip("]").strip() + if name not in shipped: + external.add(name) + if external: + print("external dependencies expected from the environment:") + for name in sorted(external): + print(f" {name}") + + +def test_shipped_libraries_load() -> None: + """Every shipped library must depend only on things that exist. + + The symbol checks prove each component is defined exactly once, but a library + can still be unloadable if it needs something nothing provides, which is a + packaging bug rather than a duplication bug. + + A dependency the wheel ships elsewhere is fine even when `ldd` cannot resolve + it: some extensions are loaded after `import torch` has already brought their + dependencies into the process, so they intentionally carry no path to them. + Only a name nothing in the wheel provides is a real problem. + """ + if shutil.which("ldd") is None: + print("- ldd not available, skipping the load check") + return + + package_dir = _installed_package_dir() + libraries = _shipped_shared_objects(package_dir) + shipped = {library.name for library in libraries} + + # A dependency is only excusable when the wheel ships it AND the loader can + # actually reach it from the library that needs it. Loaded-later extensions + # such as the Torch libraries are the real exception: they resolve once the + # Python package that owns them is imported. Anything the wheel itself ships + # must resolve here, because a RUNPATH applies to the library carrying it and + # is not inherited on behalf of a dependency's own dependencies. + broken = {} + unreachable = {} + unresolved = {} + for library in libraries: + resolved = subprocess.run( + # -r resolves data and function symbols too, not just the NEEDED + # entries. A SHARED link does not error on undefined symbols, so + # without this an under-linked library passes here and fails at first + # use instead. + ["ldd", "-r", str(library)], + capture_output=True, + text=True, + check=False, + # Any LD_LIBRARY_PATH in the build environment would paper over a + # RUNPATH the shipped library is actually missing. + env={ + key: value + for key, value in os.environ.items() + if key != "LD_LIBRARY_PATH" + }, + ) + # ldd reports missing libraries on stdout but undefined symbols on stderr, + # so both streams matter. + combined = resolved.stdout + resolved.stderr + missing = [ + line.split("=>")[0].strip() + for line in combined.splitlines() + if "not found" in line + ] + # A Python extension module resolves the interpreter's symbols only once + # the interpreter loads it, so unresolved symbols are normal there and say + # nothing about packaging. Whether those modules import at all is covered + # separately. The missing-library checks below still apply to them. + is_python_extension = ".cpython-" in library.name or library.name.endswith( + (".pyd", ".abi3.so") + ) + undefined = ( + [] + if is_python_extension + else [ + line.strip() + for line in combined.splitlines() + if "undefined symbol" in line + ] + ) + if undefined: + unresolved[str(library.relative_to(package_dir))] = undefined[:5] + absent = [name for name in missing if name not in shipped] + present_but_unreachable = [name for name in missing if name in shipped] + if absent: + broken[str(library.relative_to(package_dir))] = absent + if present_but_unreachable: + unreachable[str(library.relative_to(package_dir))] = present_but_unreachable + + assert not broken, ( + "shipped libraries need dependencies that nothing provides, so they will " + f"fail to load: {broken}" + ) + assert not unreachable, ( + "shipped libraries need dependencies the wheel ships but the loader " + "cannot reach from them, which usually means a missing RUNPATH entry: " + f"{unreachable}" + ) + assert not unresolved, ( + "shipped libraries reference symbols nothing provides, so they will fail " + f"at first use rather than at load: {unresolved}" + ) + print("✓ every shipped library resolves every dependency it needs") + + +def test_shipped_libraries_resolve_without_build_tree() -> None: + """A shipped library must resolve using only its relative runtime paths. + + Packaging copies binaries out of the build directory, so they still carry the + absolute paths they were linked with. On the machine that produced the wheel + those paths exist, which means a library whose relative path is wrong can still + resolve and look correct. Anywhere else it would fail. + + Copy each library and its wheel-provided dependencies into a fresh tree that + mirrors the wheel layout, drop every absolute runtime path, and check what is + left is enough. + """ + if shutil.which("ldd") is None or shutil.which("patchelf") is None: + print("- ldd or patchelf unavailable, skipping the relocated load check") + return + + package_dir = _installed_package_dir() + libraries = _shipped_shared_objects(package_dir) + environment = { + key: value for key, value in os.environ.items() if key != "LD_LIBRARY_PATH" + } + + with tempfile.TemporaryDirectory() as work_dir: + root = Path(work_dir) / package_dir.name + # Mirror the layout so a relative path such as $ORIGIN/../../lib still + # points where it would in a real install. + for library in libraries: + target = root / library.relative_to(package_dir) + target.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(library, target) + + broken = {} + for library in libraries: + target = root / library.relative_to(package_dir) + current = subprocess.run( + ["patchelf", "--print-rpath", str(target)], + capture_output=True, + text=True, + check=False, + ).stdout.strip() + relative = [ + entry for entry in current.split(":") if entry.startswith("$ORIGIN") + ] + subprocess.run( + ["patchelf", "--set-rpath", ":".join(relative), str(target)], + # A failure here would leave the original absolute build paths in + # place, and the check below would then pass by resolving through + # them, which is exactly what this test exists to rule out. + check=True, + ) + resolved = subprocess.run( + ["ldd", str(target)], + capture_output=True, + text=True, + check=False, + env=environment, + ).stdout + shipped = {item.name for item in libraries} + missing = [ + line.split("=>")[0].strip() + for line in resolved.splitlines() + if "not found" in line and line.split("=>")[0].strip() in shipped + ] + if missing: + broken[str(library.relative_to(package_dir))] = missing + + assert not broken, ( + "shipped libraries only resolve their wheel-provided dependencies " + "through absolute build paths, so they would fail on any other " + f"machine: {broken}" + ) + print("✓ every shipped library resolves without the build tree") + + +def _assert_single_definer(symbols, what: str, optional: bool = False) -> None: + """Exactly one shipped library may define each of `symbols`. + + `optional` allows a component that is only present in some wheel flavors, + such as an accelerator delegate, to be absent without failing. + """ assert shutil.which("nm") is not None, "nm is required to inspect the wheel" package_dir = _installed_package_dir() libraries = _shipped_shared_objects(package_dir) assert libraries, f"no shared libraries found under {package_dir}" - for symbol in symbols: - definers = [lib for lib in libraries if _defines_symbol(lib, symbol)] + # Resolve every symbol first, so a component that is only half present is + # reported rather than being mistaken for one that is absent entirely. + found = { + symbol: [lib for lib in libraries if _defines_symbol(lib, symbol)] + for symbol in symbols + } + if optional and not any(found.values()): + print(f"- no {what} in this wheel, skipping") + return + + for symbol, definers in found.items(): pretty = [str(lib.relative_to(package_dir)) for lib in definers] assert len(definers) == 1, ( f"expected exactly one library to define {symbol}, found " @@ -174,6 +414,20 @@ def test_single_xnnpack_delegate() -> None: _assert_single_definer(_XNNPACK_SYMBOLS, "XNNPACK delegate") +def test_single_cuda_delegate() -> None: + """Exactly one shipped library may define the CUDA delegate, if present. + + Presence is decided from the shipped library file, not from whether the symbol + resolves. Inferring absence from a missing symbol means a rename on a CUDA + wheel would skip the check instead of failing it. + """ + package_dir = _installed_package_dir() + if not list(package_dir.rglob("libexecutorch_cuda_backend.so*")): + print("- no CUDA delegate in this wheel, skipping") + return + _assert_single_definer(_CUDA_SYMBOLS, "CUDA delegate") + + def test_cpp_consumer(work_dir: Path) -> None: """A standalone C++ app builds and runs against the installed wheel.""" assert shutil.which("cmake") is not None, "cmake is required to build a consumer" @@ -319,9 +573,13 @@ def test_python_extensions_import() -> None: def run_tests(work_dir: Path) -> None: + report_wheel_composition() + test_shipped_libraries_load() + test_shipped_libraries_resolve_without_build_tree() test_single_backend_registry() test_python_extensions_import() test_single_threadpool() test_single_kernel_registration() test_single_xnnpack_delegate() + test_single_cuda_delegate() test_cpp_consumer(work_dir) diff --git a/CMakeLists.txt b/CMakeLists.txt index 1f0bca2f53e..93a9279cdbe 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1273,7 +1273,9 @@ if(EXECUTORCH_BUILD_PYBIND) # initializer, so nothing here references a symbol from them and some linkers # drop them from DT_NEEDED. That surfaces at runtime as a missing kernel or an # unregistered backend rather than as a link error. - foreach(_retained_component optimized_native_cpu_ops_lib xnnpack_backend) + foreach(_retained_component optimized_native_cpu_ops_lib xnnpack_backend + aoti_cuda_backend + ) if(TARGET ${_retained_component}) executorch_target_retain_shared_library( portable_lib ${_retained_component} diff --git a/backends/cuda/CMakeLists.txt b/backends/cuda/CMakeLists.txt index 06990692428..8bec5ca548d 100644 --- a/backends/cuda/CMakeLists.txt +++ b/backends/cuda/CMakeLists.txt @@ -93,8 +93,17 @@ target_compile_options( PUBLIC "$<$:${_cuda_cxx_compile_options}>" ) -# Link against ExecuTorch core libraries -target_link_libraries(cuda_platform PRIVATE executorch_core ${CMAKE_DL_LIBS}) +# Link against ExecuTorch core libraries. Resolve them from the shared runtime +# when there is one, so this does not carry a second copy of the backend +# registry. +if(EXECUTORCH_BUILD_SHARED) + target_link_libraries( + cuda_platform PRIVATE executorch_shared ${CMAKE_DL_LIBS} + ) + executorch_target_link_shared_runtime(cuda_platform) +else() + target_link_libraries(cuda_platform PRIVATE executorch_core ${CMAKE_DL_LIBS}) +endif() install( TARGETS cuda_platform @@ -169,14 +178,9 @@ if(_cuda_is_msvc_toolchain) else() target_link_libraries( aoti_cuda_shims - PRIVATE cuda_platform - PUBLIC -Wl,--whole-archive - aoti_common_shims_slim - -Wl,--no-whole-archive - CUDA::cudart - CUDA::curand - extension_cuda - ${CMAKE_DL_LIBS} + PRIVATE cuda_platform -Wl,--whole-archive aoti_common_shims_slim + -Wl,--no-whole-archive + PUBLIC CUDA::cudart CUDA::curand extension_cuda ${CMAKE_DL_LIBS} ) endif() @@ -200,7 +204,52 @@ if(_cuda_is_msvc_toolchain) list(APPEND _aoti_cuda_backend_sources runtime/cuda_allocator.cpp) endif() -add_library(aoti_cuda_backend STATIC ${_aoti_cuda_backend_sources}) +# Build the delegate as a shared library for the wheel so a process has one copy +# of it, and keep it static everywhere else so no other build changes. +if(EXECUTORCH_BUILD_SHARED) + set(_aoti_cuda_backend_library_type SHARED) +else() + set(_aoti_cuda_backend_library_type STATIC) +endif() +add_library( + aoti_cuda_backend ${_aoti_cuda_backend_library_type} + ${_aoti_cuda_backend_sources} +) +if(EXECUTORCH_BUILD_SHARED) + set_target_properties( + aoti_cuda_backend + PROPERTIES OUTPUT_NAME executorch_cuda_backend + VERSION "${PROJECT_VERSION}" + SOVERSION "${PROJECT_VERSION_MAJOR}" + ) + if(NOT APPLE) + # Ships in the wheel's lib/ directory, but the CUDA shim library it links + # lives under backends/cuda, so both locations have to be searchable. The + # CUDA runtime itself comes from the environment and is not bundled. + set(_cuda_backend_rpath "$ORIGIN:$ORIGIN/../backends/cuda") + set_target_properties( + aoti_cuda_backend PROPERTIES BUILD_RPATH "${_cuda_backend_rpath}" + INSTALL_RPATH "${_cuda_backend_rpath}" + ) + endif() +endif() + +# Outside the shared-runtime guard on purpose: the shim and the extension +# library it links are packaged for any CUDA build, so the path that lets the +# shim find that library has to be set whenever both exist, not only alongside a +# shared runtime. +if(NOT APPLE AND TARGET aoti_cuda_shims) + # The shim needs its own entry rather than relying on the backend's. A RUNPATH + # applies to the library that carries it, not to what its own dependencies + # need, so loading the shim first, or on its own, would fail to find the + # extension library it links. The shim ships under backends/cuda while that + # library ships in the wheel's lib/ directory. + set(_cuda_shims_rpath "$ORIGIN:$ORIGIN/../../lib") + set_target_properties( + aoti_cuda_shims PROPERTIES BUILD_RPATH "${_cuda_shims_rpath}" + INSTALL_RPATH "${_cuda_shims_rpath}" + ) +endif() target_include_directories( aoti_cuda_backend diff --git a/setup.py b/setup.py index 3349f4bf2fc..097227ff09a 100644 --- a/setup.py +++ b/setup.py @@ -1192,6 +1192,36 @@ def run(self): # noqa C901 "EXECUTORCH_BUILD_XNNPACK", ], ), + # Install the CUDA delegate beside them when it is built. The CUDA + # runtime itself is not bundled; it comes from the environment. + BuiltFile( + src_dir="%CMAKE_CACHE_DIR%/backends/cuda/", + src_name=( + "libexecutorch_cuda_backend.so." + f"{get_runtime_soname_major()}.*" + ), + dst=( + "executorch/lib/libexecutorch_cuda_backend.so." + f"{get_runtime_soname_major()}" + ), + dependent_cmake_flags=[ + "EXECUTORCH_BUILD_SHARED", + "EXECUTORCH_BUILD_CUDA", + ], + ), + # The CUDA delegate calls into this for stream handling, so an + # application that links the delegate from the wheel cannot + # resolve it unless this ships too. It carries no SONAME version, + # so the name is used as built. The target is always built shared + # whenever CUDA is on, so gating on the shared runtime as well + # would drop it from a CUDA wheel built with a static runtime. + BuiltFile( + src_dir="%CMAKE_CACHE_DIR%/extension/cuda/%BUILD_TYPE%/", + src_name="extension_cuda", + dst="executorch/lib/", + is_dynamic_lib=True, + dependent_cmake_flags=["EXECUTORCH_BUILD_CUDA"], + ), # Install the prebuilt pybindings extension wrapper for the runtime, # portable kernels, and a selection of backends. This lets users # load and execute .pte files from python.