diff --git a/devops/scripts/benchmarks/README.md b/devops/scripts/benchmarks/README.md index 532cb78704325..e643ea37f843d 100644 --- a/devops/scripts/benchmarks/README.md +++ b/devops/scripts/benchmarks/README.md @@ -112,6 +112,16 @@ For example `--filter "graph_api_*"` `--offline` - skips rebuilding projects, oneAPI updates, and benchmark data downloads. This is useful when you want to run benchmarks with existing builds and data without fetching updates or recompiling. Note that if build artifacts or data don't exist, the benchmarks will fail to run. +### Local development options + +`--benchmarks-source-dir ` - use an existing Compute Benchmarks source tree instead of cloning the repository into the working directory. This is intended for local development against a checkout you are editing. + +When a build finishes successfully, a JSON build-complete marker (`benchmark_build_complete.json`) is written into the project's build directory recording the source commit and `git status --porcelain` fingerprint. On subsequent runs the build is skipped when the fingerprint still matches, so editing a tracked source file triggers a rebuild automatically. If `--benchmarks-source-dir` points at a directory that is **not** a git repository, the benchmarks are always rebuilt. + +`--offload-install-dir ` / `--offload-include-dir ` - directories providing liboffload: the library dir containing `libLLVMOffload` (passed as `OFFLOAD_INSTALL_DIR`) and the header dir containing `OffloadAPI.h` (passed as `OFFLOAD_INCLUDE_DIR`). Both must be set together. When set, the OL (liboffload) `SubmitKernel` benchmark is built (via `-DBUILD_OL=ON`) and enabled. For an LLVM install tree built with `LLVM_ENABLE_PER_TARGET_RUNTIME_DIR=OFF` these are typically `/lib` and `/include/offload`. + +`--force-offload-plugin ` - backend name (`level_zero`, `cuda`, `amdgpu`, or `host`) exported as the `FORCE_OFFLOAD_PLUGIN` environment variable for the benchmark executable process. The OL benchmark uses it to select the offload device by backend. + ## Running in CI The benchmarks scripts are used in a GitHub Actions workflow, and can be automatically executed on a preconfigured system against any Pull Request. diff --git a/devops/scripts/benchmarks/benches/compute/compute.py b/devops/scripts/benchmarks/benches/compute/compute.py index 8681ff105922a..9c21afcf524b2 100644 --- a/devops/scripts/benchmarks/benches/compute/compute.py +++ b/devops/scripts/benchmarks/benches/compute/compute.py @@ -24,6 +24,7 @@ def runtime_to_name(runtime: RUNTIMES) -> str: RUNTIMES.SYCL: "SYCL", RUNTIMES.LEVEL_ZERO: "Level Zero", RUNTIMES.UR: "Unified Runtime", + RUNTIMES.OL: "Offload", }[runtime] @@ -47,12 +48,18 @@ def setup(self) -> None: return if self._project is None: + src_override = ( + Path(options.benchmarks_source_dir) + if options.benchmarks_source_dir + else None + ) self._project = GitProject( self.git_url(), self.git_hash(), Path(options.workdir), "compute-benchmarks", use_installdir=False, + src_dir_override=src_override, ) if not self._project.needs_rebuild(): @@ -83,6 +90,13 @@ def setup(self) -> None: "-DBUILD_OCL=OFF", ] + if options.offload_install_dir and options.offload_include_dir: + extra_args += [ + "-DBUILD_OL=ON", + f"-DOFFLOAD_INSTALL_DIR={options.offload_install_dir}", + f"-DOFFLOAD_INCLUDE_DIR={options.offload_include_dir}", + ] + self._project.configure(extra_args, add_sycl=True) self._project.build(add_sycl=True) @@ -845,7 +859,12 @@ def range(self) -> tuple[float, float]: return (0.0, None) def _supported_runtimes(self) -> list[RUNTIMES]: - return super()._supported_runtimes() + [RUNTIMES.SYCL_PREVIEW] + runtimes = super()._supported_runtimes() + [RUNTIMES.SYCL_PREVIEW] + # OL (liboffload) SubmitKernel is only available when built via + # --offload-install-dir / --offload-include-dir. + if options.offload_install_dir and options.offload_include_dir: + runtimes.append(RUNTIMES.OL) + return runtimes def _bin_args(self, flamegraph_enabled: bool = False) -> list[str]: iters = self._get_iters(flamegraph_enabled) diff --git a/devops/scripts/benchmarks/benches/compute/compute_benchmark.py b/devops/scripts/benchmarks/benches/compute/compute_benchmark.py index e35c5ea2ce31a..96acad115bdce 100644 --- a/devops/scripts/benchmarks/benches/compute/compute_benchmark.py +++ b/devops/scripts/benchmarks/benches/compute/compute_benchmark.py @@ -124,8 +124,10 @@ def _get_iters(self, flamegraph_enabled: bool): def _supported_runtimes(self) -> list[RUNTIMES]: """Base runtimes supported by this benchmark, can be overridden.""" - # By default, support all runtimes except SYCL_PREVIEW - return [r for r in RUNTIMES if r != RUNTIMES.SYCL_PREVIEW] + # By default, support all runtimes except SYCL_PREVIEW and OL. + # OL (liboffload) only provides a SubmitKernel binary, so it is opted + # into explicitly by that benchmark rather than supported by default. + return [r for r in RUNTIMES if r not in (RUNTIMES.SYCL_PREVIEW, RUNTIMES.OL)] def _extra_env_vars(self) -> dict: return {} diff --git a/devops/scripts/benchmarks/benches/compute/compute_enums.py b/devops/scripts/benchmarks/benches/compute/compute_enums.py index b65c464dbb2f4..f29d6afa4aac3 100644 --- a/devops/scripts/benchmarks/benches/compute/compute_enums.py +++ b/devops/scripts/benchmarks/benches/compute/compute_enums.py @@ -10,6 +10,7 @@ class RUNTIMES(Enum): SYCL = "sycl" LEVEL_ZERO = "l0" UR = "ur" + OL = "ol" class PROFILERS(Enum): @@ -29,4 +30,5 @@ def runtime_to_tag_name(runtime: RUNTIMES) -> str: RUNTIMES.SYCL: "SYCL", RUNTIMES.LEVEL_ZERO: "L0", RUNTIMES.UR: "UR", + RUNTIMES.OL: "OL", }[runtime] diff --git a/devops/scripts/benchmarks/git_project.py b/devops/scripts/benchmarks/git_project.py index 78454c406e865..6f0f229a89e6c 100644 --- a/devops/scripts/benchmarks/git_project.py +++ b/devops/scripts/benchmarks/git_project.py @@ -2,6 +2,7 @@ # See https://llvm.org/LICENSE.txt for license information. # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +import json import os from pathlib import Path import shutil @@ -10,6 +11,10 @@ from utils.utils import run from options import options +# Marker file written into build_dir after a successful build. Records a +# fingerprint of the source tree so reruns can skip rebuilding. +BUILD_COMPLETE_MARKER = "benchmark_build_complete.json" + class GitProject: def __init__( @@ -21,6 +26,7 @@ def __init__( use_installdir: bool = True, no_suffix_src: bool = False, shallow_clone: bool = True, + src_dir_override: Path | None = None, ) -> None: self._url = url self._ref = ref @@ -29,6 +35,7 @@ def __init__( self._use_installdir = use_installdir self._no_suffix_src = no_suffix_src self._shallow_clone = shallow_clone + self._src_dir_override = src_dir_override self._rebuild_needed = self._setup_repo() @property @@ -37,6 +44,8 @@ def name(self): @property def src_dir(self) -> Path: + if self._src_dir_override is not None: + return self._src_dir_override suffix = "" if self._no_suffix_src else "-src" return self._directory / f"{self._name}{suffix}" @@ -48,15 +57,56 @@ def build_dir(self) -> Path: def install_dir(self) -> Path: return self._directory / f"{self._name}-install" + @property + def _build_marker_path(self) -> Path: + return self.build_dir / BUILD_COMPLETE_MARKER + + def _source_fingerprint(self) -> dict | None: + """Fingerprint the source tree for build-completion tracking. + + Returns a dict with the current commit and `git status --porcelain` + output, or None if src_dir is not a git repository (in which case the + caller should always rebuild). + """ + try: + commit = run("git rev-parse HEAD", cwd=self.src_dir).stdout.decode().strip() + status = ( + run("git status --porcelain", cwd=self.src_dir).stdout.decode().strip() + ) + except Exception as e: + log.debug( + f"Could not fingerprint source for {self._name} at {self.src_dir}: {e}" + ) + return None + return {"commit": commit, "status": status} + + def mark_build_complete(self) -> None: + """Record a build-complete marker so reruns can skip rebuilding. + + No-op when the source is not a git repository (nothing to fingerprint). + """ + fingerprint = self._source_fingerprint() + if fingerprint is None: + return + try: + self.build_dir.mkdir(parents=True, exist_ok=True) + with open(self._build_marker_path, "w") as f: + json.dump(fingerprint, f) + log.debug(f"Wrote build-complete marker to {self._build_marker_path}") + except Exception as e: + log.debug(f"Failed to write build-complete marker for {self._name}: {e}") + + def _read_build_marker(self) -> dict | None: + try: + with open(self._build_marker_path) as f: + return json.load(f) + except Exception: + return None + def needs_rebuild(self) -> bool: if options.offline: log.debug("Rebuild is disabled due to --offline option.") return False - if self._rebuild_needed: - log.debug( - f"Rebuild needed because new sources were detected for project {self._name}." - ) - return True dir_to_check = self.install_dir if self._use_installdir else self.build_dir @@ -68,8 +118,25 @@ def needs_rebuild(self) -> bool: f"{dir_to_check} does not exist or does not contain any file, rebuild needed." ) return True - log.debug(f"{dir_to_check} exists and is not empty, no rebuild needed.") - return False + + fingerprint = self._source_fingerprint() + if fingerprint is None: + log.debug( + f"Source for {self._name} is not a git repository, rebuild needed." + ) + return True + + marker = self._read_build_marker() + if marker == fingerprint: + log.debug( + f"Build-complete marker matches current source for {self._name}, no rebuild needed." + ) + return False + + log.debug( + f"Build-complete marker missing or stale for {self._name}, rebuild needed." + ) + return True def configure( self, @@ -109,6 +176,9 @@ def build( ld_library=ld_library, timeout=timeout, ) + # run() raises on non-zero exit, so reaching here means the build + # succeeded. Record a marker so identical reruns can skip rebuilding. + self.mark_build_complete() def install(self) -> None: """Installs the project.""" @@ -179,6 +249,11 @@ def _setup_repo(self) -> bool: Returns: bool: True if the repository was cloned or updated, False if it was already up-to-date. """ + if self._src_dir_override is not None: + log.debug( + f"Using provided source directory {self.src_dir} for {self._name}, skipping git operations." + ) + return False if os.environ.get("LLVM_BENCHMARKS_UNIT_TESTING") == "1": log.debug( f"Skipping git operations during unit testing of {self._name} (LLVM_BENCHMARKS_UNIT_TESTING=1)." diff --git a/devops/scripts/benchmarks/html/scripts.js b/devops/scripts/benchmarks/html/scripts.js index 04b43f1718930..738369c088fc1 100644 --- a/devops/scripts/benchmarks/html/scripts.js +++ b/devops/scripts/benchmarks/html/scripts.js @@ -1048,7 +1048,7 @@ function getLayerTags(metadata) { const layerTags = new Set(); if (metadata?.tags) { metadata.tags.forEach(tag => { - if (tag.startsWith('SYCL') || tag.startsWith('UR') || tag === 'L0') { + if (tag.startsWith('SYCL') || tag.startsWith('UR') || tag === 'L0' || tag === 'OL') { layerTags.add(tag); } }); diff --git a/devops/scripts/benchmarks/main.py b/devops/scripts/benchmarks/main.py index d39e870e9ae3e..1cbbdf9a45437 100755 --- a/devops/scripts/benchmarks/main.py +++ b/devops/scripts/benchmarks/main.py @@ -4,6 +4,18 @@ # See https://llvm.org/LICENSE.txt for license information. # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +# /// script +# requires-python = ">=3.10" +# dependencies = [ +# "matplotlib==3.9.2", +# "mpld3==0.5.10", +# "dataclasses-json==0.6.7", +# "PyYAML==6.0.1", +# "Mako==1.3.12", +# "psutil>=7.0.0", +# ] +# /// + import argparse import re import statistics @@ -523,6 +535,34 @@ def validate_and_parse_env_args(env_args): help="Always download benchmark data dependencies, even if they already exist.", action="store_true", ) + parser.add_argument( + "--benchmarks-source-dir", + type=str, + help="Use this compute-benchmarks source dir instead of cloning the repository. " + "If the dir is not a git repository, benchmarks are always rebuilt.", + default=options.benchmarks_source_dir, + ) + parser.add_argument( + "--offload-install-dir", + type=str, + help="Directory containing libLLVMOffload (-> OFFLOAD_INSTALL_DIR). " + "Set together with --offload-include-dir to enable the OL SubmitKernel benchmark.", + default=options.offload_install_dir, + ) + parser.add_argument( + "--offload-include-dir", + type=str, + help="Directory containing OffloadAPI.h (-> OFFLOAD_INCLUDE_DIR). " + "Set together with --offload-install-dir to enable the OL SubmitKernel benchmark.", + default=options.offload_include_dir, + ) + parser.add_argument( + "--force-offload-plugin", + type=str, + help="Backend name (level_zero/cuda/amdgpu/host) exported as FORCE_OFFLOAD_PLUGIN " + "for the benchmark executable process.", + default=options.force_offload_plugin, + ) parser.add_argument( "--env", type=str, @@ -784,6 +824,10 @@ def validate_and_parse_env_args(env_args): options.workdir = args.benchmark_directory options.offline = args.offline options.redownload = args.redownload + options.benchmarks_source_dir = args.benchmarks_source_dir + options.offload_install_dir = args.offload_install_dir + options.offload_include_dir = args.offload_include_dir + options.force_offload_plugin = args.force_offload_plugin options.sycl = args.sycl options.iterations = args.iterations options.timeout = args.timeout @@ -826,6 +870,23 @@ def validate_and_parse_env_args(env_args): if not os.path.isdir(args.output_dir): parser.error("Specified --output-dir is not a valid path") options.output_directory = os.path.abspath(args.output_dir) + if args.benchmarks_source_dir is not None: + if not os.path.isdir(args.benchmarks_source_dir): + parser.error("Specified --benchmarks-source-dir is not a valid path") + options.benchmarks_source_dir = os.path.abspath(args.benchmarks_source_dir) + if args.offload_install_dir is not None or args.offload_include_dir is not None: + if args.offload_install_dir is None or args.offload_include_dir is None: + parser.error( + "--offload-install-dir and --offload-include-dir must both be defined together" + ) + if not os.path.isdir(args.offload_install_dir): + parser.error("Specified --offload-install-dir is not a valid path") + if not os.path.isdir(args.offload_include_dir): + parser.error("Specified --offload-include-dir is not a valid path") + options.offload_install_dir = os.path.abspath(args.offload_install_dir) + options.offload_include_dir = os.path.abspath(args.offload_include_dir) + if options.force_offload_plugin: + options.extra_env_vars["FORCE_OFFLOAD_PLUGIN"] = options.force_offload_plugin # Initialize GitHub summary tracking execution_stats = { diff --git a/devops/scripts/benchmarks/options.py b/devops/scripts/benchmarks/options.py index f7186ac436062..243f30bef2caa 100644 --- a/devops/scripts/benchmarks/options.py +++ b/devops/scripts/benchmarks/options.py @@ -55,6 +55,17 @@ class Options: pytorch_root: str = None offline: bool = False redownload: bool = False + # Use an existing compute-benchmarks source dir instead of cloning. When the + # dir is not a git repository, the benchmarks are always rebuilt. + benchmarks_source_dir: str = None + # liboffload library dir (-> OFFLOAD_INSTALL_DIR) and header dir + # (-> OFFLOAD_INCLUDE_DIR). When both are set, the OL SubmitKernel benchmark + # is built (via -DBUILD_OL=ON) and enabled. + offload_install_dir: str = None + offload_include_dir: str = None + # Backend name (e.g. level_zero/cuda/amdgpu/host) exported as + # FORCE_OFFLOAD_PLUGIN for the benchmark executable process. + force_offload_plugin: str = None benchmark_cwd: str = "INVALID" timeout: float = 600 iterations: int = 3 diff --git a/unified-runtime/source/adapters/offload/enqueue.cpp b/unified-runtime/source/adapters/offload/enqueue.cpp index 4405e21cc2918..e51fd3bc765df 100644 --- a/unified-runtime/source/adapters/offload/enqueue.cpp +++ b/unified-runtime/source/adapters/offload/enqueue.cpp @@ -41,7 +41,8 @@ ol_result_t makeEvent(ur_command_t Type, ol_queue_handle_t OlQueue, ur_queue_handle_t UrQueue, ur_event_handle_t *UrEvent) { if (UrEvent) { auto *Event = new ur_event_handle_t_(Type, UrQueue); - if (auto Res = olCreateEvent(OlQueue, &Event->OffloadEvent)) { + if (auto Res = + olCreateEvent(OlQueue, OL_EVENT_FLAGS_NONE, &Event->OffloadEvent)) { delete Event; return Res; }; @@ -84,7 +85,8 @@ ur_result_t doWait(ur_queue_handle_t hQueue, uint32_t numEventsInWaitList, if (Q == TargetQueue) { continue; } - OL_RETURN_ON_ERR(olCreateEvent(Q, &OffloadHandles.emplace_back())); + OL_RETURN_ON_ERR(olCreateEvent(Q, OL_EVENT_FLAGS_NONE, + &OffloadHandles.emplace_back())); } OL_RETURN_ON_ERR(olWaitEvents(TargetQueue, OffloadHandles.data(), OffloadHandles.size())); @@ -201,8 +203,9 @@ static ur_result_t urEnqueueKernelLaunch( LaunchArgs.DynSharedMemory = 0; OL_RETURN_ON_ERR(olLaunchKernel( - Queue, hQueue->OffloadDevice, hKernel->OffloadKernel, - hKernel->Args.getStorage(), hKernel->Args.getStorageSize(), &LaunchArgs)); + Queue, hQueue->OffloadDevice, hKernel->OffloadKernel, &LaunchArgs, + /*Properties=*/nullptr, hKernel->Args.Pointers.size(), + hKernel->Args.Pointers.data(), hKernel->Args.ParamSizes.data())); OL_RETURN_ON_ERR(makeEvent(UR_COMMAND_KERNEL_LAUNCH, Queue, hQueue, phEvent)); return UR_RESULT_SUCCESS; @@ -261,7 +264,8 @@ ur_result_t doMemcpy(ur_command_t Command, ur_queue_handle_t hQueue, olMemcpy(Queue, DestPtr, DestDevice, SrcPtr, SrcDevice, size)); if (phEvent) { auto *Event = new ur_event_handle_t_(Command, hQueue); - if (auto Res = olCreateEvent(Queue, &Event->OffloadEvent)) { + if (auto Res = + olCreateEvent(Queue, OL_EVENT_FLAGS_NONE, &Event->OffloadEvent)) { delete Event; return offloadResultToUR(Res); };