[Bench] Add support for liboffload

devops/scripts/benchmarks/README.md

@@ -112,6 +112,16 @@ For example `--filter "graph_api_*"`
 
 `--offline` - skips rebuilding projects, oneAPI updates, and benchmark data downloads. This is useful when you want to run benchmarks with existing builds and data without fetching updates or recompiling. Note that if build artifacts or data don't exist, the benchmarks will fail to run.
 
+### Local development options
+
+`--benchmarks-source-dir <dir>` - use an existing Compute Benchmarks source tree instead of cloning the repository into the working directory. This is intended for local development against a checkout you are editing.
+
+When a build finishes successfully, a JSON build-complete marker (`benchmark_build_complete.json`) is written into the project's build directory recording the source commit and `git status --porcelain` fingerprint. On subsequent runs the build is skipped when the fingerprint still matches, so editing a tracked source file triggers a rebuild automatically. If `--benchmarks-source-dir` points at a directory that is **not** a git repository, the benchmarks are always rebuilt.
+
+`--offload-install-dir <dir>` / `--offload-include-dir <dir>` - directories providing liboffload: the library dir containing `libLLVMOffload` (passed as `OFFLOAD_INSTALL_DIR`) and the header dir containing `OffloadAPI.h` (passed as `OFFLOAD_INCLUDE_DIR`). Both must be set together. When set, the OL (liboffload) `SubmitKernel` benchmark is built (via `-DBUILD_OL=ON`) and enabled. For an LLVM install tree built with `LLVM_ENABLE_PER_TARGET_RUNTIME_DIR=OFF` these are typically `<install>/lib` and `<install>/include/offload`.
+
+`--force-offload-plugin <name>` - backend name (`level_zero`, `cuda`, `amdgpu`, or `host`) exported as the `FORCE_OFFLOAD_PLUGIN` environment variable for the benchmark executable process. The OL benchmark uses it to select the offload device by backend.
+
 ## Running in CI
 
 The benchmarks scripts are used in a GitHub Actions workflow, and can be automatically executed on a preconfigured system against any Pull Request.

devops/scripts/benchmarks/benches/compute/compute.py

@@ -24,6 +24,7 @@ def runtime_to_name(runtime: RUNTIMES) -> str:
         RUNTIMES.SYCL: "SYCL",
         RUNTIMES.LEVEL_ZERO: "Level Zero",
         RUNTIMES.UR: "Unified Runtime",
+        RUNTIMES.OL: "Offload",
     }[runtime]
 
 
@@ -47,12 +48,18 @@ def setup(self) -> None:
             return
 
         if self._project is None:
+            src_override = (
+                Path(options.benchmarks_source_dir)
+                if options.benchmarks_source_dir
+                else None
+            )
             self._project = GitProject(
                 self.git_url(),
                 self.git_hash(),
                 Path(options.workdir),
                 "compute-benchmarks",
                 use_installdir=False,
+                src_dir_override=src_override,
             )
 
         if not self._project.needs_rebuild():
@@ -83,6 +90,13 @@ def setup(self) -> None:
                 "-DBUILD_OCL=OFF",
             ]
 
+        if options.offload_install_dir and options.offload_include_dir:
+            extra_args += [
+                "-DBUILD_OL=ON",
+                f"-DOFFLOAD_INSTALL_DIR={options.offload_install_dir}",
+                f"-DOFFLOAD_INCLUDE_DIR={options.offload_include_dir}",
+            ]
+
         self._project.configure(extra_args, add_sycl=True)
         self._project.build(add_sycl=True)
 
@@ -845,7 +859,12 @@ def range(self) -> tuple[float, float]:
         return (0.0, None)
 
     def _supported_runtimes(self) -> list[RUNTIMES]:
-        return super()._supported_runtimes() + [RUNTIMES.SYCL_PREVIEW]
+        runtimes = super()._supported_runtimes() + [RUNTIMES.SYCL_PREVIEW]
+        # OL (liboffload) SubmitKernel is only available when built via
+        # --offload-install-dir / --offload-include-dir.
+        if options.offload_install_dir and options.offload_include_dir:
+            runtimes.append(RUNTIMES.OL)
+        return runtimes
 
     def _bin_args(self, flamegraph_enabled: bool = False) -> list[str]:
         iters = self._get_iters(flamegraph_enabled)

devops/scripts/benchmarks/benches/compute/compute_benchmark.py

@@ -124,8 +124,10 @@ def _get_iters(self, flamegraph_enabled: bool):
 
     def _supported_runtimes(self) -> list[RUNTIMES]:
         """Base runtimes supported by this benchmark, can be overridden."""
-        # By default, support all runtimes except SYCL_PREVIEW
-        return [r for r in RUNTIMES if r != RUNTIMES.SYCL_PREVIEW]
+        # By default, support all runtimes except SYCL_PREVIEW and OL.
+        # OL (liboffload) only provides a SubmitKernel binary, so it is opted
+        # into explicitly by that benchmark rather than supported by default.
+        return [r for r in RUNTIMES if r not in (RUNTIMES.SYCL_PREVIEW, RUNTIMES.OL)]
 
     def _extra_env_vars(self) -> dict:
         return {}

devops/scripts/benchmarks/benches/compute/compute_enums.py

@@ -10,6 +10,7 @@ class RUNTIMES(Enum):
     SYCL = "sycl"
     LEVEL_ZERO = "l0"
     UR = "ur"
+    OL = "ol"
 
 
 class PROFILERS(Enum):
@@ -29,4 +30,5 @@ def runtime_to_tag_name(runtime: RUNTIMES) -> str:
         RUNTIMES.SYCL: "SYCL",
         RUNTIMES.LEVEL_ZERO: "L0",
         RUNTIMES.UR: "UR",
+        RUNTIMES.OL: "OL",
     }[runtime]

devops/scripts/benchmarks/git_project.py

@@ -2,6 +2,7 @@
 # See https://llvm.org/LICENSE.txt for license information.
 # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 
+import json
 import os
 from pathlib import Path
 import shutil
@@ -11,6 +12,11 @@
 from options import options
 
 
+# Marker file written into build_dir after a successful build. Records a
+# fingerprint of the source tree so reruns can skip rebuilding.
+BUILD_COMPLETE_MARKER = "benchmark_build_complete.json"
+
+
 class GitProject:
     def __init__(
         self,
@@ -21,6 +27,7 @@ def __init__(
         use_installdir: bool = True,
         no_suffix_src: bool = False,
         shallow_clone: bool = True,
+        src_dir_override: Path | None = None,
     ) -> None:
         self._url = url
         self._ref = ref
@@ -29,6 +36,7 @@ def __init__(
         self._use_installdir = use_installdir
         self._no_suffix_src = no_suffix_src
         self._shallow_clone = shallow_clone
+        self._src_dir_override = src_dir_override
         self._rebuild_needed = self._setup_repo()
 
     @property
@@ -37,6 +45,8 @@ def name(self):
 
     @property
     def src_dir(self) -> Path:
+        if self._src_dir_override is not None:
+            return self._src_dir_override
         suffix = "" if self._no_suffix_src else "-src"
         return self._directory / f"{self._name}{suffix}"
 
@@ -48,15 +58,58 @@ def build_dir(self) -> Path:
     def install_dir(self) -> Path:
         return self._directory / f"{self._name}-install"
 
+    @property
+    def _build_marker_path(self) -> Path:
+        return self.build_dir / BUILD_COMPLETE_MARKER
+
+    def _source_fingerprint(self) -> dict | None:
+        """Fingerprint the source tree for build-completion tracking.
+
+        Returns a dict with the current commit and `git status --porcelain`
+        output, or None if src_dir is not a git repository (in which case the
+        caller should always rebuild).
+        """
+        try:
+            commit = (
+                run("git rev-parse HEAD", cwd=self.src_dir).stdout.decode().strip()
+            )
+            status = (
+                run("git status --porcelain", cwd=self.src_dir).stdout.decode().strip()
+            )
+        except Exception as e:
+            log.debug(
+                f"Could not fingerprint source for {self._name} at {self.src_dir}: {e}"
+            )
+            return None
+        return {"commit": commit, "status": status}
+
+    def mark_build_complete(self) -> None:
+        """Record a build-complete marker so reruns can skip rebuilding.
+
+        No-op when the source is not a git repository (nothing to fingerprint).
+        """
+        fingerprint = self._source_fingerprint()
+        if fingerprint is None:
+            return
+        try:
+            self.build_dir.mkdir(parents=True, exist_ok=True)
+            with open(self._build_marker_path, "w") as f:
+                json.dump(fingerprint, f)
+            log.debug(f"Wrote build-complete marker to {self._build_marker_path}")
+        except Exception as e:
+            log.debug(f"Failed to write build-complete marker for {self._name}: {e}")
+
+    def _read_build_marker(self) -> dict | None:
+        try:
+            with open(self._build_marker_path) as f:
+                return json.load(f)
+        except Exception:
+            return None
+
     def needs_rebuild(self) -> bool:
         if options.offline:
             log.debug("Rebuild is disabled due to --offline option.")
             return False
-        if self._rebuild_needed:
-            log.debug(
-                f"Rebuild needed because new sources were detected for project {self._name}."
-            )
-            return True
 
         dir_to_check = self.install_dir if self._use_installdir else self.build_dir
 
@@ -68,8 +121,25 @@ def needs_rebuild(self) -> bool:
                 f"{dir_to_check} does not exist or does not contain any file, rebuild needed."
             )
             return True
-        log.debug(f"{dir_to_check} exists and is not empty, no rebuild needed.")
-        return False
+
+        fingerprint = self._source_fingerprint()
+        if fingerprint is None:
+            log.debug(
+                f"Source for {self._name} is not a git repository, rebuild needed."
+            )
+            return True
+
+        marker = self._read_build_marker()
+        if marker == fingerprint:
+            log.debug(
+                f"Build-complete marker matches current source for {self._name}, no rebuild needed."
+            )
+            return False
+
+        log.debug(
+            f"Build-complete marker missing or stale for {self._name}, rebuild needed."
+        )
+        return True
 
     def configure(
         self,
@@ -109,6 +179,9 @@ def build(
             ld_library=ld_library,
             timeout=timeout,
         )
+        # run() raises on non-zero exit, so reaching here means the build
+        # succeeded. Record a marker so identical reruns can skip rebuilding.
+        self.mark_build_complete()
 
     def install(self) -> None:
         """Installs the project."""
@@ -179,6 +252,11 @@ def _setup_repo(self) -> bool:
         Returns:
             bool: True if the repository was cloned or updated, False if it was already up-to-date.
         """
+        if self._src_dir_override is not None:
+            log.debug(
+                f"Using provided source directory {self.src_dir} for {self._name}, skipping git operations."
+            )
+            return False
         if os.environ.get("LLVM_BENCHMARKS_UNIT_TESTING") == "1":
             log.debug(
                 f"Skipping git operations during unit testing of {self._name} (LLVM_BENCHMARKS_UNIT_TESTING=1)."

devops/scripts/benchmarks/html/scripts.js

@@ -1048,7 +1048,7 @@ function getLayerTags(metadata) {
     const layerTags = new Set();
     if (metadata?.tags) {
         metadata.tags.forEach(tag => {
-            if (tag.startsWith('SYCL') || tag.startsWith('UR') || tag === 'L0') {
+            if (tag.startsWith('SYCL') || tag.startsWith('UR') || tag === 'L0' || tag === 'OL') {
                 layerTags.add(tag);
             }
         });

devops/scripts/benchmarks/main.py

@@ -4,6 +4,18 @@
 # See https://llvm.org/LICENSE.txt for license information.
 # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 
+# /// script
+# requires-python = ">=3.10"
+# dependencies = [
+#   "matplotlib==3.9.2",
+#   "mpld3==0.5.10",
+#   "dataclasses-json==0.6.7",
+#   "PyYAML==6.0.1",
+#   "Mako==1.3.12",
+#   "psutil>=7.0.0",
+# ]
+# ///
+
 import argparse
 import re
 import statistics
@@ -523,6 +535,34 @@ def validate_and_parse_env_args(env_args):
         help="Always download benchmark data dependencies, even if they already exist.",
         action="store_true",
     )
+    parser.add_argument(
+        "--benchmarks-source-dir",
+        type=str,
+        help="Use this compute-benchmarks source dir instead of cloning the repository. "
+        "If the dir is not a git repository, benchmarks are always rebuilt.",
+        default=options.benchmarks_source_dir,
+    )
+    parser.add_argument(
+        "--offload-install-dir",
+        type=str,
+        help="Directory containing libLLVMOffload (-> OFFLOAD_INSTALL_DIR). "
+        "Set together with --offload-include-dir to enable the OL SubmitKernel benchmark.",
+        default=options.offload_install_dir,
+    )
+    parser.add_argument(
+        "--offload-include-dir",
+        type=str,
+        help="Directory containing OffloadAPI.h (-> OFFLOAD_INCLUDE_DIR). "
+        "Set together with --offload-install-dir to enable the OL SubmitKernel benchmark.",
+        default=options.offload_include_dir,
+    )
+    parser.add_argument(
+        "--force-offload-plugin",
+        type=str,
+        help="Backend name (level_zero/cuda/amdgpu/host) exported as FORCE_OFFLOAD_PLUGIN "
+        "for the benchmark executable process.",
+        default=options.force_offload_plugin,
+    )
     parser.add_argument(
         "--env",
         type=str,
@@ -784,6 +824,10 @@ def validate_and_parse_env_args(env_args):
     options.workdir = args.benchmark_directory
     options.offline = args.offline
     options.redownload = args.redownload
+    options.benchmarks_source_dir = args.benchmarks_source_dir
+    options.offload_install_dir = args.offload_install_dir
+    options.offload_include_dir = args.offload_include_dir
+    options.force_offload_plugin = args.force_offload_plugin
     options.sycl = args.sycl
     options.iterations = args.iterations
     options.timeout = args.timeout
@@ -826,6 +870,23 @@ def validate_and_parse_env_args(env_args):
         if not os.path.isdir(args.output_dir):
             parser.error("Specified --output-dir is not a valid path")
         options.output_directory = os.path.abspath(args.output_dir)
+    if args.benchmarks_source_dir is not None:
+        if not os.path.isdir(args.benchmarks_source_dir):
+            parser.error("Specified --benchmarks-source-dir is not a valid path")
+        options.benchmarks_source_dir = os.path.abspath(args.benchmarks_source_dir)
+    if args.offload_install_dir is not None or args.offload_include_dir is not None:
+        if args.offload_install_dir is None or args.offload_include_dir is None:
+            parser.error(
+                "--offload-install-dir and --offload-include-dir must both be defined together"
+            )
+        if not os.path.isdir(args.offload_install_dir):
+            parser.error("Specified --offload-install-dir is not a valid path")
+        if not os.path.isdir(args.offload_include_dir):
+            parser.error("Specified --offload-include-dir is not a valid path")
+        options.offload_install_dir = os.path.abspath(args.offload_install_dir)
+        options.offload_include_dir = os.path.abspath(args.offload_include_dir)
+    if options.force_offload_plugin:
+        options.extra_env_vars["FORCE_OFFLOAD_PLUGIN"] = options.force_offload_plugin
 
     # Initialize GitHub summary tracking
     execution_stats = {

devops/scripts/benchmarks/options.py

@@ -55,6 +55,17 @@ class Options:
     pytorch_root: str = None
     offline: bool = False
     redownload: bool = False
+    # Use an existing compute-benchmarks source dir instead of cloning. When the
+    # dir is not a git repository, the benchmarks are always rebuilt.
+    benchmarks_source_dir: str = None
+    # liboffload library dir (-> OFFLOAD_INSTALL_DIR) and header dir
+    # (-> OFFLOAD_INCLUDE_DIR). When both are set, the OL SubmitKernel benchmark
+    # is built (via -DBUILD_OL=ON) and enabled.
+    offload_install_dir: str = None
+    offload_include_dir: str = None
+    # Backend name (e.g. level_zero/cuda/amdgpu/host) exported as
+    # FORCE_OFFLOAD_PLUGIN for the benchmark executable process.
+    force_offload_plugin: str = None
     benchmark_cwd: str = "INVALID"
     timeout: float = 600
     iterations: int = 3