Add combo test cases

.buildkite/benchmark/cases/daily/Gemma4-26B-dataset_custom-inlen_1024-outlen_500.json

@@ -24,16 +24,17 @@
           "VLLM_USE_V1": "1",
           "MODEL_IMPL_TYPE": "flax_nnx",
           "USE_BATCHED_RPA_KERNEL": "1",
-          "MOE_REQUANTIZE_WEIGHT_DTYPE": "float8_e4m3fn"
+          "MOE_REQUANTIZE_WEIGHT_DTYPE": "float8_e4m3fn",
+          "VLLM_ENGINE_READY_TIMEOUT_S": "1800"
         },
         "args": {
           "model": "google/gemma-4-26B-A4B-it",
           "max-num-seqs": 256,
           "max-num-batched-tokens": 4096,
           "data-parallel-size": {
-              "v6e-8": 2,
-              "v7x-2": 4,
-              "default": 1
+            "v6e-8": 2,
+            "v7x-2": 1,
+            "default": 1
           },
           "tensor-parallel-size": {
             "v7x-2": 2,
@@ -79,16 +80,17 @@
           "VLLM_USE_V1": "1",
           "MODEL_IMPL_TYPE": "flax_nnx",
           "USE_BATCHED_RPA_KERNEL": "1",
-          "MOE_REQUANTIZE_WEIGHT_DTYPE": "float8_e4m3fn"
+          "MOE_REQUANTIZE_WEIGHT_DTYPE": "float8_e4m3fn",
+          "VLLM_ENGINE_READY_TIMEOUT_S": "1800"
         },
         "args": {
           "model": "google/gemma-4-26B-A4B-it",
           "max-num-seqs": 256,
           "max-num-batched-tokens": 4096,
           "data-parallel-size": {
-              "v6e-8": 2,
-              "v7x-2": 4,
-              "default": 1
+            "v6e-8": 2,
+            "v7x-2": 1,
+            "default": 1
           },
           "tensor-parallel-size": {
             "v7x-2": 2,
@@ -121,4 +123,4 @@
       }
     }
   ]
-}
+}

.buildkite/benchmark/cases/daily/Gemma4-31B-dataset_custom-inlen_1024-outlen_500.json

@@ -24,15 +24,16 @@
         "env": {
           "VLLM_USE_V1": "1",
           "MODEL_IMPL_TYPE": "flax_nnx",
-          "USE_BATCHED_RPA_KERNEL": "1"
+          "USE_BATCHED_RPA_KERNEL": "1",
+          "VLLM_ENGINE_READY_TIMEOUT_S": "1800"
         },
         "args": {
           "model": "google/gemma-4-31B-it",
           "max-num-seqs": 256,
           "max-num-batched-tokens": 4096,
           "data-parallel-size": {
               "v6e-8": 2,
-              "v7x-2": 4,
+              "v7x-2": 1,
               "default": 1
           },
           "tensor-parallel-size": {
@@ -80,15 +81,16 @@
         "env": {
           "VLLM_USE_V1": "1",
           "MODEL_IMPL_TYPE": "flax_nnx",
-          "USE_BATCHED_RPA_KERNEL": "1"
+          "USE_BATCHED_RPA_KERNEL": "1",
+          "VLLM_ENGINE_READY_TIMEOUT_S": "1800"
         },
         "args": {
           "model": "google/gemma-4-31B-it",
           "max-num-seqs": 256,
           "max-num-batched-tokens": 4096,
           "data-parallel-size": {
               "v6e-8": 2,
-              "v7x-2": 4,
+              "v7x-2": 1,
               "default": 1
           },
           "tensor-parallel-size": {

.buildkite/benchmark/cases/daily/deepseek.json

@@ -0,0 +1,67 @@
+{
+  "global_env": {
+    "GCP_PROJECT_ID": "cloud-tpu-inference-test",
+    "GCS_BUCKET": "vllm-cb-storage2",
+    "GCP_INSTANCE_ID": "vllm-bm-inst",
+    "GCP_DATABASE_ID": "vllm-bm-bk-runs"
+  },
+  "benchmark_cases": [
+    {
+      "case_name": "deepseek-r1-dataset_custom-inlen_1024-outlen_4096-prelen_0",
+      "ci_queue": [
+        "tpu_v7x_8_queue"
+      ],
+      "env": {
+        "RUN_TYPE": "DAILY",
+        "MODELTAG": "NEW",
+        "EXPECTED_ETEL": 43200000,
+        "INPUT_LEN": 1024,
+        "OUTPUT_LEN": 4096,
+        "PREFIX_LEN": 0
+      },
+      "server_command_options": {
+        "command_type": "vllm_serve",
+        "env": {
+          "VLLM_USE_V1": "1",
+          "MODEL_IMPL_TYPE": "vllm",
+          "NEW_MODEL_DESIGN": "1",
+          "TPU_BACKEND_TYPE": "jax",
+          "VLLM_MLA_DISABLE": "0",
+          "MOE_REQUANTIZE_BLOCK_SIZE": "512",
+          "MOE_REQUANTIZE_WEIGHT_DTYPE": "fp4"
+        },
+        "args": {
+          "model": "deepseek-ai/DeepSeek-R1",
+          "seed": 42,
+          "max-num-seqs": 16,
+          "max-num-batched-tokens": 1024,
+          "tensor-parallel-size": {
+            "v7x-8": 8
+          },
+          "max-model-len": 5120,
+          "kv-cache-dtype": "fp8",
+          "gpu-memory-utilization": 0.95,
+          "enable-expert-parallel": true,
+          "no-enable-prefix-caching": true,
+          "async-scheduling": true,
+          "additional-config": "{\"sharding\": {\"sharding_strategy\": {\"enable_dp_attention\": true}}}"
+        }
+      },
+      "client_command_options": {
+        "command_type": "vllm_bench_serve",
+        "args": {
+          "model": "deepseek-ai/DeepSeek-R1",
+          "backend": "vllm",
+          "request-rate": "inf",
+          "dataset-name": "custom",
+          "dataset-path": "/workspace/tpu_inference/artifacts/dataset/DeepSeek-R1/inlen1024_outlen4096_prefixlen0.jsonl",
+          "num-prompts": 64,
+          "percentile-metrics": "ttft,tpot,itl,e2el",
+          "ignore-eos": true,
+          "custom-output-len": 4096,
+          "skip-chat-template": true
+        }
+      }
+    }
+  ]
+}

.buildkite/benchmark/cases/dev/mlcompass-example.json

@@ -0,0 +1,62 @@
+{
+  "global_env": {
+    "GCP_PROJECT_ID": "cloud-tpu-inference-test",
+    "GCS_BUCKET": "vllm-cb-storage2",
+    "GCP_INSTANCE_ID": "vllm-bm-inst",
+    "GCP_DATABASE_ID": "vllm-bm-bk-runs",
+    "MLCOMPASS_EXPORT_ENABLED": "true",
+    "MODELTAG": "NEW",
+    "EXPECTED_ETEL": 10800000
+  },
+  "benchmark_cases": [
+    {
+      "case_name": "Qwen3-32B-dataset_random-inlen_1024-outlen_1024-max-concurrency-64",
+      "ci_queue": [
+        "tpu_v7x_2_queue"
+      ],
+      "env": {
+        "MODELTAG": "NEW",
+        "EXPECTED_ETEL": 10800000,
+        "INPUT_LEN": 1024,
+        "OUTPUT_LEN": 1024,
+        "PREFIX_LEN": 0
+      },
+      "server_command_options": {
+        "command_type": "vllm_serve",
+        "env": {
+          "VLLM_USE_V1": "1",
+          "MODEL_IMPL_TYPE": "vllm"
+        },
+        "args": {
+          "model": "Qwen/Qwen3-32B",
+          "max-num-seqs": 256,
+          "max-num-batched-tokens": 4096,
+          "tensor-parallel-size": {
+            "v7x-2": 2
+          },
+          "max-model-len": 2048,
+          "kv-cache-dtype": "fp8",
+          "gpu-memory-utilization": 0.98,
+          "additional-config": "{\"quantization\": { \"qwix\": { \"rules\": [{ \"module_path\": \".*\", \"weight_qtype\": \"float8_e4m3fn\", \"act_qtype\": \"float8_e4m3fn\"}]}}}",
+          "no-enable-prefix-caching": true,
+          "async-scheduling": true
+        }
+      },
+      "client_command_options": {
+        "command_type": "vllm_bench_serve",
+        "args": {
+          "model": "Qwen/Qwen3-32B",
+          "backend": "vllm",
+          "dataset-name": "random",
+          "num-prompts": 320,
+          "random-input-len": 1024,
+          "random-output-len": 1024,
+          "max-concurrency": 64,
+          "request-rate": "inf",
+          "percentile-metrics": "ttft,tpot,itl,e2el",
+          "ignore-eos": true
+        }
+      }
+    }
+  ]
+}

.buildkite/benchmark/scripts/benchmark_bootstrap.sh

@@ -47,7 +47,7 @@ buildkite-agent meta-data set "JOB_REFERENCE" "${JOB_REFERENCE}"
 upload_benchmark_pipeline() {
     local target_case_type="$BM_CASE_TYPE"
 
-    VLLM_COMMIT_HASH=$(get_vllm_commit_hash)
+    VLLM_COMMIT_HASH=${VLLM_COMMIT_HASH:-$(get_vllm_commit_hash)}
     buildkite-agent meta-data set "VLLM_COMMIT_HASH" "${VLLM_COMMIT_HASH}"
     TPU_COMMIT_HASH=$(git rev-parse HEAD)
     CODE_HASH="${VLLM_COMMIT_HASH}-${TPU_COMMIT_HASH}-"

.buildkite/benchmark/scripts/generate_bk_pipeline.py

@@ -223,11 +223,14 @@ def create_benchmark_steps(
             # Include parent_dir in label for uniqueness
             step_label = f"[{parent_dir}] {agent} {file_basename}"
             case_parameter = f"{file_path}"
+            benchmark_name = file_basename
         else:
             step_env["TARGET_CASE_NAME"] = case_name
             # Include parent_dir in label for uniqueness
             step_label = f"[{parent_dir}] {agent} {file_basename} {case_name}"
             case_parameter = f"{file_path} {case_name}"
+            benchmark_name = case_name
+        step_env["MLCOMPASS_TEST_NAME"] = f"vllm:{agent}:{benchmark_name}"
 
         # Define step key and check for internal collisions
         step_safe_key = clean_key_string(step_label)

.buildkite/benchmark/scripts/mlcompass_export.py

@@ -0,0 +1,127 @@
+# Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import argparse
+import json
+import os
+import re
+import time
+import uuid
+
+from google.cloud import bigquery
+
+_GCP_PROJECT = os.getenv('MLCOMPASS_GCP_PROJECT',
+                         'google.com:ml-compass-benchmarks')
+_BIGQUERY_DATASET = os.getenv('MLCOMPASS_BIGQUERY_DATASET',
+                              'benchmarks_dataset')
+_BIGQUERY_TABLE = os.getenv('MLCOMPASS_BIGQUERY_TABLE', 'benchmarks_dev')
+
+
+def get_env_commit_map() -> dict[str, dict[str, str]]:
+    code_hash = os.getenv('CODE_HASH')
+    match = re.match('^([0-9a-fA-F]+)-([0-9a-fA-F]+)-$', code_hash)
+    if not match:
+        raise RuntimeError(f'CODE_HASH value error: "{code_hash}"')
+    return {
+        'vllm': {
+            'repo': 'vllm-project/vllm',
+            'commit': match[1],
+            'branch': 'main',
+        },
+        'tpu-inference': {
+            'repo': 'vllm-project/tpu-inference',
+            'commit': match[2],
+            'branch': 'main',
+        },
+    }
+
+
+def read_metrics_file(file_path: str) -> dict[str, float]:
+    metrics_dict: dict[str, float] = {}
+    with open(file_path, 'r') as file:
+        for line in file:
+            # Strip whitespace from the beginning and end of the line
+            line = line.strip()
+            # Skip empty lines and ensure the line contains '='
+            if line and '=' in line:
+                key, value = line.split('=', 1)
+                # Strip extra spaces from key/value and convert value to float
+                metrics_dict[key.strip()] = float(value.strip())
+    return metrics_dict
+
+
+def get_links() -> dict[str, str]:
+    result = {}
+    build_number = os.getenv('BUILDKITE_BUILD_NUMBER')
+    job_id = os.getenv('BUILDKITE_JOB_ID')
+    if build_number and job_id:
+        result[
+            'buildkite'] = f'https://buildkite.com/organizations/tpu-commons/pipelines/tpu-inference-benchmark/builds/{build_number}/jobs/{job_id}/log'
+    return result
+
+
+def export(metrics: dict[str, float]) -> None:
+    row_id = uuid.uuid4().hex
+
+    test_name = os.getenv('MLCOMPASS_TEST_NAME')
+    if not test_name:
+        raise RuntimeError('MLCOMPASS_TEST_NAME env var is not found.')
+
+    env_commit_map = get_env_commit_map()
+    client_info = {
+        'github_commit': env_commit_map['tpu-inference']['commit'],
+        'commit_branch_name': env_commit_map['tpu-inference']['branch'],
+    }
+    mlcompass_tracking_id = os.getenv('MLCOMPASS_TRACKING_ID', row_id)
+    mlcompass_execution_mode = os.getenv('MLCOMPASS_EXECUTION_MODE', 'oneshot')
+
+    row = {
+        'entry_id': row_id,
+        'sponge_id': os.getenv('MLCOMPASS_SPONGE_ID'),
+        'test_name': test_name,
+        'succeeded': bool(metrics),
+        'metrics': json.dumps(metrics),
+        'link_map': json.dumps(get_links()),
+        'exc_timestamp_millis': int(time.time() * 1000),
+        'client_info': client_info,
+        'env_commit_map': json.dumps(env_commit_map),
+        'mlcompass_tracking_id': mlcompass_tracking_id,
+        'mlcompass_execution_mode': mlcompass_execution_mode,
+    }
+    print(f'Exporting row: {row}')
+
+    client = bigquery.Client(project=_GCP_PROJECT)
+    table_ref = client.dataset(_BIGQUERY_DATASET).table(_BIGQUERY_TABLE)
+    table_obj = client.get_table(table_ref)
+    errors = client.insert_rows(table_obj, [row])
+    if errors:
+        raise RuntimeError(
+            f'Failed to insert row into MLCompass table: {str(errors)}')
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(
+        description="Export datapoint to MLCompass")
+    parser.add_argument(
+        "--result_file",
+        type=str,
+        help="Path to result file generated by report_result.sh",
+    )
+    args = parser.parse_args()
+    metrics = read_metrics_file(args.result_file)
+    export(metrics)
+
+
+if __name__ == "__main__":
+    main()

.buildkite/benchmark/scripts/report_result.sh

@@ -375,3 +375,10 @@ else
     echo "Warning: $RESULT_FILE not found. No results to display."
   fi
 fi
+
+if [[ "${MLCOMPASS_EXPORT_ENABLED:-false}" == "true" ]]; then
+  echo "--- Reporting to MLCompass"
+  python3 "$(dirname "${BASH_SOURCE[0]}")/mlcompass_export.py" --result_file="$RESULT_FILE"
+else
+  echo "--- Reporting to MLCompass (skipped)"
+fi

.buildkite/benchmark/scripts/run_job.sh

@@ -102,7 +102,14 @@ declare -a BENCHMARK_DOCKER_ARGS=(
   "-e" "BUILDKITE=${BUILDKITE}"
   "-e" "BUILDKITE_AGENT_NAME=${BUILDKITE_AGENT_NAME}"
   "-e" "BUILDKITE_AGENT_META_DATA_QUEUE=${BUILDKITE_AGENT_META_DATA_QUEUE}"
+  "-e" "BUILDKITE_BUILD_NUMBER=${BUILDKITE_BUILD_NUMBER}"
+  "-e" "BUILDKITE_JOB_ID=${BUILDKITE_JOB_ID}"
   "-e" "UPLOAD_DB=${UPLOAD_DB:-true}"
+  "-e" "MLCOMPASS_EXECUTION_MODE=${MLCOMPASS_EXECUTION_MODE:-}"
+  "-e" "MLCOMPASS_EXPORT_ENABLED=${MLCOMPASS_EXPORT_ENABLED:-}"
+  "-e" "MLCOMPASS_TEST_NAME=${MLCOMPASS_TEST_NAME:-}"
+  "-e" "MLCOMPASS_TRACKING_ID=${MLCOMPASS_TRACKING_ID:-}"
+  "-e" "MLCOMPASS_SPONGE_ID=${MLCOMPASS_SPONGE_ID:-}"
 )
 
 BENCHMARK_DOCKER_ARGS_STR="$(printf '%s\n' "${BENCHMARK_DOCKER_ARGS[@]}")"