REF: cc/frontier

.gitignore

@@ -14,6 +14,11 @@ __pycache__/
 # Ignore virtual environment directories
 *venv*/
 .venv*/
+# Keep OpenSpec artifacts trackable even when names include ignored words.
+!/openspec/changes/
+!/openspec/changes/**
+!/openspec/specs/
+!/openspec/specs/**
 .ssh/
 .direnv/
 
@@ -100,8 +105,12 @@ recording*.db
 
 /misc/fresh-ubuntu-tests/cache
 
-# openspec
-/openspec/changes/archive/
-
 # Deepwork local progress state
 .slim/deepwork/
+
+# Benchmark demo output artifacts
+artifacts/
+
+# LIBERO-PRO downloaded benchmark assets
+/libero/
+.codegraph/

AGENTS.md

@@ -44,7 +44,7 @@ dimos restart          # stop + re-run with same original args
 | `xarm-perception-sim-agent` | xArm | sim | GPT-4o | — | Manipulation + perception + agent, sim |
 | `xarm7-planner-coordinator` | xArm7 | real | — | — | Trajectory planner coordinator |
 | `teleop-quest-xarm7` | xArm7 | real | — | — | Quest VR teleop |
-| `dual-xarm6-planner` | xArm6×2 | real | — | — | Dual-arm motion planner |
+| `dual-xarm6-planner-coordinator` | xArm6×2 | real | — | — | Dual-arm motion planner + coordinator |
 
 Run `dimos list` for the full list.
 

dimos/benchmark/runtime/__init__.py

@@ -0,0 +1 @@
+"""Benchmark runtime orchestration helpers."""

dimos/benchmark/runtime/artifacts.py

@@ -0,0 +1,32 @@
+# Copyright 2026 Dimensional Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Small artifact helpers for runtime demos."""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+
+
+def write_json(path: Path, payload: object) -> None:
+    """Write JSON artifact with parent directory creation."""
+
+    path.parent.mkdir(parents=True, exist_ok=True)
+    model_dump = getattr(payload, "model_dump", None)
+    if callable(model_dump):
+        value = model_dump(mode="json")
+    else:
+        value = payload
+    path.write_text(json.dumps(value, indent=2, sort_keys=True) + "\n")

dimos/benchmark/runtime/config.py

@@ -0,0 +1,145 @@
+# Copyright 2026 Dimensional Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Config and resolved plan models for benchmark runtime demos."""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Literal
+
+from dimos_runtime_protocol import CommandMode, RuntimeDescription, check_compatible
+from dimos_runtime_protocol.types import JsonObject
+from pydantic import BaseModel, ConfigDict, Field, NonNegativeInt, PositiveInt, model_validator
+
+
+class StrictModel(BaseModel):
+    model_config = ConfigDict(extra="forbid")
+
+
+class BenchmarkEpisodeConfig(StrictModel):
+    """Backend-facing declaration of one benchmark episode intent."""
+
+    backend: str = "fake"
+    episode_id: str = "fake-smoke"
+    task_id: str = "fake-motor-smoke"
+    runtime_host: str = "127.0.0.1"
+    runtime_port: PositiveInt = 8765
+    robot_id: str = "fakebot"
+    dof: PositiveInt = 3
+    control_step_hz: PositiveInt = 100
+    ticks: PositiveInt = 30
+    target_position: float = 0.2
+    seed: int | None = None
+    artifact_dir: Path = Path("artifacts/benchmark/fake-runtime-smoke")
+    env_name: str = "Lift"
+    robot_model: str = "Panda"
+    controller: str = "JOINT_POSITION"
+    horizon: PositiveInt = 200
+    camera_name: str = "agentview"
+    visualize: bool = False
+    backend_options: JsonObject = Field(default_factory=dict)
+
+
+class LiberoProBackendOptions(StrictModel):
+    """Typed backend-specific options for registered LIBERO-PRO tasks."""
+
+    benchmark_name: str
+    task_order_index: NonNegativeInt = 0
+    task_index: NonNegativeInt = 0
+    init_state_index: NonNegativeInt = 0
+    controller: str = "JOINT_POSITION"
+    camera_names: list[str] = Field(default_factory=lambda: ["agentview"], min_length=1)
+    horizon: PositiveInt = 1000
+    bddl_root: Path
+    init_states_root: Path
+    allow_asset_bootstrap: bool = False
+    perturbation_mode: Literal["registered"] = "registered"
+
+    @model_validator(mode="after")
+    def _validate_non_empty_names(self) -> LiberoProBackendOptions:
+        if not self.benchmark_name.strip():
+            raise ValueError("benchmark_name must be non-empty")
+        if not self.controller.strip():
+            raise ValueError("controller must be non-empty")
+        empty_cameras = [
+            camera_name for camera_name in self.camera_names if not camera_name.strip()
+        ]
+        if empty_cameras:
+            raise ValueError("camera_names must not contain empty names")
+        return self
+
+
+def validate_libero_pro_backend_options(config: BenchmarkEpisodeConfig) -> LiberoProBackendOptions:
+    """Validate and return typed LIBERO-PRO backend options for an episode config."""
+
+    if config.backend != "libero-pro":
+        raise ValueError(
+            "LIBERO-PRO backend options can only be validated for backend 'libero-pro'"
+        )
+    return LiberoProBackendOptions.model_validate(config.backend_options)
+
+
+class ResolvedRuntimePlan(StrictModel):
+    """Concrete launch material derived from a benchmark episode config."""
+
+    episode_id: str
+    task_id: str
+    backend: str
+    runtime_base_url: str
+    shm_key: str
+    robot_id: str
+    motor_names: list[str]
+    control_step_hz: PositiveInt
+    ticks: PositiveInt
+    target_position: float
+    artifact_dir: Path
+
+
+def resolve_runtime_plan(
+    config: BenchmarkEpisodeConfig,
+    description: RuntimeDescription,
+) -> ResolvedRuntimePlan:
+    """Validate sidecar metadata and derive a concrete runtime plan."""
+
+    compatibility = check_compatible(description.protocol)
+    if not compatibility.compatible:
+        raise ValueError(f"incompatible sidecar protocol: {compatibility.reason}")
+    if description.backend != config.backend:
+        raise ValueError(f"backend mismatch: config={config.backend} sidecar={description.backend}")
+    matching_surfaces = [
+        surface for surface in description.robot_surfaces if surface.robot_id == config.robot_id
+    ]
+    if not matching_surfaces:
+        raise ValueError(f"sidecar did not report robot surface {config.robot_id!r}")
+    surface = matching_surfaces[0]
+    motor_names = [motor.name for motor in sorted(surface.motors, key=lambda motor: motor.index)]
+    if len(motor_names) != config.dof:
+        raise ValueError(f"expected {config.dof} motors, sidecar reported {len(motor_names)}")
+    if CommandMode.POSITION not in surface.supported_command_modes:
+        supported = ", ".join(mode.value for mode in surface.supported_command_modes)
+        raise ValueError(f"sidecar robot surface does not support position commands: {supported}")
+    return ResolvedRuntimePlan(
+        episode_id=config.episode_id,
+        task_id=config.task_id,
+        backend=config.backend,
+        runtime_base_url=f"http://{config.runtime_host}:{config.runtime_port}",
+        shm_key=f"{config.episode_id}-{config.robot_id}",
+        robot_id=config.robot_id,
+        motor_names=motor_names,
+        control_step_hz=config.control_step_hz,
+        ticks=config.ticks,
+        target_position=config.target_position,
+        artifact_dir=config.artifact_dir,
+    )

dimos/benchmark/runtime/configs/fake_runtime_smoke.json

@@ -0,0 +1,14 @@
+{
+  "backend": "fake",
+  "episode_id": "fake-runtime-smoke",
+  "task_id": "fake-motor-smoke",
+  "runtime_host": "127.0.0.1",
+  "runtime_port": 8765,
+  "robot_id": "fakebot",
+  "dof": 3,
+  "control_step_hz": 100,
+  "ticks": 30,
+  "target_position": 0.2,
+  "seed": 7,
+  "artifact_dir": "artifacts/benchmark/fake-runtime-smoke"
+}

dimos/benchmark/runtime/configs/libero_pro_goal_task0.json

@@ -0,0 +1,27 @@
+{
+  "backend": "libero-pro",
+  "episode_id": "libero-pro-goal-task0",
+  "task_id": "libero_goal_task/0",
+  "runtime_host": "127.0.0.1",
+  "runtime_port": 8767,
+  "robot_id": "panda",
+  "dof": 8,
+  "control_step_hz": 20,
+  "ticks": 200,
+  "target_position": 0.03,
+  "seed": 7,
+  "artifact_dir": "artifacts/benchmark/libero-pro-goal-task0",
+  "backend_options": {
+    "benchmark_name": "libero_goal_task",
+    "task_order_index": 0,
+    "task_index": 0,
+    "init_state_index": 0,
+    "controller": "JOINT_POSITION",
+    "camera_names": [
+      "agentview"
+    ],
+    "horizon": 1000,
+    "bddl_root": "libero/libero/bddl_files",
+    "init_states_root": "libero/libero/init_files"
+  }
+}

dimos/benchmark/runtime/configs/robosuite_panda_lift.json

@@ -0,0 +1,19 @@
+{
+  "backend": "robosuite",
+  "episode_id": "robosuite-panda-lift",
+  "task_id": "Lift",
+  "runtime_host": "127.0.0.1",
+  "runtime_port": 8766,
+  "robot_id": "panda",
+  "dof": 8,
+  "control_step_hz": 100,
+  "ticks": 30,
+  "target_position": 0.05,
+  "seed": 7,
+  "artifact_dir": "artifacts/benchmark/robosuite-panda-lift",
+  "env_name": "Lift",
+  "robot_model": "Panda",
+  "controller": "JOINT_POSITION",
+  "horizon": 200,
+  "camera_name": "agentview"
+}