From 1597582ef4aae48b0d8cd9aade129c13e2f84900 Mon Sep 17 00:00:00 2001 From: z1050297972-spec Date: Sun, 17 May 2026 00:05:24 +0800 Subject: [PATCH 01/36] refactor(go2): split blueprint architecture layers --- .../go2/blueprints/agentic/_common_agentic.py | 16 +------ .../blueprints/agentic/unitree_go2_agentic.py | 22 +++++---- .../agentic/unitree_go2_temporal_memory.py | 10 ++-- .../unitree/go2/blueprints/layers/README.md | 20 ++++++++ .../blueprints/layers/layer_3_agent_brain.py | 33 +++++++++++++ .../blueprints/layers/layer_4_world_state.py | 39 +++++++++++++++ .../layers/layer_5_skill_interface.py | 48 +++++++++++++++++++ .../blueprints/layers/layer_6_robot_body.py | 20 ++++++++ .../blueprints/smart/unitree_go2_spatial.py | 19 ++++---- 9 files changed, 189 insertions(+), 38 deletions(-) create mode 100644 dimos/robot/unitree/go2/blueprints/layers/README.md create mode 100644 dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain.py create mode 100644 dimos/robot/unitree/go2/blueprints/layers/layer_4_world_state.py create mode 100644 dimos/robot/unitree/go2/blueprints/layers/layer_5_skill_interface.py create mode 100644 dimos/robot/unitree/go2/blueprints/layers/layer_6_robot_body.py diff --git a/dimos/robot/unitree/go2/blueprints/agentic/_common_agentic.py b/dimos/robot/unitree/go2/blueprints/agentic/_common_agentic.py index 93312225bc..f0a04aa53c 100644 --- a/dimos/robot/unitree/go2/blueprints/agentic/_common_agentic.py +++ b/dimos/robot/unitree/go2/blueprints/agentic/_common_agentic.py @@ -13,20 +13,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -from dimos.agents.skills.navigation import NavigationSkillContainer -from dimos.agents.skills.person_follow import PersonFollowSkillContainer -from dimos.agents.skills.speak_skill import SpeakSkill -from dimos.agents.web_human_input import WebInput -from dimos.core.coordination.blueprints import autoconnect -from dimos.robot.unitree.go2.connection import GO2Connection -from dimos.robot.unitree.unitree_skill_container import UnitreeSkillContainer - -_common_agentic = autoconnect( - NavigationSkillContainer.blueprint(), - PersonFollowSkillContainer.blueprint(camera_info=GO2Connection.camera_info_static), - UnitreeSkillContainer.blueprint(), - WebInput.blueprint(), - SpeakSkill.blueprint(), +from dimos.robot.unitree.go2.blueprints.layers.layer_5_skill_interface import ( + _go2_agentic_skill_interface as _common_agentic, ) __all__ = ["_common_agentic"] diff --git a/dimos/robot/unitree/go2/blueprints/agentic/unitree_go2_agentic.py b/dimos/robot/unitree/go2/blueprints/agentic/unitree_go2_agentic.py index b9716842fa..f27a88c7b7 100644 --- a/dimos/robot/unitree/go2/blueprints/agentic/unitree_go2_agentic.py +++ b/dimos/robot/unitree/go2/blueprints/agentic/unitree_go2_agentic.py @@ -13,17 +13,21 @@ # See the License for the specific language governing permissions and # limitations under the License. -from dimos.agents.mcp.mcp_client import McpClient -from dimos.agents.mcp.mcp_server import McpServer from dimos.core.coordination.blueprints import autoconnect -from dimos.robot.unitree.go2.blueprints.agentic._common_agentic import _common_agentic -from dimos.robot.unitree.go2.blueprints.smart.unitree_go2_spatial import unitree_go2_spatial +from dimos.robot.unitree.go2.blueprints.layers.layer_3_agent_brain import _go2_agent_brain +from dimos.robot.unitree.go2.blueprints.layers.layer_4_world_state import ( + _go2_spatial_world_state, +) +from dimos.robot.unitree.go2.blueprints.layers.layer_5_skill_interface import ( + _go2_skill_interface, +) +from dimos.robot.unitree.go2.blueprints.layers.layer_6_robot_body import _go2_robot_body unitree_go2_agentic = autoconnect( - unitree_go2_spatial, - McpServer.blueprint(), - McpClient.blueprint(), - _common_agentic, -) + _go2_robot_body, + _go2_spatial_world_state, + _go2_skill_interface, + _go2_agent_brain, +).global_config(n_workers=8) __all__ = ["unitree_go2_agentic"] diff --git a/dimos/robot/unitree/go2/blueprints/agentic/unitree_go2_temporal_memory.py b/dimos/robot/unitree/go2/blueprints/agentic/unitree_go2_temporal_memory.py index ba3be30c9d..9978502763 100644 --- a/dimos/robot/unitree/go2/blueprints/agentic/unitree_go2_temporal_memory.py +++ b/dimos/robot/unitree/go2/blueprints/agentic/unitree_go2_temporal_memory.py @@ -14,18 +14,16 @@ # limitations under the License. from dimos.core.coordination.blueprints import autoconnect -from dimos.core.global_config import global_config -from dimos.perception.experimental.temporal_memory.temporal_memory import ( - TemporalMemory, - TemporalMemoryConfig, -) from dimos.robot.unitree.go2.blueprints.agentic.unitree_go2_agentic import unitree_go2_agentic +from dimos.robot.unitree.go2.blueprints.layers.layer_4_world_state import ( + _go2_temporal_memory_world_state, +) # This module is imported lazily by `get_by_name()` in the CLI run command, # AFTER global_config.update() has applied CLI flags like --new-memory. unitree_go2_temporal_memory = autoconnect( unitree_go2_agentic, - TemporalMemory.blueprint(config=TemporalMemoryConfig(new_memory=global_config.new_memory)), + _go2_temporal_memory_world_state(), ) __all__ = ["unitree_go2_temporal_memory"] diff --git a/dimos/robot/unitree/go2/blueprints/layers/README.md b/dimos/robot/unitree/go2/blueprints/layers/README.md new file mode 100644 index 0000000000..f1fa565a8a --- /dev/null +++ b/dimos/robot/unitree/go2/blueprints/layers/README.md @@ -0,0 +1,20 @@ +# Go2 Architecture Layers + +This directory names the Go2 blueprint composition boundaries that map to +layers 3 through 6 of the project architecture diagram. These files do not move +implementation modules; they only group existing blueprints so the runtime +structure can evolve without breaking current import paths. + +## Layer Mapping + +- Layer 3, Agent Brain: MCP server/client and the LLM agent loop. +- Layer 4, World State: spatial memory today, with a lazy temporal-memory + factory for the existing temporal-memory blueprint. +- Layer 5, Skill Interface: MCP-callable skills and skill containers that + bridge the agent to navigation, perception, speech, and Unitree commands. +- Layer 6, Robot Body / Local Policy: the existing Go2 robot stack, including + perception, mapping, navigation, movement management, and hardware connection. + +All blueprint variables in this directory are intentionally prefixed with `_`. +The registry generator skips those names, so these internal layer nodes do not +become new CLI blueprints. diff --git a/dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain.py b/dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain.py new file mode 100644 index 0000000000..f36ec20549 --- /dev/null +++ b/dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain.py @@ -0,0 +1,33 @@ +#!/usr/bin/env python3 +# Copyright 2025-2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import Any + +from dimos.agents.mcp.mcp_client import McpClient +from dimos.agents.mcp.mcp_server import McpServer +from dimos.core.coordination.blueprints import Blueprint, autoconnect + + +def _go2_agent_brain_with_client(**mcp_client_kwargs: Any) -> Blueprint: + """Layer 3: MCP tool server plus the LLM/VLM agent client.""" + return autoconnect( + McpServer.blueprint(), + McpClient.blueprint(**mcp_client_kwargs), + ) + + +_go2_agent_brain = _go2_agent_brain_with_client() + +__all__ = ["_go2_agent_brain", "_go2_agent_brain_with_client"] diff --git a/dimos/robot/unitree/go2/blueprints/layers/layer_4_world_state.py b/dimos/robot/unitree/go2/blueprints/layers/layer_4_world_state.py new file mode 100644 index 0000000000..ca26730882 --- /dev/null +++ b/dimos/robot/unitree/go2/blueprints/layers/layer_4_world_state.py @@ -0,0 +1,39 @@ +#!/usr/bin/env python3 +# Copyright 2025-2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from dimos.core.coordination.blueprints import Blueprint, autoconnect +from dimos.perception.spatial_perception import SpatialMemory + +_go2_spatial_world_state = autoconnect( + SpatialMemory.blueprint(), +) + + +def _go2_temporal_memory_world_state() -> Blueprint: + """Layer 4: temporal memory, imported lazily after CLI config is resolved.""" + from dimos.core.global_config import global_config + from dimos.perception.experimental.temporal_memory.temporal_memory import ( + TemporalMemory, + TemporalMemoryConfig, + ) + + return autoconnect( + TemporalMemory.blueprint( + config=TemporalMemoryConfig(new_memory=global_config.new_memory) + ), + ) + + +__all__ = ["_go2_spatial_world_state", "_go2_temporal_memory_world_state"] diff --git a/dimos/robot/unitree/go2/blueprints/layers/layer_5_skill_interface.py b/dimos/robot/unitree/go2/blueprints/layers/layer_5_skill_interface.py new file mode 100644 index 0000000000..ec6f980a86 --- /dev/null +++ b/dimos/robot/unitree/go2/blueprints/layers/layer_5_skill_interface.py @@ -0,0 +1,48 @@ +#!/usr/bin/env python3 +# Copyright 2025-2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from dimos.agents.skills.navigation import NavigationSkillContainer +from dimos.agents.skills.person_follow import PersonFollowSkillContainer +from dimos.agents.skills.speak_skill import SpeakSkill +from dimos.agents.web_human_input import WebInput +from dimos.core.coordination.blueprints import autoconnect +from dimos.experimental.security_demo.security_module import SecurityModule +from dimos.perception.perceive_loop_skill import PerceiveLoopSkill +from dimos.robot.unitree.go2.connection import GO2Connection +from dimos.robot.unitree.unitree_skill_container import UnitreeSkillContainer + +_go2_spatial_skill_interface = autoconnect( + PerceiveLoopSkill.blueprint(), + SecurityModule.blueprint(camera_info=GO2Connection.camera_info_static), +) + +_go2_agentic_skill_interface = autoconnect( + NavigationSkillContainer.blueprint(), + PersonFollowSkillContainer.blueprint(camera_info=GO2Connection.camera_info_static), + UnitreeSkillContainer.blueprint(), + WebInput.blueprint(), + SpeakSkill.blueprint(), +) + +_go2_skill_interface = autoconnect( + _go2_spatial_skill_interface, + _go2_agentic_skill_interface, +) + +__all__ = [ + "_go2_agentic_skill_interface", + "_go2_skill_interface", + "_go2_spatial_skill_interface", +] diff --git a/dimos/robot/unitree/go2/blueprints/layers/layer_6_robot_body.py b/dimos/robot/unitree/go2/blueprints/layers/layer_6_robot_body.py new file mode 100644 index 0000000000..5c054ed523 --- /dev/null +++ b/dimos/robot/unitree/go2/blueprints/layers/layer_6_robot_body.py @@ -0,0 +1,20 @@ +#!/usr/bin/env python3 +# Copyright 2025-2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from dimos.robot.unitree.go2.blueprints.smart.unitree_go2 import unitree_go2 + +_go2_robot_body = unitree_go2 + +__all__ = ["_go2_robot_body"] diff --git a/dimos/robot/unitree/go2/blueprints/smart/unitree_go2_spatial.py b/dimos/robot/unitree/go2/blueprints/smart/unitree_go2_spatial.py index 97a0803b98..514e07a71e 100644 --- a/dimos/robot/unitree/go2/blueprints/smart/unitree_go2_spatial.py +++ b/dimos/robot/unitree/go2/blueprints/smart/unitree_go2_spatial.py @@ -14,17 +14,18 @@ # limitations under the License. from dimos.core.coordination.blueprints import autoconnect -from dimos.experimental.security_demo.security_module import SecurityModule -from dimos.perception.perceive_loop_skill import PerceiveLoopSkill -from dimos.perception.spatial_perception import SpatialMemory -from dimos.robot.unitree.go2.blueprints.smart.unitree_go2 import unitree_go2 -from dimos.robot.unitree.go2.connection import GO2Connection +from dimos.robot.unitree.go2.blueprints.layers.layer_4_world_state import ( + _go2_spatial_world_state, +) +from dimos.robot.unitree.go2.blueprints.layers.layer_5_skill_interface import ( + _go2_spatial_skill_interface, +) +from dimos.robot.unitree.go2.blueprints.layers.layer_6_robot_body import _go2_robot_body unitree_go2_spatial = autoconnect( - unitree_go2, - SpatialMemory.blueprint(), - PerceiveLoopSkill.blueprint(), - SecurityModule.blueprint(camera_info=GO2Connection.camera_info_static), + _go2_robot_body, + _go2_spatial_world_state, + _go2_spatial_skill_interface, ).global_config(n_workers=8) __all__ = ["unitree_go2_spatial"] From 2045b6ab9cf264444caea191b109f5c38e00f013 Mon Sep 17 00:00:00 2001 From: z1050297972-spec Date: Sun, 17 May 2026 20:26:07 +0800 Subject: [PATCH 02/36] feat(go2): add layer 3 context routing --- AGENTS.md | 14 + PROJECT_CHANGELOG.md | 84 +++++ dimos/perception/temporal_memory_spec.py | 28 ++ .../unitree/go2/blueprints/layers/README.md | 52 ++- .../go2/blueprints/layers/context_provider.py | 286 +++++++++++++++++ .../go2/blueprints/layers/expert_router.py | 302 ++++++++++++++++++ .../blueprints/layers/layer_3_agent_brain.py | 6 +- .../layers/test_context_provider.py | 98 ++++++ .../blueprints/layers/test_expert_router.py | 57 ++++ 9 files changed, 925 insertions(+), 2 deletions(-) create mode 100644 PROJECT_CHANGELOG.md create mode 100644 dimos/perception/temporal_memory_spec.py create mode 100644 dimos/robot/unitree/go2/blueprints/layers/context_provider.py create mode 100644 dimos/robot/unitree/go2/blueprints/layers/expert_router.py create mode 100644 dimos/robot/unitree/go2/blueprints/layers/test_context_provider.py create mode 100644 dimos/robot/unitree/go2/blueprints/layers/test_expert_router.py diff --git a/AGENTS.md b/AGENTS.md index bc410d6b88..19c3babbc0 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -358,6 +358,20 @@ Code style rules: --- +## Change Documentation + +Maintain `PROJECT_CHANGELOG.md` at the repository root for project-level work in +this fork/branch. Add a reverse-chronological entry after substantive changes: +architecture migrations, feature implementation, test fixes, blueprint rewiring, +or upstream sync work. + +Each entry should include the date, branch, change summary, affected +files/modules, validation performed, and open follow-up items. Small exploratory +queries, pure discussion, and local command output that does not change the +project do not need a changelog entry. + +--- + ## `all_blueprints.py` is auto-generated `dimos/robot/all_blueprints.py` is generated by `test_all_blueprints_generation.py`. After adding or renaming blueprints: diff --git a/PROJECT_CHANGELOG.md b/PROJECT_CHANGELOG.md new file mode 100644 index 0000000000..62a895414a --- /dev/null +++ b/PROJECT_CHANGELOG.md @@ -0,0 +1,84 @@ +# Project Change Log + +This file tracks project-level changes made in this fork/branch. It is not a +release changelog; it records architecture, code, testing, and upstream-sync +work so future contributors can understand why the fork differs from upstream. + +Entries are listed in reverse chronological order. + +## 2026-05-17 + +- Branch: `refactor/go2-architecture-layers` +- Summary: Implemented the first Go2 Layer 3 ExpertRouter and documented the + Layer 3 v1-v4 roadmap. +- Files/modules: + - `dimos/robot/unitree/go2/blueprints/layers/expert_router.py` + - `dimos/robot/unitree/go2/blueprints/layers/test_expert_router.py` + - `dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain.py` + - `dimos/robot/unitree/go2/blueprints/layers/README.md` +- Validation: + - Ran `python -m py_compile` for the new ExpertRouter, test, and Layer 3 + blueprint files. + - Ran `git diff --check`. + - Attempted the targeted pytest file; local Windows environment still fails + during repo `conftest.py` import because the Unix-only `resource` module is + unavailable. +- Open items: + - Feed route results into the LLM prompt or prompt policy so the agent calls + `route_task` and `get_context` consistently. + - Implement `SkillOutcomePredictor` after a shared skill-outcome store exists. + +## 2026-05-17 + +- Branch: `refactor/go2-architecture-layers` +- Summary: Implemented the first Go2 Layer 3 ContextProvider and wired it into + the internal Agent Brain layer. +- Files/modules: + - `dimos/robot/unitree/go2/blueprints/layers/context_provider.py` + - `dimos/perception/temporal_memory_spec.py` + - `dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain.py` +- Validation: + - Ran `python -m py_compile` for the new ContextProvider, spec, test, and + Layer 3 blueprint files. + - Attempted the targeted pytest file; local Windows environment still fails + during repo `conftest.py` import because the Unix-only `resource` module is + unavailable. +- Open items: + - Add a shared skill-outcome store so ContextProvider can report recent + Layer 5 outcomes. + - Decide whether to promote the Go2-specific provider into a robot-agnostic + `dimos.agents` module after the API stabilizes. + +## 2026-05-17 + +- Branch: `refactor/go2-architecture-layers` +- Summary: Added the project change-log convention and documented the intended + ContextProvider scope for the Go2 architecture-layer migration. +- Files/modules: + - `PROJECT_CHANGELOG.md` + - `AGENTS.md` + - `dimos/robot/unitree/go2/blueprints/layers/README.md` +- Validation: + - Documentation-only change; no runtime tests required. +- Open items: + - Implement `ContextProvider` as a Layer 3 module exposed through MCP. + - Define the first stable context payload returned by `get_context(task)`. + +## 2026-05-17 + +- Branch: `refactor/go2-architecture-layers` +- Summary: Split the Go2 blueprint composition into architecture-layer + boundaries for layers 3 through 6 without moving underlying implementation + modules or changing public blueprint names. +- Files/modules: + - `dimos/robot/unitree/go2/blueprints/layers/` + - `dimos/robot/unitree/go2/blueprints/smart/unitree_go2_spatial.py` + - `dimos/robot/unitree/go2/blueprints/agentic/` +- Validation: + - Ran Python compile checks for the new/changed Go2 blueprint files. + - Attempted blueprint-registry tests; local Windows dependency/platform + blockers prevented a full local run. +- Open items: + - Fill Layer 3 with ContextProvider, ExpertRouter, outcome prediction, and + causal world-model behavior. + - Gradually convert key skills to structured `SkillResult` returns. diff --git a/dimos/perception/temporal_memory_spec.py b/dimos/perception/temporal_memory_spec.py new file mode 100644 index 0000000000..d29f2eab47 --- /dev/null +++ b/dimos/perception/temporal_memory_spec.py @@ -0,0 +1,28 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import Any, Protocol + +from dimos.spec.utils import Spec + + +class TemporalMemorySpec(Spec, Protocol): + def query(self, question: str) -> str: ... + def get_state(self) -> dict[str, Any]: ... + def get_entity_roster(self) -> list[dict[str, Any]]: ... + def get_rolling_summary(self) -> str: ... + def get_graph_db_stats(self) -> dict[str, Any]: ... + + +__all__ = ["TemporalMemorySpec"] diff --git a/dimos/robot/unitree/go2/blueprints/layers/README.md b/dimos/robot/unitree/go2/blueprints/layers/README.md index f1fa565a8a..4db3ad893e 100644 --- a/dimos/robot/unitree/go2/blueprints/layers/README.md +++ b/dimos/robot/unitree/go2/blueprints/layers/README.md @@ -7,7 +7,8 @@ structure can evolve without breaking current import paths. ## Layer Mapping -- Layer 3, Agent Brain: MCP server/client and the LLM agent loop. +- Layer 3, Agent Brain: ExpertRouter, ContextProvider, MCP server/client, and + the LLM agent loop. - Layer 4, World State: spatial memory today, with a lazy temporal-memory factory for the existing temporal-memory blueprint. - Layer 5, Skill Interface: MCP-callable skills and skill containers that @@ -18,3 +19,52 @@ structure can evolve without breaking current import paths. All blueprint variables in this directory are intentionally prefixed with `_`. The registry generator skips those names, so these internal layer nodes do not become new CLI blueprints. + +## ContextProvider Direction + +`ContextProvider` belongs in Layer 3 because it prepares context for the LLM +agent. It should not own the underlying robot or world state. Instead, it should +aggregate and compress context from these sources: + +- LLM/task context: user goal, decomposed subtasks, dialogue summary, and current + tool-call intent. +- Layer 4 world state: `SpatialMemory`, `TemporalMemory`, and later structured + world-state or semantic-temporal map modules. +- Layer 5 skill state: recent skill outcomes, failure reasons, risks, and + recovery suggestions. +- Layer 6 robot state: odometry, navigation state, connection state, safety + state, and perception summaries. +- Runtime context: replay/simulation/hardware mode, blueprint/config values, and + available modules. +- External context: web input, human input, external LLM/VLM results, and future + connected services. + +The current implementation exposes a compact MCP skill, +`get_context(task: str, focus: str = "", spatial_limit: int = 3)`, returning +the minimum context needed for the agent's next decision. Planning and tool +execution remain with the agent and Layer 5 skills. + +## ExpertRouter Direction + +`ExpertRouter` belongs in Layer 3 because it routes an LLM task toward the +right expert domain before tool selection. It does not execute robot actions or +own world state. The current implementation exposes +`route_task(task: str, context: str = "")` as an MCP skill. It uses deterministic +rules to return a domain, confidence, matched keywords, recommended tools, and a +flag indicating whether the agent should call `get_context` first. + +## Layer 3 Version Roadmap + +- V1: Keep Layer 3 as explicit, testable MCP tools. `ContextProvider` aggregates + task, world, robot, runtime, and external-context placeholders. + `ExpertRouter` performs deterministic domain routing for Go2 skills. The + existing `McpClient` remains the LLM/VLM agent loop. +- V2: Add a shared skill-outcome store and `SkillOutcomePredictor` so Layer 3 + can warn about likely failures before tool execution. Upgrade routing to use + recent skill outcomes, navigation state, and memory availability. +- V3: Add `CausalWorldModel` as an event/transition recorder that links + before-context, chosen tool, tool result, after-context, inferred cause, and + recovery suggestion. Use it to summarize repeated failure patterns. +- V4: Promote stable Go2-specific Layer 3 pieces into robot-agnostic modules + under `dimos.agents`, while keeping robot-specific routing rules in the + robot blueprint layer. diff --git a/dimos/robot/unitree/go2/blueprints/layers/context_provider.py b/dimos/robot/unitree/go2/blueprints/layers/context_provider.py new file mode 100644 index 0000000000..a4341a913e --- /dev/null +++ b/dimos/robot/unitree/go2/blueprints/layers/context_provider.py @@ -0,0 +1,286 @@ +#!/usr/bin/env python3 +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import annotations + +import math +from typing import Any + +from reactivex.disposable import Disposable + +from dimos.agents.annotation import skill +from dimos.agents.skill_result import SkillResult +from dimos.core.core import rpc +from dimos.core.global_config import global_config +from dimos.core.module import Module +from dimos.core.stream import In +from dimos.msgs.geometry_msgs.PoseStamped import PoseStamped +from dimos.navigation.navigation_spec import NavigationInterfaceSpec +from dimos.perception.spatial_memory_spec import SpatialMemorySpec +from dimos.perception.temporal_memory_spec import TemporalMemorySpec +from dimos.utils.logging_config import setup_logger + +logger = setup_logger() + + +class _Go2ContextProvider(Module): + """Layer 3 context aggregator for the Go2 agent brain.""" + + _spatial_memory: SpatialMemorySpec | None = None + _temporal_memory: TemporalMemorySpec | None = None + _navigation: NavigationInterfaceSpec | None = None + _latest_odom: PoseStamped | None = None + + odom: In[PoseStamped] + + @rpc + def start(self) -> None: + super().start() + self.register_disposable(Disposable(self.odom.subscribe(self._on_odom))) + + def _on_odom(self, odom: PoseStamped) -> None: + self._latest_odom = odom + + @skill + def get_context(self, task: str, focus: str = "", spatial_limit: int = 3) -> SkillResult: + """Get compact context for the agent's next decision. + + This tool gathers task, world-state, robot-state, and runtime context + without planning or executing robot actions. Use it before choosing + navigation, perception, speech, or recovery tools. + + Args: + task: The current user goal or agent subtask. + focus: Optional area to emphasize, such as navigation, memory, + safety, perception, or recovery. + spatial_limit: Maximum number of spatial-memory matches to include. + """ + task = task.strip() + focus = focus.strip() + if not task: + return SkillResult.fail("INVALID_INPUT", "task is required") + + spatial_limit = max(0, min(spatial_limit, 10)) + errors: list[str] = [] + metadata: dict[str, Any] = { + "task": task, + "focus": focus, + "sources": self._source_status(), + "runtime": self._runtime_context(), + "robot_state": self._robot_context(errors), + "world_state": self._world_context(task, spatial_limit, errors), + "skill_state": { + "available": False, + "reason": "No shared skill-outcome store is wired into ContextProvider yet.", + }, + "external_context": { + "available": False, + "reason": "External context sources are not wired into ContextProvider yet.", + }, + } + if errors: + metadata["errors"] = errors + + message = self._format_message(metadata) + return SkillResult(success=True, message=message, metadata=_to_jsonable(metadata)) + + def _source_status(self) -> dict[str, bool]: + return { + "task": True, + "spatial_memory": self._spatial_memory is not None, + "temporal_memory": self._temporal_memory is not None, + "odom": self._latest_odom is not None, + "navigation": self._navigation is not None, + "runtime": True, + } + + def _runtime_context(self) -> dict[str, Any]: + simulation = bool(getattr(global_config, "simulation", False)) + replay = bool(getattr(global_config, "replay", False)) + mode = "simulation" if simulation else "replay" if replay else "hardware" + return { + "mode": mode, + "simulation": simulation, + "replay": replay, + "robot_ip": getattr(global_config, "robot_ip", None), + "viewer": getattr(global_config, "viewer", None), + "mcp_port": getattr(global_config, "mcp_port", None), + "n_workers": getattr(global_config, "n_workers", None), + } + + def _robot_context(self, errors: list[str]) -> dict[str, Any]: + context: dict[str, Any] = { + "odom": self._pose_to_dict(self._latest_odom), + "navigation": None, + } + + if self._navigation is None: + return context + + try: + state = self._navigation.get_state() + context["navigation"] = { + "state": getattr(state, "value", str(state)), + "goal_reached": self._navigation.is_goal_reached(), + } + except Exception as exc: + logger.warning("Failed to read navigation context", exc_info=True) + errors.append(f"navigation: {exc}") + + return context + + def _world_context(self, task: str, spatial_limit: int, errors: list[str]) -> dict[str, Any]: + return { + "spatial": self._spatial_context(task, spatial_limit, errors), + "temporal": self._temporal_context(errors), + } + + def _spatial_context( + self, task: str, spatial_limit: int, errors: list[str] + ) -> dict[str, Any]: + if self._spatial_memory is None: + return {"available": False, "matches": []} + if spatial_limit == 0: + return {"available": True, "matches": []} + + try: + matches = self._spatial_memory.query_by_text(task, limit=spatial_limit) + except Exception as exc: + logger.warning("Failed to query spatial memory", exc_info=True) + errors.append(f"spatial_memory: {exc}") + return {"available": True, "matches": []} + + return { + "available": True, + "matches": [_summarize_spatial_match(match) for match in matches], + } + + def _temporal_context(self, errors: list[str]) -> dict[str, Any]: + if self._temporal_memory is None: + return {"available": False} + + context: dict[str, Any] = {"available": True} + try: + context["rolling_summary"] = self._temporal_memory.get_rolling_summary() + except Exception as exc: + logger.warning("Failed to read temporal rolling summary", exc_info=True) + errors.append(f"temporal_memory.summary: {exc}") + + try: + state = self._temporal_memory.get_state() + context["state"] = _to_jsonable(state) + except Exception as exc: + logger.warning("Failed to read temporal state", exc_info=True) + errors.append(f"temporal_memory.state: {exc}") + + return context + + def _pose_to_dict(self, pose: PoseStamped | None) -> dict[str, Any] | None: + if pose is None: + return None + return { + "frame_id": pose.frame_id, + "timestamp": pose.ts, + "position": { + "x": round(pose.position.x, 3), + "y": round(pose.position.y, 3), + "z": round(pose.position.z, 3), + }, + "yaw_degrees": round(math.degrees(pose.yaw), 1), + } + + def _format_message(self, metadata: dict[str, Any]) -> str: + task = metadata["task"] + focus = metadata["focus"] or "general" + runtime = metadata["runtime"] + robot_state = metadata["robot_state"] + world_state = metadata["world_state"] + + lines = [ + f"Task: {task}", + f"Focus: {focus}", + f"Runtime: {runtime['mode']}", + ] + + odom = robot_state.get("odom") + if odom: + pos = odom["position"] + lines.append( + "Robot pose: " + f"x={pos['x']}, y={pos['y']}, z={pos['z']}, yaw={odom['yaw_degrees']}deg" + ) + else: + lines.append("Robot pose: unavailable") + + navigation = robot_state.get("navigation") + if navigation: + lines.append( + "Navigation: " + f"state={navigation['state']}, goal_reached={navigation['goal_reached']}" + ) + + spatial = world_state.get("spatial", {}) + matches = spatial.get("matches") or [] + if matches: + lines.append(f"Spatial memory: {len(matches)} relevant match(es)") + elif spatial.get("available"): + lines.append("Spatial memory: available, no relevant matches") + else: + lines.append("Spatial memory: unavailable") + + temporal = world_state.get("temporal", {}) + summary = temporal.get("rolling_summary") + if summary: + lines.append(f"Temporal summary: {summary}") + elif temporal.get("available"): + lines.append("Temporal memory: available, no rolling summary") + else: + lines.append("Temporal memory: unavailable") + + if metadata.get("errors"): + lines.append(f"Context warnings: {metadata['errors']}") + + return "\n".join(lines) + + +def _summarize_spatial_match(match: dict[str, Any]) -> dict[str, Any]: + summary: dict[str, Any] = {} + for key in ("distance", "score", "id", "text"): + if key in match: + summary[key] = match[key] + if "metadata" in match: + summary["metadata"] = _to_jsonable(match["metadata"]) + if not summary: + summary["keys"] = sorted(match.keys()) + return _to_jsonable(summary) + + +def _to_jsonable(value: Any, max_string_length: int = 500) -> Any: + if value is None or isinstance(value, bool | int | float): + return value + if isinstance(value, str): + return value[:max_string_length] + if isinstance(value, dict): + return { + str(key): _to_jsonable(item, max_string_length) + for key, item in value.items() + if not str(key).startswith("_") + } + if isinstance(value, list | tuple): + return [_to_jsonable(item, max_string_length) for item in value[:10]] + return str(value)[:max_string_length] + + +__all__ = ["_Go2ContextProvider"] diff --git a/dimos/robot/unitree/go2/blueprints/layers/expert_router.py b/dimos/robot/unitree/go2/blueprints/layers/expert_router.py new file mode 100644 index 0000000000..cdf59d61de --- /dev/null +++ b/dimos/robot/unitree/go2/blueprints/layers/expert_router.py @@ -0,0 +1,302 @@ +#!/usr/bin/env python3 +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import annotations + +from dataclasses import dataclass + +from dimos.agents.annotation import skill +from dimos.agents.skill_result import SkillResult +from dimos.core.module import Module + + +@dataclass(frozen=True) +class _RouteRule: + domain: str + keywords: tuple[str, ...] + tools: tuple[str, ...] + reason: str + needs_context: bool = True + + +_ROUTE_RULES: tuple[_RouteRule, ...] = ( + _RouteRule( + domain="safety", + keywords=( + "stop", + "halt", + "cancel", + "emergency", + "danger", + "unsafe", + "\u505c\u6b62", + "\u505c\u4e0b", + "\u53d6\u6d88", + "\u5371\u9669", + "\u7d27\u6025", + "\u907f\u9669", + ), + tools=("stop_navigation", "stop_following", "stop_looking_out", "stop_security_patrol"), + reason="The task describes stopping, cancelling, or handling a safety condition.", + needs_context=False, + ), + _RouteRule( + domain="speech", + keywords=( + "say", + "speak", + "tell", + "announce", + "reply", + "\u8bf4", + "\u544a\u8bc9", + "\u56de\u7b54", + "\u64ad\u62a5", + ), + tools=("speak",), + reason="The task asks the robot to communicate with a person.", + needs_context=False, + ), + _RouteRule( + domain="memory", + keywords=( + "remember", + "memory", + "recall", + "where did", + "previous", + "last time", + "\u8bb0\u4f4f", + "\u8bb0\u5f97", + "\u56de\u5fc6", + "\u4e4b\u524d", + "\u521a\u624d", + "\u4e0a\u6b21", + "\u54ea\u91cc", + ), + tools=("get_context", "query", "tag_location"), + reason="The task depends on remembered locations, events, or prior observations.", + ), + _RouteRule( + domain="person_follow", + keywords=( + "follow", + "track person", + "follow person", + "person wearing", + "\u8ddf\u968f", + "\u8ddf\u7740", + "\u8ffd\u8e2a\u4eba", + "\u8ddf\u4eba", + "\u7a7f", + ), + tools=("get_context", "look_out_for", "follow_person", "stop_following"), + reason="The task asks the robot to visually identify and follow a person.", + ), + _RouteRule( + domain="perception", + keywords=( + "look", + "watch", + "detect", + "find object", + "find person", + "see", + "\u770b\u5230", + "\u5bfb\u627e", + "\u68c0\u6d4b", + "\u89c2\u5bdf", + "\u8bc6\u522b", + "\u627e\u4eba", + "\u627e\u7269\u4f53", + ), + tools=("get_context", "look_out_for", "stop_looking_out"), + reason="The task primarily requires visual perception or object/person detection.", + ), + _RouteRule( + domain="navigation", + keywords=( + "go to", + "navigate", + "walk to", + "move to", + "go find", + "room", + "hallway", + "kitchen", + "\u5230", + "\u53bb", + "\u5bfc\u822a", + "\u8d70\u5230", + "\u79fb\u52a8\u5230", + "\u623f\u95f4", + "\u8d70\u5eca", + "\u53a8\u623f", + ), + tools=("get_context", "navigate_with_text", "relative_move", "stop_navigation"), + reason="The task asks the robot to reach a place or move toward a target.", + ), + _RouteRule( + domain="security", + keywords=( + "patrol", + "guard", + "security", + "\u5de1\u903b", + "\u5b89\u4fdd", + "\u770b\u5b88", + "\u8b66\u6212", + ), + tools=("get_context", "start_security_patrol", "stop_security_patrol"), + reason="The task maps to the security patrol workflow.", + ), + _RouteRule( + domain="robot_motion", + keywords=( + "forward", + "back", + "left", + "right", + "turn", + "rotate", + "jump", + "dance", + "\u524d\u8fdb", + "\u540e\u9000", + "\u5de6", + "\u53f3", + "\u8f6c", + "\u8df3", + "\u8df3\u821e", + ), + tools=("relative_move", "execute_sport_command", "wait"), + reason="The task is a direct robot-body movement or Unitree sport command.", + needs_context=False, + ), + _RouteRule( + domain="utility", + keywords=( + "time", + "wait", + "sleep", + "\u73b0\u5728\u51e0\u70b9", + "\u65f6\u95f4", + "\u7b49\u5f85", + ), + tools=("current_time", "wait"), + reason="The task asks for a utility action that does not require world context.", + needs_context=False, + ), + _RouteRule( + domain="human_help", + keywords=( + "ask human", + "human help", + "clarify", + "confirm", + "\u95ee\u4eba", + "\u786e\u8ba4", + "\u6f84\u6e05", + ), + tools=("speak",), + reason="The task is underspecified or needs explicit human confirmation.", + needs_context=False, + ), +) + + +class _Go2ExpertRouter(Module): + """Rule-based Layer 3 router for Go2 agent tasks.""" + + @skill + def route_task(self, task: str, context: str = "") -> SkillResult: + """Route a task to the most relevant Go2 expert domain. + + This tool classifies the current task and recommends MCP tools. It does + not execute robot actions. Use it before selecting navigation, + perception, speech, safety, memory, or direct movement tools. + + Args: + task: The current user goal or agent subtask. + context: Optional context summary from get_context or the dialogue. + """ + task = task.strip() + context = context.strip() + if not task: + return SkillResult.fail("INVALID_INPUT", "task is required") + + route, matches = _select_route(task, context) + needs_context = route.needs_context and not context + tools = _recommended_tools(route, needs_context) + confidence = _confidence(matches) + + metadata = { + "task": task, + "context_used": bool(context), + "domain": route.domain, + "confidence": confidence, + "matched_keywords": matches, + "recommended_tools": tools, + "needs_context": needs_context, + "reason": route.reason, + } + + message = ( + f"Route domain={route.domain}, confidence={confidence}. " + f"Recommended tools: {', '.join(tools)}. {route.reason}" + ) + return SkillResult(success=True, message=message, metadata=metadata) + + +def _select_route(task: str, context: str) -> tuple[_RouteRule, list[str]]: + text = f"{task} {context}".casefold() + best_rule = _default_route() + best_matches: list[str] = [] + + for rule in _ROUTE_RULES: + matches = [keyword for keyword in rule.keywords if keyword.casefold() in text] + if len(matches) > len(best_matches): + best_rule = rule + best_matches = matches + + return best_rule, best_matches + + +def _default_route() -> _RouteRule: + return _RouteRule( + domain="general", + keywords=(), + tools=("get_context",), + reason="No domain-specific rule matched; gather context before choosing tools.", + needs_context=True, + ) + + +def _recommended_tools(route: _RouteRule, needs_context: bool) -> list[str]: + tools = list(route.tools) + if needs_context and "get_context" not in tools: + tools.insert(0, "get_context") + return tools + + +def _confidence(matches: list[str]) -> str: + if len(matches) >= 2: + return "high" + if len(matches) == 1: + return "medium" + return "low" + + +__all__ = ["_Go2ExpertRouter"] diff --git a/dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain.py b/dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain.py index f36ec20549..b0d1104a54 100644 --- a/dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain.py +++ b/dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain.py @@ -18,11 +18,15 @@ from dimos.agents.mcp.mcp_client import McpClient from dimos.agents.mcp.mcp_server import McpServer from dimos.core.coordination.blueprints import Blueprint, autoconnect +from dimos.robot.unitree.go2.blueprints.layers.context_provider import _Go2ContextProvider +from dimos.robot.unitree.go2.blueprints.layers.expert_router import _Go2ExpertRouter def _go2_agent_brain_with_client(**mcp_client_kwargs: Any) -> Blueprint: - """Layer 3: MCP tool server plus the LLM/VLM agent client.""" + """Layer 3: expert routing, context, MCP tools, and the LLM/VLM agent.""" return autoconnect( + _Go2ExpertRouter.blueprint(), + _Go2ContextProvider.blueprint(), McpServer.blueprint(), McpClient.blueprint(**mcp_client_kwargs), ) diff --git a/dimos/robot/unitree/go2/blueprints/layers/test_context_provider.py b/dimos/robot/unitree/go2/blueprints/layers/test_context_provider.py new file mode 100644 index 0000000000..ef865dca9b --- /dev/null +++ b/dimos/robot/unitree/go2/blueprints/layers/test_context_provider.py @@ -0,0 +1,98 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import Any +import json + +from dimos.msgs.geometry_msgs.PoseStamped import PoseStamped +from dimos.navigation.base import NavigationState +from dimos.robot.unitree.go2.blueprints.layers.context_provider import _Go2ContextProvider + + +class StubSpatialMemory: + def query_by_text(self, text: str, limit: int = 5) -> list[dict[str, Any]]: + return [ + { + "distance": 0.12, + "metadata": [{"pos_x": 1.5, "pos_y": -0.2, "label": text, "_raw": b"bytes"}], + } + ][:limit] + + +class StubTemporalMemory: + def query(self, question: str) -> str: + return f"answer to {question}" + + def get_state(self) -> dict[str, Any]: + return {"entity_count": 2, "entities": [{"id": "person_1"}]} + + def get_entity_roster(self) -> list[dict[str, Any]]: + return [{"id": "person_1"}] + + def get_rolling_summary(self) -> str: + return "A person was recently visible near the hallway." + + def get_graph_db_stats(self) -> dict[str, Any]: + return {"stats": {"entities": 1}} + + +class StubNavigation: + def get_state(self) -> NavigationState: + return NavigationState.FOLLOWING_PATH + + def is_goal_reached(self) -> bool: + return False + + +def test_get_context_aggregates_available_sources() -> None: + provider = _Go2ContextProvider() + provider._spatial_memory = StubSpatialMemory() # type: ignore[assignment] + provider._temporal_memory = StubTemporalMemory() # type: ignore[assignment] + provider._navigation = StubNavigation() # type: ignore[assignment] + provider._latest_odom = PoseStamped(position=[1.0, 2.0, 0.0], frame_id="map") + + result = provider.get_context("find the person", focus="navigation") + + assert result.success is True + assert "Task: find the person" in result.message + assert result.metadata["sources"]["spatial_memory"] is True + assert result.metadata["sources"]["temporal_memory"] is True + assert result.metadata["robot_state"]["navigation"]["state"] == "following_path" + assert result.metadata["world_state"]["spatial"]["matches"][0]["distance"] == 0.12 + assert result.metadata["world_state"]["temporal"]["rolling_summary"] + + encoded = json.loads(result.agent_encode()[0]["text"]) + assert encoded["success"] is True + assert encoded["metadata"]["task"] == "find the person" + + +def test_get_context_handles_missing_optional_sources() -> None: + provider = _Go2ContextProvider() + + result = provider.get_context("walk forward") + + assert result.success is True + assert result.metadata["sources"]["spatial_memory"] is False + assert result.metadata["sources"]["temporal_memory"] is False + assert result.metadata["robot_state"]["odom"] is None + assert "Spatial memory: unavailable" in result.message + + +def test_get_context_requires_task() -> None: + provider = _Go2ContextProvider() + + result = provider.get_context(" ") + + assert result.success is False + assert result.error_code == "INVALID_INPUT" diff --git a/dimos/robot/unitree/go2/blueprints/layers/test_expert_router.py b/dimos/robot/unitree/go2/blueprints/layers/test_expert_router.py new file mode 100644 index 0000000000..adf700a776 --- /dev/null +++ b/dimos/robot/unitree/go2/blueprints/layers/test_expert_router.py @@ -0,0 +1,57 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from dimos.robot.unitree.go2.blueprints.layers.expert_router import _Go2ExpertRouter + + +def test_route_task_recommends_navigation_with_context() -> None: + router = _Go2ExpertRouter() + + result = router.route_task("go to the kitchen") + + assert result.success is True + assert result.metadata["domain"] == "navigation" + assert result.metadata["needs_context"] is True + assert result.metadata["recommended_tools"][0] == "get_context" + assert "navigate_with_text" in result.metadata["recommended_tools"] + + +def test_route_task_prioritizes_safety_without_context() -> None: + router = _Go2ExpertRouter() + + result = router.route_task("stop following and cancel navigation") + + assert result.success is True + assert result.metadata["domain"] == "safety" + assert result.metadata["needs_context"] is False + assert "stop_navigation" in result.metadata["recommended_tools"] + + +def test_route_task_supports_chinese_person_follow() -> None: + router = _Go2ExpertRouter() + + result = router.route_task("\u8ddf\u968f\u7a7f\u84dd\u8272\u8863\u670d\u7684\u4eba") + + assert result.success is True + assert result.metadata["domain"] == "person_follow" + assert "follow_person" in result.metadata["recommended_tools"] + + +def test_route_task_requires_task() -> None: + router = _Go2ExpertRouter() + + result = router.route_task(" ") + + assert result.success is False + assert result.error_code == "INVALID_INPUT" From 33ddbeb23bb3d0aabaacb45f48d3f40c49b49ca7 Mon Sep 17 00:00:00 2001 From: z1050297972-spec Date: Tue, 19 May 2026 09:50:11 +0800 Subject: [PATCH 03/36] feat(go2): organize architecture layers --- AGENTS.md | 10 +- .../robot/unitree/go2/PROJECT_CHANGELOG.md | 80 ++++- .../unitree/go2/blueprints/layers/GOAL_1.md | 109 ++++++ .../unitree/go2/blueprints/layers/README.md | 70 ---- .../layers/layer_3_agent_brain/DESIGN.md | 310 ++++++++++++++++++ .../__init__.py} | 16 +- .../context_provider.py | 33 +- .../expert_router.py | 0 .../skill_outcome_predictor.py | 249 ++++++++++++++ .../skill_outcome_store.py | 219 +++++++++++++ .../test_context_provider.py | 20 +- .../test_expert_router.py | 4 +- .../test_skill_outcomes.py | 82 +++++ .../__init__.py} | 0 .../__init__.py} | 0 .../__init__.py} | 0 16 files changed, 1113 insertions(+), 89 deletions(-) rename PROJECT_CHANGELOG.md => dimos/robot/unitree/go2/PROJECT_CHANGELOG.md (51%) create mode 100644 dimos/robot/unitree/go2/blueprints/layers/GOAL_1.md delete mode 100644 dimos/robot/unitree/go2/blueprints/layers/README.md create mode 100644 dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/DESIGN.md rename dimos/robot/unitree/go2/blueprints/layers/{layer_3_agent_brain.py => layer_3_agent_brain/__init__.py} (68%) rename dimos/robot/unitree/go2/blueprints/layers/{ => layer_3_agent_brain}/context_provider.py (87%) rename dimos/robot/unitree/go2/blueprints/layers/{ => layer_3_agent_brain}/expert_router.py (100%) create mode 100644 dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/skill_outcome_predictor.py create mode 100644 dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/skill_outcome_store.py rename dimos/robot/unitree/go2/blueprints/layers/{ => layer_3_agent_brain}/test_context_provider.py (83%) rename dimos/robot/unitree/go2/blueprints/layers/{ => layer_3_agent_brain}/test_expert_router.py (94%) create mode 100644 dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/test_skill_outcomes.py rename dimos/robot/unitree/go2/blueprints/layers/{layer_4_world_state.py => layer_4_world_state/__init__.py} (100%) rename dimos/robot/unitree/go2/blueprints/layers/{layer_5_skill_interface.py => layer_5_skill_interface/__init__.py} (100%) rename dimos/robot/unitree/go2/blueprints/layers/{layer_6_robot_body.py => layer_6_robot_body/__init__.py} (100%) diff --git a/AGENTS.md b/AGENTS.md index 19c3babbc0..81d65976e2 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -360,10 +360,12 @@ Code style rules: ## Change Documentation -Maintain `PROJECT_CHANGELOG.md` at the repository root for project-level work in -this fork/branch. Add a reverse-chronological entry after substantive changes: -architecture migrations, feature implementation, test fixes, blueprint rewiring, -or upstream sync work. +Maintain the nearest scoped `PROJECT_CHANGELOG.md` for project-level work in +this fork/branch. Use the repository root changelog for repo-wide changes, or a +subsystem changelog such as `dimos/robot/unitree/go2/PROJECT_CHANGELOG.md` for +focused subsystem architecture work. Add a reverse-chronological entry after +substantive changes: architecture migrations, feature implementation, test +fixes, blueprint rewiring, or upstream sync work. Each entry should include the date, branch, change summary, affected files/modules, validation performed, and open follow-up items. Small exploratory diff --git a/PROJECT_CHANGELOG.md b/dimos/robot/unitree/go2/PROJECT_CHANGELOG.md similarity index 51% rename from PROJECT_CHANGELOG.md rename to dimos/robot/unitree/go2/PROJECT_CHANGELOG.md index 62a895414a..038eaace7b 100644 --- a/PROJECT_CHANGELOG.md +++ b/dimos/robot/unitree/go2/PROJECT_CHANGELOG.md @@ -6,16 +6,83 @@ work so future contributors can understand why the fork differs from upstream. Entries are listed in reverse chronological order. +## 2026-05-19 + +- Branch: `refactor/go2-architecture-layers` +- Summary: Expanded the Layer 3 design document with a required + function-level implementation-note standard and detailed notes for the + current ContextProvider, ExpertRouter, SkillOutcomeStore, and + SkillOutcomePredictor entry points. +- Files/modules: + - `dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/DESIGN.md` + - `dimos/robot/unitree/go2/PROJECT_CHANGELOG.md` +- Validation: + - Documentation-only change; runtime tests were not required. +- Open items: + - Keep adding implementation notes whenever Layer 3 storage, data shape, + decision logic, or public MCP/RPC behavior changes. + +## 2026-05-19 + +- Branch: `refactor/go2-architecture-layers` +- Summary: Renamed the Go2 layer stage document to `GOAL_1.md` and added a + Layer 3 implementation design document for readable change rationale. +- Files/modules: + - `dimos/robot/unitree/go2/blueprints/layers/GOAL_1.md` + - `dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/DESIGN.md` +- Validation: + - Ran `python -m py_compile` for the reorganized layer package entrypoints. + - Ran `git diff --check`. +- Open items: + - Keep `DESIGN.md` updated whenever Layer 3 data flow or module boundaries + change. + +## 2026-05-19 + +- Branch: `refactor/go2-architecture-layers` +- Summary: Reorganized the Go2 layers package so each architecture layer has + its own directory, while preserving stable layer-level import paths. +- Files/modules: + - `dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/` + - `dimos/robot/unitree/go2/blueprints/layers/layer_4_world_state/` + - `dimos/robot/unitree/go2/blueprints/layers/layer_5_skill_interface/` + - `dimos/robot/unitree/go2/blueprints/layers/layer_6_robot_body/` +- Validation: + - Ran `python -m py_compile` for the reorganized layer packages and focused + Layer 3 tests. + - Ran `git diff --check`. +- Open items: + - Decide later whether layer directories should grow separate README files + once Layer 4/5/6 contain more than blueprint composition. + +## 2026-05-19 + +- Branch: `refactor/go2-architecture-layers` +- Summary: Added the first Layer 3 skill-outcome loop for Go2: a recent outcome + store, a rule-based preflight predictor, and ContextProvider integration. +- Files/modules: + - `dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/` +- Validation: + - Ran `python -m py_compile` for Layer 3 provider/router/outcome modules, + blueprint wiring, and focused tests. + - Ran `git diff --check`. + - Attempted the focused pytest files; local Windows environment still fails + during repo `conftest.py` import because the Unix-only `resource` module is + unavailable. +- Open items: + - Decide whether `McpClient` prompt policy should explicitly require + `record_skill_outcome` after important tool calls. + - Add automatic outcome capture later if the MCP server/tool wrapper gains a + safe cross-module event sink. + ## 2026-05-17 - Branch: `refactor/go2-architecture-layers` - Summary: Implemented the first Go2 Layer 3 ExpertRouter and documented the Layer 3 v1-v4 roadmap. - Files/modules: - - `dimos/robot/unitree/go2/blueprints/layers/expert_router.py` - - `dimos/robot/unitree/go2/blueprints/layers/test_expert_router.py` - - `dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain.py` - - `dimos/robot/unitree/go2/blueprints/layers/README.md` + - `dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/` + - `dimos/robot/unitree/go2/blueprints/layers/GOAL_1.md` - Validation: - Ran `python -m py_compile` for the new ExpertRouter, test, and Layer 3 blueprint files. @@ -34,9 +101,8 @@ Entries are listed in reverse chronological order. - Summary: Implemented the first Go2 Layer 3 ContextProvider and wired it into the internal Agent Brain layer. - Files/modules: - - `dimos/robot/unitree/go2/blueprints/layers/context_provider.py` + - `dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/` - `dimos/perception/temporal_memory_spec.py` - - `dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain.py` - Validation: - Ran `python -m py_compile` for the new ContextProvider, spec, test, and Layer 3 blueprint files. @@ -57,7 +123,7 @@ Entries are listed in reverse chronological order. - Files/modules: - `PROJECT_CHANGELOG.md` - `AGENTS.md` - - `dimos/robot/unitree/go2/blueprints/layers/README.md` + - `dimos/robot/unitree/go2/blueprints/layers/GOAL_1.md` - Validation: - Documentation-only change; no runtime tests required. - Open items: diff --git a/dimos/robot/unitree/go2/blueprints/layers/GOAL_1.md b/dimos/robot/unitree/go2/blueprints/layers/GOAL_1.md new file mode 100644 index 0000000000..0f2a1ec23b --- /dev/null +++ b/dimos/robot/unitree/go2/blueprints/layers/GOAL_1.md @@ -0,0 +1,109 @@ +# GOAL_1: Go2 Architecture Layers + +This file records the first-stage architecture goal for the Go2 blueprint +layers. The stage-1 goal is to name and organize the Layer 3 through Layer 6 +composition boundaries without moving underlying implementation modules or +breaking existing public blueprint names. + +## Layer Mapping + +- Layer 3, Agent Brain: ExpertRouter, ContextProvider, SkillOutcomeStore, + SkillOutcomePredictor, MCP server/client, and the LLM agent loop. +- Layer 4, World State: spatial memory today, with a lazy temporal-memory + factory for the existing temporal-memory blueprint. +- Layer 5, Skill Interface: MCP-callable skills and skill containers that + bridge the agent to navigation, perception, speech, and Unitree commands. +- Layer 6, Robot Body / Local Policy: the existing Go2 robot stack, including + perception, mapping, navigation, movement management, and hardware connection. + +All blueprint variables in this directory are intentionally prefixed with `_`. +The registry generator skips those names, so these internal layer nodes do not +become new CLI blueprints. + +## Directory Layout + +Each architecture layer has its own directory: + +```text +layers/ + layer_3_agent_brain/ + __init__.py # internal Layer 3 blueprint composition + context_provider.py # context aggregation for the LLM agent + expert_router.py # deterministic expert-domain routing + skill_outcome_store.py # recent skill outcome memory + skill_outcome_predictor.py # rule-based preflight risk checks + test_*.py # focused Layer 3 tests + layer_4_world_state/ + __init__.py # spatial/temporal memory blueprint pieces + layer_5_skill_interface/ + __init__.py # Go2 skill-interface blueprint pieces + layer_6_robot_body/ + __init__.py # Go2 robot-body/local-policy blueprint piece +``` + +The public internal import paths intentionally stay stable at the layer level, +for example `dimos.robot.unitree.go2.blueprints.layers.layer_3_agent_brain`. + +## ContextProvider Direction + +`ContextProvider` belongs in Layer 3 because it prepares context for the LLM +agent. It should not own the underlying robot or world state. Instead, it should +aggregate and compress context from these sources: + +- LLM/task context: user goal, decomposed subtasks, dialogue summary, and current + tool-call intent. +- Layer 4 world state: `SpatialMemory`, `TemporalMemory`, and later structured + world-state or semantic-temporal map modules. +- Layer 5 skill state: recent skill outcomes, failure reasons, risks, and + recovery suggestions. +- Layer 6 robot state: odometry, navigation state, connection state, safety + state, and perception summaries. +- Runtime context: replay/simulation/hardware mode, blueprint/config values, and + available modules. +- External context: web input, human input, external LLM/VLM results, and future + connected services. + +The current implementation exposes a compact MCP skill, +`get_context(task: str, focus: str = "", spatial_limit: int = 3)`, returning +the minimum context needed for the agent's next decision. Planning and tool +execution remain with the agent and Layer 5 skills. + +## ExpertRouter Direction + +`ExpertRouter` belongs in Layer 3 because it routes an LLM task toward the +right expert domain before tool selection. It does not execute robot actions or +own world state. The current implementation exposes +`route_task(task: str, context: str = "")` as an MCP skill. It uses deterministic +rules to return a domain, confidence, matched keywords, recommended tools, and a +flag indicating whether the agent should call `get_context` first. + +## Layer 3 Version Roadmap + +- V1 implemented: Keep Layer 3 as explicit, testable MCP tools. + `ContextProvider` aggregates task, world, robot, runtime, skill-outcome, and + external-context placeholders. + `ExpertRouter` performs deterministic domain routing for Go2 skills. The + existing `McpClient` remains the LLM/VLM agent loop. +- V2 first pass implemented: Add a shared skill-outcome store and + `SkillOutcomePredictor` so Layer 3 can warn about likely failures before tool + execution. Later V2 work should make the agent record outcomes consistently + after important tool calls. +- V3 planned: Add `CausalWorldModel` as an event/transition recorder that links + before-context, chosen tool, tool result, after-context, inferred cause, and + recovery suggestion. Use it to summarize repeated failure patterns. +- V4 planned: Promote stable Go2-specific Layer 3 pieces into robot-agnostic + modules under `dimos.agents`, while keeping robot-specific routing rules in + the robot blueprint layer. + +## Skill Outcome Direction + +`SkillOutcomeStore` is the Layer 3 memory for recent skill calls. The current +implementation exposes `record_skill_outcome(...)` and +`summarize_skill_outcomes(...)` as MCP skills, and exposes an internal +`get_recent_outcomes(...)` RPC for other Layer 3 modules. + +`SkillOutcomePredictor` performs conservative preflight checks with +`predict_skill_outcome(skill_name: str, args_json: str = "", context: str = "")`. +It uses deterministic rules plus recent outcomes from `SkillOutcomeStore`. +This is not a trained model yet; it is a risk gate that helps the agent avoid +obvious retries, missing arguments, or movement-sensitive calls without context. diff --git a/dimos/robot/unitree/go2/blueprints/layers/README.md b/dimos/robot/unitree/go2/blueprints/layers/README.md deleted file mode 100644 index 4db3ad893e..0000000000 --- a/dimos/robot/unitree/go2/blueprints/layers/README.md +++ /dev/null @@ -1,70 +0,0 @@ -# Go2 Architecture Layers - -This directory names the Go2 blueprint composition boundaries that map to -layers 3 through 6 of the project architecture diagram. These files do not move -implementation modules; they only group existing blueprints so the runtime -structure can evolve without breaking current import paths. - -## Layer Mapping - -- Layer 3, Agent Brain: ExpertRouter, ContextProvider, MCP server/client, and - the LLM agent loop. -- Layer 4, World State: spatial memory today, with a lazy temporal-memory - factory for the existing temporal-memory blueprint. -- Layer 5, Skill Interface: MCP-callable skills and skill containers that - bridge the agent to navigation, perception, speech, and Unitree commands. -- Layer 6, Robot Body / Local Policy: the existing Go2 robot stack, including - perception, mapping, navigation, movement management, and hardware connection. - -All blueprint variables in this directory are intentionally prefixed with `_`. -The registry generator skips those names, so these internal layer nodes do not -become new CLI blueprints. - -## ContextProvider Direction - -`ContextProvider` belongs in Layer 3 because it prepares context for the LLM -agent. It should not own the underlying robot or world state. Instead, it should -aggregate and compress context from these sources: - -- LLM/task context: user goal, decomposed subtasks, dialogue summary, and current - tool-call intent. -- Layer 4 world state: `SpatialMemory`, `TemporalMemory`, and later structured - world-state or semantic-temporal map modules. -- Layer 5 skill state: recent skill outcomes, failure reasons, risks, and - recovery suggestions. -- Layer 6 robot state: odometry, navigation state, connection state, safety - state, and perception summaries. -- Runtime context: replay/simulation/hardware mode, blueprint/config values, and - available modules. -- External context: web input, human input, external LLM/VLM results, and future - connected services. - -The current implementation exposes a compact MCP skill, -`get_context(task: str, focus: str = "", spatial_limit: int = 3)`, returning -the minimum context needed for the agent's next decision. Planning and tool -execution remain with the agent and Layer 5 skills. - -## ExpertRouter Direction - -`ExpertRouter` belongs in Layer 3 because it routes an LLM task toward the -right expert domain before tool selection. It does not execute robot actions or -own world state. The current implementation exposes -`route_task(task: str, context: str = "")` as an MCP skill. It uses deterministic -rules to return a domain, confidence, matched keywords, recommended tools, and a -flag indicating whether the agent should call `get_context` first. - -## Layer 3 Version Roadmap - -- V1: Keep Layer 3 as explicit, testable MCP tools. `ContextProvider` aggregates - task, world, robot, runtime, and external-context placeholders. - `ExpertRouter` performs deterministic domain routing for Go2 skills. The - existing `McpClient` remains the LLM/VLM agent loop. -- V2: Add a shared skill-outcome store and `SkillOutcomePredictor` so Layer 3 - can warn about likely failures before tool execution. Upgrade routing to use - recent skill outcomes, navigation state, and memory availability. -- V3: Add `CausalWorldModel` as an event/transition recorder that links - before-context, chosen tool, tool result, after-context, inferred cause, and - recovery suggestion. Use it to summarize repeated failure patterns. -- V4: Promote stable Go2-specific Layer 3 pieces into robot-agnostic modules - under `dimos.agents`, while keeping robot-specific routing rules in the - robot blueprint layer. diff --git a/dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/DESIGN.md b/dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/DESIGN.md new file mode 100644 index 0000000000..10f67196e7 --- /dev/null +++ b/dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/DESIGN.md @@ -0,0 +1,310 @@ +# Layer 3 Agent Brain Design + +This document explains how the Go2 Layer 3 implementation works and why it is +shaped this way. It is the companion to `GOAL_1.md`: `GOAL_1.md` records the +stage goal, while this file explains the implementation logic. + +## Current Scope + +Layer 3 is the decision layer above world state and skill execution. It does +not move the robot directly. It prepares context, routes tasks, estimates risk, +and exposes tools through MCP so the LLM agent can choose safer next actions. + +Current Layer 3 modules: + +- `McpClient`: existing LLM agent loop. +- `McpServer`: exposes `@skill` methods as MCP tools. +- `ContextProvider`: gathers compact task/world/robot/runtime/skill context. +- `ExpertRouter`: maps a task to an expert domain and recommended tools. +- `SkillOutcomeStore`: records recent skill outcomes in memory. +- `SkillOutcomePredictor`: checks planned tool calls for obvious risk. + +## Runtime Flow + +The intended task flow is: + +```text +user task + -> McpClient / LLM + -> route_task(task) + -> get_context(task, focus) + -> predict_skill_outcome(skill_name, args_json, context) + -> call a Layer 5 skill + -> record_skill_outcome(skill_name, success, ...) +``` + +The final `record_skill_outcome(...)` call is still manual in this version. The +agent can call it after important skill calls. Automatic capture is deferred +until the MCP/tool wrapper has a safe event sink. + +## Module Responsibilities + +`ContextProvider` + +- Reads task text passed by the agent. +- Optionally reads `SpatialMemory`, `TemporalMemory`, navigation state, odom, + runtime config, and recent skill outcomes. +- Returns a compact `SkillResult` with text plus structured metadata. +- Does not plan, execute skills, or own persistent state. + +`ExpertRouter` + +- Uses deterministic keyword rules. +- Returns a domain such as `navigation`, `perception`, `safety`, `memory`, or + `person_follow`. +- Recommends existing MCP tools. +- Does not call those tools. + +`SkillOutcomeStore` + +- Stores recent outcomes in a bounded in-memory `deque`. +- Keeps at most 100 records. +- Clears when the blueprint/process restarts. +- Is intentionally not backed by a database in this stage. + +`SkillOutcomePredictor` + +- Is not a trained model. +- Uses rule checks over `skill_name`, `args_json`, context text, and recent + same-skill outcomes. +- Returns `low`, `medium`, or `high` risk, with reasons and recovery + suggestions. + +## Implementation Documentation Standard + +Every new Layer 3 requirement that changes behavior should add or update a +function-level implementation note in this document. The goal is that a reader +can understand what changed without reverse-engineering every branch from the +code. + +Each implementation note should cover: + +- Entry point: class, function, file, and whether it is an MCP skill, RPC-only + helper, stream callback, or private helper. +- Purpose: what problem it solves and what it deliberately does not do. +- Inputs: user arguments, injected Specs, streams, global config, and helper + state. +- Storage: whether it writes memory, a file, a database, or no persistent state. +- Data shape: the exact fields written or returned. +- Algorithm: the branch/order of operations used to produce the result. +- Error handling: validation failures, caught dependency errors, and fallback + behavior. +- Return shape: success/failure `SkillResult`, RPC return, metadata keys, and + important message text. +- Tests: focused tests or compile checks that protect the behavior. +- Limits and next version: known shortcuts and what should change in V2/V3. + +## Function Implementation Notes + +### `_Go2ContextProvider.get_context(...)` + +- File: `layer_3_agent_brain/context_provider.py` +- Entry point: MCP skill exposed by `@skill`. +- Purpose: aggregate compact context for the LLM before it selects a tool. + It does not plan, execute robot actions, or own world-state storage. +- Inputs: + - Direct arguments: `task`, `focus`, `spatial_limit`. + - Stream state: latest `odom` message cached by `_on_odom`. + - Optional injected Specs: `SpatialMemorySpec`, `TemporalMemorySpec`, + `NavigationInterfaceSpec`, `SkillOutcomeStoreSpec`. + - Runtime config: `global_config.simulation`, `replay`, `robot_ip`, `viewer`, + `mcp_port`, and `n_workers`. +- Storage: + - No database. + - No persistent file writes. + - Only keeps the latest odom message in `_latest_odom` while the process is + running. +- Data read: + - Spatial memory: `query_by_text(task, limit=spatial_limit)`. + - Temporal memory: `get_rolling_summary()` and `get_state()`. + - Navigation: `get_state()` and `is_goal_reached()`. + - Skill history: `get_recent_outcomes(limit=5)`. +- Algorithm: + - Trim `task` and `focus`. + - Reject empty `task` with `SkillResult.fail("INVALID_INPUT", ...)`. + - Clamp `spatial_limit` to `0..10`. + - Build a metadata dictionary with `sources`, `runtime`, `robot_state`, + `world_state`, `skill_state`, and `external_context`. + - Catch dependency failures per source and append readable warnings to + `errors` instead of failing the whole context request. + - Convert metadata through `_to_jsonable(...)` so MCP responses stay compact + and serializable. + - Format a short text summary for the LLM. +- Return shape: + - `SkillResult(success=True, message=, metadata=)`. + - Metadata keys: `task`, `focus`, `sources`, `runtime`, `robot_state`, + `world_state`, `skill_state`, `external_context`, and optional `errors`. +- Current limits: + - External context is represented as unavailable. + - Context compression is deterministic formatting, not learned retrieval. + - It reads existing stores but does not decide whether a skill should run. + +### `_Go2ExpertRouter.route_task(...)` + +- File: `layer_3_agent_brain/expert_router.py` +- Entry point: MCP skill exposed by `@skill`. +- Purpose: classify a task into a Go2 expert domain and recommend candidate MCP + tools. It does not call those tools. +- Inputs: + - Direct arguments: `task`, optional `context`. + - Static rule table: `_ROUTE_RULES`. +- Storage: + - No database. + - No persistent state. + - Rules are Python constants in the module. +- Data shape: + - Each `_RouteRule` contains `domain`, `keywords`, `tools`, `reason`, and + `needs_context`. + - Domains currently include `safety`, `speech`, `memory`, `person_follow`, + `perception`, `navigation`, `security`, `robot_motion`, `utility`, + `human_help`, and fallback `general`. +- Algorithm: + - Trim inputs and reject empty `task`. + - Combine `task` and `context`, then `casefold()` for matching. + - For each rule, collect keywords contained in the combined text. + - Select the rule with the highest keyword-match count. + - Use the fallback `general` rule when no domain-specific keyword matches. + - If the selected rule needs context and no context was supplied, mark + `needs_context=True` and ensure `get_context` is recommended. + - Map keyword count to confidence: `0=low`, `1=medium`, `2+=high`. +- Return shape: + - `SkillResult(success=True, message=, metadata=)`. + - Metadata keys: `task`, `context_used`, `domain`, `confidence`, + `matched_keywords`, `recommended_tools`, `needs_context`, and `reason`. +- Current limits: + - Routing is deterministic keyword matching, not semantic classification. + - Rule order only matters when match counts tie; the current implementation + keeps the earlier best route. + +### `_Go2SkillOutcomeStore.record_skill_outcome(...)` + +- File: `layer_3_agent_brain/skill_outcome_store.py` +- Entry point: MCP skill exposed by `@skill`. +- Purpose: record recent Layer 5 tool results so Layer 3 can include history in + context and prediction. It does not execute, retry, or repair the skill. +- Inputs: + - `skill_name`, `success`, `domain`, `error_code`, `message`, `risk`, and + `recovery`. +- Storage: + - Storage backend is an in-memory `collections.deque`. + - The deque lives at `self._outcomes` inside `_Go2SkillOutcomeStore`. + - Maximum retained records: `_max_outcomes = 100`. + - No SQL database, vector database, file, Redis, or external service is used. + - Restarting the blueprint or worker process clears the history. +- Data written: + - Each record is a `_SkillOutcome` dataclass with `timestamp`, `skill_name`, + `success`, `domain`, `error_code`, `message`, `risk`, and `recovery`. + - `timestamp` is recorded with `time.time()` when the outcome is written. + - `to_dict()` rounds timestamp to three decimals for JSON-shaped responses. +- Algorithm: + - Strip text inputs. + - Reject empty `skill_name`. + - Validate `risk` against `low`, `medium`, `high`, and `unknown`. + - Create `_SkillOutcome`. + - Append to `self._outcomes`; the oldest item is dropped automatically after + the 100-record cap. + - Return the written outcome and current count. +- Related read APIs: + - `summarize_skill_outcomes(...)` is an MCP skill for human/LLM inspection. + - `get_recent_outcomes(...)` is RPC-only and returns newest records first. + - Filtering uses exact `skill_name` and `domain` matches. +- Return shape: + - Success: `SkillResult.ok("Recorded outcome for ...", outcome=, + total_outcomes=)`. + - Failure: `SkillResult.fail("INVALID_INPUT", ...)`. +- Current limits: + - Recording is manual; the LLM or prompt policy must call it after important + tool calls. + - There is no durable audit log yet. + +### `_Go2SkillOutcomePredictor.predict_skill_outcome(...)` + +- File: `layer_3_agent_brain/skill_outcome_predictor.py` +- Entry point: MCP skill exposed by `@skill`. +- Purpose: perform a conservative preflight risk check before the agent calls a + physical or recovery-sensitive skill. It does not execute the skill and it is + not a trained ML model. +- Inputs: + - Direct arguments: `skill_name`, `args_json`, `context`. + - Optional injected Spec: `SkillOutcomeStoreSpec` for recent same-skill + outcomes. +- Storage: + - No database. + - No persistent state. + - Reads recent outcomes if the store is wired, but does not write them. +- Data read: + - `args_json` is parsed as a JSON object string because MCP tool schemas are + simpler with primitive string arguments. + - Recent history is read with `get_recent_outcomes(limit=5, + skill_name=skill_name)`. +- Algorithm: + - Trim `skill_name` and `context`. + - Reject empty `skill_name`. + - Parse `args_json`; reject invalid JSON and non-object JSON. + - Build risk reasons from four signals: skill name, parsed args, context + text, and recent same-skill outcomes. + - Add history reasons for one recent failure or repeated recent failures. + - Add movement-context reasons for `navigate_with_text`, `relative_move`, and + `follow_person` when context or odom appears unavailable. + - Add argument-specific blockers: + - `navigate_with_text` requires a non-empty `query`. + - `follow_person` requires a non-empty `query`. + - `look_out_for` requires `description_of_things`. + - Downgrade stop, utility, and speech tools so missing robot context does not + block urgent or simple actions. + - Map reasons to risk: hard blockers or repeated failures become `high`, any + other reason becomes `medium`, and no reasons becomes `low`. + - Produce recovery suggestions based on skill family. +- Return shape: + - `SkillResult(success=True, message=, metadata=)`. + - Metadata keys: `skill_name`, `args`, `risk`, `predicted_success`, + `failure_reasons`, `recovery_suggestions`, `recent_outcomes`, and + `outcome_store_available`. +- Current limits: + - `predicted_success` is `risk != "high"`, not a probability. + - Risk rules are handcrafted and should be revised after real outcome data is + available. + +## Version Boundaries + +V1 implemented: + +- Layer 3 is explicit MCP tools rather than hidden agent internals. +- `ContextProvider` gives the agent a compact context view. +- `ExpertRouter` gives deterministic task-domain routing. +- Existing `McpClient` remains the LLM/VLM agent loop. + +V2 first pass implemented: + +- `SkillOutcomeStore` records recent skill results. +- `SkillOutcomePredictor` performs preflight risk checks. +- `ContextProvider` includes recent skill outcomes when available. + +V2 remaining: + +- Update agent prompt/policy so important tool calls are followed by + `record_skill_outcome(...)`. +- Decide whether some outcomes should be captured automatically by MCP. + +V3 planned: + +- Add `CausalWorldModel` as an event transition recorder: + before-context, action, result, after-context, inferred cause, recovery. + +V4 planned: + +- Promote stable Go2-specific Layer 3 pieces into robot-agnostic modules under + `dimos.agents`, while keeping robot-specific routing rules near Go2. + +## Design Rules + +- Keep layer files internal by using `_`-prefixed blueprint variables/classes. +- Preserve existing public blueprint names and imports at the layer level. +- Do not make Layer 3 execute physical actions directly. +- Prefer deterministic rules first; introduce models only after data and tests + are stable. +- Update this file when Layer 3 behavior changes in a way that affects task + flow, stored data, prediction logic, or version boundaries. +- Update the implementation notes above whenever a new requirement adds a + function, changes storage, changes returned metadata, or changes decision + logic. diff --git a/dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain.py b/dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/__init__.py similarity index 68% rename from dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain.py rename to dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/__init__.py index b0d1104a54..c19eec348c 100644 --- a/dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain.py +++ b/dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/__init__.py @@ -18,14 +18,26 @@ from dimos.agents.mcp.mcp_client import McpClient from dimos.agents.mcp.mcp_server import McpServer from dimos.core.coordination.blueprints import Blueprint, autoconnect -from dimos.robot.unitree.go2.blueprints.layers.context_provider import _Go2ContextProvider -from dimos.robot.unitree.go2.blueprints.layers.expert_router import _Go2ExpertRouter +from dimos.robot.unitree.go2.blueprints.layers.layer_3_agent_brain.context_provider import ( + _Go2ContextProvider, +) +from dimos.robot.unitree.go2.blueprints.layers.layer_3_agent_brain.expert_router import ( + _Go2ExpertRouter, +) +from dimos.robot.unitree.go2.blueprints.layers.layer_3_agent_brain.skill_outcome_predictor import ( + _Go2SkillOutcomePredictor, +) +from dimos.robot.unitree.go2.blueprints.layers.layer_3_agent_brain.skill_outcome_store import ( + _Go2SkillOutcomeStore, +) def _go2_agent_brain_with_client(**mcp_client_kwargs: Any) -> Blueprint: """Layer 3: expert routing, context, MCP tools, and the LLM/VLM agent.""" return autoconnect( _Go2ExpertRouter.blueprint(), + _Go2SkillOutcomeStore.blueprint(), + _Go2SkillOutcomePredictor.blueprint(), _Go2ContextProvider.blueprint(), McpServer.blueprint(), McpClient.blueprint(**mcp_client_kwargs), diff --git a/dimos/robot/unitree/go2/blueprints/layers/context_provider.py b/dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/context_provider.py similarity index 87% rename from dimos/robot/unitree/go2/blueprints/layers/context_provider.py rename to dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/context_provider.py index a4341a913e..c595962ff5 100644 --- a/dimos/robot/unitree/go2/blueprints/layers/context_provider.py +++ b/dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/context_provider.py @@ -30,6 +30,9 @@ from dimos.navigation.navigation_spec import NavigationInterfaceSpec from dimos.perception.spatial_memory_spec import SpatialMemorySpec from dimos.perception.temporal_memory_spec import TemporalMemorySpec +from dimos.robot.unitree.go2.blueprints.layers.layer_3_agent_brain.skill_outcome_store import ( + SkillOutcomeStoreSpec, +) from dimos.utils.logging_config import setup_logger logger = setup_logger() @@ -41,6 +44,7 @@ class _Go2ContextProvider(Module): _spatial_memory: SpatialMemorySpec | None = None _temporal_memory: TemporalMemorySpec | None = None _navigation: NavigationInterfaceSpec | None = None + _skill_outcomes: SkillOutcomeStoreSpec | None = None _latest_odom: PoseStamped | None = None odom: In[PoseStamped] @@ -81,10 +85,7 @@ def get_context(self, task: str, focus: str = "", spatial_limit: int = 3) -> Ski "runtime": self._runtime_context(), "robot_state": self._robot_context(errors), "world_state": self._world_context(task, spatial_limit, errors), - "skill_state": { - "available": False, - "reason": "No shared skill-outcome store is wired into ContextProvider yet.", - }, + "skill_state": self._skill_context(errors), "external_context": { "available": False, "reason": "External context sources are not wired into ContextProvider yet.", @@ -103,6 +104,7 @@ def _source_status(self) -> dict[str, bool]: "temporal_memory": self._temporal_memory is not None, "odom": self._latest_odom is not None, "navigation": self._navigation is not None, + "skill_outcomes": self._skill_outcomes is not None, "runtime": True, } @@ -147,6 +149,17 @@ def _world_context(self, task: str, spatial_limit: int, errors: list[str]) -> di "temporal": self._temporal_context(errors), } + def _skill_context(self, errors: list[str]) -> dict[str, Any]: + if self._skill_outcomes is None: + return {"available": False, "recent_outcomes": []} + try: + recent = self._skill_outcomes.get_recent_outcomes(limit=5) + except Exception as exc: + logger.warning("Failed to read skill-outcome context", exc_info=True) + errors.append(f"skill_outcomes: {exc}") + return {"available": True, "recent_outcomes": []} + return {"available": True, "recent_outcomes": recent} + def _spatial_context( self, task: str, spatial_limit: int, errors: list[str] ) -> dict[str, Any]: @@ -252,6 +265,18 @@ def _format_message(self, metadata: dict[str, Any]) -> str: if metadata.get("errors"): lines.append(f"Context warnings: {metadata['errors']}") + skill_state = metadata.get("skill_state", {}) + recent_outcomes = skill_state.get("recent_outcomes") or [] + if recent_outcomes: + failures = [outcome for outcome in recent_outcomes if not outcome.get("success")] + lines.append( + f"Skill outcomes: {len(recent_outcomes)} recent, {len(failures)} failure(s)" + ) + elif skill_state.get("available"): + lines.append("Skill outcomes: available, no recorded outcomes") + else: + lines.append("Skill outcomes: unavailable") + return "\n".join(lines) diff --git a/dimos/robot/unitree/go2/blueprints/layers/expert_router.py b/dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/expert_router.py similarity index 100% rename from dimos/robot/unitree/go2/blueprints/layers/expert_router.py rename to dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/expert_router.py diff --git a/dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/skill_outcome_predictor.py b/dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/skill_outcome_predictor.py new file mode 100644 index 0000000000..1d8c197134 --- /dev/null +++ b/dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/skill_outcome_predictor.py @@ -0,0 +1,249 @@ +#!/usr/bin/env python3 +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import annotations + +import json +from typing import Any + +from dimos.agents.annotation import skill +from dimos.agents.skill_result import SkillResult +from dimos.core.module import Module +from dimos.robot.unitree.go2.blueprints.layers.layer_3_agent_brain.skill_outcome_store import ( + SkillOutcomeStoreSpec, +) + + +class _Go2SkillOutcomePredictor(Module): + """Rule-based Layer 3 predictor for pre-skill risk checks. + + What "prediction" means in this version: + This is not a trained model and does not estimate a numeric success + probability. It is a deterministic risk gate. It checks planned skill + arguments, text context from ContextProvider, and recent outcomes from + SkillOutcomeStore, then returns low/medium/high risk plus reasons. + + Why this is in Layer 3: + It helps the LLM agent decide whether to proceed, gather more context, + or choose a recovery tool before calling a physical skill. + """ + + # Optional Spec injection. If the store is present in the blueprint, + # autoconnect wires this ref to _Go2SkillOutcomeStore. If not present, the + # predictor still works using only args_json and context. + _skill_outcomes: SkillOutcomeStoreSpec | None = None + + @skill + def predict_skill_outcome( + self, skill_name: str, args_json: str = "", context: str = "" + ) -> SkillResult: + """Predict whether a skill is likely to succeed in the current context. + + This tool performs a conservative preflight risk check. It does not + execute robot actions. Use it before navigation, perception, following, + or recovery-sensitive tools when the current context is uncertain. + + Args: + skill_name: Name of the skill or MCP tool the agent is considering. + args_json: Optional JSON object string containing planned arguments. + context: Optional context summary from get_context or route_task. + """ + skill_name = skill_name.strip() + context = context.strip() + if not skill_name: + return SkillResult.fail("INVALID_INPUT", "skill_name is required") + + # MCP skill schemas do not handle arbitrary nested dict parameters as + # consistently as strings. Accept JSON text, parse it here, and reject + # non-object payloads. Example: '{"query": "kitchen"}'. + parsed_args = _parse_args_json(args_json) + if isinstance(parsed_args, str): + return SkillResult.fail("INVALID_INPUT", parsed_args) + + # Inputs to the V2 predictor: + # 1. skill_name: which tool the agent wants to call. + # 2. parsed_args: planned tool arguments. + # 3. context: text summary from get_context/route_task/dialogue. + # 4. recent: same-skill outcomes from SkillOutcomeStore. + recent = self._recent_outcomes(skill_name) + reasons = _risk_reasons(skill_name, parsed_args, context, recent) + risk = _risk_level(reasons) + predicted_success = risk != "high" + suggestions = _recovery_suggestions(skill_name, reasons) + + metadata = { + "skill_name": skill_name, + "args": parsed_args, + "risk": risk, + "predicted_success": predicted_success, + "failure_reasons": reasons, + "recovery_suggestions": suggestions, + "recent_outcomes": recent, + "outcome_store_available": self._skill_outcomes is not None, + } + message = ( + f"Predicted risk={risk}, predicted_success={predicted_success}. " + f"Reasons: {reasons or ['no blocking risk detected']}" + ) + return SkillResult(success=True, message=message, metadata=metadata) + + def _recent_outcomes(self, skill_name: str) -> list[dict[str, Any]]: + """Fetch recent same-skill outcomes if the outcome store is wired.""" + if self._skill_outcomes is None: + return [] + return self._skill_outcomes.get_recent_outcomes(limit=5, skill_name=skill_name) + + +def _parse_args_json(args_json: str) -> dict[str, Any] | str: + """Parse the planned skill arguments. + + Returns a dict on success and an error string on failure. Keeping errors as + strings avoids exceptions bubbling through the MCP skill response path. + """ + args_json = args_json.strip() + if not args_json: + return {} + try: + parsed = json.loads(args_json) + except json.JSONDecodeError as exc: + return f"args_json must be a JSON object: {exc}" + if not isinstance(parsed, dict): + return "args_json must be a JSON object" + return parsed + + +def _risk_reasons( + skill_name: str, + args: dict[str, Any], + context: str, + recent: list[dict[str, Any]], +) -> list[str]: + """Collect human-readable risk reasons. + + The rules are intentionally conservative: + - repeated recent failures increase risk even if arguments look valid; + - movement-sensitive skills need context/odom; + - semantic navigation needs a query and benefits from spatial memory; + - person following needs a query and benefits from memory/perception context. + """ + name = skill_name.casefold() + context_text = context.casefold() + reasons: list[str] = [] + + # Recent history is the only "learned" signal in this version. It is still + # rule-based: one failure is medium concern; repeated failures become high + # risk in _risk_level. + failures = [outcome for outcome in recent if not outcome.get("success", False)] + if len(failures) >= 2: + reasons.append("same skill failed repeatedly in recent outcomes") + elif len(failures) == 1: + reasons.append("same skill has one recent failure") + + if name in {"navigate_with_text", "relative_move", "follow_person"}: + # ContextProvider formats missing odom as "Robot pose: unavailable". + # Some structured summaries may instead include an odom=false-style + # marker, so keep the broader fallback check. + odom_unavailable = "odom" in context_text and "false" in context_text + if "robot pose: unavailable" in context_text or odom_unavailable: + reasons.append("robot pose or odometry may be unavailable") + if not context: + reasons.append("no context provided for movement-sensitive skill") + + if name == "navigate_with_text": + # Navigation without a target query is a hard blocker. Missing or empty + # spatial memory is not always fatal, but it should push the agent to + # gather context or use visual perception first. + query = str(args.get("query", "")).strip() + if not query: + reasons.append("navigation query is missing") + if "spatial memory: unavailable" in context_text: + reasons.append("spatial memory is unavailable for semantic navigation") + if "no relevant matches" in context_text: + reasons.append("spatial memory has no relevant matches") + + if name == "follow_person": + # Person following is visual, but a concrete query still matters. Memory + # availability is used as a weak risk signal when the target identity is + # likely ambiguous. + query = str(args.get("query", "")).strip() + if not query: + reasons.append("person-follow query is missing") + no_memory_context = ( + "temporal memory: unavailable" in context_text + and "spatial memory: unavailable" in context_text + ) + if no_memory_context: + reasons.append("no memory source is available to help identify the target") + + if name == "look_out_for" and not args.get("description_of_things"): + reasons.append("look_out_for descriptions are missing") + + # Stop/utility/speech tools should remain easy to call. Do not penalize them + # for missing robot context; only preserve recent-failure concerns. + if name.startswith("stop_") or name in {"current_time", "wait", "speak"}: + return [reason for reason in reasons if "recent failure" in reason] + + return reasons + + +def _risk_level(reasons: list[str]) -> str: + """Map reasons to coarse risk. + + V2 uses three buckets rather than a fake numeric probability: + - high: missing required args or repeated same-skill failures; + - medium: context/history warnings; + - low: no blocking risk detected. + """ + high_markers = ( + "failed repeatedly", + "navigation query is missing", + "person-follow query is missing", + "descriptions are missing", + ) + if any(any(marker in reason for marker in high_markers) for reason in reasons): + return "high" + if reasons: + return "medium" + return "low" + + +def _recovery_suggestions(skill_name: str, reasons: list[str]) -> list[str]: + """Translate risk reasons into actionable next steps for the LLM agent.""" + if not reasons: + return [] + + suggestions: list[str] = [] + name = skill_name.casefold() + if "navigate" in name or name == "relative_move": + suggestions.extend( + [ + "call get_context with focus='navigation'", + "consider look_out_for before navigation if the target is visual", + "call stop_navigation before retrying if navigation is already active", + ] + ) + if "follow" in name: + suggestions.extend( + [ + "call look_out_for to detect the person first", + "include a specific visual description in the query", + ] + ) + if not suggestions: + suggestions.append("gather context and retry with more specific arguments") + return suggestions + + +__all__ = ["_Go2SkillOutcomePredictor"] diff --git a/dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/skill_outcome_store.py b/dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/skill_outcome_store.py new file mode 100644 index 0000000000..c5b970f0a3 --- /dev/null +++ b/dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/skill_outcome_store.py @@ -0,0 +1,219 @@ +#!/usr/bin/env python3 +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import annotations + +from collections import deque +from dataclasses import dataclass +import time +from typing import Any, Protocol + +from dimos.agents.annotation import skill +from dimos.agents.skill_result import SkillResult +from dimos.core.core import rpc +from dimos.core.module import Module +from dimos.spec.utils import Spec + +RiskLevel = str + + +class SkillOutcomeStoreSpec(Spec, Protocol): + """RPC surface used by Layer 3 modules that need recent skill history. + + The store itself is Go2-specific for now, but consumers depend on this + small Spec instead of the concrete class. That keeps ContextProvider and + SkillOutcomePredictor decoupled from the storage implementation. + """ + + def get_recent_outcomes( + self, limit: int = 5, skill_name: str = "", domain: str = "" + ) -> list[dict[str, Any]]: ... + + +@dataclass(frozen=True) +class _SkillOutcome: + """One recorded skill result. + + This is intentionally a small, JSON-shaped event. We store only the fields + needed by Layer 3 routing/prediction, not raw robot messages or large + artifacts. The event is converted to a dict before crossing RPC/MCP + boundaries. + """ + + timestamp: float + skill_name: str + success: bool + domain: str + error_code: str + message: str + risk: RiskLevel + recovery: str + + def to_dict(self) -> dict[str, Any]: + return { + "timestamp": round(self.timestamp, 3), + "skill_name": self.skill_name, + "success": self.success, + "domain": self.domain, + "error_code": self.error_code, + "message": self.message, + "risk": self.risk, + "recovery": self.recovery, + } + + +class _Go2SkillOutcomeStore(Module): + """In-memory Layer 3 store for recent skill outcomes. + + Where data lives: + Outcomes are stored in ``self._outcomes``, a bounded in-memory deque + inside this module instance. In the normal DimOS runtime this module + runs in a worker process, so the data is process-local. + + Persistence: + V2 first pass is deliberately non-persistent. Restarting the blueprint + clears the history. This avoids introducing database/schema decisions + before the outcome payload stabilizes. + + How data is written: + The agent can call the MCP skill ``record_skill_outcome(...)`` after an + important tool call. Later we can add automatic capture in the MCP/tool + wrapper, but manual recording is safer for the first version. + """ + + _max_outcomes = 100 + + def __init__(self, **kwargs: Any) -> None: + super().__init__(**kwargs) + # Bounded memory: when the 101st event is appended, the oldest one is + # dropped automatically. This keeps the store useful for recent context + # without becoming an unbounded log. + self._outcomes: deque[_SkillOutcome] = deque(maxlen=self._max_outcomes) + + @skill + def record_skill_outcome( + self, + skill_name: str, + success: bool, + domain: str = "", + error_code: str = "", + message: str = "", + risk: str = "unknown", + recovery: str = "", + ) -> SkillResult: + """Record the outcome of a skill call for later routing and prediction. + + Use this after a robot skill returns, especially when the result failed + or contains useful recovery information. This tool only records the + outcome; it does not execute or retry the skill. + + Args: + skill_name: Name of the tool or skill that just ran. + success: Whether the skill succeeded. + domain: Optional expert domain, such as navigation or perception. + error_code: Optional structured failure code. + message: Short result message from the skill. + risk: Optional observed risk level. + recovery: Optional recovery suggestion. + """ + skill_name = skill_name.strip() + domain = domain.strip() + error_code = error_code.strip() + message = message.strip() + recovery = recovery.strip() + + # Keep the MCP-facing API simple and defensive. A missing skill name + # would make later filtering/prediction meaningless. + if not skill_name: + return SkillResult.fail("INVALID_INPUT", "skill_name is required") + + # ``risk`` is a string instead of Literal[...] because MCP/tool schema + # generation is more reliable with primitive annotations. Validate the + # accepted vocabulary manually here. + if risk not in ("low", "medium", "high", "unknown"): + return SkillResult.fail("INVALID_INPUT", f"invalid risk level: {risk}") + + # Timestamp at record time, not at skill start. This is enough for + # "recent outcome" reasoning; CausalWorldModel can add richer timing + # later if it needs before/after transitions. + outcome = _SkillOutcome( + timestamp=time.time(), + skill_name=skill_name, + success=success, + domain=domain, + error_code=error_code, + message=message, + risk=risk, + recovery=recovery, + ) + + # The only write path in V2 first pass: append to the bounded deque. + # There is no file/db write here. + self._outcomes.append(outcome) + return SkillResult.ok( + f"Recorded outcome for {skill_name}", + outcome=outcome.to_dict(), + total_outcomes=len(self._outcomes), + ) + + @skill + def summarize_skill_outcomes( + self, limit: int = 5, skill_name: str = "", domain: str = "" + ) -> SkillResult: + """Summarize recent skill outcomes. + + Args: + limit: Maximum number of recent outcomes to include. + skill_name: Optional skill name filter. + domain: Optional expert domain filter. + """ + outcomes = self.get_recent_outcomes(limit=limit, skill_name=skill_name, domain=domain) + if not outcomes: + return SkillResult.ok("No recorded skill outcomes", outcomes=[]) + + failures = [outcome for outcome in outcomes if not outcome["success"]] + message = ( + f"{len(outcomes)} recent outcome(s), " + f"{len(failures)} failure(s), " + f"filters skill_name={skill_name or '*'} domain={domain or '*'}" + ) + return SkillResult.ok(message, outcomes=outcomes) + + @rpc + def get_recent_outcomes( + self, limit: int = 5, skill_name: str = "", domain: str = "" + ) -> list[dict[str, Any]]: + """Return newest outcomes first, optionally filtered. + + This method is RPC-only, not a skill. Other Layer 3 modules call it via + Spec injection. Human/LLM users should normally use + ``summarize_skill_outcomes`` instead. + """ + limit = max(0, min(limit, self._max_outcomes)) + skill_name = skill_name.strip() + domain = domain.strip() + + # Iterate newest-to-oldest so predictors see the most relevant failures + # first. Filters are exact-match to avoid surprising fuzzy behavior. + outcomes = [ + outcome + for outcome in reversed(self._outcomes) + if (not skill_name or outcome.skill_name == skill_name) + and (not domain or outcome.domain == domain) + ] + return [outcome.to_dict() for outcome in outcomes[:limit]] + + +__all__ = ["SkillOutcomeStoreSpec", "_Go2SkillOutcomeStore"] diff --git a/dimos/robot/unitree/go2/blueprints/layers/test_context_provider.py b/dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/test_context_provider.py similarity index 83% rename from dimos/robot/unitree/go2/blueprints/layers/test_context_provider.py rename to dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/test_context_provider.py index ef865dca9b..7753e7b3b1 100644 --- a/dimos/robot/unitree/go2/blueprints/layers/test_context_provider.py +++ b/dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/test_context_provider.py @@ -17,7 +17,9 @@ from dimos.msgs.geometry_msgs.PoseStamped import PoseStamped from dimos.navigation.base import NavigationState -from dimos.robot.unitree.go2.blueprints.layers.context_provider import _Go2ContextProvider +from dimos.robot.unitree.go2.blueprints.layers.layer_3_agent_brain.context_provider import ( + _Go2ContextProvider, +) class StubSpatialMemory: @@ -55,11 +57,26 @@ def is_goal_reached(self) -> bool: return False +class StubSkillOutcomes: + def get_recent_outcomes( + self, limit: int = 5, skill_name: str = "", domain: str = "" + ) -> list[dict[str, Any]]: + return [ + { + "skill_name": "navigate_with_text", + "success": False, + "domain": "navigation", + "error_code": "EXECUTION_FAILED", + } + ][:limit] + + def test_get_context_aggregates_available_sources() -> None: provider = _Go2ContextProvider() provider._spatial_memory = StubSpatialMemory() # type: ignore[assignment] provider._temporal_memory = StubTemporalMemory() # type: ignore[assignment] provider._navigation = StubNavigation() # type: ignore[assignment] + provider._skill_outcomes = StubSkillOutcomes() # type: ignore[assignment] provider._latest_odom = PoseStamped(position=[1.0, 2.0, 0.0], frame_id="map") result = provider.get_context("find the person", focus="navigation") @@ -71,6 +88,7 @@ def test_get_context_aggregates_available_sources() -> None: assert result.metadata["robot_state"]["navigation"]["state"] == "following_path" assert result.metadata["world_state"]["spatial"]["matches"][0]["distance"] == 0.12 assert result.metadata["world_state"]["temporal"]["rolling_summary"] + assert result.metadata["skill_state"]["recent_outcomes"][0]["success"] is False encoded = json.loads(result.agent_encode()[0]["text"]) assert encoded["success"] is True diff --git a/dimos/robot/unitree/go2/blueprints/layers/test_expert_router.py b/dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/test_expert_router.py similarity index 94% rename from dimos/robot/unitree/go2/blueprints/layers/test_expert_router.py rename to dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/test_expert_router.py index adf700a776..f86e389151 100644 --- a/dimos/robot/unitree/go2/blueprints/layers/test_expert_router.py +++ b/dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/test_expert_router.py @@ -12,7 +12,9 @@ # See the License for the specific language governing permissions and # limitations under the License. -from dimos.robot.unitree.go2.blueprints.layers.expert_router import _Go2ExpertRouter +from dimos.robot.unitree.go2.blueprints.layers.layer_3_agent_brain.expert_router import ( + _Go2ExpertRouter, +) def test_route_task_recommends_navigation_with_context() -> None: diff --git a/dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/test_skill_outcomes.py b/dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/test_skill_outcomes.py new file mode 100644 index 0000000000..c6a3663bba --- /dev/null +++ b/dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/test_skill_outcomes.py @@ -0,0 +1,82 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from dimos.robot.unitree.go2.blueprints.layers.layer_3_agent_brain.skill_outcome_predictor import ( + _Go2SkillOutcomePredictor, +) +from dimos.robot.unitree.go2.blueprints.layers.layer_3_agent_brain.skill_outcome_store import ( + _Go2SkillOutcomeStore, +) + + +def test_record_and_summarize_skill_outcomes() -> None: + store = _Go2SkillOutcomeStore() + + result = store.record_skill_outcome( + skill_name="navigate_with_text", + success=False, + domain="navigation", + error_code="EXECUTION_FAILED", + message="No matching location", + risk="high", + recovery="call get_context", + ) + + assert result.success is True + outcomes = store.get_recent_outcomes(limit=5) + assert len(outcomes) == 1 + assert outcomes[0]["skill_name"] == "navigate_with_text" + assert outcomes[0]["success"] is False + + summary = store.summarize_skill_outcomes(domain="navigation") + assert summary.success is True + assert summary.metadata["outcomes"][0]["error_code"] == "EXECUTION_FAILED" + + +def test_predictor_uses_recent_failures() -> None: + store = _Go2SkillOutcomeStore() + store.record_skill_outcome("navigate_with_text", False) + store.record_skill_outcome("navigate_with_text", False) + + predictor = _Go2SkillOutcomePredictor() + predictor._skill_outcomes = store + + result = predictor.predict_skill_outcome( + "navigate_with_text", + args_json='{"query": "kitchen"}', + context="Robot pose: x=1.0, y=2.0", + ) + + assert result.success is True + assert result.metadata["risk"] == "high" + assert result.metadata["predicted_success"] is False + assert "same skill failed repeatedly in recent outcomes" in result.metadata["failure_reasons"] + + +def test_predictor_rejects_invalid_args_json() -> None: + predictor = _Go2SkillOutcomePredictor() + + result = predictor.predict_skill_outcome("navigate_with_text", args_json="[1, 2]") + + assert result.success is False + assert result.error_code == "INVALID_INPUT" + + +def test_predictor_requires_skill_name() -> None: + predictor = _Go2SkillOutcomePredictor() + + result = predictor.predict_skill_outcome(" ") + + assert result.success is False + assert result.error_code == "INVALID_INPUT" diff --git a/dimos/robot/unitree/go2/blueprints/layers/layer_4_world_state.py b/dimos/robot/unitree/go2/blueprints/layers/layer_4_world_state/__init__.py similarity index 100% rename from dimos/robot/unitree/go2/blueprints/layers/layer_4_world_state.py rename to dimos/robot/unitree/go2/blueprints/layers/layer_4_world_state/__init__.py diff --git a/dimos/robot/unitree/go2/blueprints/layers/layer_5_skill_interface.py b/dimos/robot/unitree/go2/blueprints/layers/layer_5_skill_interface/__init__.py similarity index 100% rename from dimos/robot/unitree/go2/blueprints/layers/layer_5_skill_interface.py rename to dimos/robot/unitree/go2/blueprints/layers/layer_5_skill_interface/__init__.py diff --git a/dimos/robot/unitree/go2/blueprints/layers/layer_6_robot_body.py b/dimos/robot/unitree/go2/blueprints/layers/layer_6_robot_body/__init__.py similarity index 100% rename from dimos/robot/unitree/go2/blueprints/layers/layer_6_robot_body.py rename to dimos/robot/unitree/go2/blueprints/layers/layer_6_robot_body/__init__.py From 75143482e0de7c9c9d4c151fd2c4328a6cdfe031 Mon Sep 17 00:00:00 2001 From: z1050297972-spec Date: Tue, 19 May 2026 12:33:47 +0800 Subject: [PATCH 04/36] feat(go2): complete layer 3 causal loop --- dimos/agents/mcp/mcp_client.py | 166 ++++++- dimos/agents/mcp/test_mcp_client_unit.py | 66 +++ dimos/robot/unitree/go2/PROJECT_CHANGELOG.md | 75 ++++ .../unitree/go2/blueprints/layers/GOAL_1.md | 42 +- .../layers/layer_3_agent_brain/DESIGN.md | 206 ++++++++- .../layers/layer_3_agent_brain/__init__.py | 13 +- .../layer_3_agent_brain/causal_world_model.py | 413 ++++++++++++++++++ .../layer_3_agent_brain/context_provider.py | 34 ++ .../layer_3_agent_brain/prompt_policy.py | 61 +++ .../skill_outcome_predictor.py | 45 +- .../skill_outcome_store.py | 7 +- .../test_causal_world_model.py | 124 ++++++ .../test_context_provider.py | 20 + .../layer_3_agent_brain/test_prompt_policy.py | 37 ++ 14 files changed, 1266 insertions(+), 43 deletions(-) create mode 100644 dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/causal_world_model.py create mode 100644 dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/prompt_policy.py create mode 100644 dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/test_causal_world_model.py create mode 100644 dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/test_prompt_policy.py diff --git a/dimos/agents/mcp/mcp_client.py b/dimos/agents/mcp/mcp_client.py index 75b532e9cc..76394c6509 100644 --- a/dimos/agents/mcp/mcp_client.py +++ b/dimos/agents/mcp/mcp_client.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +import json from collections.abc import Callable from queue import Empty, Queue from threading import Event, RLock, Thread @@ -40,6 +41,36 @@ logger = setup_logger() +_RECORD_TOOL_NAME = "record_skill_outcome" +_OUTCOME_SKIP_TOOLS = frozenset( + { + "get_context", + "route_task", + "predict_skill_outcome", + "record_skill_outcome", + "summarize_skill_outcomes", + "server_status", + "list_modules", + "agent_send", + } +) +_TOOL_DOMAINS = { + "navigate_with_text": "navigation", + "stop_navigation": "navigation", + "relative_move": "robot_motion", + "execute_sport_command": "robot_motion", + "follow_person": "person_follow", + "stop_following": "person_follow", + "look_out_for": "perception", + "stop_looking_out": "perception", + "tag_location": "memory", + "start_security_patrol": "security", + "stop_security_patrol": "security", + "speak": "speech", + "wait": "utility", + "current_time": "utility", +} + class McpClientConfig(ModuleConfig): system_prompt: str | None = SYSTEM_PROMPT @@ -104,17 +135,57 @@ def _mcp_request(self, method: str, params: dict[str, Any] | None = None) -> dic result: dict[str, Any] = data.get("result") return result - def _mcp_tool_call(self, name: str, arguments: dict[str, Any]) -> dict[str, Any]: + def _mcp_tool_call( + self, name: str, arguments: dict[str, Any], record_outcome: bool = True + ) -> dict[str, Any]: progress_token = str(uuid.uuid4()) - return self._mcp_request( - "tools/call", + try: + result = self._mcp_request( + "tools/call", + { + "name": name, + "arguments": arguments, + "_meta": {"progressToken": progress_token}, + }, + ) + except Exception as exc: + if record_outcome: + self._record_tool_exception(name, exc) + raise + + if record_outcome: + self._record_tool_outcome(name, result) + return result + + def _should_record_tool_outcome(self, name: str) -> bool: + return name not in _OUTCOME_SKIP_TOOLS and _RECORD_TOOL_NAME in self._tool_registry + + def _record_tool_exception(self, name: str, exc: Exception) -> None: + if not self._should_record_tool_outcome(name): + return + self._record_tool_payload( { - "name": name, - "arguments": arguments, - "_meta": {"progressToken": progress_token}, - }, + "skill_name": name, + "success": False, + "domain": _infer_tool_domain(name), + "error_code": "EXECUTION_FAILED", + "message": _truncate(f"MCP request failed: {exc}"), + "risk": "high", + "recovery": "Inspect the tool error before retrying.", + } ) + def _record_tool_outcome(self, name: str, result: dict[str, Any]) -> None: + if not self._should_record_tool_outcome(name): + return + self._record_tool_payload(_outcome_payload_from_result(name, result)) + + def _record_tool_payload(self, payload: dict[str, Any]) -> None: + try: + self._mcp_tool_call(_RECORD_TOOL_NAME, payload, record_outcome=False) + except Exception: + logger.warning("Failed to record MCP tool outcome", tool=payload["skill_name"]) + def _on_tool_stream_message(self, msg: dict[str, Any]) -> None: method = msg.get("method") params = msg.get("params") or {} @@ -348,3 +419,84 @@ def _append_image_to_history( ] ) ) + + +def _outcome_payload_from_result(name: str, result: dict[str, Any]) -> dict[str, Any]: + text = _content_text(result) + structured = _parse_skill_result_text(text) + if structured is not None: + success = bool(structured.get("success", False)) + message = str(structured.get("message") or "") + error_code = str(structured.get("error_code") or "") + raw_metadata = structured.get("metadata") + metadata = raw_metadata if isinstance(raw_metadata, dict) else {} + risk = str(metadata.get("risk") or ("unknown" if success else "medium")) + recovery = _recovery_text(metadata) + else: + message = text + failure = _looks_like_failure_text(text) + success = not failure + error_code = "EXECUTION_FAILED" if failure else "" + risk = "high" if failure else "unknown" + recovery = "Inspect the tool error before retrying." if failure else "" + + return { + "skill_name": name, + "success": success, + "domain": _infer_tool_domain(name), + "error_code": error_code, + "message": _truncate(message), + "risk": _normalize_risk(risk), + "recovery": _truncate(recovery), + } + + +def _content_text(result: dict[str, Any]) -> str: + content = result.get("content", []) + if not isinstance(content, list): + return "" + parts = [item.get("text", "") for item in content if isinstance(item, dict)] + return "\n".join(part for part in parts if part) + + +def _parse_skill_result_text(text: str) -> dict[str, Any] | None: + text = text.strip() + if not text.startswith("{"): + return None + try: + parsed = json.loads(text) + except json.JSONDecodeError: + return None + if not isinstance(parsed, dict) or "success" not in parsed: + return None + return parsed + + +def _looks_like_failure_text(text: str) -> bool: + lowered = text.strip().casefold() + return lowered.startswith("error running tool") or lowered.startswith("tool not found") + + +def _infer_tool_domain(name: str) -> str: + return _TOOL_DOMAINS.get(name, "") + + +def _normalize_risk(risk: str) -> str: + risk = risk.strip().casefold() + if risk in {"low", "medium", "high", "unknown"}: + return risk + return "unknown" + + +def _recovery_text(metadata: dict[str, Any]) -> str: + recovery = metadata.get("recovery") + if isinstance(recovery, str): + return recovery + suggestions = metadata.get("recovery_suggestions") + if isinstance(suggestions, list): + return "; ".join(str(item) for item in suggestions) + return "" + + +def _truncate(text: str, limit: int = 500) -> str: + return text[:limit] diff --git a/dimos/agents/mcp/test_mcp_client_unit.py b/dimos/agents/mcp/test_mcp_client_unit.py index ea26f7c54f..a1b8d7577a 100644 --- a/dimos/agents/mcp/test_mcp_client_unit.py +++ b/dimos/agents/mcp/test_mcp_client_unit.py @@ -229,3 +229,69 @@ def fake_request(method: str, params: dict[str, object] | None = None) -> dict[s assert isinstance(meta, dict) token = meta["progressToken"] assert isinstance(token, str) and len(token) > 0 + + +def test_mcp_tool_call_records_structured_skill_outcome(mcp_client: McpClient) -> None: + """When the outcome store is available, agent tool calls are recorded.""" + calls: list[dict[str, object]] = [] + + def fake_request(method: str, params: dict[str, object] | None = None) -> dict[str, object]: + assert params is not None + calls.append(params) + if params["name"] == "navigate_with_text": + return { + "content": [ + { + "type": "text", + "text": json.dumps( + { + "success": False, + "message": "No matching location found", + "error_code": "EXECUTION_FAILED", + "duration_ms": 12.0, + } + ), + } + ] + } + if params["name"] == "record_skill_outcome": + return {"content": [{"type": "text", "text": "recorded"}]} + raise AssertionError(f"Unexpected tool call: {params['name']}") + + mcp_client._mcp_request = fake_request + mcp_client._tool_registry = { + "navigate_with_text": {}, + "record_skill_outcome": {}, + } + + result = mcp_client._mcp_tool_call("navigate_with_text", {"query": "kitchen"}) + + assert result["content"][0]["type"] == "text" + assert [call["name"] for call in calls] == ["navigate_with_text", "record_skill_outcome"] + record_args = calls[1]["arguments"] + assert isinstance(record_args, dict) + assert record_args["skill_name"] == "navigate_with_text" + assert record_args["success"] is False + assert record_args["domain"] == "navigation" + assert record_args["error_code"] == "EXECUTION_FAILED" + assert record_args["risk"] == "medium" + + +def test_mcp_tool_call_skips_layer_3_internal_outcome_recording(mcp_client: McpClient) -> None: + """Layer 3 support tools should not pollute the skill-outcome store.""" + calls: list[dict[str, object]] = [] + + def fake_request(method: str, params: dict[str, object] | None = None) -> dict[str, object]: + assert params is not None + calls.append(params) + return {"content": [{"type": "text", "text": "context"}]} + + mcp_client._mcp_request = fake_request + mcp_client._tool_registry = { + "get_context": {}, + "record_skill_outcome": {}, + } + + mcp_client._mcp_tool_call("get_context", {"task": "go to the kitchen"}) + + assert [call["name"] for call in calls] == ["get_context"] diff --git a/dimos/robot/unitree/go2/PROJECT_CHANGELOG.md b/dimos/robot/unitree/go2/PROJECT_CHANGELOG.md index 038eaace7b..a0c6204396 100644 --- a/dimos/robot/unitree/go2/PROJECT_CHANGELOG.md +++ b/dimos/robot/unitree/go2/PROJECT_CHANGELOG.md @@ -8,6 +8,81 @@ Entries are listed in reverse chronological order. ## 2026-05-19 +- Branch: `refactor/go2-architecture-layers` +- Summary: Implemented the first Go2 Layer 3 `CausalWorldModel` for V3, + including in-memory causal transitions, repeated failure-pattern summaries, + ContextProvider integration, Predictor risk escalation, prompt policy updates, + and focused tests. +- Files/modules: + - `dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/causal_world_model.py` + - `dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/test_causal_world_model.py` + - `dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/context_provider.py` + - `dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/test_context_provider.py` + - `dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/skill_outcome_predictor.py` + - `dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/prompt_policy.py` + - `dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/test_prompt_policy.py` + - `dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/__init__.py` + - `dimos/robot/unitree/go2/blueprints/layers/GOAL_1.md` + - `dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/DESIGN.md` + - `dimos/robot/unitree/go2/PROJECT_CHANGELOG.md` +- Validation: + - Ran `python -m py_compile` for the changed MCP client, Layer 3 modules, + prompt policy, and focused tests. + - Ran `git diff --check`. + - Attempted targeted pytest with `.venv\\Scripts\\python.exe -m pytest`; + local Windows environment still fails during repo `conftest.py` import + because the Unix-only `resource` module is unavailable. +- Open items: + - Decide later whether causal transitions should be persisted to JSONL, + SQLite, TemporalMemory, or another durable event store. + +## 2026-05-19 + +- Branch: `refactor/go2-architecture-layers` +- Summary: Completed the remaining Go2 Layer 3 V2 behavior by adding a default + Layer 3 decision prompt policy and automatic `McpClient` outcome recording + for non-internal MCP tool calls. +- Files/modules: + - `dimos/agents/mcp/mcp_client.py` + - `dimos/agents/mcp/test_mcp_client_unit.py` + - `dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/__init__.py` + - `dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/prompt_policy.py` + - `dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/test_prompt_policy.py` + - `dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/skill_outcome_store.py` + - `dimos/robot/unitree/go2/blueprints/layers/GOAL_1.md` + - `dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/DESIGN.md` + - `dimos/robot/unitree/go2/PROJECT_CHANGELOG.md` +- Validation: + - Ran `python -m py_compile` for the changed MCP client, tests, prompt + policy, Layer 3 blueprint entrypoint, and outcome store. + - Ran `git diff --check`. + - Attempted targeted pytest with `.venv\\Scripts\\python.exe -m pytest`; + local Windows environment still fails during repo `conftest.py` import + because the Unix-only `resource` module is unavailable. + - Attempted `uv run pytest`; local Windows dependency resolution is blocked + because `dimos-viewer==0.30.0a6.dev99` has no Windows wheel. +- Open items: + - V3 should add `CausalWorldModel` for before/action/result/after causal + transition records. + +## 2026-05-19 + +- Branch: `refactor/go2-architecture-layers` +- Summary: Narrowed the Go2 Layer 3 roadmap to stop at V3 for the current + architecture stage and deferred robot-agnostic Layer 3 extraction until + Layers 4, 5, and 6 are clearer. +- Files/modules: + - `dimos/robot/unitree/go2/blueprints/layers/GOAL_1.md` + - `dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/DESIGN.md` + - `dimos/robot/unitree/go2/PROJECT_CHANGELOG.md` +- Validation: + - Documentation-only change; runtime tests were not required. +- Open items: + - Finish V2 remaining prompt/outcome-recording behavior, then implement the + V3 `CausalWorldModel` before revisiting robot-agnostic extraction. + +## 2026-05-19 + - Branch: `refactor/go2-architecture-layers` - Summary: Expanded the Layer 3 design document with a required function-level implementation-note standard and detailed notes for the diff --git a/dimos/robot/unitree/go2/blueprints/layers/GOAL_1.md b/dimos/robot/unitree/go2/blueprints/layers/GOAL_1.md index 0f2a1ec23b..9333267263 100644 --- a/dimos/robot/unitree/go2/blueprints/layers/GOAL_1.md +++ b/dimos/robot/unitree/go2/blueprints/layers/GOAL_1.md @@ -8,7 +8,8 @@ breaking existing public blueprint names. ## Layer Mapping - Layer 3, Agent Brain: ExpertRouter, ContextProvider, SkillOutcomeStore, - SkillOutcomePredictor, MCP server/client, and the LLM agent loop. + SkillOutcomePredictor, CausalWorldModel, MCP server/client, and the LLM + agent loop. - Layer 4, World State: spatial memory today, with a lazy temporal-memory factory for the existing temporal-memory blueprint. - Layer 5, Skill Interface: MCP-callable skills and skill containers that @@ -30,8 +31,10 @@ layers/ __init__.py # internal Layer 3 blueprint composition context_provider.py # context aggregation for the LLM agent expert_router.py # deterministic expert-domain routing + prompt_policy.py # Layer 3 tool-use policy for McpClient skill_outcome_store.py # recent skill outcome memory skill_outcome_predictor.py # rule-based preflight risk checks + causal_world_model.py # before/action/result/after causal memory test_*.py # focused Layer 3 tests layer_4_world_state/ __init__.py # spatial/temporal memory blueprint pieces @@ -84,26 +87,47 @@ flag indicating whether the agent should call `get_context` first. external-context placeholders. `ExpertRouter` performs deterministic domain routing for Go2 skills. The existing `McpClient` remains the LLM/VLM agent loop. -- V2 first pass implemented: Add a shared skill-outcome store and +- V2 implemented: Add a shared skill-outcome store and `SkillOutcomePredictor` so Layer 3 can warn about likely failures before tool - execution. Later V2 work should make the agent record outcomes consistently - after important tool calls. -- V3 planned: Add `CausalWorldModel` as an event/transition recorder that links + execution. The Go2 Layer 3 prompt now asks the agent to use + `route_task -> get_context -> predict_skill_outcome` for non-trivial tasks, + and `McpClient` automatically records non-internal MCP tool outcomes when + `record_skill_outcome` is available. +- V3 implemented: Add `CausalWorldModel` as an event/transition recorder that links before-context, chosen tool, tool result, after-context, inferred cause, and recovery suggestion. Use it to summarize repeated failure patterns. -- V4 planned: Promote stable Go2-specific Layer 3 pieces into robot-agnostic - modules under `dimos.agents`, while keeping robot-specific routing rules in - the robot blueprint layer. +- Out of scope for this stage: promoting Layer 3 pieces into robot-agnostic + modules under `dimos.agents`. Do this only after Layers 4, 5, and 6 have + clearer Go2 boundaries and the V3 causal loop has been validated. ## Skill Outcome Direction `SkillOutcomeStore` is the Layer 3 memory for recent skill calls. The current implementation exposes `record_skill_outcome(...)` and `summarize_skill_outcomes(...)` as MCP skills, and exposes an internal -`get_recent_outcomes(...)` RPC for other Layer 3 modules. +`get_recent_outcomes(...)` RPC for other Layer 3 modules. Agent-driven tool +calls are recorded automatically by `McpClient` when the outcome store is +present; manual recording is still available for external events or outcomes +outside the agent's MCP tool path. `SkillOutcomePredictor` performs conservative preflight checks with `predict_skill_outcome(skill_name: str, args_json: str = "", context: str = "")`. It uses deterministic rules plus recent outcomes from `SkillOutcomeStore`. This is not a trained model yet; it is a risk gate that helps the agent avoid obvious retries, missing arguments, or movement-sensitive calls without context. + +## Causal World Model Direction + +`CausalWorldModel` records compact Layer 3 transitions: + +```text +task + before_context + chosen skill + args + prediction + outcome + -> after_context + inferred_cause + recovery suggestion +``` + +The current implementation exposes `record_causal_transition(...)` and +`summarize_causal_patterns(...)` as MCP skills, and exposes +`get_recent_transitions(...)` as an internal RPC for other Layer 3 modules. +It stores recent transitions in memory only. `ContextProvider` includes recent +causal transitions in its context payload, and `SkillOutcomePredictor` raises +risk when the same skill repeatedly fails for the same inferred cause. diff --git a/dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/DESIGN.md b/dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/DESIGN.md index 10f67196e7..ca8676eb2b 100644 --- a/dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/DESIGN.md +++ b/dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/DESIGN.md @@ -16,8 +16,11 @@ Current Layer 3 modules: - `McpServer`: exposes `@skill` methods as MCP tools. - `ContextProvider`: gathers compact task/world/robot/runtime/skill context. - `ExpertRouter`: maps a task to an expert domain and recommended tools. +- `PromptPolicy`: appends Layer 3 tool-use rules to the Go2 MCP client prompt. - `SkillOutcomeStore`: records recent skill outcomes in memory. - `SkillOutcomePredictor`: checks planned tool calls for obvious risk. +- `CausalWorldModel`: records before/action/result/after transitions and + summarizes repeated failure causes. ## Runtime Flow @@ -30,12 +33,16 @@ user task -> get_context(task, focus) -> predict_skill_outcome(skill_name, args_json, context) -> call a Layer 5 skill - -> record_skill_outcome(skill_name, success, ...) + -> McpClient auto-records the outcome through record_skill_outcome(...) + -> get_context(task, focus) for after-context + -> record_causal_transition(...) + -> summarize_causal_patterns(...) before risky retries ``` -The final `record_skill_outcome(...)` call is still manual in this version. The -agent can call it after important skill calls. Automatic capture is deferred -until the MCP/tool wrapper has a safe event sink. +The final `record_skill_outcome(...)` call is automatic for agent-driven, +non-internal MCP tool calls when the outcome store is present. Manual recording +is still useful for external events or important outcomes that did not flow +through `McpClient`. ## Module Responsibilities @@ -61,6 +68,7 @@ until the MCP/tool wrapper has a safe event sink. - Keeps at most 100 records. - Clears when the blueprint/process restarts. - Is intentionally not backed by a database in this stage. +- Receives automatic records from `McpClient` for non-internal MCP tool calls. `SkillOutcomePredictor` @@ -70,6 +78,14 @@ until the MCP/tool wrapper has a safe event sink. - Returns `low`, `medium`, or `high` risk, with reasons and recovery suggestions. +`CausalWorldModel` + +- Stores recent causal transitions in a bounded in-memory `deque`. +- Links task, before-context, chosen skill, args, prediction, outcome, + after-context, inferred cause, and recovery suggestion. +- Uses deterministic cause rules, not a learned causal model. +- Does not execute skills or own world state. + ## Implementation Documentation Standard Every new Layer 3 requirement that changes behavior should add or update a @@ -185,6 +201,11 @@ Each implementation note should cover: - Inputs: - `skill_name`, `success`, `domain`, `error_code`, `message`, `risk`, and `recovery`. +- Writers: + - `McpClient._mcp_tool_call(...)` writes automatically after non-internal MCP + tool calls when `record_skill_outcome` is available. + - LLM or human operators may still call this skill manually for external + events or non-MCP outcomes. - Storage: - Storage backend is an in-memory `collections.deque`. - The deque lives at `self._outcomes` inside `_Go2SkillOutcomeStore`. @@ -213,8 +234,9 @@ Each implementation note should cover: total_outcomes=)`. - Failure: `SkillResult.fail("INVALID_INPUT", ...)`. - Current limits: - - Recording is manual; the LLM or prompt policy must call it after important - tool calls. + - Automatic recording only covers the agent's `McpClient` tool path. + Direct external MCP clients can still call `record_skill_outcome(...)` + manually if they need history. - There is no durable audit log yet. ### `_Go2SkillOutcomePredictor.predict_skill_outcome(...)` @@ -232,6 +254,7 @@ Each implementation note should cover: - No database. - No persistent state. - Reads recent outcomes if the store is wired, but does not write them. + - Reads recent causal transitions if `CausalWorldModel` is wired. - Data read: - `args_json` is parsed as a JSON object string because MCP tool schemas are simpler with primitive string arguments. @@ -241,8 +264,8 @@ Each implementation note should cover: - Trim `skill_name` and `context`. - Reject empty `skill_name`. - Parse `args_json`; reject invalid JSON and non-object JSON. - - Build risk reasons from four signals: skill name, parsed args, context - text, and recent same-skill outcomes. + - Build risk reasons from five signals: skill name, parsed args, context + text, recent same-skill outcomes, and recent same-skill causal transitions. - Add history reasons for one recent failure or repeated recent failures. - Add movement-context reasons for `navigate_with_text`, `relative_move`, and `follow_person` when context or odom appears unavailable. @@ -258,13 +281,149 @@ Each implementation note should cover: - Return shape: - `SkillResult(success=True, message=, metadata=)`. - Metadata keys: `skill_name`, `args`, `risk`, `predicted_success`, - `failure_reasons`, `recovery_suggestions`, `recent_outcomes`, and - `outcome_store_available`. + `failure_reasons`, `recovery_suggestions`, `recent_outcomes`, + `recent_causal_transitions`, `outcome_store_available`, and + `causal_world_model_available`. - Current limits: - `predicted_success` is `risk != "high"`, not a probability. - Risk rules are handcrafted and should be revised after real outcome data is available. +### `_go2_layer_3_system_prompt(...)` + +- File: `layer_3_agent_brain/prompt_policy.py` +- Entry point: private helper used by `_go2_agent_brain_with_client(...)`. +- Purpose: append the Go2 Layer 3 decision policy to the MCP client's system + prompt without changing the public `McpClient` API. +- Inputs: + - Optional `base_prompt`; defaults to `dimos.agents.system_prompt.SYSTEM_PROMPT`. +- Storage: + - No database. + - No runtime state. + - Returns a composed prompt string. +- Algorithm: + - Trim the base prompt. + - If the Layer 3 policy header is already present, return the prompt + unchanged. + - Otherwise append a policy block telling the agent to use + `route_task`, `get_context`, and `predict_skill_outcome` before risky + physical tools. + - Tell the agent that normal MCP tool outcomes are recorded automatically and + manual `record_skill_outcome(...)` is only for external/non-MCP events. +- Return shape: + - `str`. +- Current limits: + - This is prompt guidance, not a hard planner. It improves consistency but + cannot prove the LLM will always follow the intended flow. + +### `McpClient._mcp_tool_call(...)` Outcome Recording + +- File: `dimos/agents/mcp/mcp_client.py` +- Entry point: internal MCP client helper used by LangChain tools and + continuation execution. +- Purpose: automatically write recent outcomes into `SkillOutcomeStore` after + agent-driven MCP tool calls. +- Inputs: + - Tool name and arguments. + - MCP tool response content. + - `_tool_registry`, used to detect whether `record_skill_outcome` is + available. +- Storage: + - Does not store data locally. + - Calls the MCP `record_skill_outcome` tool with `record_outcome=False` to + avoid recursive self-recording. +- Data shape written: + - `skill_name`, `success`, `domain`, `error_code`, `message`, `risk`, and + `recovery`. +- Algorithm: + - Send the original `tools/call` request with a progress token. + - If the call raises, record a failed outcome with `risk="high"` when the + store is available, then re-raise. + - If the call succeeds, skip Layer 3/internal tools such as `get_context`, + `route_task`, `predict_skill_outcome`, `record_skill_outcome`, server + status, and utility agent-send tools. + - For `SkillResult` JSON responses, preserve `success`, `message`, and + `error_code`. + - For plain string responses, treat normal MCP responses as successful and + obvious server error text as failed. + - Infer a coarse domain from known Go2 tool names when possible. +- Return shape: + - Returns the original MCP response unchanged. +- Current limits: + - Plain string tools cannot provide precise success/failure semantics unless + they migrate to `SkillResult`. + - Direct external MCP calls that bypass `McpClient` are not auto-recorded. + +### `_Go2CausalWorldModel.record_causal_transition(...)` + +- File: `layer_3_agent_brain/causal_world_model.py` +- Entry point: MCP skill exposed by `@skill`. +- Purpose: record one compact causal transition after an important skill call. + It does not execute, retry, or repair the skill. +- Inputs: + - Direct arguments: `task`, `skill_name`, `args_json`, `before_context`, + `after_context`, `prediction_json`, `outcome_json`, and `domain`. + - Optional injected Spec: `SkillOutcomeStoreSpec` for latest same-skill + outcome fallback when `outcome_json` is omitted. +- Storage: + - Storage backend is an in-memory `collections.deque`. + - The deque lives at `self._transitions` inside `_Go2CausalWorldModel`. + - Maximum retained records: `_max_transitions = 200`. + - No SQL database, vector database, file, Redis, or external service is used. + - Restarting the blueprint or worker process clears the history. +- Data written: + - Each record is a `_CausalTransition` dataclass with `timestamp`, `task`, + `domain`, `skill_name`, `args`, `before_context`, `prediction_risk`, + `prediction_reasons`, `outcome_success`, `outcome_error_code`, + `outcome_message`, `after_context`, `inferred_cause`, `recovery`, and + `confidence`. +- Algorithm: + - Trim `task`, `skill_name`, context strings, and domain. + - Reject empty `task` or `skill_name`. + - Parse `args_json`, `prediction_json`, and `outcome_json` as JSON objects. + - If `outcome_json` is omitted, read the latest same-skill outcome from + `SkillOutcomeStore` when available. + - Flatten `metadata` from SkillResult-shaped JSON so prediction/outcome fields + are easy to read. + - Infer cause with deterministic rules: + missing args, repeated failures, unavailable odom, missing map target, + unavailable world state, unavailable visual target, explicit error code, + unknown failure, or success. + - Append the transition to the bounded deque. +- Return shape: + - Success: `SkillResult.ok("Recorded causal transition ...", + transition=, total_transitions=)`. + - Failure: `SkillResult.fail("INVALID_INPUT", ...)`. +- Current limits: + - Cause inference is rule-based and coarse. + - The agent must explicitly call this after important physical/recovery tool + calls; `McpClient` only auto-records simple skill outcomes, not full causal + transitions. + - Context snapshots are compact strings, not full world-state copies. + +### `_Go2CausalWorldModel.summarize_causal_patterns(...)` + +- File: `layer_3_agent_brain/causal_world_model.py` +- Entry point: MCP skill exposed by `@skill`. +- Purpose: summarize repeated recent failure causes before the agent retries a + skill. +- Inputs: + - Optional filters: `skill_name`, `domain`, and `limit`. +- Storage: + - Reads the in-memory causal transition deque. + - Does not write new data. +- Algorithm: + - Read newest transitions first through `get_recent_transitions(...)`. + - Keep failure transitions where `outcome_success is False`. + - Count transitions by `inferred_cause`. + - Return ordered patterns with cause, count, affected skill names, latest + message, and recovery suggestion. +- Return shape: + - `SkillResult.ok(, patterns=, transitions=)`. +- Current limits: + - Pattern summaries only cover recent in-memory transitions. + - It does not yet aggregate across process restarts. + ## Version Boundaries V1 implemented: @@ -274,27 +433,30 @@ V1 implemented: - `ExpertRouter` gives deterministic task-domain routing. - Existing `McpClient` remains the LLM/VLM agent loop. -V2 first pass implemented: +V2 implemented: - `SkillOutcomeStore` records recent skill results. - `SkillOutcomePredictor` performs preflight risk checks. - `ContextProvider` includes recent skill outcomes when available. +- Go2 Layer 3 prompt policy asks the LLM to route, gather context, and predict + risk before non-trivial or movement-sensitive tools. +- `McpClient` automatically records non-internal MCP tool outcomes when + `record_skill_outcome` is available. -V2 remaining: - -- Update agent prompt/policy so important tool calls are followed by - `record_skill_outcome(...)`. -- Decide whether some outcomes should be captured automatically by MCP. - -V3 planned: +V3 implemented: -- Add `CausalWorldModel` as an event transition recorder: +- Added `CausalWorldModel` as an event transition recorder: before-context, action, result, after-context, inferred cause, recovery. +- `ContextProvider` includes recent causal transitions when available. +- `SkillOutcomePredictor` raises risk when same-skill causal failures repeat. +- `PromptPolicy` asks the agent to record causal transitions after important + physical or recovery-sensitive tool calls. -V4 planned: +Out of scope for this stage: -- Promote stable Go2-specific Layer 3 pieces into robot-agnostic modules under - `dimos.agents`, while keeping robot-specific routing rules near Go2. +- Do not promote Layer 3 pieces into robot-agnostic `dimos.agents` modules yet. + The current architecture work should stop at V3 until Layers 4, 5, and 6 have + clearer Go2 boundaries and the causal loop has been validated. ## Design Rules diff --git a/dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/__init__.py b/dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/__init__.py index c19eec348c..0cfe6d3fe7 100644 --- a/dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/__init__.py +++ b/dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/__init__.py @@ -18,12 +18,18 @@ from dimos.agents.mcp.mcp_client import McpClient from dimos.agents.mcp.mcp_server import McpServer from dimos.core.coordination.blueprints import Blueprint, autoconnect +from dimos.robot.unitree.go2.blueprints.layers.layer_3_agent_brain.causal_world_model import ( + _Go2CausalWorldModel, +) from dimos.robot.unitree.go2.blueprints.layers.layer_3_agent_brain.context_provider import ( _Go2ContextProvider, ) from dimos.robot.unitree.go2.blueprints.layers.layer_3_agent_brain.expert_router import ( _Go2ExpertRouter, ) +from dimos.robot.unitree.go2.blueprints.layers.layer_3_agent_brain.prompt_policy import ( + _go2_layer_3_system_prompt, +) from dimos.robot.unitree.go2.blueprints.layers.layer_3_agent_brain.skill_outcome_predictor import ( _Go2SkillOutcomePredictor, ) @@ -34,13 +40,18 @@ def _go2_agent_brain_with_client(**mcp_client_kwargs: Any) -> Blueprint: """Layer 3: expert routing, context, MCP tools, and the LLM/VLM agent.""" + client_kwargs = dict(mcp_client_kwargs) + client_kwargs["system_prompt"] = _go2_layer_3_system_prompt( + client_kwargs.get("system_prompt") + ) return autoconnect( _Go2ExpertRouter.blueprint(), _Go2SkillOutcomeStore.blueprint(), + _Go2CausalWorldModel.blueprint(), _Go2SkillOutcomePredictor.blueprint(), _Go2ContextProvider.blueprint(), McpServer.blueprint(), - McpClient.blueprint(**mcp_client_kwargs), + McpClient.blueprint(**client_kwargs), ) diff --git a/dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/causal_world_model.py b/dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/causal_world_model.py new file mode 100644 index 0000000000..77149dbe65 --- /dev/null +++ b/dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/causal_world_model.py @@ -0,0 +1,413 @@ +#!/usr/bin/env python3 +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import annotations + +from collections import Counter, deque +from dataclasses import dataclass +import json +import time +from typing import Any, Protocol + +from dimos.agents.annotation import skill +from dimos.agents.skill_result import SkillResult +from dimos.core.core import rpc +from dimos.core.module import Module +from dimos.robot.unitree.go2.blueprints.layers.layer_3_agent_brain.skill_outcome_store import ( + SkillOutcomeStoreSpec, +) +from dimos.spec.utils import Spec + + +class CausalWorldModelSpec(Spec, Protocol): + """RPC surface for Layer 3 modules that need recent causal history.""" + + def get_recent_transitions( + self, + limit: int = 5, + skill_name: str = "", + domain: str = "", + cause: str = "", + ) -> list[dict[str, Any]]: ... + + +@dataclass(frozen=True) +class _CausalTransition: + """One observed action transition with inferred cause and recovery.""" + + timestamp: float + task: str + domain: str + skill_name: str + args: dict[str, Any] + before_context: str + prediction_risk: str + prediction_reasons: list[str] + outcome_success: bool | None + outcome_error_code: str + outcome_message: str + after_context: str + inferred_cause: str + recovery: str + confidence: str + + def to_dict(self) -> dict[str, Any]: + return { + "timestamp": round(self.timestamp, 3), + "task": self.task, + "domain": self.domain, + "skill_name": self.skill_name, + "args": self.args, + "before_context": self.before_context, + "prediction_risk": self.prediction_risk, + "prediction_reasons": self.prediction_reasons, + "outcome_success": self.outcome_success, + "outcome_error_code": self.outcome_error_code, + "outcome_message": self.outcome_message, + "after_context": self.after_context, + "inferred_cause": self.inferred_cause, + "recovery": self.recovery, + "confidence": self.confidence, + } + + +class _Go2CausalWorldModel(Module): + """In-memory Layer 3 causal transition recorder for Go2.""" + + _skill_outcomes: SkillOutcomeStoreSpec | None = None + _max_transitions = 200 + + def __init__(self, **kwargs: Any) -> None: + super().__init__(**kwargs) + self._transitions: deque[_CausalTransition] = deque(maxlen=self._max_transitions) + + @skill + def record_causal_transition( + self, + task: str, + skill_name: str, + args_json: str = "", + before_context: str = "", + after_context: str = "", + prediction_json: str = "", + outcome_json: str = "", + domain: str = "", + ) -> SkillResult: + """Record one before/action/result/after causal transition. + + Use this after an important physical or recovery-sensitive tool call. + The tool stores a compact event, infers a coarse cause, and returns a + recovery suggestion. It does not execute or retry robot actions. + + Args: + task: User goal or agent subtask that led to the action. + skill_name: Tool or skill that was executed. + args_json: Optional JSON object string with the executed arguments. + before_context: Context summary captured before the action. + after_context: Context summary captured after the action. + prediction_json: Optional JSON object from predict_skill_outcome. + outcome_json: Optional JSON object from the tool result or outcome + store. If omitted, the latest same-skill outcome is used when + SkillOutcomeStore is wired. + domain: Optional expert domain override. + """ + task = task.strip() + skill_name = skill_name.strip() + domain = domain.strip() + before_context = before_context.strip() + after_context = after_context.strip() + + if not task: + return SkillResult.fail("INVALID_INPUT", "task is required") + if not skill_name: + return SkillResult.fail("INVALID_INPUT", "skill_name is required") + + parsed_args = _parse_json_object(args_json, "args_json") + if isinstance(parsed_args, str): + return SkillResult.fail("INVALID_INPUT", parsed_args) + + prediction = _parse_json_object(prediction_json, "prediction_json") + if isinstance(prediction, str): + return SkillResult.fail("INVALID_INPUT", prediction) + + outcome = _parse_json_object(outcome_json, "outcome_json") + if isinstance(outcome, str): + return SkillResult.fail("INVALID_INPUT", outcome) + if not outcome: + outcome = self._latest_outcome(skill_name) + + prediction_view = _metadata_view(prediction) + outcome_view = _metadata_view(outcome) + prediction_reasons = _prediction_reasons(prediction_view) + outcome_success = _optional_bool(outcome_view.get("success")) + outcome_error_code = str(outcome_view.get("error_code") or "") + outcome_message = str(outcome_view.get("message") or "") + prediction_risk = str(prediction_view.get("risk") or "unknown") + resolved_domain = domain or str(prediction_view.get("domain") or _infer_domain(skill_name)) + + cause, recovery, confidence = _infer_cause( + skill_name=skill_name, + before_context=before_context, + after_context=after_context, + prediction_reasons=prediction_reasons, + outcome_success=outcome_success, + outcome_message=outcome_message, + outcome_error_code=outcome_error_code, + ) + + transition = _CausalTransition( + timestamp=time.time(), + task=task, + domain=resolved_domain, + skill_name=skill_name, + args=parsed_args, + before_context=before_context[:500], + prediction_risk=prediction_risk, + prediction_reasons=prediction_reasons, + outcome_success=outcome_success, + outcome_error_code=outcome_error_code, + outcome_message=outcome_message[:500], + after_context=after_context[:500], + inferred_cause=cause, + recovery=recovery, + confidence=confidence, + ) + self._transitions.append(transition) + + return SkillResult.ok( + f"Recorded causal transition for {skill_name}: {cause}", + transition=transition.to_dict(), + total_transitions=len(self._transitions), + ) + + @skill + def summarize_causal_patterns( + self, skill_name: str = "", domain: str = "", limit: int = 10 + ) -> SkillResult: + """Summarize recent causal failure patterns. + + Args: + skill_name: Optional skill-name filter. + domain: Optional expert-domain filter. + limit: Maximum number of recent transitions to inspect. + """ + transitions = self.get_recent_transitions( + limit=limit, + skill_name=skill_name, + domain=domain, + ) + if not transitions: + return SkillResult.ok("No recorded causal transitions", patterns=[], transitions=[]) + + patterns = _summarize_patterns(transitions) + if not patterns: + return SkillResult.ok( + f"{len(transitions)} transition(s), no repeated failure pattern", + patterns=[], + transitions=transitions, + ) + + top = patterns[0] + message = ( + f"{len(transitions)} transition(s), top failure cause " + f"{top['cause']} occurred {top['count']} time(s)" + ) + return SkillResult.ok(message, patterns=patterns, transitions=transitions) + + @rpc + def get_recent_transitions( + self, + limit: int = 5, + skill_name: str = "", + domain: str = "", + cause: str = "", + ) -> list[dict[str, Any]]: + """Return newest causal transitions first, optionally filtered.""" + limit = max(0, min(limit, self._max_transitions)) + skill_name = skill_name.strip() + domain = domain.strip() + cause = cause.strip() + + transitions = [ + transition + for transition in reversed(self._transitions) + if (not skill_name or transition.skill_name == skill_name) + and (not domain or transition.domain == domain) + and (not cause or transition.inferred_cause == cause) + ] + return [transition.to_dict() for transition in transitions[:limit]] + + def _latest_outcome(self, skill_name: str) -> dict[str, Any]: + if self._skill_outcomes is None: + return {} + outcomes = self._skill_outcomes.get_recent_outcomes(limit=1, skill_name=skill_name) + return outcomes[0] if outcomes else {} + + +def _parse_json_object(value: str, field_name: str) -> dict[str, Any] | str: + value = value.strip() + if not value: + return {} + try: + parsed = json.loads(value) + except json.JSONDecodeError as exc: + return f"{field_name} must be a JSON object: {exc}" + if not isinstance(parsed, dict): + return f"{field_name} must be a JSON object" + return parsed + + +def _metadata_view(value: dict[str, Any]) -> dict[str, Any]: + metadata = value.get("metadata") + if isinstance(metadata, dict): + merged = dict(value) + merged.update(metadata) + return merged + return value + + +def _prediction_reasons(prediction: dict[str, Any]) -> list[str]: + reasons = prediction.get("failure_reasons") or prediction.get("reasons") or [] + if isinstance(reasons, list): + return [str(reason)[:300] for reason in reasons] + if isinstance(reasons, str): + return [reasons[:300]] + return [] + + +def _optional_bool(value: Any) -> bool | None: + if isinstance(value, bool): + return value + if value is None: + return None + if isinstance(value, str): + lowered = value.casefold() + if lowered == "true": + return True + if lowered == "false": + return False + return None + + +def _infer_cause( + skill_name: str, + before_context: str, + after_context: str, + prediction_reasons: list[str], + outcome_success: bool | None, + outcome_message: str, + outcome_error_code: str, +) -> tuple[str, str, str]: + text = " ".join([before_context, after_context, outcome_message]).casefold() + reasons = " ".join(prediction_reasons).casefold() + + if outcome_success is True: + return ("success_no_failure", "", "high") + + if "query is missing" in reasons or "descriptions are missing" in reasons: + return ( + "invalid_or_missing_arguments", + "Retry with complete, specific tool arguments.", + "high", + ) + if "failed repeatedly" in reasons: + return ( + "repeated_failure_pattern", + "Call summarize_causal_patterns before retrying this skill.", + "high", + ) + if "robot pose: unavailable" in text or "odometry may be unavailable" in reasons: + return ( + "robot_state_unavailable", + "Call get_context with focus='navigation' and wait for odometry.", + "high", + ) + if "no relevant matches" in text or "no matching location" in text: + return ( + "semantic_map_missing_target", + "Use perception, ask the user, or tag the location before navigating.", + "medium", + ) + if "spatial memory is unavailable" in reasons: + return ( + "world_state_unavailable", + "Wait for world-state modules or use a direct perception skill.", + "medium", + ) + if "could not find" in text or "no image available" in text: + return ( + "visual_target_unavailable", + "Call look_out_for or ask for a more specific visual description.", + "medium", + ) + if outcome_error_code: + return ( + f"tool_error:{outcome_error_code}", + "Inspect the tool error and gather context before retrying.", + "medium", + ) + if outcome_success is False: + return ( + "unknown_failure", + "Gather context and retry with more specific arguments.", + "low", + ) + return ("unknown_transition", "Record a clearer outcome before reasoning from it.", "low") + + +def _summarize_patterns(transitions: list[dict[str, Any]]) -> list[dict[str, Any]]: + failures = [ + transition + for transition in transitions + if transition.get("outcome_success") is False + and transition.get("inferred_cause") not in {"success_no_failure"} + ] + counts = Counter(str(transition.get("inferred_cause")) for transition in failures) + patterns: list[dict[str, Any]] = [] + for cause, count in counts.most_common(): + examples = [ + transition for transition in failures if transition.get("inferred_cause") == cause + ] + skills = sorted({str(transition.get("skill_name")) for transition in examples}) + recovery = str(examples[0].get("recovery") or "") + patterns.append( + { + "cause": cause, + "count": count, + "skill_names": skills, + "latest_message": str(examples[0].get("outcome_message") or ""), + "recovery": recovery, + } + ) + return patterns + + +def _infer_domain(skill_name: str) -> str: + if skill_name in {"navigate_with_text", "stop_navigation"}: + return "navigation" + if skill_name in {"relative_move", "execute_sport_command"}: + return "robot_motion" + if skill_name in {"follow_person", "stop_following"}: + return "person_follow" + if skill_name in {"look_out_for", "stop_looking_out"}: + return "perception" + if skill_name in {"start_security_patrol", "stop_security_patrol"}: + return "security" + if skill_name == "speak": + return "speech" + return "" + + +__all__ = ["CausalWorldModelSpec", "_Go2CausalWorldModel"] diff --git a/dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/context_provider.py b/dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/context_provider.py index c595962ff5..d24cba1317 100644 --- a/dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/context_provider.py +++ b/dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/context_provider.py @@ -30,6 +30,9 @@ from dimos.navigation.navigation_spec import NavigationInterfaceSpec from dimos.perception.spatial_memory_spec import SpatialMemorySpec from dimos.perception.temporal_memory_spec import TemporalMemorySpec +from dimos.robot.unitree.go2.blueprints.layers.layer_3_agent_brain.causal_world_model import ( + CausalWorldModelSpec, +) from dimos.robot.unitree.go2.blueprints.layers.layer_3_agent_brain.skill_outcome_store import ( SkillOutcomeStoreSpec, ) @@ -45,6 +48,7 @@ class _Go2ContextProvider(Module): _temporal_memory: TemporalMemorySpec | None = None _navigation: NavigationInterfaceSpec | None = None _skill_outcomes: SkillOutcomeStoreSpec | None = None + _causal_world_model: CausalWorldModelSpec | None = None _latest_odom: PoseStamped | None = None odom: In[PoseStamped] @@ -86,6 +90,7 @@ def get_context(self, task: str, focus: str = "", spatial_limit: int = 3) -> Ski "robot_state": self._robot_context(errors), "world_state": self._world_context(task, spatial_limit, errors), "skill_state": self._skill_context(errors), + "causal_state": self._causal_context(errors), "external_context": { "available": False, "reason": "External context sources are not wired into ContextProvider yet.", @@ -105,6 +110,7 @@ def _source_status(self) -> dict[str, bool]: "odom": self._latest_odom is not None, "navigation": self._navigation is not None, "skill_outcomes": self._skill_outcomes is not None, + "causal_world_model": self._causal_world_model is not None, "runtime": True, } @@ -160,6 +166,17 @@ def _skill_context(self, errors: list[str]) -> dict[str, Any]: return {"available": True, "recent_outcomes": []} return {"available": True, "recent_outcomes": recent} + def _causal_context(self, errors: list[str]) -> dict[str, Any]: + if self._causal_world_model is None: + return {"available": False, "recent_transitions": []} + try: + recent = self._causal_world_model.get_recent_transitions(limit=5) + except Exception as exc: + logger.warning("Failed to read causal-world context", exc_info=True) + errors.append(f"causal_world_model: {exc}") + return {"available": True, "recent_transitions": []} + return {"available": True, "recent_transitions": recent} + def _spatial_context( self, task: str, spatial_limit: int, errors: list[str] ) -> dict[str, Any]: @@ -277,6 +294,23 @@ def _format_message(self, metadata: dict[str, Any]) -> str: else: lines.append("Skill outcomes: unavailable") + causal_state = metadata.get("causal_state", {}) + recent_transitions = causal_state.get("recent_transitions") or [] + if recent_transitions: + failures = [ + transition + for transition in recent_transitions + if transition.get("outcome_success") is False + ] + lines.append( + f"Causal memory: {len(recent_transitions)} recent, " + f"{len(failures)} failure transition(s)" + ) + elif causal_state.get("available"): + lines.append("Causal memory: available, no recorded transitions") + else: + lines.append("Causal memory: unavailable") + return "\n".join(lines) diff --git a/dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/prompt_policy.py b/dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/prompt_policy.py new file mode 100644 index 0000000000..fcd70a627a --- /dev/null +++ b/dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/prompt_policy.py @@ -0,0 +1,61 @@ +#!/usr/bin/env python3 +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import annotations + +from dimos.agents.system_prompt import SYSTEM_PROMPT + +_POLICY_HEADER = "# LAYER 3 DECISION POLICY" + +_GO2_LAYER_3_DECISION_POLICY = f""" +{_POLICY_HEADER} + +Use the Layer 3 tools as the decision scaffold before calling physical Go2 +skills. + +1. For any non-trivial user goal, call `route_task(task)` first. +2. If the route says context is needed, or if the task may involve movement, + perception, memory, safety, or recovery, call `get_context(task, focus)`. +3. Before calling movement-sensitive or recovery-sensitive tools such as + `navigate_with_text`, `relative_move`, `follow_person`, `look_out_for`, or + security patrol tools, call `predict_skill_outcome(skill_name, args_json, + context)`. +4. If prediction returns high risk, gather more context, ask the user for + clarification, or choose a safer recovery tool instead of blindly retrying. +5. Skill outcomes are recorded automatically when the outcome store is present. + Only call `record_skill_outcome(...)` manually for external events or + important results that were not produced by an MCP tool call. +6. After important physical or recovery-sensitive tool calls, call + `get_context(task, focus)` again for after-context, then call + `record_causal_transition(...)` with the before-context, prediction, tool + result, and after-context. +7. Before retrying a failed skill, call `summarize_causal_patterns(...)` to + check whether the same failure cause is repeating. + +Do not use Layer 3 tools as a substitute for physical safety checks. Stop or +cancel active motion immediately when the user asks for it or when the task is +unsafe. +""".strip() + + +def _go2_layer_3_system_prompt(base_prompt: str | None = None) -> str: + """Append the Go2 Layer 3 decision policy to an MCP client prompt.""" + prompt = (base_prompt or SYSTEM_PROMPT).rstrip() + if _POLICY_HEADER in prompt: + return prompt + return f"{prompt}\n\n{_GO2_LAYER_3_DECISION_POLICY}" + + +__all__ = ["_go2_layer_3_system_prompt"] diff --git a/dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/skill_outcome_predictor.py b/dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/skill_outcome_predictor.py index 1d8c197134..8bc6d30b61 100644 --- a/dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/skill_outcome_predictor.py +++ b/dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/skill_outcome_predictor.py @@ -21,6 +21,9 @@ from dimos.agents.annotation import skill from dimos.agents.skill_result import SkillResult from dimos.core.module import Module +from dimos.robot.unitree.go2.blueprints.layers.layer_3_agent_brain.causal_world_model import ( + CausalWorldModelSpec, +) from dimos.robot.unitree.go2.blueprints.layers.layer_3_agent_brain.skill_outcome_store import ( SkillOutcomeStoreSpec, ) @@ -44,6 +47,7 @@ class _Go2SkillOutcomePredictor(Module): # autoconnect wires this ref to _Go2SkillOutcomeStore. If not present, the # predictor still works using only args_json and context. _skill_outcomes: SkillOutcomeStoreSpec | None = None + _causal_world_model: CausalWorldModelSpec | None = None @skill def predict_skill_outcome( @@ -77,8 +81,10 @@ def predict_skill_outcome( # 2. parsed_args: planned tool arguments. # 3. context: text summary from get_context/route_task/dialogue. # 4. recent: same-skill outcomes from SkillOutcomeStore. + # 5. causal_recent: same-skill transitions from CausalWorldModel. recent = self._recent_outcomes(skill_name) - reasons = _risk_reasons(skill_name, parsed_args, context, recent) + causal_recent = self._recent_causal_transitions(skill_name) + reasons = _risk_reasons(skill_name, parsed_args, context, recent, causal_recent) risk = _risk_level(reasons) predicted_success = risk != "high" suggestions = _recovery_suggestions(skill_name, reasons) @@ -91,7 +97,9 @@ def predict_skill_outcome( "failure_reasons": reasons, "recovery_suggestions": suggestions, "recent_outcomes": recent, + "recent_causal_transitions": causal_recent, "outcome_store_available": self._skill_outcomes is not None, + "causal_world_model_available": self._causal_world_model is not None, } message = ( f"Predicted risk={risk}, predicted_success={predicted_success}. " @@ -105,6 +113,12 @@ def _recent_outcomes(self, skill_name: str) -> list[dict[str, Any]]: return [] return self._skill_outcomes.get_recent_outcomes(limit=5, skill_name=skill_name) + def _recent_causal_transitions(self, skill_name: str) -> list[dict[str, Any]]: + """Fetch recent same-skill causal transitions if the model is wired.""" + if self._causal_world_model is None: + return [] + return self._causal_world_model.get_recent_transitions(limit=5, skill_name=skill_name) + def _parse_args_json(args_json: str) -> dict[str, Any] | str: """Parse the planned skill arguments. @@ -129,6 +143,7 @@ def _risk_reasons( args: dict[str, Any], context: str, recent: list[dict[str, Any]], + causal_recent: list[dict[str, Any]], ) -> list[str]: """Collect human-readable risk reasons. @@ -151,6 +166,19 @@ def _risk_reasons( elif len(failures) == 1: reasons.append("same skill has one recent failure") + causal_failures = [ + transition + for transition in causal_recent + if transition.get("outcome_success") is False + and transition.get("inferred_cause") not in {"success_no_failure"} + ] + repeated_cause = _repeated_causal_cause(causal_failures) + if repeated_cause: + reasons.append(f"same skill repeatedly failed from causal cause: {repeated_cause}") + elif causal_failures: + cause = causal_failures[0].get("inferred_cause") or "unknown_failure" + reasons.append(f"same skill has a recent causal failure: {cause}") + if name in {"navigate_with_text", "relative_move", "follow_person"}: # ContextProvider formats missing odom as "Robot pose: unavailable". # Some structured summaries may instead include an odom=false-style @@ -208,6 +236,7 @@ def _risk_level(reasons: list[str]) -> str: """ high_markers = ( "failed repeatedly", + "repeatedly failed from causal cause", "navigation query is missing", "person-follow query is missing", "descriptions are missing", @@ -219,6 +248,18 @@ def _risk_level(reasons: list[str]) -> str: return "low" +def _repeated_causal_cause(transitions: list[dict[str, Any]]) -> str: + counts: dict[str, int] = {} + for transition in transitions: + cause = str(transition.get("inferred_cause") or "") + if not cause: + continue + counts[cause] = counts.get(cause, 0) + 1 + if counts[cause] >= 2: + return cause + return "" + + def _recovery_suggestions(skill_name: str, reasons: list[str]) -> list[str]: """Translate risk reasons into actionable next steps for the LLM agent.""" if not reasons: @@ -234,6 +275,8 @@ def _recovery_suggestions(skill_name: str, reasons: list[str]) -> list[str]: "call stop_navigation before retrying if navigation is already active", ] ) + if any("causal" in reason for reason in reasons): + suggestions.append("call summarize_causal_patterns before retrying") if "follow" in name: suggestions.extend( [ diff --git a/dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/skill_outcome_store.py b/dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/skill_outcome_store.py index c5b970f0a3..b6bb095c1c 100644 --- a/dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/skill_outcome_store.py +++ b/dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/skill_outcome_store.py @@ -88,9 +88,10 @@ class _Go2SkillOutcomeStore(Module): before the outcome payload stabilizes. How data is written: - The agent can call the MCP skill ``record_skill_outcome(...)`` after an - important tool call. Later we can add automatic capture in the MCP/tool - wrapper, but manual recording is safer for the first version. + McpClient automatically calls ``record_skill_outcome(...)`` after + non-internal tool calls when this store is present in the MCP tool + registry. The skill remains available for manual recording of external + events or outcomes that did not flow through the agent's MCP tool path. """ _max_outcomes = 100 diff --git a/dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/test_causal_world_model.py b/dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/test_causal_world_model.py new file mode 100644 index 0000000000..e63b195fc6 --- /dev/null +++ b/dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/test_causal_world_model.py @@ -0,0 +1,124 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from dimos.robot.unitree.go2.blueprints.layers.layer_3_agent_brain.causal_world_model import ( + _Go2CausalWorldModel, +) +from dimos.robot.unitree.go2.blueprints.layers.layer_3_agent_brain.skill_outcome_predictor import ( + _Go2SkillOutcomePredictor, +) +from dimos.robot.unitree.go2.blueprints.layers.layer_3_agent_brain.skill_outcome_store import ( + _Go2SkillOutcomeStore, +) + + +def test_record_causal_transition_infers_semantic_map_failure() -> None: + model = _Go2CausalWorldModel() + + result = model.record_causal_transition( + task="go to the kitchen", + skill_name="navigate_with_text", + args_json='{"query": "kitchen"}', + before_context="Spatial memory: available, no relevant matches", + prediction_json='{"risk": "medium", "failure_reasons": []}', + outcome_json='{"success": false, "message": "No matching location found"}', + ) + + assert result.success is True + transition = model.get_recent_transitions(limit=1)[0] + assert transition["skill_name"] == "navigate_with_text" + assert transition["outcome_success"] is False + assert transition["inferred_cause"] == "semantic_map_missing_target" + assert transition["domain"] == "navigation" + + +def test_summarize_causal_patterns_counts_repeated_failures() -> None: + model = _Go2CausalWorldModel() + + for target in ("kitchen", "office"): + model.record_causal_transition( + task=f"go to the {target}", + skill_name="navigate_with_text", + args_json=f'{{"query": "{target}"}}', + before_context="Spatial memory: available, no relevant matches", + outcome_json='{"success": false, "message": "No matching location found"}', + ) + + summary = model.summarize_causal_patterns(skill_name="navigate_with_text") + + assert summary.success is True + assert summary.metadata["patterns"][0]["cause"] == "semantic_map_missing_target" + assert summary.metadata["patterns"][0]["count"] == 2 + + +def test_record_causal_transition_can_use_latest_skill_outcome() -> None: + store = _Go2SkillOutcomeStore() + store.record_skill_outcome( + skill_name="navigate_with_text", + success=False, + domain="navigation", + message="No matching location found", + ) + + model = _Go2CausalWorldModel() + model._skill_outcomes = store + + result = model.record_causal_transition( + task="go to the kitchen", + skill_name="navigate_with_text", + args_json='{"query": "kitchen"}', + ) + + assert result.success is True + transition = model.get_recent_transitions(limit=1)[0] + assert transition["outcome_message"] == "No matching location found" + assert transition["inferred_cause"] == "semantic_map_missing_target" + + +def test_record_causal_transition_rejects_invalid_json() -> None: + model = _Go2CausalWorldModel() + + result = model.record_causal_transition( + task="go to the kitchen", + skill_name="navigate_with_text", + args_json="[1, 2]", + ) + + assert result.success is False + assert result.error_code == "INVALID_INPUT" + + +def test_predictor_uses_repeated_causal_failures() -> None: + model = _Go2CausalWorldModel() + for target in ("kitchen", "office"): + model.record_causal_transition( + task=f"go to the {target}", + skill_name="navigate_with_text", + args_json=f'{{"query": "{target}"}}', + outcome_json='{"success": false, "message": "No matching location found"}', + ) + + predictor = _Go2SkillOutcomePredictor() + predictor._causal_world_model = model + + result = predictor.predict_skill_outcome( + "navigate_with_text", + args_json='{"query": "kitchen"}', + context="Robot pose: x=1.0, y=2.0", + ) + + assert result.success is True + assert result.metadata["risk"] == "high" + assert result.metadata["predicted_success"] is False + assert any("causal cause" in reason for reason in result.metadata["failure_reasons"]) diff --git a/dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/test_context_provider.py b/dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/test_context_provider.py index 7753e7b3b1..b56e2d6b82 100644 --- a/dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/test_context_provider.py +++ b/dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/test_context_provider.py @@ -71,12 +71,31 @@ def get_recent_outcomes( ][:limit] +class StubCausalWorldModel: + def get_recent_transitions( + self, + limit: int = 5, + skill_name: str = "", + domain: str = "", + cause: str = "", + ) -> list[dict[str, Any]]: + return [ + { + "skill_name": "navigate_with_text", + "domain": "navigation", + "outcome_success": False, + "inferred_cause": "semantic_map_missing_target", + } + ][:limit] + + def test_get_context_aggregates_available_sources() -> None: provider = _Go2ContextProvider() provider._spatial_memory = StubSpatialMemory() # type: ignore[assignment] provider._temporal_memory = StubTemporalMemory() # type: ignore[assignment] provider._navigation = StubNavigation() # type: ignore[assignment] provider._skill_outcomes = StubSkillOutcomes() # type: ignore[assignment] + provider._causal_world_model = StubCausalWorldModel() # type: ignore[assignment] provider._latest_odom = PoseStamped(position=[1.0, 2.0, 0.0], frame_id="map") result = provider.get_context("find the person", focus="navigation") @@ -89,6 +108,7 @@ def test_get_context_aggregates_available_sources() -> None: assert result.metadata["world_state"]["spatial"]["matches"][0]["distance"] == 0.12 assert result.metadata["world_state"]["temporal"]["rolling_summary"] assert result.metadata["skill_state"]["recent_outcomes"][0]["success"] is False + assert result.metadata["causal_state"]["recent_transitions"][0]["outcome_success"] is False encoded = json.loads(result.agent_encode()[0]["text"]) assert encoded["success"] is True diff --git a/dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/test_prompt_policy.py b/dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/test_prompt_policy.py new file mode 100644 index 0000000000..170a85cfb3 --- /dev/null +++ b/dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/test_prompt_policy.py @@ -0,0 +1,37 @@ +#!/usr/bin/env python3 +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from dimos.robot.unitree.go2.blueprints.layers.layer_3_agent_brain.prompt_policy import ( + _go2_layer_3_system_prompt, +) + + +def test_go2_layer_3_system_prompt_appends_decision_policy() -> None: + prompt = _go2_layer_3_system_prompt("Base prompt.") + + assert "Base prompt." in prompt + assert "# LAYER 3 DECISION POLICY" in prompt + assert "route_task(task)" in prompt + assert "get_context(task, focus)" in prompt + assert "predict_skill_outcome" in prompt + assert "recorded automatically" in prompt + assert "record_causal_transition" in prompt + assert "summarize_causal_patterns" in prompt + + +def test_go2_layer_3_system_prompt_is_idempotent() -> None: + prompt = _go2_layer_3_system_prompt("Base prompt.") + + assert _go2_layer_3_system_prompt(prompt) == prompt From 367971e92341f3b11752c4032eab928f74636a35 Mon Sep 17 00:00:00 2001 From: z1050297972-spec Date: Wed, 20 May 2026 15:00:01 +0800 Subject: [PATCH 05/36] feat(go2): add layer 4 world state --- dimos/robot/unitree/go2/PROJECT_CHANGELOG.md | 38 +++ .../unitree/go2/blueprints/layers/GOAL_1.md | 26 +- .../layer_3_agent_brain/context_provider.py | 69 ++++- .../test_causal_world_model.py | 163 +++++----- .../test_context_provider.py | 140 +++++++-- .../layer_3_agent_brain/test_expert_router.py | 59 ++-- .../test_skill_outcomes.py | 98 +++--- .../layers/layer_4_world_state/DESIGN.md | 126 ++++++++ .../layers/layer_4_world_state/__init__.py | 8 + .../semantic_temporal_map.py | 160 ++++++++++ .../structured_world_state.py | 278 ++++++++++++++++++ .../layer_4_world_state/test_world_state.py | 127 ++++++++ .../layer_4_world_state/world_state_spec.py | 33 +++ 13 files changed, 1145 insertions(+), 180 deletions(-) create mode 100644 dimos/robot/unitree/go2/blueprints/layers/layer_4_world_state/DESIGN.md create mode 100644 dimos/robot/unitree/go2/blueprints/layers/layer_4_world_state/semantic_temporal_map.py create mode 100644 dimos/robot/unitree/go2/blueprints/layers/layer_4_world_state/structured_world_state.py create mode 100644 dimos/robot/unitree/go2/blueprints/layers/layer_4_world_state/test_world_state.py create mode 100644 dimos/robot/unitree/go2/blueprints/layers/layer_4_world_state/world_state_spec.py diff --git a/dimos/robot/unitree/go2/PROJECT_CHANGELOG.md b/dimos/robot/unitree/go2/PROJECT_CHANGELOG.md index a0c6204396..2922126469 100644 --- a/dimos/robot/unitree/go2/PROJECT_CHANGELOG.md +++ b/dimos/robot/unitree/go2/PROJECT_CHANGELOG.md @@ -6,6 +6,44 @@ work so future contributors can understand why the fork differs from upstream. Entries are listed in reverse chronological order. +## 2026-05-20 + +- Branch: `refactor/go2-architecture-layers` +- Summary: Started Go2 Layer 4 construction by adding RPC-only + semantic-temporal memory and structured world-state facades, then connected + Layer 3 `ContextProvider` to prefer the Layer 4 world snapshot when present. +- Files/modules: + - `dimos/robot/unitree/go2/blueprints/layers/layer_4_world_state/semantic_temporal_map.py` + - `dimos/robot/unitree/go2/blueprints/layers/layer_4_world_state/structured_world_state.py` + - `dimos/robot/unitree/go2/blueprints/layers/layer_4_world_state/world_state_spec.py` + - `dimos/robot/unitree/go2/blueprints/layers/layer_4_world_state/test_world_state.py` + - `dimos/robot/unitree/go2/blueprints/layers/layer_4_world_state/DESIGN.md` + - `dimos/robot/unitree/go2/blueprints/layers/layer_4_world_state/__init__.py` + - `dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/test_causal_world_model.py` + - `dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/context_provider.py` + - `dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/test_context_provider.py` + - `dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/test_expert_router.py` + - `dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/test_skill_outcomes.py` + - `dimos/robot/unitree/go2/blueprints/layers/GOAL_1.md` + - `dimos/robot/unitree/go2/PROJECT_CHANGELOG.md` +- Validation: + - Ran `python -m py_compile` for the new Layer 4 modules, focused tests, and + changed Layer 3 ContextProvider files. + - Ran `git diff --check`. + - Ran static Layer 4 connectivity assertions confirming the path: + Go2 blueprint -> `_go2_spatial_world_state` -> `_Go2StructuredWorldState` + -> `_Go2SemanticTemporalMap` -> existing spatial/temporal memory, and + Layer 3 `ContextProvider` -> `WorldStateSpec.get_world_snapshot(...)`. + - Installed WSL user-level `uv` and WSL system build dependencies needed for + Python packages that compile extensions. + - Copied the working tree to `~/dimos-layer-test` in WSL and ran the focused + Layer 3 + Layer 4 test suite with a minimal pytest dependency set: + `21 passed, 1 warning`. +- Open items: + - Add safety and connection status to `robot_state`. + - Decide whether stable world snapshots should be persisted in a later Layer + 4 version. + ## 2026-05-19 - Branch: `refactor/go2-architecture-layers` diff --git a/dimos/robot/unitree/go2/blueprints/layers/GOAL_1.md b/dimos/robot/unitree/go2/blueprints/layers/GOAL_1.md index 9333267263..e6d5b73969 100644 --- a/dimos/robot/unitree/go2/blueprints/layers/GOAL_1.md +++ b/dimos/robot/unitree/go2/blueprints/layers/GOAL_1.md @@ -38,6 +38,11 @@ layers/ test_*.py # focused Layer 3 tests layer_4_world_state/ __init__.py # spatial/temporal memory blueprint pieces + structured_world_state.py # normalized Layer 4 world snapshot + semantic_temporal_map.py # spatial/temporal memory evidence view + world_state_spec.py # Layer 4 RPC specs + DESIGN.md # Layer 4 implementation logic + test_*.py # focused Layer 4 tests layer_5_skill_interface/ __init__.py # Go2 skill-interface blueprint pieces layer_6_robot_body/ @@ -68,9 +73,28 @@ aggregate and compress context from these sources: The current implementation exposes a compact MCP skill, `get_context(task: str, focus: str = "", spatial_limit: int = 3)`, returning -the minimum context needed for the agent's next decision. Planning and tool +the minimum context needed for the agent's next decision. It now prefers the +Layer 4 `WorldStateSpec.get_world_snapshot(...)` RPC when available, and keeps +direct spatial/temporal/odom reads as fallback behavior. Planning and tool execution remain with the agent and Layer 5 skills. +## Layer 4 Direction + +Layer 4 owns the structured robot/world-state view. It should normalize state +from memory, odom, navigation, runtime config, and later safety/connection +sources, then expose RPC methods that upper layers can read. + +Current Layer 4 modules: + +- `_Go2SemanticTemporalMap`: combines spatial-memory matches and temporal-memory + summaries/answers for one query. It does not create a new database. +- `_Go2StructuredWorldState`: returns one `get_world_snapshot(...)` payload with + `runtime`, `robot_state`, `memory_state`, and `semantic_temporal_map` fields. + +Layer 4 V1 is intentionally a facade over existing modules. The existing +`SpatialMemory` and lazy `TemporalMemory` implementations stay in their current +packages. + ## ExpertRouter Direction `ExpertRouter` belongs in Layer 3 because it routes an LLM task toward the diff --git a/dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/context_provider.py b/dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/context_provider.py index d24cba1317..1675f109f4 100644 --- a/dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/context_provider.py +++ b/dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/context_provider.py @@ -36,6 +36,9 @@ from dimos.robot.unitree.go2.blueprints.layers.layer_3_agent_brain.skill_outcome_store import ( SkillOutcomeStoreSpec, ) +from dimos.robot.unitree.go2.blueprints.layers.layer_4_world_state.world_state_spec import ( + WorldStateSpec, +) from dimos.utils.logging_config import setup_logger logger = setup_logger() @@ -44,6 +47,7 @@ class _Go2ContextProvider(Module): """Layer 3 context aggregator for the Go2 agent brain.""" + _world_state: WorldStateSpec | None = None _spatial_memory: SpatialMemorySpec | None = None _temporal_memory: TemporalMemorySpec | None = None _navigation: NavigationInterfaceSpec | None = None @@ -82,13 +86,14 @@ def get_context(self, task: str, focus: str = "", spatial_limit: int = 3) -> Ski spatial_limit = max(0, min(spatial_limit, 10)) errors: list[str] = [] + world_snapshot = self._layer4_snapshot(task, spatial_limit, errors) metadata: dict[str, Any] = { "task": task, "focus": focus, - "sources": self._source_status(), - "runtime": self._runtime_context(), - "robot_state": self._robot_context(errors), - "world_state": self._world_context(task, spatial_limit, errors), + "sources": self._source_status(world_snapshot), + "runtime": self._runtime_context(world_snapshot), + "robot_state": self._robot_context(errors, world_snapshot), + "world_state": self._world_context(task, spatial_limit, errors, world_snapshot), "skill_state": self._skill_context(errors), "causal_state": self._causal_context(errors), "external_context": { @@ -102,19 +107,38 @@ def get_context(self, task: str, focus: str = "", spatial_limit: int = 3) -> Ski message = self._format_message(metadata) return SkillResult(success=True, message=message, metadata=_to_jsonable(metadata)) - def _source_status(self) -> dict[str, bool]: + def _source_status(self, world_snapshot: dict[str, Any] | None) -> dict[str, bool]: + layer4_sources = (world_snapshot or {}).get("sources", {}) return { "task": True, - "spatial_memory": self._spatial_memory is not None, - "temporal_memory": self._temporal_memory is not None, - "odom": self._latest_odom is not None, - "navigation": self._navigation is not None, + "structured_world_state": self._world_state is not None, + "spatial_memory": self._spatial_memory is not None + or bool(layer4_sources.get("spatial_memory")), + "temporal_memory": self._temporal_memory is not None + or bool(layer4_sources.get("temporal_memory")), + "odom": self._latest_odom is not None or bool(layer4_sources.get("odom")), + "navigation": self._navigation is not None or bool(layer4_sources.get("navigation")), "skill_outcomes": self._skill_outcomes is not None, "causal_world_model": self._causal_world_model is not None, "runtime": True, } - def _runtime_context(self) -> dict[str, Any]: + def _layer4_snapshot( + self, task: str, spatial_limit: int, errors: list[str] + ) -> dict[str, Any] | None: + if self._world_state is None: + return None + try: + return self._world_state.get_world_snapshot(task=task, spatial_limit=spatial_limit) + except Exception as exc: + logger.warning("Failed to read Layer 4 world state", exc_info=True) + errors.append(f"world_state: {exc}") + return None + + def _runtime_context(self, world_snapshot: dict[str, Any] | None) -> dict[str, Any]: + if world_snapshot is not None and isinstance(world_snapshot.get("runtime"), dict): + return world_snapshot["runtime"] + simulation = bool(getattr(global_config, "simulation", False)) replay = bool(getattr(global_config, "replay", False)) mode = "simulation" if simulation else "replay" if replay else "hardware" @@ -128,7 +152,12 @@ def _runtime_context(self) -> dict[str, Any]: "n_workers": getattr(global_config, "n_workers", None), } - def _robot_context(self, errors: list[str]) -> dict[str, Any]: + def _robot_context( + self, errors: list[str], world_snapshot: dict[str, Any] | None + ) -> dict[str, Any]: + if world_snapshot is not None and isinstance(world_snapshot.get("robot_state"), dict): + return world_snapshot["robot_state"] + context: dict[str, Any] = { "odom": self._pose_to_dict(self._latest_odom), "navigation": None, @@ -149,7 +178,23 @@ def _robot_context(self, errors: list[str]) -> dict[str, Any]: return context - def _world_context(self, task: str, spatial_limit: int, errors: list[str]) -> dict[str, Any]: + def _world_context( + self, + task: str, + spatial_limit: int, + errors: list[str], + world_snapshot: dict[str, Any] | None, + ) -> dict[str, Any]: + if world_snapshot is not None: + memory_state = world_snapshot.get("memory_state") or {} + return { + "source": "structured_world_state", + "spatial": memory_state.get("spatial", {"available": False, "matches": []}), + "temporal": memory_state.get("temporal", {"available": False}), + "semantic_temporal": world_snapshot.get("semantic_temporal_map", {}), + "snapshot": world_snapshot, + } + return { "spatial": self._spatial_context(task, spatial_limit, errors), "temporal": self._temporal_context(errors), diff --git a/dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/test_causal_world_model.py b/dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/test_causal_world_model.py index e63b195fc6..8154e97f7e 100644 --- a/dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/test_causal_world_model.py +++ b/dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/test_causal_world_model.py @@ -23,102 +23,121 @@ ) -def test_record_causal_transition_infers_semantic_map_failure() -> None: - model = _Go2CausalWorldModel() - - result = model.record_causal_transition( - task="go to the kitchen", - skill_name="navigate_with_text", - args_json='{"query": "kitchen"}', - before_context="Spatial memory: available, no relevant matches", - prediction_json='{"risk": "medium", "failure_reasons": []}', - outcome_json='{"success": false, "message": "No matching location found"}', - ) - - assert result.success is True - transition = model.get_recent_transitions(limit=1)[0] - assert transition["skill_name"] == "navigate_with_text" - assert transition["outcome_success"] is False - assert transition["inferred_cause"] == "semantic_map_missing_target" - assert transition["domain"] == "navigation" +def _stop_modules(*modules: object) -> None: + for module in modules: + stop = getattr(module, "stop", None) + if stop is not None: + stop() -def test_summarize_causal_patterns_counts_repeated_failures() -> None: +def test_record_causal_transition_infers_semantic_map_failure() -> None: model = _Go2CausalWorldModel() - - for target in ("kitchen", "office"): - model.record_causal_transition( - task=f"go to the {target}", + try: + result = model.record_causal_transition( + task="go to the kitchen", skill_name="navigate_with_text", - args_json=f'{{"query": "{target}"}}', + args_json='{"query": "kitchen"}', before_context="Spatial memory: available, no relevant matches", + prediction_json='{"risk": "medium", "failure_reasons": []}', outcome_json='{"success": false, "message": "No matching location found"}', ) - summary = model.summarize_causal_patterns(skill_name="navigate_with_text") + assert result.success is True + transition = model.get_recent_transitions(limit=1)[0] + assert transition["skill_name"] == "navigate_with_text" + assert transition["outcome_success"] is False + assert transition["inferred_cause"] == "semantic_map_missing_target" + assert transition["domain"] == "navigation" + finally: + _stop_modules(model) - assert summary.success is True - assert summary.metadata["patterns"][0]["cause"] == "semantic_map_missing_target" - assert summary.metadata["patterns"][0]["count"] == 2 + +def test_summarize_causal_patterns_counts_repeated_failures() -> None: + model = _Go2CausalWorldModel() + try: + for target in ("kitchen", "office"): + model.record_causal_transition( + task=f"go to the {target}", + skill_name="navigate_with_text", + args_json=f'{{"query": "{target}"}}', + before_context="Spatial memory: available, no relevant matches", + outcome_json='{"success": false, "message": "No matching location found"}', + ) + + summary = model.summarize_causal_patterns(skill_name="navigate_with_text") + + assert summary.success is True + assert summary.metadata["patterns"][0]["cause"] == "semantic_map_missing_target" + assert summary.metadata["patterns"][0]["count"] == 2 + finally: + _stop_modules(model) def test_record_causal_transition_can_use_latest_skill_outcome() -> None: store = _Go2SkillOutcomeStore() - store.record_skill_outcome( - skill_name="navigate_with_text", - success=False, - domain="navigation", - message="No matching location found", - ) - model = _Go2CausalWorldModel() - model._skill_outcomes = store + try: + store.record_skill_outcome( + skill_name="navigate_with_text", + success=False, + domain="navigation", + message="No matching location found", + ) + + model._skill_outcomes = store - result = model.record_causal_transition( - task="go to the kitchen", - skill_name="navigate_with_text", - args_json='{"query": "kitchen"}', - ) + result = model.record_causal_transition( + task="go to the kitchen", + skill_name="navigate_with_text", + args_json='{"query": "kitchen"}', + ) - assert result.success is True - transition = model.get_recent_transitions(limit=1)[0] - assert transition["outcome_message"] == "No matching location found" - assert transition["inferred_cause"] == "semantic_map_missing_target" + assert result.success is True + transition = model.get_recent_transitions(limit=1)[0] + assert transition["outcome_message"] == "No matching location found" + assert transition["inferred_cause"] == "semantic_map_missing_target" + finally: + _stop_modules(model, store) def test_record_causal_transition_rejects_invalid_json() -> None: model = _Go2CausalWorldModel() + try: + result = model.record_causal_transition( + task="go to the kitchen", + skill_name="navigate_with_text", + args_json="[1, 2]", + ) - result = model.record_causal_transition( - task="go to the kitchen", - skill_name="navigate_with_text", - args_json="[1, 2]", - ) - - assert result.success is False - assert result.error_code == "INVALID_INPUT" + assert result.success is False + assert result.error_code == "INVALID_INPUT" + finally: + _stop_modules(model) def test_predictor_uses_repeated_causal_failures() -> None: model = _Go2CausalWorldModel() - for target in ("kitchen", "office"): - model.record_causal_transition( - task=f"go to the {target}", - skill_name="navigate_with_text", - args_json=f'{{"query": "{target}"}}', - outcome_json='{"success": false, "message": "No matching location found"}', + predictor = _Go2SkillOutcomePredictor() + try: + for target in ("kitchen", "office"): + model.record_causal_transition( + task=f"go to the {target}", + skill_name="navigate_with_text", + args_json=f'{{"query": "{target}"}}', + outcome_json='{"success": false, "message": "No matching location found"}', + ) + + predictor._causal_world_model = model + + result = predictor.predict_skill_outcome( + "navigate_with_text", + args_json='{"query": "kitchen"}', + context="Robot pose: x=1.0, y=2.0", ) - predictor = _Go2SkillOutcomePredictor() - predictor._causal_world_model = model - - result = predictor.predict_skill_outcome( - "navigate_with_text", - args_json='{"query": "kitchen"}', - context="Robot pose: x=1.0, y=2.0", - ) - - assert result.success is True - assert result.metadata["risk"] == "high" - assert result.metadata["predicted_success"] is False - assert any("causal cause" in reason for reason in result.metadata["failure_reasons"]) + assert result.success is True + assert result.metadata["risk"] == "high" + assert result.metadata["predicted_success"] is False + assert any("causal cause" in reason for reason in result.metadata["failure_reasons"]) + finally: + _stop_modules(predictor, model) diff --git a/dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/test_context_provider.py b/dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/test_context_provider.py index b56e2d6b82..b4327ae535 100644 --- a/dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/test_context_provider.py +++ b/dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/test_context_provider.py @@ -22,6 +22,13 @@ ) +def _stop_modules(*modules: object) -> None: + for module in modules: + stop = getattr(module, "stop", None) + if stop is not None: + stop() + + class StubSpatialMemory: def query_by_text(self, text: str, limit: int = 5) -> list[dict[str, Any]]: return [ @@ -89,48 +96,117 @@ def get_recent_transitions( ][:limit] +class StubWorldState: + def get_world_snapshot(self, task: str = "", spatial_limit: int = 3) -> dict[str, Any]: + return { + "task": task, + "sources": { + "odom": True, + "navigation": True, + "spatial_memory": True, + "temporal_memory": True, + "semantic_temporal_map": True, + "runtime": True, + }, + "runtime": {"mode": "replay", "simulation": False, "replay": True}, + "robot_state": { + "odom": { + "frame_id": "map", + "timestamp": 1.0, + "position": {"x": 3.0, "y": 4.0, "z": 0.0}, + "yaw_degrees": 0.0, + }, + "navigation": {"state": "following_path", "goal_reached": False}, + }, + "memory_state": { + "spatial": {"available": True, "matches": [{"distance": 0.3}]}, + "temporal": { + "available": True, + "rolling_summary": "The hallway was recently observed.", + }, + }, + "semantic_temporal_map": { + "fused": {"available": True, "spatial_match_count": 1} + }, + } + + def get_robot_state(self) -> dict[str, Any]: + return {} + + def get_runtime_state(self) -> dict[str, Any]: + return {} + + def get_memory_state(self, task: str = "", spatial_limit: int = 3) -> dict[str, Any]: + return {} + + def test_get_context_aggregates_available_sources() -> None: provider = _Go2ContextProvider() - provider._spatial_memory = StubSpatialMemory() # type: ignore[assignment] - provider._temporal_memory = StubTemporalMemory() # type: ignore[assignment] - provider._navigation = StubNavigation() # type: ignore[assignment] - provider._skill_outcomes = StubSkillOutcomes() # type: ignore[assignment] - provider._causal_world_model = StubCausalWorldModel() # type: ignore[assignment] - provider._latest_odom = PoseStamped(position=[1.0, 2.0, 0.0], frame_id="map") - - result = provider.get_context("find the person", focus="navigation") - - assert result.success is True - assert "Task: find the person" in result.message - assert result.metadata["sources"]["spatial_memory"] is True - assert result.metadata["sources"]["temporal_memory"] is True - assert result.metadata["robot_state"]["navigation"]["state"] == "following_path" - assert result.metadata["world_state"]["spatial"]["matches"][0]["distance"] == 0.12 - assert result.metadata["world_state"]["temporal"]["rolling_summary"] - assert result.metadata["skill_state"]["recent_outcomes"][0]["success"] is False - assert result.metadata["causal_state"]["recent_transitions"][0]["outcome_success"] is False - - encoded = json.loads(result.agent_encode()[0]["text"]) - assert encoded["success"] is True - assert encoded["metadata"]["task"] == "find the person" + try: + provider._spatial_memory = StubSpatialMemory() # type: ignore[assignment] + provider._temporal_memory = StubTemporalMemory() # type: ignore[assignment] + provider._navigation = StubNavigation() # type: ignore[assignment] + provider._skill_outcomes = StubSkillOutcomes() # type: ignore[assignment] + provider._causal_world_model = StubCausalWorldModel() # type: ignore[assignment] + provider._latest_odom = PoseStamped(position=[1.0, 2.0, 0.0], frame_id="map") + + result = provider.get_context("find the person", focus="navigation") + + assert result.success is True + assert "Task: find the person" in result.message + assert result.metadata["sources"]["spatial_memory"] is True + assert result.metadata["sources"]["temporal_memory"] is True + assert result.metadata["robot_state"]["navigation"]["state"] == "following_path" + assert result.metadata["world_state"]["spatial"]["matches"][0]["distance"] == 0.12 + assert result.metadata["world_state"]["temporal"]["rolling_summary"] + assert result.metadata["skill_state"]["recent_outcomes"][0]["success"] is False + assert result.metadata["causal_state"]["recent_transitions"][0]["outcome_success"] is False + + encoded = json.loads(result.agent_encode()[0]["text"]) + assert encoded["success"] is True + assert encoded["metadata"]["task"] == "find the person" + finally: + _stop_modules(provider) def test_get_context_handles_missing_optional_sources() -> None: provider = _Go2ContextProvider() + try: + result = provider.get_context("walk forward") - result = provider.get_context("walk forward") - - assert result.success is True - assert result.metadata["sources"]["spatial_memory"] is False - assert result.metadata["sources"]["temporal_memory"] is False - assert result.metadata["robot_state"]["odom"] is None - assert "Spatial memory: unavailable" in result.message + assert result.success is True + assert result.metadata["sources"]["spatial_memory"] is False + assert result.metadata["sources"]["temporal_memory"] is False + assert result.metadata["robot_state"]["odom"] is None + assert "Spatial memory: unavailable" in result.message + finally: + _stop_modules(provider) def test_get_context_requires_task() -> None: provider = _Go2ContextProvider() + try: + result = provider.get_context(" ") + + assert result.success is False + assert result.error_code == "INVALID_INPUT" + finally: + _stop_modules(provider) - result = provider.get_context(" ") - assert result.success is False - assert result.error_code == "INVALID_INPUT" +def test_get_context_prefers_layer4_structured_world_state() -> None: + provider = _Go2ContextProvider() + try: + provider._world_state = StubWorldState() # type: ignore[assignment] + + result = provider.get_context("find the hallway") + + assert result.success is True + assert result.metadata["sources"]["structured_world_state"] is True + assert result.metadata["sources"]["spatial_memory"] is True + assert result.metadata["runtime"]["mode"] == "replay" + assert result.metadata["robot_state"]["odom"]["position"]["x"] == 3.0 + assert result.metadata["world_state"]["source"] == "structured_world_state" + assert result.metadata["world_state"]["spatial"]["matches"][0]["distance"] == 0.3 + finally: + _stop_modules(provider) diff --git a/dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/test_expert_router.py b/dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/test_expert_router.py index f86e389151..98e76a5d2a 100644 --- a/dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/test_expert_router.py +++ b/dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/test_expert_router.py @@ -17,43 +17,58 @@ ) +def _stop_modules(*modules: object) -> None: + for module in modules: + stop = getattr(module, "stop", None) + if stop is not None: + stop() + + def test_route_task_recommends_navigation_with_context() -> None: router = _Go2ExpertRouter() + try: + result = router.route_task("go to the kitchen") - result = router.route_task("go to the kitchen") - - assert result.success is True - assert result.metadata["domain"] == "navigation" - assert result.metadata["needs_context"] is True - assert result.metadata["recommended_tools"][0] == "get_context" - assert "navigate_with_text" in result.metadata["recommended_tools"] + assert result.success is True + assert result.metadata["domain"] == "navigation" + assert result.metadata["needs_context"] is True + assert result.metadata["recommended_tools"][0] == "get_context" + assert "navigate_with_text" in result.metadata["recommended_tools"] + finally: + _stop_modules(router) def test_route_task_prioritizes_safety_without_context() -> None: router = _Go2ExpertRouter() + try: + result = router.route_task("stop following and cancel navigation") - result = router.route_task("stop following and cancel navigation") - - assert result.success is True - assert result.metadata["domain"] == "safety" - assert result.metadata["needs_context"] is False - assert "stop_navigation" in result.metadata["recommended_tools"] + assert result.success is True + assert result.metadata["domain"] == "safety" + assert result.metadata["needs_context"] is False + assert "stop_navigation" in result.metadata["recommended_tools"] + finally: + _stop_modules(router) def test_route_task_supports_chinese_person_follow() -> None: router = _Go2ExpertRouter() + try: + result = router.route_task("\u8ddf\u968f\u7a7f\u84dd\u8272\u8863\u670d\u7684\u4eba") - result = router.route_task("\u8ddf\u968f\u7a7f\u84dd\u8272\u8863\u670d\u7684\u4eba") - - assert result.success is True - assert result.metadata["domain"] == "person_follow" - assert "follow_person" in result.metadata["recommended_tools"] + assert result.success is True + assert result.metadata["domain"] == "person_follow" + assert "follow_person" in result.metadata["recommended_tools"] + finally: + _stop_modules(router) def test_route_task_requires_task() -> None: router = _Go2ExpertRouter() + try: + result = router.route_task(" ") - result = router.route_task(" ") - - assert result.success is False - assert result.error_code == "INVALID_INPUT" + assert result.success is False + assert result.error_code == "INVALID_INPUT" + finally: + _stop_modules(router) diff --git a/dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/test_skill_outcomes.py b/dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/test_skill_outcomes.py index c6a3663bba..a5df9cde8a 100644 --- a/dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/test_skill_outcomes.py +++ b/dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/test_skill_outcomes.py @@ -20,63 +20,79 @@ ) -def test_record_and_summarize_skill_outcomes() -> None: - store = _Go2SkillOutcomeStore() - - result = store.record_skill_outcome( - skill_name="navigate_with_text", - success=False, - domain="navigation", - error_code="EXECUTION_FAILED", - message="No matching location", - risk="high", - recovery="call get_context", - ) +def _stop_modules(*modules: object) -> None: + for module in modules: + stop = getattr(module, "stop", None) + if stop is not None: + stop() - assert result.success is True - outcomes = store.get_recent_outcomes(limit=5) - assert len(outcomes) == 1 - assert outcomes[0]["skill_name"] == "navigate_with_text" - assert outcomes[0]["success"] is False - summary = store.summarize_skill_outcomes(domain="navigation") - assert summary.success is True - assert summary.metadata["outcomes"][0]["error_code"] == "EXECUTION_FAILED" +def test_record_and_summarize_skill_outcomes() -> None: + store = _Go2SkillOutcomeStore() + try: + result = store.record_skill_outcome( + skill_name="navigate_with_text", + success=False, + domain="navigation", + error_code="EXECUTION_FAILED", + message="No matching location", + risk="high", + recovery="call get_context", + ) + + assert result.success is True + outcomes = store.get_recent_outcomes(limit=5) + assert len(outcomes) == 1 + assert outcomes[0]["skill_name"] == "navigate_with_text" + assert outcomes[0]["success"] is False + + summary = store.summarize_skill_outcomes(domain="navigation") + assert summary.success is True + assert summary.metadata["outcomes"][0]["error_code"] == "EXECUTION_FAILED" + finally: + _stop_modules(store) def test_predictor_uses_recent_failures() -> None: store = _Go2SkillOutcomeStore() - store.record_skill_outcome("navigate_with_text", False) - store.record_skill_outcome("navigate_with_text", False) - predictor = _Go2SkillOutcomePredictor() - predictor._skill_outcomes = store + try: + store.record_skill_outcome("navigate_with_text", False) + store.record_skill_outcome("navigate_with_text", False) + + predictor._skill_outcomes = store - result = predictor.predict_skill_outcome( - "navigate_with_text", - args_json='{"query": "kitchen"}', - context="Robot pose: x=1.0, y=2.0", - ) + result = predictor.predict_skill_outcome( + "navigate_with_text", + args_json='{"query": "kitchen"}', + context="Robot pose: x=1.0, y=2.0", + ) - assert result.success is True - assert result.metadata["risk"] == "high" - assert result.metadata["predicted_success"] is False - assert "same skill failed repeatedly in recent outcomes" in result.metadata["failure_reasons"] + assert result.success is True + assert result.metadata["risk"] == "high" + assert result.metadata["predicted_success"] is False + assert "same skill failed repeatedly in recent outcomes" in result.metadata["failure_reasons"] + finally: + _stop_modules(predictor, store) def test_predictor_rejects_invalid_args_json() -> None: predictor = _Go2SkillOutcomePredictor() + try: + result = predictor.predict_skill_outcome("navigate_with_text", args_json="[1, 2]") - result = predictor.predict_skill_outcome("navigate_with_text", args_json="[1, 2]") - - assert result.success is False - assert result.error_code == "INVALID_INPUT" + assert result.success is False + assert result.error_code == "INVALID_INPUT" + finally: + _stop_modules(predictor) def test_predictor_requires_skill_name() -> None: predictor = _Go2SkillOutcomePredictor() + try: + result = predictor.predict_skill_outcome(" ") - result = predictor.predict_skill_outcome(" ") - - assert result.success is False - assert result.error_code == "INVALID_INPUT" + assert result.success is False + assert result.error_code == "INVALID_INPUT" + finally: + _stop_modules(predictor) diff --git a/dimos/robot/unitree/go2/blueprints/layers/layer_4_world_state/DESIGN.md b/dimos/robot/unitree/go2/blueprints/layers/layer_4_world_state/DESIGN.md new file mode 100644 index 0000000000..29abdd557e --- /dev/null +++ b/dimos/robot/unitree/go2/blueprints/layers/layer_4_world_state/DESIGN.md @@ -0,0 +1,126 @@ +# Layer 4 World State Design + +This document explains the first Go2 Layer 4 implementation. Layer 4 owns the +robot-facing world-state view. Layer 3 may read and compress this state for an +LLM, but Layer 3 should not be the only place where spatial memory, temporal +memory, odom, navigation state, and runtime mode are normalized. + +## Current Scope + +Layer 4 currently contains: + +- Existing `SpatialMemory`: persistent visual/spatial memory backed by ChromaDB + and visual-memory files through the existing perception module. +- Existing lazy `TemporalMemory`: imported only by the temporal-memory + blueprint after CLI config is resolved. +- `_Go2SemanticTemporalMap`: a lightweight RPC view that combines spatial and + temporal memory evidence for one query. +- `_Go2StructuredWorldState`: a lightweight RPC facade that returns a single + structured world snapshot. + +## Runtime Flow + +```text +Layer 6 odom/navigation/perception + -> SpatialMemory / TemporalMemory + -> _Go2SemanticTemporalMap.query_semantic_temporal_map(...) + -> _Go2StructuredWorldState.get_world_snapshot(...) + -> Layer 3 ContextProvider.get_context(...) +``` + +`ContextProvider` now prefers `WorldStateSpec.get_world_snapshot(...)` when the +Layer 4 module is wired. Its older direct reads of spatial memory, temporal +memory, navigation, and odom remain as a fallback. + +## Implementation Notes + +### `_Go2SemanticTemporalMap.query_semantic_temporal_map(...)` + +- File: `layer_4_world_state/semantic_temporal_map.py` +- Entry point: RPC-only helper; not an MCP skill. +- Purpose: produce one memory-evidence payload from spatial and temporal + sources. It does not create a new store, plan actions, or call skills. +- Inputs: + - Direct arguments: `query`, `spatial_limit`. + - Optional injected Specs: `SpatialMemorySpec`, `TemporalMemorySpec`. +- Storage: + - No new database. + - No persistent state. + - Reads existing spatial/temporal memory modules only. +- Data read: + - Spatial memory: `query_by_text(query, limit=spatial_limit)`. + - Temporal memory: `query(query)`, `get_rolling_summary()`, `get_state()`, + and `get_entity_roster()`. +- Return shape: + - `query` + - `sources` + - `spatial` + - `temporal` + - `fused` + - optional `errors` +- Current limits: + - Fusion is deterministic metadata packaging. + - It does not resolve contradictions between spatial and temporal evidence. + +### `_Go2StructuredWorldState.get_world_snapshot(...)` + +- File: `layer_4_world_state/structured_world_state.py` +- Entry point: RPC-only helper; not an MCP skill. +- Purpose: provide one normalized Layer 4 snapshot for Layer 3 and future + callers. +- Inputs: + - Direct arguments: `task`, `spatial_limit`. + - Stream state: latest `odom` message cached by `_on_odom`. + - Optional injected Specs: `SemanticTemporalMapSpec`, + `NavigationInterfaceSpec`, `SpatialMemorySpec`, and `TemporalMemorySpec`. + - Runtime config: `global_config`. +- Storage: + - No new database. + - No persistent files. + - Keeps only latest odom in memory while the process is running. +- Return shape: + - `task` + - `sources` + - `runtime` + - `robot_state` + - `memory_state` + - `semantic_temporal_map` + - optional `errors` +- Current limits: + - Robot safety state and connection health are not normalized yet. + - The module currently reads navigation state by RPC when available. + +## Version Boundaries + +V1 implemented: + +- Keep existing `SpatialMemory` and `TemporalMemory` implementation files in + place. +- Add Layer 4 RPC facades for semantic-temporal memory and structured world + snapshots. +- Make Layer 3 `ContextProvider` prefer Layer 4 world snapshots while keeping + direct-read fallback behavior. + +V2 planned: + +- Add connection/safety/local-control status to `robot_state`. +- Add named object/location summaries to `memory_state`. +- Make semantic-temporal map entries more explicit: entity, location, time, + evidence source, and confidence. + +V3 planned: + +- Decide whether stable world snapshots should be written to a durable store + such as `memory2` SQLite, JSONL, or TemporalMemory. +- Add tests that build the full Go2 agentic blueprint and verify the injected + Layer 3 -> Layer 4 RPC path. + +## Design Rules + +- Layer 4 owns normalized state, not LLM prompting or tool choice. +- Layer 4 modules should be RPC helpers unless there is a clear reason to + expose a method directly as an MCP skill. +- Do not move existing perception or memory implementation files during this + stage. +- Update this document when Layer 4 storage, snapshot fields, or RPC behavior + changes. diff --git a/dimos/robot/unitree/go2/blueprints/layers/layer_4_world_state/__init__.py b/dimos/robot/unitree/go2/blueprints/layers/layer_4_world_state/__init__.py index ca26730882..d1dda8082a 100644 --- a/dimos/robot/unitree/go2/blueprints/layers/layer_4_world_state/__init__.py +++ b/dimos/robot/unitree/go2/blueprints/layers/layer_4_world_state/__init__.py @@ -15,9 +15,17 @@ from dimos.core.coordination.blueprints import Blueprint, autoconnect from dimos.perception.spatial_perception import SpatialMemory +from dimos.robot.unitree.go2.blueprints.layers.layer_4_world_state.semantic_temporal_map import ( + _Go2SemanticTemporalMap, +) +from dimos.robot.unitree.go2.blueprints.layers.layer_4_world_state.structured_world_state import ( + _Go2StructuredWorldState, +) _go2_spatial_world_state = autoconnect( SpatialMemory.blueprint(), + _Go2SemanticTemporalMap.blueprint(), + _Go2StructuredWorldState.blueprint(), ) diff --git a/dimos/robot/unitree/go2/blueprints/layers/layer_4_world_state/semantic_temporal_map.py b/dimos/robot/unitree/go2/blueprints/layers/layer_4_world_state/semantic_temporal_map.py new file mode 100644 index 0000000000..a44ff0001c --- /dev/null +++ b/dimos/robot/unitree/go2/blueprints/layers/layer_4_world_state/semantic_temporal_map.py @@ -0,0 +1,160 @@ +#!/usr/bin/env python3 +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import annotations + +from typing import Any + +from dimos.core.core import rpc +from dimos.core.module import Module +from dimos.perception.spatial_memory_spec import SpatialMemorySpec +from dimos.perception.temporal_memory_spec import TemporalMemorySpec +from dimos.utils.logging_config import setup_logger + +logger = setup_logger() + + +class _Go2SemanticTemporalMap(Module): + """Layer 4 semantic-temporal memory view for Go2. + + This module does not create a new database. It is a thin normalization layer + over the existing spatial and temporal memory modules, so callers can ask + one Layer 4 component for memory evidence instead of separately knowing + every underlying memory implementation. + """ + + _spatial_memory: SpatialMemorySpec | None = None + _temporal_memory: TemporalMemorySpec | None = None + + @rpc + def query_semantic_temporal_map( + self, query: str = "", spatial_limit: int = 3 + ) -> dict[str, Any]: + """Return spatial and temporal memory evidence for a task or query.""" + query = query.strip() + spatial_limit = max(0, min(spatial_limit, 10)) + errors: list[str] = [] + + result: dict[str, Any] = { + "query": query, + "sources": { + "spatial_memory": self._spatial_memory is not None, + "temporal_memory": self._temporal_memory is not None, + }, + "spatial": self._spatial_section(query, spatial_limit, errors), + "temporal": self._temporal_section(query, errors), + } + result["fused"] = self._fused_section(result) + if errors: + result["errors"] = errors + return _to_jsonable(result) + + def _spatial_section( + self, query: str, spatial_limit: int, errors: list[str] + ) -> dict[str, Any]: + if self._spatial_memory is None: + return {"available": False, "matches": []} + if not query: + return {"available": True, "matches": [], "reason": "query is empty"} + if spatial_limit == 0: + return {"available": True, "matches": [], "reason": "spatial_limit is 0"} + + try: + matches = self._spatial_memory.query_by_text(query, limit=spatial_limit) + except Exception as exc: + logger.warning("Failed to query spatial memory", exc_info=True) + errors.append(f"spatial_memory: {exc}") + return {"available": True, "matches": []} + + return { + "available": True, + "matches": [_summarize_spatial_match(match) for match in matches], + } + + def _temporal_section(self, query: str, errors: list[str]) -> dict[str, Any]: + if self._temporal_memory is None: + return {"available": False} + + context: dict[str, Any] = {"available": True} + if query: + try: + context["answer"] = self._temporal_memory.query(query) + except Exception as exc: + logger.warning("Failed to query temporal memory", exc_info=True) + errors.append(f"temporal_memory.query: {exc}") + + try: + context["rolling_summary"] = self._temporal_memory.get_rolling_summary() + except Exception as exc: + logger.warning("Failed to read temporal rolling summary", exc_info=True) + errors.append(f"temporal_memory.summary: {exc}") + + try: + context["state"] = self._temporal_memory.get_state() + except Exception as exc: + logger.warning("Failed to read temporal state", exc_info=True) + errors.append(f"temporal_memory.state: {exc}") + + try: + context["entity_roster"] = self._temporal_memory.get_entity_roster() + except Exception as exc: + logger.warning("Failed to read temporal entity roster", exc_info=True) + errors.append(f"temporal_memory.roster: {exc}") + + return context + + def _fused_section(self, result: dict[str, Any]) -> dict[str, Any]: + spatial = result["spatial"] + temporal = result["temporal"] + spatial_matches = spatial.get("matches") or [] + temporal_answer = temporal.get("answer") + temporal_summary = temporal.get("rolling_summary") + return { + "available": bool(spatial.get("available") or temporal.get("available")), + "spatial_match_count": len(spatial_matches), + "has_temporal_answer": bool(temporal_answer), + "has_temporal_summary": bool(temporal_summary), + } + + +def _summarize_spatial_match(match: dict[str, Any]) -> dict[str, Any]: + summary: dict[str, Any] = {} + for key in ("distance", "score", "id", "text"): + if key in match: + summary[key] = match[key] + if "metadata" in match: + summary["metadata"] = _to_jsonable(match["metadata"]) + if not summary: + summary["keys"] = sorted(match.keys()) + return _to_jsonable(summary) + + +def _to_jsonable(value: Any, max_string_length: int = 500) -> Any: + if value is None or isinstance(value, bool | int | float): + return value + if isinstance(value, str): + return value[:max_string_length] + if isinstance(value, dict): + return { + str(key): _to_jsonable(item, max_string_length) + for key, item in value.items() + if not str(key).startswith("_") + } + if isinstance(value, list | tuple): + return [_to_jsonable(item, max_string_length) for item in value[:10]] + return str(value)[:max_string_length] + + +__all__ = ["_Go2SemanticTemporalMap"] diff --git a/dimos/robot/unitree/go2/blueprints/layers/layer_4_world_state/structured_world_state.py b/dimos/robot/unitree/go2/blueprints/layers/layer_4_world_state/structured_world_state.py new file mode 100644 index 0000000000..2937cb0c9e --- /dev/null +++ b/dimos/robot/unitree/go2/blueprints/layers/layer_4_world_state/structured_world_state.py @@ -0,0 +1,278 @@ +#!/usr/bin/env python3 +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import annotations + +import math +from typing import Any + +from reactivex.disposable import Disposable + +from dimos.core.core import rpc +from dimos.core.global_config import global_config +from dimos.core.module import Module +from dimos.core.stream import In +from dimos.msgs.geometry_msgs.PoseStamped import PoseStamped +from dimos.navigation.navigation_spec import NavigationInterfaceSpec +from dimos.perception.spatial_memory_spec import SpatialMemorySpec +from dimos.perception.temporal_memory_spec import TemporalMemorySpec +from dimos.robot.unitree.go2.blueprints.layers.layer_4_world_state.world_state_spec import ( + SemanticTemporalMapSpec, +) +from dimos.utils.logging_config import setup_logger + +logger = setup_logger() + + +class _Go2StructuredWorldState(Module): + """Layer 4 structured world-state facade for Go2. + + Layer 3 should not need to know which low-level module owns odom, + navigation, spatial memory, or temporal memory. This module provides a + compact RPC snapshot with stable keys while keeping the underlying modules + unchanged. + """ + + _semantic_temporal_map: SemanticTemporalMapSpec | None = None + _spatial_memory: SpatialMemorySpec | None = None + _temporal_memory: TemporalMemorySpec | None = None + _navigation: NavigationInterfaceSpec | None = None + _latest_odom: PoseStamped | None = None + + odom: In[PoseStamped] + + @rpc + def start(self) -> None: + super().start() + self.register_disposable(Disposable(self.odom.subscribe(self._on_odom))) + + def _on_odom(self, odom: PoseStamped) -> None: + self._latest_odom = odom + + @rpc + def get_world_snapshot(self, task: str = "", spatial_limit: int = 3) -> dict[str, Any]: + """Return the current normalized Layer 4 snapshot.""" + task = task.strip() + spatial_limit = max(0, min(spatial_limit, 10)) + errors: list[str] = [] + semantic_temporal = self._semantic_temporal_section(task, spatial_limit, errors) + + snapshot: dict[str, Any] = { + "task": task, + "sources": self._source_status(semantic_temporal), + "runtime": self.get_runtime_state(), + "robot_state": self.get_robot_state(), + "memory_state": self._memory_state(task, spatial_limit, semantic_temporal, errors), + "semantic_temporal_map": semantic_temporal, + } + if errors: + snapshot["errors"] = errors + return _to_jsonable(snapshot) + + @rpc + def get_robot_state(self) -> dict[str, Any]: + """Return robot state owned or normalized by Layer 4.""" + state: dict[str, Any] = { + "odom": self._pose_to_dict(self._latest_odom), + "navigation": None, + } + + if self._navigation is None: + return state + + try: + navigation_state = self._navigation.get_state() + state["navigation"] = { + "state": getattr(navigation_state, "value", str(navigation_state)), + "goal_reached": self._navigation.is_goal_reached(), + } + except Exception as exc: + logger.warning("Failed to read navigation state", exc_info=True) + state["navigation"] = {"error": str(exc)} + + return _to_jsonable(state) + + @rpc + def get_runtime_state(self) -> dict[str, Any]: + """Return runtime mode and relevant blueprint config values.""" + simulation = bool(getattr(global_config, "simulation", False)) + replay = bool(getattr(global_config, "replay", False)) + mode = "simulation" if simulation else "replay" if replay else "hardware" + return { + "mode": mode, + "simulation": simulation, + "replay": replay, + "robot_ip": getattr(global_config, "robot_ip", None), + "viewer": getattr(global_config, "viewer", None), + "mcp_port": getattr(global_config, "mcp_port", None), + "n_workers": getattr(global_config, "n_workers", None), + } + + @rpc + def get_memory_state(self, task: str = "", spatial_limit: int = 3) -> dict[str, Any]: + """Return Layer 4 memory state without robot or runtime state.""" + task = task.strip() + spatial_limit = max(0, min(spatial_limit, 10)) + errors: list[str] = [] + semantic_temporal = self._semantic_temporal_section(task, spatial_limit, errors) + state = self._memory_state(task, spatial_limit, semantic_temporal, errors) + if errors: + state["errors"] = errors + return _to_jsonable(state) + + def _source_status(self, semantic_temporal: dict[str, Any]) -> dict[str, bool]: + semantic_sources = semantic_temporal.get("sources", {}) + return { + "odom": self._latest_odom is not None, + "navigation": self._navigation is not None, + "spatial_memory": self._spatial_memory is not None + or bool(semantic_sources.get("spatial_memory")), + "temporal_memory": self._temporal_memory is not None + or bool(semantic_sources.get("temporal_memory")), + "semantic_temporal_map": self._semantic_temporal_map is not None, + "runtime": True, + } + + def _semantic_temporal_section( + self, task: str, spatial_limit: int, errors: list[str] + ) -> dict[str, Any]: + if self._semantic_temporal_map is None: + return { + "available": False, + "query": task, + "spatial": self._spatial_fallback(task, spatial_limit, errors), + "temporal": self._temporal_fallback(errors), + "fused": {"available": False}, + } + + try: + return self._semantic_temporal_map.query_semantic_temporal_map( + query=task, + spatial_limit=spatial_limit, + ) + except Exception as exc: + logger.warning("Failed to query semantic-temporal map", exc_info=True) + errors.append(f"semantic_temporal_map: {exc}") + return { + "available": True, + "query": task, + "spatial": self._spatial_fallback(task, spatial_limit, errors), + "temporal": self._temporal_fallback(errors), + "fused": {"available": False}, + } + + def _memory_state( + self, + task: str, + spatial_limit: int, + semantic_temporal: dict[str, Any], + errors: list[str], + ) -> dict[str, Any]: + if semantic_temporal.get("spatial") or semantic_temporal.get("temporal"): + return { + "query": task, + "spatial": semantic_temporal.get("spatial", {"available": False, "matches": []}), + "temporal": semantic_temporal.get("temporal", {"available": False}), + } + + return { + "query": task, + "spatial": self._spatial_fallback(task, spatial_limit, errors), + "temporal": self._temporal_fallback(errors), + } + + def _spatial_fallback( + self, task: str, spatial_limit: int, errors: list[str] + ) -> dict[str, Any]: + if self._spatial_memory is None: + return {"available": False, "matches": []} + if not task or spatial_limit == 0: + return {"available": True, "matches": []} + + try: + matches = self._spatial_memory.query_by_text(task, limit=spatial_limit) + except Exception as exc: + logger.warning("Failed to query spatial memory", exc_info=True) + errors.append(f"spatial_memory: {exc}") + return {"available": True, "matches": []} + + return { + "available": True, + "matches": [_summarize_spatial_match(match) for match in matches], + } + + def _temporal_fallback(self, errors: list[str]) -> dict[str, Any]: + if self._temporal_memory is None: + return {"available": False} + + context: dict[str, Any] = {"available": True} + try: + context["rolling_summary"] = self._temporal_memory.get_rolling_summary() + except Exception as exc: + logger.warning("Failed to read temporal rolling summary", exc_info=True) + errors.append(f"temporal_memory.summary: {exc}") + + try: + context["state"] = self._temporal_memory.get_state() + except Exception as exc: + logger.warning("Failed to read temporal state", exc_info=True) + errors.append(f"temporal_memory.state: {exc}") + + return context + + def _pose_to_dict(self, pose: PoseStamped | None) -> dict[str, Any] | None: + if pose is None: + return None + return { + "frame_id": pose.frame_id, + "timestamp": pose.ts, + "position": { + "x": round(pose.position.x, 3), + "y": round(pose.position.y, 3), + "z": round(pose.position.z, 3), + }, + "yaw_degrees": round(math.degrees(pose.yaw), 1), + } + + +def _summarize_spatial_match(match: dict[str, Any]) -> dict[str, Any]: + summary: dict[str, Any] = {} + for key in ("distance", "score", "id", "text"): + if key in match: + summary[key] = match[key] + if "metadata" in match: + summary["metadata"] = _to_jsonable(match["metadata"]) + if not summary: + summary["keys"] = sorted(match.keys()) + return _to_jsonable(summary) + + +def _to_jsonable(value: Any, max_string_length: int = 500) -> Any: + if value is None or isinstance(value, bool | int | float): + return value + if isinstance(value, str): + return value[:max_string_length] + if isinstance(value, dict): + return { + str(key): _to_jsonable(item, max_string_length) + for key, item in value.items() + if not str(key).startswith("_") + } + if isinstance(value, list | tuple): + return [_to_jsonable(item, max_string_length) for item in value[:10]] + return str(value)[:max_string_length] + + +__all__ = ["_Go2StructuredWorldState"] diff --git a/dimos/robot/unitree/go2/blueprints/layers/layer_4_world_state/test_world_state.py b/dimos/robot/unitree/go2/blueprints/layers/layer_4_world_state/test_world_state.py new file mode 100644 index 0000000000..27e718083e --- /dev/null +++ b/dimos/robot/unitree/go2/blueprints/layers/layer_4_world_state/test_world_state.py @@ -0,0 +1,127 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import Any + +from dimos.msgs.geometry_msgs.PoseStamped import PoseStamped +from dimos.navigation.base import NavigationState +from dimos.robot.unitree.go2.blueprints.layers.layer_4_world_state.semantic_temporal_map import ( + _Go2SemanticTemporalMap, +) +from dimos.robot.unitree.go2.blueprints.layers.layer_4_world_state.structured_world_state import ( + _Go2StructuredWorldState, +) + + +def _stop_modules(*modules: object) -> None: + for module in modules: + stop = getattr(module, "stop", None) + if stop is not None: + stop() + + +class StubSpatialMemory: + def query_by_text(self, text: str, limit: int = 5) -> list[dict[str, Any]]: + return [ + { + "distance": 0.2, + "metadata": [{"pos_x": 1.0, "pos_y": 2.0, "label": text}], + } + ][:limit] + + +class StubTemporalMemory: + def query(self, question: str) -> str: + return f"temporal answer for {question}" + + def get_state(self) -> dict[str, Any]: + return {"entity_count": 1} + + def get_entity_roster(self) -> list[dict[str, Any]]: + return [{"id": "person_1"}] + + def get_rolling_summary(self) -> str: + return "A person was recently near the hallway." + + def get_graph_db_stats(self) -> dict[str, Any]: + return {"stats": {"entities": 1}} + + +class StubNavigation: + def get_state(self) -> NavigationState: + return NavigationState.FOLLOWING_PATH + + def is_goal_reached(self) -> bool: + return False + + +class StubSemanticTemporalMap: + def query_semantic_temporal_map( + self, query: str = "", spatial_limit: int = 3 + ) -> dict[str, Any]: + return { + "query": query, + "sources": {"spatial_memory": True, "temporal_memory": True}, + "spatial": { + "available": True, + "matches": [{"distance": 0.2, "metadata": [{"label": query}]}], + }, + "temporal": { + "available": True, + "rolling_summary": "A person was recently near the hallway.", + }, + "fused": { + "available": True, + "spatial_match_count": 1, + "has_temporal_answer": False, + "has_temporal_summary": True, + }, + } + + +def test_semantic_temporal_map_combines_memory_sources() -> None: + semantic_map = _Go2SemanticTemporalMap() + try: + semantic_map._spatial_memory = StubSpatialMemory() # type: ignore[assignment] + semantic_map._temporal_memory = StubTemporalMemory() # type: ignore[assignment] + + result = semantic_map.query_semantic_temporal_map("find the hallway", spatial_limit=3) + + assert result["sources"]["spatial_memory"] is True + assert result["sources"]["temporal_memory"] is True + assert result["spatial"]["matches"][0]["distance"] == 0.2 + assert result["temporal"]["answer"] == "temporal answer for find the hallway" + assert result["fused"]["spatial_match_count"] == 1 + assert result["fused"]["has_temporal_summary"] is True + finally: + _stop_modules(semantic_map) + + +def test_structured_world_state_returns_snapshot() -> None: + world_state = _Go2StructuredWorldState() + try: + world_state._semantic_temporal_map = StubSemanticTemporalMap() # type: ignore[assignment] + world_state._navigation = StubNavigation() # type: ignore[assignment] + world_state._latest_odom = PoseStamped(position=[1.0, 2.0, 0.0], frame_id="map") + + snapshot = world_state.get_world_snapshot("find the hallway", spatial_limit=2) + + assert snapshot["sources"]["semantic_temporal_map"] is True + assert snapshot["sources"]["spatial_memory"] is True + assert snapshot["robot_state"]["navigation"]["state"] == "following_path" + assert snapshot["robot_state"]["odom"]["position"]["x"] == 1.0 + assert snapshot["memory_state"]["spatial"]["matches"][0]["distance"] == 0.2 + assert snapshot["semantic_temporal_map"]["fused"]["spatial_match_count"] == 1 + finally: + _stop_modules(world_state) diff --git a/dimos/robot/unitree/go2/blueprints/layers/layer_4_world_state/world_state_spec.py b/dimos/robot/unitree/go2/blueprints/layers/layer_4_world_state/world_state_spec.py new file mode 100644 index 0000000000..7e77666e82 --- /dev/null +++ b/dimos/robot/unitree/go2/blueprints/layers/layer_4_world_state/world_state_spec.py @@ -0,0 +1,33 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import Any, Protocol + +from dimos.spec.utils import Spec + + +class SemanticTemporalMapSpec(Spec, Protocol): + def query_semantic_temporal_map( + self, query: str = "", spatial_limit: int = 3 + ) -> dict[str, Any]: ... + + +class WorldStateSpec(Spec, Protocol): + def get_world_snapshot(self, task: str = "", spatial_limit: int = 3) -> dict[str, Any]: ... + def get_robot_state(self) -> dict[str, Any]: ... + def get_runtime_state(self) -> dict[str, Any]: ... + def get_memory_state(self, task: str = "", spatial_limit: int = 3) -> dict[str, Any]: ... + + +__all__ = ["SemanticTemporalMapSpec", "WorldStateSpec"] From ca009ebf550e04aad8bf4d426d53d0af22a6fba5 Mon Sep 17 00:00:00 2001 From: z1050297972-spec Date: Wed, 20 May 2026 23:05:33 +0800 Subject: [PATCH 06/36] feat(go2): add layer 5 skill interface registry --- dimos/robot/unitree/go2/PROJECT_CHANGELOG.md | 32 ++ .../unitree/go2/blueprints/layers/GOAL_1.md | 21 + .../layers/layer_3_agent_brain/DESIGN.md | 8 +- .../layer_3_agent_brain/context_provider.py | 42 +- .../test_context_provider.py | 32 +- .../layers/layer_5_skill_interface/DESIGN.md | 158 ++++++++ .../layer_5_skill_interface/__init__.py | 85 ++-- .../skill_interface_registry.py | 362 ++++++++++++++++++ .../skill_interface_spec.py | 26 ++ .../test_skill_interface_registry.py | 105 +++++ 10 files changed, 839 insertions(+), 32 deletions(-) create mode 100644 dimos/robot/unitree/go2/blueprints/layers/layer_5_skill_interface/DESIGN.md create mode 100644 dimos/robot/unitree/go2/blueprints/layers/layer_5_skill_interface/skill_interface_registry.py create mode 100644 dimos/robot/unitree/go2/blueprints/layers/layer_5_skill_interface/skill_interface_spec.py create mode 100644 dimos/robot/unitree/go2/blueprints/layers/layer_5_skill_interface/test_skill_interface_registry.py diff --git a/dimos/robot/unitree/go2/PROJECT_CHANGELOG.md b/dimos/robot/unitree/go2/PROJECT_CHANGELOG.md index 2922126469..7145f7c490 100644 --- a/dimos/robot/unitree/go2/PROJECT_CHANGELOG.md +++ b/dimos/robot/unitree/go2/PROJECT_CHANGELOG.md @@ -8,6 +8,38 @@ Entries are listed in reverse chronological order. ## 2026-05-20 +- Branch: `refactor/go2-architecture-layers` +- Summary: Started Go2 Layer 5 construction by adding a static skill-interface + contract registry, connecting it into the full Go2 skill-interface layer, and + exposing the contract summary to Layer 3 `ContextProvider`. The Layer 5 + package entrypoint now lazily builds blueprint variables so lightweight spec + imports do not pull in the full skill dependency stack. +- Files/modules: + - `dimos/robot/unitree/go2/blueprints/layers/layer_5_skill_interface/__init__.py` + - `dimos/robot/unitree/go2/blueprints/layers/layer_5_skill_interface/skill_interface_registry.py` + - `dimos/robot/unitree/go2/blueprints/layers/layer_5_skill_interface/skill_interface_spec.py` + - `dimos/robot/unitree/go2/blueprints/layers/layer_5_skill_interface/test_skill_interface_registry.py` + - `dimos/robot/unitree/go2/blueprints/layers/layer_5_skill_interface/DESIGN.md` + - `dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/context_provider.py` + - `dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/test_context_provider.py` + - `dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/DESIGN.md` + - `dimos/robot/unitree/go2/blueprints/layers/GOAL_1.md` + - `dimos/robot/unitree/go2/PROJECT_CHANGELOG.md` +- Validation: + - Ran `python -m py_compile` for the new Layer 5 spec/registry/test files, + the Layer 5 package entrypoint, and the changed Layer 3 ContextProvider + files. + - Copied the working tree to `~/dimos-layer-test` in WSL and ran the focused + Layer 3 + Layer 4 + Layer 5 test suite with a minimal pytest dependency + set: `27 passed, 1 warning`. +- Open items: + - Compare the static Layer 5 contracts against the MCP server tool list in a + later version so renamed or missing skills are detected automatically. + - Add more precise constraints for Unitree sport commands and perception + callback payloads. + +## 2026-05-20 + - Branch: `refactor/go2-architecture-layers` - Summary: Started Go2 Layer 4 construction by adding RPC-only semantic-temporal memory and structured world-state facades, then connected diff --git a/dimos/robot/unitree/go2/blueprints/layers/GOAL_1.md b/dimos/robot/unitree/go2/blueprints/layers/GOAL_1.md index e6d5b73969..5a185f1651 100644 --- a/dimos/robot/unitree/go2/blueprints/layers/GOAL_1.md +++ b/dimos/robot/unitree/go2/blueprints/layers/GOAL_1.md @@ -45,6 +45,10 @@ layers/ test_*.py # focused Layer 4 tests layer_5_skill_interface/ __init__.py # Go2 skill-interface blueprint pieces + skill_interface_registry.py # static Go2 skill contract registry + skill_interface_spec.py # Layer 5 RPC spec + DESIGN.md # Layer 5 implementation logic + test_*.py # focused Layer 5 tests layer_6_robot_body/ __init__.py # Go2 robot-body/local-policy blueprint piece ``` @@ -95,6 +99,23 @@ Layer 4 V1 is intentionally a facade over existing modules. The existing `SpatialMemory` and lazy `TemporalMemory` implementations stay in their current packages. +## Layer 5 Direction + +Layer 5 owns the skill-interface boundary. It should tell upper layers which +Go2 skills exist, what arguments each skill expects, which skills are +motion-sensitive, and what preflight checks should run before execution. + +Current Layer 5 V1 keeps existing skill containers unchanged and adds +`_Go2SkillInterfaceRegistry` as an RPC-only contract registry. The registry +does not execute skills. It returns static contracts for current Go2 skills, +including navigation, person following, Unitree motion/utility commands, +perception lookout, security patrol, and speech. `ContextProvider` reads this +registry through `SkillInterfaceSpec` and includes a compact skill-interface +summary in `get_context(...)`. + +Layer 5 V2 should compare the static contracts with the MCP server tool list so +renamed or missing skills are detected automatically. + ## ExpertRouter Direction `ExpertRouter` belongs in Layer 3 because it routes an LLM task toward the diff --git a/dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/DESIGN.md b/dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/DESIGN.md index ca8676eb2b..38b2c0a048 100644 --- a/dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/DESIGN.md +++ b/dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/DESIGN.md @@ -122,7 +122,8 @@ Each implementation note should cover: - Direct arguments: `task`, `focus`, `spatial_limit`. - Stream state: latest `odom` message cached by `_on_odom`. - Optional injected Specs: `SpatialMemorySpec`, `TemporalMemorySpec`, - `NavigationInterfaceSpec`, `SkillOutcomeStoreSpec`. + `NavigationInterfaceSpec`, `SkillOutcomeStoreSpec`, + `SkillInterfaceSpec`. - Runtime config: `global_config.simulation`, `replay`, `robot_ip`, `viewer`, `mcp_port`, and `n_workers`. - Storage: @@ -135,6 +136,7 @@ Each implementation note should cover: - Temporal memory: `get_rolling_summary()` and `get_state()`. - Navigation: `get_state()` and `is_goal_reached()`. - Skill history: `get_recent_outcomes(limit=5)`. + - Skill interface: `get_skill_interface_snapshot()`. - Algorithm: - Trim `task` and `focus`. - Reject empty `task` with `SkillResult.fail("INVALID_INPUT", ...)`. @@ -143,6 +145,8 @@ Each implementation note should cover: `world_state`, `skill_state`, and `external_context`. - Catch dependency failures per source and append readable warnings to `errors` instead of failing the whole context request. + - Include Layer 5 skill-interface contracts under + `skill_state.interface` when the registry is wired. - Convert metadata through `_to_jsonable(...)` so MCP responses stay compact and serializable. - Format a short text summary for the LLM. @@ -154,6 +158,8 @@ Each implementation note should cover: - External context is represented as unavailable. - Context compression is deterministic formatting, not learned retrieval. - It reads existing stores but does not decide whether a skill should run. + - Layer 5 contract details are summarized; full contract validation stays in + `SkillInterfaceRegistry`. ### `_Go2ExpertRouter.route_task(...)` diff --git a/dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/context_provider.py b/dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/context_provider.py index 1675f109f4..d1e5e80115 100644 --- a/dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/context_provider.py +++ b/dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/context_provider.py @@ -39,6 +39,9 @@ from dimos.robot.unitree.go2.blueprints.layers.layer_4_world_state.world_state_spec import ( WorldStateSpec, ) +from dimos.robot.unitree.go2.blueprints.layers.layer_5_skill_interface.skill_interface_spec import ( + SkillInterfaceSpec, +) from dimos.utils.logging_config import setup_logger logger = setup_logger() @@ -53,6 +56,7 @@ class _Go2ContextProvider(Module): _navigation: NavigationInterfaceSpec | None = None _skill_outcomes: SkillOutcomeStoreSpec | None = None _causal_world_model: CausalWorldModelSpec | None = None + _skill_interface: SkillInterfaceSpec | None = None _latest_odom: PoseStamped | None = None odom: In[PoseStamped] @@ -120,6 +124,7 @@ def _source_status(self, world_snapshot: dict[str, Any] | None) -> dict[str, boo "navigation": self._navigation is not None or bool(layer4_sources.get("navigation")), "skill_outcomes": self._skill_outcomes is not None, "causal_world_model": self._causal_world_model is not None, + "skill_interface": self._skill_interface is not None, "runtime": True, } @@ -201,15 +206,38 @@ def _world_context( } def _skill_context(self, errors: list[str]) -> dict[str, Any]: + skill_interface = self._skill_interface_context(errors) if self._skill_outcomes is None: - return {"available": False, "recent_outcomes": []} + return { + "available": False, + "recent_outcomes": [], + "interface": skill_interface, + } try: recent = self._skill_outcomes.get_recent_outcomes(limit=5) except Exception as exc: logger.warning("Failed to read skill-outcome context", exc_info=True) errors.append(f"skill_outcomes: {exc}") - return {"available": True, "recent_outcomes": []} - return {"available": True, "recent_outcomes": recent} + return { + "available": True, + "recent_outcomes": [], + "interface": skill_interface, + } + return { + "available": True, + "recent_outcomes": recent, + "interface": skill_interface, + } + + def _skill_interface_context(self, errors: list[str]) -> dict[str, Any]: + if self._skill_interface is None: + return {"available": False, "skills": []} + try: + return self._skill_interface.get_skill_interface_snapshot() + except Exception as exc: + logger.warning("Failed to read skill-interface context", exc_info=True) + errors.append(f"skill_interface: {exc}") + return {"available": True, "skills": []} def _causal_context(self, errors: list[str]) -> dict[str, Any]: if self._causal_world_model is None: @@ -339,6 +367,14 @@ def _format_message(self, metadata: dict[str, Any]) -> str: else: lines.append("Skill outcomes: unavailable") + skill_interface = skill_state.get("interface") or {} + if skill_interface.get("available"): + lines.append( + f"Skill interface: {skill_interface.get('skill_count', 0)} contract(s)" + ) + else: + lines.append("Skill interface: unavailable") + causal_state = metadata.get("causal_state", {}) recent_transitions = causal_state.get("recent_transitions") or [] if recent_transitions: diff --git a/dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/test_context_provider.py b/dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/test_context_provider.py index b4327ae535..d27a84df5c 100644 --- a/dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/test_context_provider.py +++ b/dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/test_context_provider.py @@ -12,8 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -from typing import Any import json +from typing import Any from dimos.msgs.geometry_msgs.PoseStamped import PoseStamped from dimos.navigation.base import NavigationState @@ -78,6 +78,33 @@ def get_recent_outcomes( ][:limit] +class StubSkillInterface: + def get_skill_interface_snapshot(self, domain: str = "") -> dict[str, Any]: + return { + "available": True, + "version": "v1", + "domain_filter": domain, + "domains": ["navigation"], + "skill_count": 1, + "skills": [ + { + "skill_name": "navigate_with_text", + "domain": "navigation", + "required_args": ["query"], + "motion_sensitive": True, + } + ], + } + + def get_skill_contract(self, skill_name: str) -> dict[str, Any] | None: + if skill_name == "navigate_with_text": + return self.get_skill_interface_snapshot()["skills"][0] + return None + + def validate_skill_request(self, skill_name: str, args_json: str = "{}") -> dict[str, Any]: + return {"valid": True, "skill_name": skill_name, "errors": [], "warnings": []} + + class StubCausalWorldModel: def get_recent_transitions( self, @@ -147,6 +174,7 @@ def test_get_context_aggregates_available_sources() -> None: provider._temporal_memory = StubTemporalMemory() # type: ignore[assignment] provider._navigation = StubNavigation() # type: ignore[assignment] provider._skill_outcomes = StubSkillOutcomes() # type: ignore[assignment] + provider._skill_interface = StubSkillInterface() # type: ignore[assignment] provider._causal_world_model = StubCausalWorldModel() # type: ignore[assignment] provider._latest_odom = PoseStamped(position=[1.0, 2.0, 0.0], frame_id="map") @@ -160,6 +188,8 @@ def test_get_context_aggregates_available_sources() -> None: assert result.metadata["world_state"]["spatial"]["matches"][0]["distance"] == 0.12 assert result.metadata["world_state"]["temporal"]["rolling_summary"] assert result.metadata["skill_state"]["recent_outcomes"][0]["success"] is False + assert result.metadata["skill_state"]["interface"]["skill_count"] == 1 + assert "Skill interface: 1 contract(s)" in result.message assert result.metadata["causal_state"]["recent_transitions"][0]["outcome_success"] is False encoded = json.loads(result.agent_encode()[0]["text"]) diff --git a/dimos/robot/unitree/go2/blueprints/layers/layer_5_skill_interface/DESIGN.md b/dimos/robot/unitree/go2/blueprints/layers/layer_5_skill_interface/DESIGN.md new file mode 100644 index 0000000000..59529a9306 --- /dev/null +++ b/dimos/robot/unitree/go2/blueprints/layers/layer_5_skill_interface/DESIGN.md @@ -0,0 +1,158 @@ +# Layer 5 Skill Interface Design + +This document explains the Go2 Layer 5 implementation logic. Layer 5 is the +boundary between the decision layer and executable robot skills. It describes +what tools exist, how they should be called, and what preflight checks upper +layers should run before execution. + +## Current Scope + +Layer 5 V1 keeps all existing skill implementations in place. It does not move, +wrap, or rewrite `NavigationSkillContainer`, `PersonFollowSkillContainer`, +`UnitreeSkillContainer`, `PerceiveLoopSkill`, `SecurityModule`, `WebInput`, or +`SpeakSkill`. + +The new Layer 5 module is `_Go2SkillInterfaceRegistry`. It is an RPC-only +contract registry for the current Go2 MCP-callable skills. + +The package entrypoint uses lazy blueprint construction. Importing +`skill_interface_spec.py` or `skill_interface_registry.py` should stay light and +must not import the full skill stack. The heavy skill containers are imported +only when a blueprint asks for `_go2_spatial_skill_interface`, +`_go2_agentic_skill_interface`, or `_go2_skill_interface`. + +## Runtime Flow + +The intended task flow is: + +```text +Layer 3 ContextProvider + -> SkillInterfaceSpec.get_skill_interface_snapshot() + -> compact skill list in get_context(...) + -> LLM chooses a Layer 5 skill + -> Layer 3 predictor validates likely risk + -> existing Layer 5 skill container executes the skill +``` + +The registry is read-only. It does not execute robot actions and does not +replace the existing MCP skill containers. + +## Module Responsibilities + +`_Go2SkillInterfaceRegistry` + +- Stores static skill contracts for the Go2 agentic blueprint. +- Returns all contracts or contracts filtered by domain. +- Returns one contract by `skill_name`. +- Validates planned arguments against required fields and simple JSON types. +- Does not call, retry, interrupt, or monitor the skill itself. + +Existing skill containers + +- Continue to expose the actual `@skill` methods. +- Continue to own their injected specs, streams, background loops, and robot + side effects. +- Keep their current import paths and public skill names. + +## Function Implementation Notes + +### `_Go2SkillInterfaceRegistry.get_skill_interface_snapshot(...)` + +- File: `layer_5_skill_interface/skill_interface_registry.py` +- Entry point: RPC-only helper. +- Purpose: expose a structured list of Go2 skill contracts to upper layers. +- Inputs: + - Direct argument: optional `domain` filter. + - Static data: `_SKILL_CONTRACTS`. +- Storage: + - No database. + - No persistent file writes. + - Contracts are Python constants in the module. +- Data returned: + - `available`, `version`, `source`, `domain_filter`, `domains`, + `skill_count`, and `skills`. + - Each skill includes domain, module, summary, required/optional arguments, + argument types, context requirements, risk class, and recommended preflight + tools. +- Algorithm: + - Strip the domain filter. + - Select all contracts if no domain is given, otherwise exact-match the + domain. + - Convert contracts to JSON-shaped dictionaries. + - Derive sorted domains from the selected contracts. +- Current limits: + - Contracts are manually maintained and should be updated when Go2 skills are + added, removed, or renamed. + +### `_Go2SkillInterfaceRegistry.get_skill_contract(...)` + +- File: `layer_5_skill_interface/skill_interface_registry.py` +- Entry point: RPC-only helper. +- Purpose: return one skill contract for a planned MCP tool name. +- Inputs: + - Direct argument: `skill_name`. +- Storage: + - No database or persistent state. +- Algorithm: + - Strip `skill_name`. + - Find the exact contract in `_SKILL_CONTRACTS`. + - Return `None` when no contract exists. +- Return shape: + - Contract dictionary or `None`. + +### `_Go2SkillInterfaceRegistry.validate_skill_request(...)` + +- File: `layer_5_skill_interface/skill_interface_registry.py` +- Entry point: RPC-only helper. +- Purpose: catch obvious skill-call mistakes before execution. +- Inputs: + - `skill_name`: the MCP tool name. + - `args_json`: JSON object string containing planned tool arguments. +- Storage: + - No database or persistent state. +- Algorithm: + - Reject unknown skills. + - Parse `args_json`; reject invalid JSON and non-object JSON. + - Check all required arguments for missing, empty-string, or empty-list + values. + - Check simple argument types: `str`, `bool`, `float`, `list[float]`, + `list[str]`, and `dict`. + - Add warnings when the skill is context-dependent or motion-sensitive. +- Return shape: + - `valid`, `skill_name`, `errors`, `warnings`, and `contract`. +- Current limits: + - This is schema/risk metadata validation, not semantic validation. For + example, it can confirm that `query` exists, but not that a target location + is actually reachable. + +## Version Boundaries + +V1 implemented: + +- A static Go2 skill contract registry. +- RPC methods for skill snapshot, single-skill lookup, and planned-call + validation. +- ContextProvider integration so Layer 3 can see the Layer 5 skill interface. +- Lazy Layer 5 package imports so Layer 3 specs/tests do not pull in the full + skill dependency stack. + +V2 planned: + +- Compare the static contract registry against the MCP server tool list during + tests or startup, so missing/renamed skills are detected automatically. +- Add more precise argument constraints for Unitree sport commands and + perception callbacks. + +V3 planned: + +- Feed validated skill contracts into a structured planner/policy so Layer 3 + can choose tools from contracts instead of relying only on prompt text. + +## Design Rules + +- Keep Layer 5 contracts close to Go2 while the skill set is still evolving. +- Do not move or wrap existing skill implementations in this stage. +- Use exact public skill names as contract keys. +- Treat physical-motion skills as context-dependent and preflight-sensitive. +- Update this document and `GOAL_1.md` whenever Layer 5 contract fields, + validation logic, or included skills change. diff --git a/dimos/robot/unitree/go2/blueprints/layers/layer_5_skill_interface/__init__.py b/dimos/robot/unitree/go2/blueprints/layers/layer_5_skill_interface/__init__.py index ec6f980a86..288b72b837 100644 --- a/dimos/robot/unitree/go2/blueprints/layers/layer_5_skill_interface/__init__.py +++ b/dimos/robot/unitree/go2/blueprints/layers/layer_5_skill_interface/__init__.py @@ -13,36 +13,67 @@ # See the License for the specific language governing permissions and # limitations under the License. -from dimos.agents.skills.navigation import NavigationSkillContainer -from dimos.agents.skills.person_follow import PersonFollowSkillContainer -from dimos.agents.skills.speak_skill import SpeakSkill -from dimos.agents.web_human_input import WebInput -from dimos.core.coordination.blueprints import autoconnect -from dimos.experimental.security_demo.security_module import SecurityModule -from dimos.perception.perceive_loop_skill import PerceiveLoopSkill -from dimos.robot.unitree.go2.connection import GO2Connection -from dimos.robot.unitree.unitree_skill_container import UnitreeSkillContainer - -_go2_spatial_skill_interface = autoconnect( - PerceiveLoopSkill.blueprint(), - SecurityModule.blueprint(camera_info=GO2Connection.camera_info_static), -) - -_go2_agentic_skill_interface = autoconnect( - NavigationSkillContainer.blueprint(), - PersonFollowSkillContainer.blueprint(camera_info=GO2Connection.camera_info_static), - UnitreeSkillContainer.blueprint(), - WebInput.blueprint(), - SpeakSkill.blueprint(), -) - -_go2_skill_interface = autoconnect( - _go2_spatial_skill_interface, - _go2_agentic_skill_interface, -) +from __future__ import annotations + +from typing import Any __all__ = [ "_go2_agentic_skill_interface", "_go2_skill_interface", "_go2_spatial_skill_interface", ] + +_LAYER_BLUEPRINTS: dict[str, Any] = {} + + +def __getattr__(name: str) -> Any: + if name not in __all__: + raise AttributeError(name) + _build_layer_blueprints() + return _LAYER_BLUEPRINTS[name] + + +def _build_layer_blueprints() -> None: + if _LAYER_BLUEPRINTS: + return + + from dimos.agents.skills.navigation import NavigationSkillContainer + from dimos.agents.skills.person_follow import PersonFollowSkillContainer + from dimos.agents.skills.speak_skill import SpeakSkill + from dimos.agents.web_human_input import WebInput + from dimos.core.coordination.blueprints import autoconnect + from dimos.experimental.security_demo.security_module import SecurityModule + from dimos.perception.perceive_loop_skill import PerceiveLoopSkill + from dimos.robot.unitree.go2.blueprints.layers.layer_5_skill_interface.skill_interface_registry import ( + _Go2SkillInterfaceRegistry, + ) + from dimos.robot.unitree.go2.connection import GO2Connection + from dimos.robot.unitree.unitree_skill_container import UnitreeSkillContainer + + spatial_skill_interface = autoconnect( + PerceiveLoopSkill.blueprint(), + SecurityModule.blueprint(camera_info=GO2Connection.camera_info_static), + ) + + agentic_skill_interface = autoconnect( + NavigationSkillContainer.blueprint(), + PersonFollowSkillContainer.blueprint(camera_info=GO2Connection.camera_info_static), + UnitreeSkillContainer.blueprint(), + WebInput.blueprint(), + SpeakSkill.blueprint(), + ) + + skill_interface = autoconnect( + spatial_skill_interface, + agentic_skill_interface, + _Go2SkillInterfaceRegistry.blueprint(), + ) + + _LAYER_BLUEPRINTS.update( + { + "_go2_spatial_skill_interface": spatial_skill_interface, + "_go2_agentic_skill_interface": agentic_skill_interface, + "_go2_skill_interface": skill_interface, + } + ) + globals().update(_LAYER_BLUEPRINTS) diff --git a/dimos/robot/unitree/go2/blueprints/layers/layer_5_skill_interface/skill_interface_registry.py b/dimos/robot/unitree/go2/blueprints/layers/layer_5_skill_interface/skill_interface_registry.py new file mode 100644 index 0000000000..62c3f6f76c --- /dev/null +++ b/dimos/robot/unitree/go2/blueprints/layers/layer_5_skill_interface/skill_interface_registry.py @@ -0,0 +1,362 @@ +#!/usr/bin/env python3 +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import annotations + +import json +from dataclasses import dataclass +from typing import Any, Literal + +from dimos.core.core import rpc +from dimos.core.module import Module + +RiskClass = Literal["stop", "low", "medium", "high"] + + +@dataclass(frozen=True) +class _SkillContract: + skill_name: str + domain: str + module: str + summary: str + required_args: tuple[str, ...] = () + optional_args: tuple[str, ...] = () + arg_types: dict[str, str] | None = None + motion_sensitive: bool = False + requires_context: bool = False + requires_world_state: bool = False + requires_robot_state: bool = False + async_updates: bool = False + risk_class: RiskClass = "low" + recommended_preflight: tuple[str, ...] = () + outcome_shape: str = "str" + + def to_dict(self) -> dict[str, Any]: + return { + "skill_name": self.skill_name, + "domain": self.domain, + "module": self.module, + "summary": self.summary, + "required_args": list(self.required_args), + "optional_args": list(self.optional_args), + "arg_types": dict(self.arg_types or {}), + "motion_sensitive": self.motion_sensitive, + "requires_context": self.requires_context, + "requires_world_state": self.requires_world_state, + "requires_robot_state": self.requires_robot_state, + "async_updates": self.async_updates, + "risk_class": self.risk_class, + "recommended_preflight": list(self.recommended_preflight), + "outcome_shape": self.outcome_shape, + } + + +_CONTEXT_PREFLIGHT = ("route_task", "get_context", "predict_skill_outcome") +_LIGHT_PREFLIGHT = ("route_task",) + +_SKILL_CONTRACTS = ( + _SkillContract( + skill_name="tag_location", + domain="memory", + module="NavigationSkillContainer", + summary="Tag the current robot location in spatial memory.", + required_args=("location_name",), + arg_types={"location_name": "str"}, + requires_context=True, + requires_world_state=True, + requires_robot_state=True, + risk_class="low", + recommended_preflight=("get_context",), + ), + _SkillContract( + skill_name="navigate_with_text", + domain="navigation", + module="NavigationSkillContainer", + summary="Navigate to a tagged, visible, or semantic-map location.", + required_args=("query",), + arg_types={"query": "str"}, + motion_sensitive=True, + requires_context=True, + requires_world_state=True, + requires_robot_state=True, + risk_class="medium", + recommended_preflight=_CONTEXT_PREFLIGHT, + ), + _SkillContract( + skill_name="stop_navigation", + domain="navigation", + module="NavigationSkillContainer", + summary="Cancel the current navigation goal.", + motion_sensitive=False, + requires_context=False, + risk_class="stop", + recommended_preflight=(), + ), + _SkillContract( + skill_name="follow_person", + domain="person_follow", + module="PersonFollowSkillContainer", + summary="Detect and follow a person matching a text description.", + required_args=("query",), + optional_args=("initial_bbox", "initial_image"), + arg_types={"query": "str", "initial_bbox": "list[float]", "initial_image": "str"}, + motion_sensitive=True, + requires_context=True, + requires_world_state=True, + requires_robot_state=True, + async_updates=True, + risk_class="medium", + recommended_preflight=_CONTEXT_PREFLIGHT, + ), + _SkillContract( + skill_name="stop_following", + domain="person_follow", + module="PersonFollowSkillContainer", + summary="Stop the active person-follow behavior.", + risk_class="stop", + recommended_preflight=(), + ), + _SkillContract( + skill_name="relative_move", + domain="robot_motion", + module="UnitreeSkillContainer", + summary="Move or rotate the robot relative to its current pose.", + optional_args=("forward", "left", "degrees"), + arg_types={"forward": "float", "left": "float", "degrees": "float"}, + motion_sensitive=True, + requires_context=True, + requires_robot_state=True, + risk_class="medium", + recommended_preflight=_CONTEXT_PREFLIGHT, + ), + _SkillContract( + skill_name="execute_sport_command", + domain="robot_motion", + module="UnitreeSkillContainer", + summary="Execute a named Unitree sport command.", + required_args=("command_name",), + arg_types={"command_name": "str"}, + motion_sensitive=True, + requires_context=True, + requires_robot_state=True, + risk_class="high", + recommended_preflight=_CONTEXT_PREFLIGHT, + ), + _SkillContract( + skill_name="wait", + domain="utility", + module="UnitreeSkillContainer", + summary="Wait for a given number of seconds.", + required_args=("seconds",), + arg_types={"seconds": "float"}, + risk_class="low", + recommended_preflight=_LIGHT_PREFLIGHT, + ), + _SkillContract( + skill_name="current_time", + domain="utility", + module="UnitreeSkillContainer", + summary="Return the current local time.", + risk_class="low", + recommended_preflight=(), + ), + _SkillContract( + skill_name="look_out_for", + domain="perception", + module="PerceiveLoopSkill", + summary="Continuously look for visual targets and optionally dispatch a follow-up tool.", + required_args=("description_of_things",), + optional_args=("then",), + arg_types={"description_of_things": "list[str]", "then": "dict"}, + requires_context=True, + requires_world_state=True, + async_updates=True, + risk_class="medium", + recommended_preflight=_CONTEXT_PREFLIGHT, + ), + _SkillContract( + skill_name="stop_looking_out", + domain="perception", + module="PerceiveLoopSkill", + summary="Stop the active visual lookout.", + risk_class="stop", + recommended_preflight=(), + ), + _SkillContract( + skill_name="start_security_patrol", + domain="security", + module="SecurityModule", + summary="Start patrol, person detection, and automatic following.", + motion_sensitive=True, + requires_context=True, + requires_world_state=True, + requires_robot_state=True, + async_updates=True, + risk_class="high", + recommended_preflight=_CONTEXT_PREFLIGHT, + ), + _SkillContract( + skill_name="stop_security_patrol", + domain="security", + module="SecurityModule", + summary="Stop the active security patrol behavior.", + risk_class="stop", + recommended_preflight=(), + ), + _SkillContract( + skill_name="speak", + domain="speech", + module="SpeakSkill", + summary="Speak text through the robot speakers.", + required_args=("text",), + optional_args=("blocking",), + arg_types={"text": "str", "blocking": "bool"}, + risk_class="low", + recommended_preflight=_LIGHT_PREFLIGHT, + ), +) + + +class _Go2SkillInterfaceRegistry(Module): + """Layer 5 registry for the Go2 MCP-callable skill interface. + + This registry is static in V1. It documents the skills that the current Go2 + blueprint composes, and gives upper layers a structured contract for + validation and planning. It does not execute skills or wrap the existing + skill containers. + """ + + @rpc + def get_skill_interface_snapshot(self, domain: str = "") -> dict[str, Any]: + """Return the Go2 Layer 5 skill contracts, optionally filtered by domain.""" + domain = domain.strip() + contracts = [ + contract.to_dict() + for contract in _SKILL_CONTRACTS + if not domain or contract.domain == domain + ] + return { + "available": True, + "version": "v1", + "source": "static_go2_layer5_contracts", + "domain_filter": domain, + "domains": sorted({contract["domain"] for contract in contracts}), + "skill_count": len(contracts), + "skills": contracts, + } + + @rpc + def get_skill_contract(self, skill_name: str) -> dict[str, Any] | None: + """Return one Go2 skill contract by MCP tool name.""" + name = skill_name.strip() + contract = _contract_by_name(name) + if contract is None: + return None + return contract.to_dict() + + @rpc + def validate_skill_request(self, skill_name: str, args_json: str = "{}") -> dict[str, Any]: + """Validate a planned skill call against the Layer 5 contract.""" + name = skill_name.strip() + contract = _contract_by_name(name) + if contract is None: + return { + "valid": False, + "skill_name": name, + "errors": [f"unknown skill: {name}"], + "warnings": [], + "contract": None, + } + + args, parse_error = _parse_args(args_json) + errors: list[str] = [] + warnings: list[str] = [] + if parse_error: + errors.append(parse_error) + + if args is not None: + for required in contract.required_args: + if _is_missing(args.get(required)): + errors.append(f"missing required argument: {required}") + + arg_types = contract.arg_types or {} + for arg_name, expected_type in arg_types.items(): + if arg_name in args and not _matches_type(args[arg_name], expected_type): + errors.append(f"argument {arg_name} should be {expected_type}") + + if contract.requires_context: + warnings.append("call get_context before this skill when task context is non-trivial") + if contract.motion_sensitive: + warnings.append("call predict_skill_outcome before executing this motion-sensitive skill") + + return { + "valid": not errors, + "skill_name": name, + "errors": errors, + "warnings": warnings, + "contract": contract.to_dict(), + } + + +def _contract_by_name(skill_name: str) -> _SkillContract | None: + for contract in _SKILL_CONTRACTS: + if contract.skill_name == skill_name: + return contract + return None + + +def _parse_args(args_json: str) -> tuple[dict[str, Any] | None, str | None]: + if not args_json.strip(): + return {}, None + try: + parsed = json.loads(args_json) + except json.JSONDecodeError as exc: + return None, f"args_json is invalid JSON: {exc.msg}" + if not isinstance(parsed, dict): + return None, "args_json must be a JSON object" + return parsed, None + + +def _is_missing(value: Any) -> bool: + if value is None: + return True + if isinstance(value, str) and not value.strip(): + return True + if isinstance(value, list) and not value: + return True + return False + + +def _matches_type(value: Any, expected_type: str) -> bool: + if _is_missing(value): + return True + if expected_type == "str": + return isinstance(value, str) + if expected_type == "bool": + return isinstance(value, bool) + if expected_type == "float": + return isinstance(value, int | float) and not isinstance(value, bool) + if expected_type == "list[float]": + return isinstance(value, list) and all( + isinstance(item, int | float) and not isinstance(item, bool) for item in value + ) + if expected_type == "list[str]": + return isinstance(value, list) and all(isinstance(item, str) for item in value) + if expected_type == "dict": + return isinstance(value, dict) + return True + + +__all__ = ["_Go2SkillInterfaceRegistry"] diff --git a/dimos/robot/unitree/go2/blueprints/layers/layer_5_skill_interface/skill_interface_spec.py b/dimos/robot/unitree/go2/blueprints/layers/layer_5_skill_interface/skill_interface_spec.py new file mode 100644 index 0000000000..02fcb39bf9 --- /dev/null +++ b/dimos/robot/unitree/go2/blueprints/layers/layer_5_skill_interface/skill_interface_spec.py @@ -0,0 +1,26 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import Any, Protocol + +from dimos.spec.utils import Spec + + +class SkillInterfaceSpec(Spec, Protocol): + def get_skill_interface_snapshot(self, domain: str = "") -> dict[str, Any]: ... + def get_skill_contract(self, skill_name: str) -> dict[str, Any] | None: ... + def validate_skill_request(self, skill_name: str, args_json: str = "{}") -> dict[str, Any]: ... + + +__all__ = ["SkillInterfaceSpec"] diff --git a/dimos/robot/unitree/go2/blueprints/layers/layer_5_skill_interface/test_skill_interface_registry.py b/dimos/robot/unitree/go2/blueprints/layers/layer_5_skill_interface/test_skill_interface_registry.py new file mode 100644 index 0000000000..5328cea261 --- /dev/null +++ b/dimos/robot/unitree/go2/blueprints/layers/layer_5_skill_interface/test_skill_interface_registry.py @@ -0,0 +1,105 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from dimos.robot.unitree.go2.blueprints.layers.layer_5_skill_interface.skill_interface_registry import ( + _Go2SkillInterfaceRegistry, +) + + +def _stop_modules(*modules: object) -> None: + for module in modules: + stop = getattr(module, "stop", None) + if stop is not None: + stop() + + +def test_snapshot_lists_go2_skill_contracts() -> None: + registry = _Go2SkillInterfaceRegistry() + try: + snapshot = registry.get_skill_interface_snapshot() + + skill_names = {skill["skill_name"] for skill in snapshot["skills"]} + assert snapshot["available"] is True + assert snapshot["version"] == "v1" + assert "navigation" in snapshot["domains"] + assert "robot_motion" in snapshot["domains"] + assert "navigate_with_text" in skill_names + assert "follow_person" in skill_names + assert "speak" in skill_names + finally: + _stop_modules(registry) + + +def test_snapshot_can_filter_by_domain() -> None: + registry = _Go2SkillInterfaceRegistry() + try: + snapshot = registry.get_skill_interface_snapshot(domain="navigation") + + assert snapshot["domain_filter"] == "navigation" + assert snapshot["domains"] == ["navigation"] + assert {skill["domain"] for skill in snapshot["skills"]} == {"navigation"} + finally: + _stop_modules(registry) + + +def test_get_skill_contract_returns_motion_sensitive_metadata() -> None: + registry = _Go2SkillInterfaceRegistry() + try: + contract = registry.get_skill_contract("follow_person") + + assert contract is not None + assert contract["domain"] == "person_follow" + assert contract["required_args"] == ["query"] + assert contract["motion_sensitive"] is True + assert "predict_skill_outcome" in contract["recommended_preflight"] + finally: + _stop_modules(registry) + + +def test_validate_skill_request_rejects_unknown_skill() -> None: + registry = _Go2SkillInterfaceRegistry() + try: + result = registry.validate_skill_request("not_a_skill", "{}") + + assert result["valid"] is False + assert result["contract"] is None + assert result["errors"] == ["unknown skill: not_a_skill"] + finally: + _stop_modules(registry) + + +def test_validate_skill_request_checks_required_arguments() -> None: + registry = _Go2SkillInterfaceRegistry() + try: + result = registry.validate_skill_request("navigate_with_text", "{}") + + assert result["valid"] is False + assert "missing required argument: query" in result["errors"] + assert any("get_context" in warning for warning in result["warnings"]) + finally: + _stop_modules(registry) + + +def test_validate_skill_request_accepts_safe_speech_call() -> None: + registry = _Go2SkillInterfaceRegistry() + try: + result = registry.validate_skill_request( + "speak", '{"text": "hello", "blocking": false}' + ) + + assert result["valid"] is True + assert result["errors"] == [] + assert result["contract"]["risk_class"] == "low" + finally: + _stop_modules(registry) From 23c4f0e5cc70e98fa757d33052feb0fdbc64207c Mon Sep 17 00:00:00 2001 From: z1050297972-spec Date: Wed, 20 May 2026 23:27:48 +0800 Subject: [PATCH 07/36] feat(go2): add layer 6 robot body state --- dimos/robot/unitree/go2/PROJECT_CHANGELOG.md | 33 +++ .../unitree/go2/blueprints/layers/GOAL_1.md | 18 ++ .../layer_3_agent_brain/context_provider.py | 15 ++ .../test_context_provider.py | 8 + .../layers/layer_4_world_state/DESIGN.md | 10 +- .../structured_world_state.py | 29 +++ .../layer_4_world_state/test_world_state.py | 26 +++ .../layers/layer_6_robot_body/DESIGN.md | 123 +++++++++++ .../layers/layer_6_robot_body/__init__.py | 32 ++- .../layer_6_robot_body/robot_body_spec.py | 27 +++ .../layer_6_robot_body/robot_body_state.py | 203 ++++++++++++++++++ .../test_robot_body_state.py | 72 +++++++ 12 files changed, 591 insertions(+), 5 deletions(-) create mode 100644 dimos/robot/unitree/go2/blueprints/layers/layer_6_robot_body/DESIGN.md create mode 100644 dimos/robot/unitree/go2/blueprints/layers/layer_6_robot_body/robot_body_spec.py create mode 100644 dimos/robot/unitree/go2/blueprints/layers/layer_6_robot_body/robot_body_state.py create mode 100644 dimos/robot/unitree/go2/blueprints/layers/layer_6_robot_body/test_robot_body_state.py diff --git a/dimos/robot/unitree/go2/PROJECT_CHANGELOG.md b/dimos/robot/unitree/go2/PROJECT_CHANGELOG.md index 7145f7c490..e3b700fd2a 100644 --- a/dimos/robot/unitree/go2/PROJECT_CHANGELOG.md +++ b/dimos/robot/unitree/go2/PROJECT_CHANGELOG.md @@ -8,6 +8,39 @@ Entries are listed in reverse chronological order. ## 2026-05-20 +- Branch: `refactor/go2-architecture-layers` +- Summary: Started Go2 Layer 6 construction by adding an observational + robot-body/local-policy state facade, connecting it into the internal Go2 + robot-body layer, and exposing the Layer 6 snapshot through Layer 4 and Layer + 3 context. +- Files/modules: + - `dimos/robot/unitree/go2/blueprints/layers/layer_6_robot_body/__init__.py` + - `dimos/robot/unitree/go2/blueprints/layers/layer_6_robot_body/robot_body_state.py` + - `dimos/robot/unitree/go2/blueprints/layers/layer_6_robot_body/robot_body_spec.py` + - `dimos/robot/unitree/go2/blueprints/layers/layer_6_robot_body/test_robot_body_state.py` + - `dimos/robot/unitree/go2/blueprints/layers/layer_6_robot_body/DESIGN.md` + - `dimos/robot/unitree/go2/blueprints/layers/layer_4_world_state/structured_world_state.py` + - `dimos/robot/unitree/go2/blueprints/layers/layer_4_world_state/test_world_state.py` + - `dimos/robot/unitree/go2/blueprints/layers/layer_4_world_state/DESIGN.md` + - `dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/context_provider.py` + - `dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/test_context_provider.py` + - `dimos/robot/unitree/go2/blueprints/layers/GOAL_1.md` + - `dimos/robot/unitree/go2/PROJECT_CHANGELOG.md` +- Validation: + - Ran `python -m py_compile` for the new Layer 6 spec/state/test files, + the Layer 6 package entrypoint, the changed Layer 4 world-state files, and + the changed Layer 3 ContextProvider files. + - Ran `git diff --check`. + - Copied the working tree to `~/dimos-layer-test` in WSL and ran the focused + Layer 3 + Layer 4 + Layer 5 + Layer 6 test suite with a minimal pytest + dependency set: `30 passed, 1 warning`. +- Open items: + - Add real connection heartbeat/health if `GO2Connection` exposes it. + - Add command-stream tracking after confirming it will not interfere with + movement-manager fan-out. + +## 2026-05-20 + - Branch: `refactor/go2-architecture-layers` - Summary: Started Go2 Layer 5 construction by adding a static skill-interface contract registry, connecting it into the full Go2 skill-interface layer, and diff --git a/dimos/robot/unitree/go2/blueprints/layers/GOAL_1.md b/dimos/robot/unitree/go2/blueprints/layers/GOAL_1.md index 5a185f1651..3c7399953f 100644 --- a/dimos/robot/unitree/go2/blueprints/layers/GOAL_1.md +++ b/dimos/robot/unitree/go2/blueprints/layers/GOAL_1.md @@ -51,6 +51,10 @@ layers/ test_*.py # focused Layer 5 tests layer_6_robot_body/ __init__.py # Go2 robot-body/local-policy blueprint piece + robot_body_state.py # Layer 6 body/local-policy state facade + robot_body_spec.py # Layer 6 RPC spec + DESIGN.md # Layer 6 implementation logic + test_*.py # focused Layer 6 tests ``` The public internal import paths intentionally stay stable at the layer level, @@ -116,6 +120,20 @@ summary in `get_context(...)`. Layer 5 V2 should compare the static contracts with the MCP server tool list so renamed or missing skills are detected automatically. +## Layer 6 Direction + +Layer 6 owns the robot body and local-policy boundary. It includes the existing +Go2 connection, sensor streams, mapping, navigation, patrolling, movement +manager, and local safety behavior. Current Layer 6 V1 keeps those +implementations in place and adds `_Go2RobotBodyState` as an RPC-only +observational facade. + +`_Go2RobotBodyState` tracks odom, color-image, and lidar stream liveness, +reports runtime connection mode/configuration, and exposes conservative +local-policy/safety context. It does not move the robot or replace physical +safety checks. Layer 4 reads it through `RobotBodyStateSpec` and includes the +body/local-policy sections in `robot_state`. + ## ExpertRouter Direction `ExpertRouter` belongs in Layer 3 because it routes an LLM task toward the diff --git a/dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/context_provider.py b/dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/context_provider.py index d1e5e80115..ab0aecb04c 100644 --- a/dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/context_provider.py +++ b/dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/context_provider.py @@ -334,6 +334,21 @@ def _format_message(self, metadata: dict[str, Any]) -> str: f"state={navigation['state']}, goal_reached={navigation['goal_reached']}" ) + connection = robot_state.get("connection") + if connection: + lines.append( + "Connection: " + f"mode={connection.get('mode')}, available={connection.get('available')}" + ) + + safety = robot_state.get("safety") + if safety: + lines.append( + "Safety: " + f"body_pose_available={safety.get('body_pose_available')}, " + f"obstacle_avoidance={safety.get('obstacle_avoidance_configured')}" + ) + spatial = world_state.get("spatial", {}) matches = spatial.get("matches") or [] if matches: diff --git a/dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/test_context_provider.py b/dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/test_context_provider.py index d27a84df5c..07f8686077 100644 --- a/dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/test_context_provider.py +++ b/dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/test_context_provider.py @@ -144,6 +144,12 @@ def get_world_snapshot(self, task: str = "", spatial_limit: int = 3) -> dict[str "yaw_degrees": 0.0, }, "navigation": {"state": "following_path", "goal_reached": False}, + "connection": {"available": True, "mode": "replay"}, + "safety": { + "available": True, + "body_pose_available": True, + "obstacle_avoidance_configured": True, + }, }, "memory_state": { "spatial": {"available": True, "matches": [{"distance": 0.3}]}, @@ -236,6 +242,8 @@ def test_get_context_prefers_layer4_structured_world_state() -> None: assert result.metadata["sources"]["spatial_memory"] is True assert result.metadata["runtime"]["mode"] == "replay" assert result.metadata["robot_state"]["odom"]["position"]["x"] == 3.0 + assert "Connection: mode=replay, available=True" in result.message + assert "Safety: body_pose_available=True, obstacle_avoidance=True" in result.message assert result.metadata["world_state"]["source"] == "structured_world_state" assert result.metadata["world_state"]["spatial"]["matches"][0]["distance"] == 0.3 finally: diff --git a/dimos/robot/unitree/go2/blueprints/layers/layer_4_world_state/DESIGN.md b/dimos/robot/unitree/go2/blueprints/layers/layer_4_world_state/DESIGN.md index 29abdd557e..40b085e622 100644 --- a/dimos/robot/unitree/go2/blueprints/layers/layer_4_world_state/DESIGN.md +++ b/dimos/robot/unitree/go2/blueprints/layers/layer_4_world_state/DESIGN.md @@ -17,6 +17,8 @@ Layer 4 currently contains: temporal memory evidence for one query. - `_Go2StructuredWorldState`: a lightweight RPC facade that returns a single structured world snapshot. +- Optional `RobotBodyStateSpec`: Layer 6 body/local-policy state read by + `_Go2StructuredWorldState` when wired. ## Runtime Flow @@ -24,6 +26,7 @@ Layer 4 currently contains: Layer 6 odom/navigation/perception -> SpatialMemory / TemporalMemory -> _Go2SemanticTemporalMap.query_semantic_temporal_map(...) + -> _Go2RobotBodyState.get_robot_body_snapshot(...) -> _Go2StructuredWorldState.get_world_snapshot(...) -> Layer 3 ContextProvider.get_context(...) ``` @@ -72,7 +75,8 @@ memory, navigation, and odom remain as a fallback. - Direct arguments: `task`, `spatial_limit`. - Stream state: latest `odom` message cached by `_on_odom`. - Optional injected Specs: `SemanticTemporalMapSpec`, - `NavigationInterfaceSpec`, `SpatialMemorySpec`, and `TemporalMemorySpec`. + `NavigationInterfaceSpec`, `RobotBodyStateSpec`, `SpatialMemorySpec`, and + `TemporalMemorySpec`. - Runtime config: `global_config`. - Storage: - No new database. @@ -87,7 +91,8 @@ memory, navigation, and odom remain as a fallback. - `semantic_temporal_map` - optional `errors` - Current limits: - - Robot safety state and connection health are not normalized yet. + - Robot safety and connection state are read from Layer 6 when wired, but + they are still descriptive context rather than physical safety guarantees. - The module currently reads navigation state by RPC when available. ## Version Boundaries @@ -103,7 +108,6 @@ V1 implemented: V2 planned: -- Add connection/safety/local-control status to `robot_state`. - Add named object/location summaries to `memory_state`. - Make semantic-temporal map entries more explicit: entity, location, time, evidence source, and confidence. diff --git a/dimos/robot/unitree/go2/blueprints/layers/layer_4_world_state/structured_world_state.py b/dimos/robot/unitree/go2/blueprints/layers/layer_4_world_state/structured_world_state.py index 2937cb0c9e..eeabcc4c0d 100644 --- a/dimos/robot/unitree/go2/blueprints/layers/layer_4_world_state/structured_world_state.py +++ b/dimos/robot/unitree/go2/blueprints/layers/layer_4_world_state/structured_world_state.py @@ -31,6 +31,9 @@ from dimos.robot.unitree.go2.blueprints.layers.layer_4_world_state.world_state_spec import ( SemanticTemporalMapSpec, ) +from dimos.robot.unitree.go2.blueprints.layers.layer_6_robot_body.robot_body_spec import ( + RobotBodyStateSpec, +) from dimos.utils.logging_config import setup_logger logger = setup_logger() @@ -49,6 +52,7 @@ class _Go2StructuredWorldState(Module): _spatial_memory: SpatialMemorySpec | None = None _temporal_memory: TemporalMemorySpec | None = None _navigation: NavigationInterfaceSpec | None = None + _robot_body: RobotBodyStateSpec | None = None _latest_odom: PoseStamped | None = None odom: In[PoseStamped] @@ -84,9 +88,15 @@ def get_world_snapshot(self, task: str = "", spatial_limit: int = 3) -> dict[str @rpc def get_robot_state(self) -> dict[str, Any]: """Return robot state owned or normalized by Layer 4.""" + body_snapshot = self._robot_body_snapshot() state: dict[str, Any] = { "odom": self._pose_to_dict(self._latest_odom), "navigation": None, + "body": body_snapshot, + "connection": _section(body_snapshot, "connection"), + "sensors": _section(body_snapshot, "sensors"), + "local_policy": _section(body_snapshot, "local_policy"), + "safety": _section(body_snapshot, "safety"), } if self._navigation is None: @@ -142,9 +152,19 @@ def _source_status(self, semantic_temporal: dict[str, Any]) -> dict[str, bool]: "temporal_memory": self._temporal_memory is not None or bool(semantic_sources.get("temporal_memory")), "semantic_temporal_map": self._semantic_temporal_map is not None, + "robot_body": self._robot_body is not None, "runtime": True, } + def _robot_body_snapshot(self) -> dict[str, Any] | None: + if self._robot_body is None: + return None + try: + return self._robot_body.get_robot_body_snapshot() + except Exception as exc: + logger.warning("Failed to read Layer 6 robot body state", exc_info=True) + return {"available": True, "error": str(exc)} + def _semantic_temporal_section( self, task: str, spatial_limit: int, errors: list[str] ) -> dict[str, Any]: @@ -259,6 +279,15 @@ def _summarize_spatial_match(match: dict[str, Any]) -> dict[str, Any]: return _to_jsonable(summary) +def _section(snapshot: dict[str, Any] | None, key: str) -> dict[str, Any] | None: + if snapshot is None: + return None + value = snapshot.get(key) + if isinstance(value, dict): + return value + return None + + def _to_jsonable(value: Any, max_string_length: int = 500) -> Any: if value is None or isinstance(value, bool | int | float): return value diff --git a/dimos/robot/unitree/go2/blueprints/layers/layer_4_world_state/test_world_state.py b/dimos/robot/unitree/go2/blueprints/layers/layer_4_world_state/test_world_state.py index 27e718083e..a8d202f055 100644 --- a/dimos/robot/unitree/go2/blueprints/layers/layer_4_world_state/test_world_state.py +++ b/dimos/robot/unitree/go2/blueprints/layers/layer_4_world_state/test_world_state.py @@ -90,6 +90,27 @@ def query_semantic_temporal_map( } +class StubRobotBody: + def get_robot_body_snapshot(self) -> dict[str, Any]: + return { + "available": True, + "version": "v1", + "connection": {"available": True, "mode": "replay"}, + "sensors": {"odom": {"available": True, "count": 3}}, + "local_policy": {"available": True, "obstacle_avoidance": True}, + "safety": {"available": True, "body_pose_available": True}, + } + + def get_connection_state(self) -> dict[str, Any]: + return {"available": True, "mode": "replay"} + + def get_sensor_state(self) -> dict[str, Any]: + return {"odom": {"available": True, "count": 3}} + + def get_local_policy_state(self) -> dict[str, Any]: + return {"available": True, "obstacle_avoidance": True} + + def test_semantic_temporal_map_combines_memory_sources() -> None: semantic_map = _Go2SemanticTemporalMap() try: @@ -113,14 +134,19 @@ def test_structured_world_state_returns_snapshot() -> None: try: world_state._semantic_temporal_map = StubSemanticTemporalMap() # type: ignore[assignment] world_state._navigation = StubNavigation() # type: ignore[assignment] + world_state._robot_body = StubRobotBody() # type: ignore[assignment] world_state._latest_odom = PoseStamped(position=[1.0, 2.0, 0.0], frame_id="map") snapshot = world_state.get_world_snapshot("find the hallway", spatial_limit=2) assert snapshot["sources"]["semantic_temporal_map"] is True + assert snapshot["sources"]["robot_body"] is True assert snapshot["sources"]["spatial_memory"] is True assert snapshot["robot_state"]["navigation"]["state"] == "following_path" assert snapshot["robot_state"]["odom"]["position"]["x"] == 1.0 + assert snapshot["robot_state"]["connection"]["mode"] == "replay" + assert snapshot["robot_state"]["sensors"]["odom"]["count"] == 3 + assert snapshot["robot_state"]["safety"]["body_pose_available"] is True assert snapshot["memory_state"]["spatial"]["matches"][0]["distance"] == 0.2 assert snapshot["semantic_temporal_map"]["fused"]["spatial_match_count"] == 1 finally: diff --git a/dimos/robot/unitree/go2/blueprints/layers/layer_6_robot_body/DESIGN.md b/dimos/robot/unitree/go2/blueprints/layers/layer_6_robot_body/DESIGN.md new file mode 100644 index 0000000000..f3866d9416 --- /dev/null +++ b/dimos/robot/unitree/go2/blueprints/layers/layer_6_robot_body/DESIGN.md @@ -0,0 +1,123 @@ +# Layer 6 Robot Body / Local Policy Design + +This document explains the first Go2 Layer 6 implementation. Layer 6 is the +robot-body and local-policy boundary: hardware connection, replay/simulation +connection, odometry, camera, lidar, mapping, navigation, movement management, +and local safety behavior. + +## Current Scope + +Layer 6 V1 keeps the existing Go2 robot stack in place. It does not move or +rewrite `GO2Connection`, `VoxelGridMapper`, `CostMapper`, +`ReplanningAStarPlanner`, `WavefrontFrontierExplorer`, `PatrollingModule`, or +`MovementManager`. + +The new module is `_Go2RobotBodyState`. It observes existing body streams and +exposes an RPC-only state snapshot for upper layers. + +The Layer 6 package entrypoint uses lazy blueprint construction. Importing +`robot_body_spec.py` or `robot_body_state.py` should stay light; the full Go2 +robot stack is imported only when a blueprint asks for `_go2_robot_body`. + +## Runtime Flow + +```text +GO2Connection + mapping/navigation/local policy modules + -> odom / color_image / lidar streams + -> _Go2RobotBodyState.get_robot_body_snapshot(...) + -> Layer 4 _Go2StructuredWorldState.get_robot_state(...) + -> Layer 3 ContextProvider.get_context(...) +``` + +`_Go2RobotBodyState` is observational. It does not publish movement commands, +call sport commands, or override local robot safety. + +## Module Responsibilities + +`_Go2RobotBodyState` + +- Tracks whether odom, color image, and lidar streams have produced messages. +- Returns connection mode and runtime connection configuration. +- Returns local-policy settings visible from `GlobalConfig`. +- Returns conservative safety metadata for upper-layer context. +- Does not execute motion, stop the robot, or own durable state. + +Existing Go2 robot-body modules + +- Continue to own hardware/replay/simulation IO. +- Continue to publish sensor streams and consume command streams. +- Continue to own local navigation, mapping, patrolling, and movement behavior. + +## Function Implementation Notes + +### `_Go2RobotBodyState.get_robot_body_snapshot(...)` + +- File: `layer_6_robot_body/robot_body_state.py` +- Entry point: RPC-only helper. +- Purpose: return one compact Layer 6 body/local-policy state payload. +- Inputs: + - Cached stream state from `odom`, `color_image`, and `lidar`. + - Optional injected `GO2ConnectionSpec`. + - Runtime config from `global_config`. +- Storage: + - No database. + - No persistent files. + - Keeps latest odom and stream counters in memory while the process runs. +- Data shape: + - `available` + - `version` + - `connection` + - `sensors` + - `local_policy` + - `safety` +- Algorithm: + - Read connection mode from runtime config. + - Convert latest odom to a JSON-shaped pose when available. + - Report stream availability through per-stream counters and last-seen age. + - Report local-policy and safety metadata as conservative context only. +- Current limits: + - Connection health is inferred from config and injected spec presence, not + active hardware heartbeat. + - Safety status is descriptive; physical safety remains in local control. + +### `_Go2RobotBodyState.get_sensor_state(...)` + +- File: `layer_6_robot_body/robot_body_state.py` +- Entry point: RPC-only helper. +- Purpose: expose stream liveness for odom, image, and lidar. +- Inputs: + - Stream callbacks `_on_odom`, `_on_color_image`, and `_on_lidar`. +- Storage: + - In-memory counters and timestamps only. +- Return shape: + - `odom`, `color_image`, and `lidar` sections, each with `available`, + `count`, and `last_seen_age_sec`. + - Odom also includes latest pose data when available. + +## Version Boundaries + +V1 implemented: + +- Add an observational Layer 6 robot-body state facade. +- Add a `RobotBodyStateSpec` so Layer 4 can read body/local-policy state. +- Connect the facade into `_go2_robot_body` without changing public Go2 + blueprint names. + +V2 planned: + +- Add real connection heartbeat/health if `GO2Connection` exposes it. +- Add local command-stream tracking after confirming it will not interfere with + movement-manager fan-out. +- Add explicit safety-stop state if a local control module exposes it. + +V3 planned: + +- Use Layer 6 body state in end-to-end Go2 blueprint tests to verify the full + Layer 3 -> Layer 4 -> Layer 6 context path. + +## Design Rules + +- Keep physical actions in existing low-level modules and Layer 5 skills. +- Layer 6 state helpers should be RPC-only, not MCP skills. +- Avoid importing the full Go2 robot stack from lightweight spec modules. +- Do not treat Layer 6 state summaries as physical safety guarantees. diff --git a/dimos/robot/unitree/go2/blueprints/layers/layer_6_robot_body/__init__.py b/dimos/robot/unitree/go2/blueprints/layers/layer_6_robot_body/__init__.py index 5c054ed523..5ab456d32e 100644 --- a/dimos/robot/unitree/go2/blueprints/layers/layer_6_robot_body/__init__.py +++ b/dimos/robot/unitree/go2/blueprints/layers/layer_6_robot_body/__init__.py @@ -13,8 +13,36 @@ # See the License for the specific language governing permissions and # limitations under the License. -from dimos.robot.unitree.go2.blueprints.smart.unitree_go2 import unitree_go2 +from __future__ import annotations -_go2_robot_body = unitree_go2 +from typing import Any __all__ = ["_go2_robot_body"] + +_LAYER_BLUEPRINTS: dict[str, Any] = {} + + +def __getattr__(name: str) -> Any: + if name not in __all__: + raise AttributeError(name) + _build_layer_blueprints() + return _LAYER_BLUEPRINTS[name] + + +def _build_layer_blueprints() -> None: + if _LAYER_BLUEPRINTS: + return + + from dimos.core.coordination.blueprints import autoconnect + from dimos.robot.unitree.go2.blueprints.layers.layer_6_robot_body.robot_body_state import ( + _Go2RobotBodyState, + ) + from dimos.robot.unitree.go2.blueprints.smart.unitree_go2 import unitree_go2 + + robot_body = autoconnect( + unitree_go2, + _Go2RobotBodyState.blueprint(), + ) + + _LAYER_BLUEPRINTS["_go2_robot_body"] = robot_body + globals().update(_LAYER_BLUEPRINTS) diff --git a/dimos/robot/unitree/go2/blueprints/layers/layer_6_robot_body/robot_body_spec.py b/dimos/robot/unitree/go2/blueprints/layers/layer_6_robot_body/robot_body_spec.py new file mode 100644 index 0000000000..653bb2f68c --- /dev/null +++ b/dimos/robot/unitree/go2/blueprints/layers/layer_6_robot_body/robot_body_spec.py @@ -0,0 +1,27 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import Any, Protocol + +from dimos.spec.utils import Spec + + +class RobotBodyStateSpec(Spec, Protocol): + def get_robot_body_snapshot(self) -> dict[str, Any]: ... + def get_connection_state(self) -> dict[str, Any]: ... + def get_sensor_state(self) -> dict[str, Any]: ... + def get_local_policy_state(self) -> dict[str, Any]: ... + + +__all__ = ["RobotBodyStateSpec"] diff --git a/dimos/robot/unitree/go2/blueprints/layers/layer_6_robot_body/robot_body_state.py b/dimos/robot/unitree/go2/blueprints/layers/layer_6_robot_body/robot_body_state.py new file mode 100644 index 0000000000..1c26b41f7e --- /dev/null +++ b/dimos/robot/unitree/go2/blueprints/layers/layer_6_robot_body/robot_body_state.py @@ -0,0 +1,203 @@ +#!/usr/bin/env python3 +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import annotations + +import math +import time +from typing import Any + +from reactivex.disposable import Disposable + +from dimos.core.core import rpc +from dimos.core.global_config import global_config +from dimos.core.module import Module +from dimos.core.stream import In +from dimos.msgs.geometry_msgs.PoseStamped import PoseStamped +from dimos.msgs.sensor_msgs.Image import Image +from dimos.msgs.sensor_msgs.PointCloud2 import PointCloud2 +from dimos.robot.unitree.go2.connection_spec import GO2ConnectionSpec + + +class _Go2RobotBodyState(Module): + """Layer 6 robot body and local-policy state facade for Go2. + + This module observes already-existing low-level streams. It does not move + the robot, publish commands, or replace `GO2Connection`, mapping, + navigation, or local-control modules. + """ + + _connection: GO2ConnectionSpec | None = None + _latest_odom: PoseStamped | None = None + _latest_odom_seen_at: float | None = None + _latest_image_seen_at: float | None = None + _latest_lidar_seen_at: float | None = None + _odom_count: int = 0 + _image_count: int = 0 + _lidar_count: int = 0 + + odom: In[PoseStamped] + color_image: In[Image] + lidar: In[PointCloud2] + + @rpc + def start(self) -> None: + super().start() + self.register_disposable(Disposable(self.odom.subscribe(self._on_odom))) + self.register_disposable(Disposable(self.color_image.subscribe(self._on_color_image))) + self.register_disposable(Disposable(self.lidar.subscribe(self._on_lidar))) + + def _on_odom(self, odom: PoseStamped) -> None: + self._latest_odom = odom + self._latest_odom_seen_at = time.time() + self._odom_count += 1 + + def _on_color_image(self, _image: Image) -> None: + self._latest_image_seen_at = time.time() + self._image_count += 1 + + def _on_lidar(self, _pointcloud: PointCloud2) -> None: + self._latest_lidar_seen_at = time.time() + self._lidar_count += 1 + + @rpc + def get_robot_body_snapshot(self) -> dict[str, Any]: + """Return the current Layer 6 robot-body and local-policy snapshot.""" + return _to_jsonable( + { + "available": True, + "version": "v1", + "connection": self.get_connection_state(), + "sensors": self.get_sensor_state(), + "local_policy": self.get_local_policy_state(), + "safety": self.get_safety_state(), + } + ) + + @rpc + def get_connection_state(self) -> dict[str, Any]: + """Return connection-mode and connection-spec state.""" + simulation = bool(getattr(global_config, "simulation", False)) + replay = bool(getattr(global_config, "replay", False)) + mode = "simulation" if simulation else "replay" if replay else "hardware" + return { + "available": self._connection is not None, + "mode": mode, + "robot_ip": getattr(global_config, "robot_ip", None), + "unitree_connection_type": getattr(global_config, "unitree_connection_type", None), + "robot_model": getattr(global_config, "robot_model", None), + "replay_db": getattr(global_config, "replay_db", None), + } + + @rpc + def get_sensor_state(self) -> dict[str, Any]: + """Return latest observed body sensor stream state.""" + return { + "odom": self._stream_state( + count=self._odom_count, + seen_at=self._latest_odom_seen_at, + latest=self._pose_to_dict(self._latest_odom), + ), + "color_image": self._stream_state( + count=self._image_count, + seen_at=self._latest_image_seen_at, + ), + "lidar": self._stream_state( + count=self._lidar_count, + seen_at=self._latest_lidar_seen_at, + ), + } + + @rpc + def get_local_policy_state(self) -> dict[str, Any]: + """Return local-control policy settings visible from runtime config.""" + return { + "available": True, + "obstacle_avoidance": bool(getattr(global_config, "obstacle_avoidance", False)), + "go2_mode": getattr(global_config, "go2_mode", None), + "unitree_connection_type": getattr(global_config, "unitree_connection_type", None), + "local_control_owner": "GO2Connection and navigation/movement modules", + "command_interface": "cmd_vel stream and GO2Connection RPCs", + } + + @rpc + def get_safety_state(self) -> dict[str, Any]: + """Return conservative Layer 6 safety state visible to upper layers.""" + return { + "available": True, + "stop_interfaces": [ + "stop_navigation", + "stop_following", + "stop_security_patrol", + "GO2Connection.liedown", + ], + "obstacle_avoidance_configured": bool( + getattr(global_config, "obstacle_avoidance", False) + ), + "body_pose_available": self._latest_odom is not None, + "notes": [ + "Layer 6 state is observational in V1.", + "Physical safety checks still belong to local robot control.", + ], + } + + def _stream_state( + self, + count: int, + seen_at: float | None, + latest: dict[str, Any] | None = None, + ) -> dict[str, Any]: + now = time.time() + state: dict[str, Any] = { + "available": count > 0, + "count": count, + "last_seen_age_sec": round(now - seen_at, 3) if seen_at else None, + } + if latest is not None: + state["latest"] = latest + return state + + def _pose_to_dict(self, pose: PoseStamped | None) -> dict[str, Any] | None: + if pose is None: + return None + return { + "frame_id": pose.frame_id, + "timestamp": pose.ts, + "position": { + "x": round(pose.position.x, 3), + "y": round(pose.position.y, 3), + "z": round(pose.position.z, 3), + }, + "yaw_degrees": round(math.degrees(pose.yaw), 1), + } + + +def _to_jsonable(value: Any, max_string_length: int = 500) -> Any: + if value is None or isinstance(value, bool | int | float): + return value + if isinstance(value, str): + return value[:max_string_length] + if isinstance(value, dict): + return { + str(key): _to_jsonable(item, max_string_length) + for key, item in value.items() + if not str(key).startswith("_") + } + if isinstance(value, list | tuple): + return [_to_jsonable(item, max_string_length) for item in value[:10]] + return str(value)[:max_string_length] + + +__all__ = ["_Go2RobotBodyState"] diff --git a/dimos/robot/unitree/go2/blueprints/layers/layer_6_robot_body/test_robot_body_state.py b/dimos/robot/unitree/go2/blueprints/layers/layer_6_robot_body/test_robot_body_state.py new file mode 100644 index 0000000000..938059ef19 --- /dev/null +++ b/dimos/robot/unitree/go2/blueprints/layers/layer_6_robot_body/test_robot_body_state.py @@ -0,0 +1,72 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from dimos.msgs.geometry_msgs.PoseStamped import PoseStamped +from dimos.robot.unitree.go2.blueprints.layers.layer_6_robot_body.robot_body_state import ( + _Go2RobotBodyState, +) + + +def _stop_modules(*modules: object) -> None: + for module in modules: + stop = getattr(module, "stop", None) + if stop is not None: + stop() + + +def test_robot_body_snapshot_reports_missing_streams() -> None: + body_state = _Go2RobotBodyState() + try: + snapshot = body_state.get_robot_body_snapshot() + + assert snapshot["available"] is True + assert snapshot["version"] == "v1" + assert snapshot["sensors"]["odom"]["available"] is False + assert snapshot["connection"]["available"] is False + assert snapshot["local_policy"]["available"] is True + assert snapshot["safety"]["body_pose_available"] is False + finally: + _stop_modules(body_state) + + +def test_robot_body_snapshot_tracks_observed_odom() -> None: + body_state = _Go2RobotBodyState() + try: + body_state._on_odom(PoseStamped(position=[1.5, -2.0, 0.0], frame_id="map")) + + snapshot = body_state.get_robot_body_snapshot() + + assert snapshot["sensors"]["odom"]["available"] is True + assert snapshot["sensors"]["odom"]["count"] == 1 + assert snapshot["sensors"]["odom"]["latest"]["position"]["x"] == 1.5 + assert snapshot["sensors"]["odom"]["latest"]["position"]["y"] == -2.0 + assert snapshot["safety"]["body_pose_available"] is True + finally: + _stop_modules(body_state) + + +def test_sensor_state_counts_image_and_lidar_events() -> None: + body_state = _Go2RobotBodyState() + try: + body_state._on_color_image(object()) # type: ignore[arg-type] + body_state._on_lidar(object()) # type: ignore[arg-type] + + sensors = body_state.get_sensor_state() + + assert sensors["color_image"]["available"] is True + assert sensors["color_image"]["count"] == 1 + assert sensors["lidar"]["available"] is True + assert sensors["lidar"]["count"] == 1 + finally: + _stop_modules(body_state) From 9ff6c98cca3698cc75ab5a47ac6aec882f529140 Mon Sep 17 00:00:00 2001 From: z1050297972-spec Date: Fri, 22 May 2026 19:01:00 +0800 Subject: [PATCH 08/36] refactor(go2): advance layer world state and skill contracts --- dimos/robot/unitree/go2/PROJECT_CHANGELOG.md | 73 +++++++ .../unitree/go2/blueprints/layers/GOAL_1.md | 27 ++- .../layers/layer_4_world_state/DESIGN.md | 56 ++++- .../semantic_temporal_map.py | 193 ++++++++++++++++++ .../structured_world_state.py | 159 ++++++++++++++- .../layer_4_world_state/test_world_state.py | 162 ++++++++++++++- .../layer_4_world_state/world_state_spec.py | 1 + .../layers/layer_5_skill_interface/DESIGN.md | 41 +++- .../skill_interface_registry.py | 129 +++++++++++- .../skill_interface_spec.py | 1 + .../test_skill_interface_registry.py | 79 ++++++- 11 files changed, 897 insertions(+), 24 deletions(-) diff --git a/dimos/robot/unitree/go2/PROJECT_CHANGELOG.md b/dimos/robot/unitree/go2/PROJECT_CHANGELOG.md index e3b700fd2a..28b0767b34 100644 --- a/dimos/robot/unitree/go2/PROJECT_CHANGELOG.md +++ b/dimos/robot/unitree/go2/PROJECT_CHANGELOG.md @@ -6,6 +6,79 @@ work so future contributors can understand why the fork differs from upstream. Entries are listed in reverse chronological order. +## 2026-05-22 + +- Branch: `refactor/go2-architecture-layers` +- Summary: Implemented the first Go2 Layer 5 V2 consistency check by comparing + static skill contracts against MCP tool names from the full Go2 agentic + blueprint, and filled previously missing contracts for MCP-exposed Go2 skills. +- Files/modules: + - `dimos/robot/unitree/go2/blueprints/layers/layer_5_skill_interface/skill_interface_registry.py` + - `dimos/robot/unitree/go2/blueprints/layers/layer_5_skill_interface/skill_interface_spec.py` + - `dimos/robot/unitree/go2/blueprints/layers/layer_5_skill_interface/test_skill_interface_registry.py` + - `dimos/robot/unitree/go2/blueprints/layers/layer_5_skill_interface/DESIGN.md` + - `dimos/robot/unitree/go2/blueprints/layers/GOAL_1.md` + - `dimos/robot/unitree/go2/PROJECT_CHANGELOG.md` +- Validation: + - Ran `python -m py_compile` for the changed Layer 5 spec, registry, and + focused test file. + - Ran focused Layer 5 pytest in WSL Ubuntu against the current Windows + working tree using the existing WSL dependency environment: + `9 passed, 1 warning`. +- Open items: + - Add more precise argument constraints for Unitree sport commands and + perception callback payloads. + +## 2026-05-22 + +- Branch: `refactor/go2-architecture-layers` +- Summary: Implemented Go2 Layer 4 V3 by making the world-snapshot storage + policy explicit and adding a full Go2 agentic blueprint static test for the + Layer 3 -> Layer 4 -> Layer 6 RPC provider path. +- Files/modules: + - `dimos/robot/unitree/go2/blueprints/layers/layer_4_world_state/world_state_spec.py` + - `dimos/robot/unitree/go2/blueprints/layers/layer_4_world_state/structured_world_state.py` + - `dimos/robot/unitree/go2/blueprints/layers/layer_4_world_state/test_world_state.py` + - `dimos/robot/unitree/go2/blueprints/layers/layer_4_world_state/DESIGN.md` + - `dimos/robot/unitree/go2/blueprints/layers/GOAL_1.md` + - `dimos/robot/unitree/go2/PROJECT_CHANGELOG.md` +- Validation: + - Ran `python -m py_compile` for the changed Layer 4 spec, + implementation, and focused test file. + - Ran focused Layer 4 pytest in WSL Ubuntu against the current Windows + working tree using the existing WSL dependency environment: + `3 passed, 1 warning`. +- Open items: + - If snapshot retention becomes necessary, revisit whether to persist + normalized snapshots to `memory2` SQLite, JSONL, or TemporalMemory. + +## 2026-05-22 + +- Branch: `refactor/go2-architecture-layers` +- Summary: Implemented Go2 Layer 4 V2 memory summaries by adding explicit + semantic-temporal evidence entries and normalized named object/location + summaries to the structured world snapshot. +- Files/modules: + - `dimos/robot/unitree/go2/blueprints/layers/layer_4_world_state/semantic_temporal_map.py` + - `dimos/robot/unitree/go2/blueprints/layers/layer_4_world_state/structured_world_state.py` + - `dimos/robot/unitree/go2/blueprints/layers/layer_4_world_state/test_world_state.py` + - `dimos/robot/unitree/go2/blueprints/layers/layer_4_world_state/DESIGN.md` + - `dimos/robot/unitree/go2/blueprints/layers/GOAL_1.md` +- Validation: + - Ran `python -m py_compile` for the changed Layer 4 implementation and + focused test file. + - Ran `git diff --check`. + - Ran focused Layer 4 pytest in WSL Ubuntu against the current Windows + working tree using the existing WSL dependency environment: + `3 passed, 1 warning`. + - Attempted focused pytest with system Python; blocked because `pytest` is + not installed. + - Attempted focused pytest with `uv run`; blocked by local Windows + environment creation/cache errors before tests could run. Removed the + partial `.venv` left by that failed attempt. +- Open items: + - Add full Go2 agentic blueprint tests for the Layer 3 -> Layer 4 RPC path. + ## 2026-05-20 - Branch: `refactor/go2-architecture-layers` diff --git a/dimos/robot/unitree/go2/blueprints/layers/GOAL_1.md b/dimos/robot/unitree/go2/blueprints/layers/GOAL_1.md index 3c7399953f..92edecccf3 100644 --- a/dimos/robot/unitree/go2/blueprints/layers/GOAL_1.md +++ b/dimos/robot/unitree/go2/blueprints/layers/GOAL_1.md @@ -103,22 +103,35 @@ Layer 4 V1 is intentionally a facade over existing modules. The existing `SpatialMemory` and lazy `TemporalMemory` implementations stay in their current packages. +Layer 4 V2 adds normalized `memory_state.named_objects` and +`memory_state.named_locations` summaries, plus explicit fused +semantic-temporal evidence entries with entity, location, time, evidence source, +and confidence fields. + +Layer 4 V3 makes the snapshot durability policy explicit: +`get_world_snapshot(...)` reports `snapshot_storage.policy = +"ephemeral_read_through"`. Stable Layer 4 snapshot writes are deferred until a +real retention/query requirement exists; existing `SpatialMemory` and +`TemporalMemory` remain the durable source stores. + ## Layer 5 Direction Layer 5 owns the skill-interface boundary. It should tell upper layers which Go2 skills exist, what arguments each skill expects, which skills are motion-sensitive, and what preflight checks should run before execution. -Current Layer 5 V1 keeps existing skill containers unchanged and adds +Current Layer 5 keeps existing skill containers unchanged and adds `_Go2SkillInterfaceRegistry` as an RPC-only contract registry. The registry does not execute skills. It returns static contracts for current Go2 skills, including navigation, person following, Unitree motion/utility commands, -perception lookout, security patrol, and speech. `ContextProvider` reads this -registry through `SkillInterfaceSpec` and includes a compact skill-interface -summary in `get_context(...)`. - -Layer 5 V2 should compare the static contracts with the MCP server tool list so -renamed or missing skills are detected automatically. +perception, exploration, patrolling, security patrol, and speech. +`ContextProvider` reads this registry through `SkillInterfaceSpec` and includes +a compact skill-interface summary in `get_context(...)`. + +Layer 5 V2 compares the static contracts with MCP tool names so renamed, +missing, or unregistered Go2 skills are detected automatically in focused +tests. Remaining Layer 5 V2 work is to add more precise constraints for Unitree +sport commands and perception callback payloads. ## Layer 6 Direction diff --git a/dimos/robot/unitree/go2/blueprints/layers/layer_4_world_state/DESIGN.md b/dimos/robot/unitree/go2/blueprints/layers/layer_4_world_state/DESIGN.md index 40b085e622..f7cb043983 100644 --- a/dimos/robot/unitree/go2/blueprints/layers/layer_4_world_state/DESIGN.md +++ b/dimos/robot/unitree/go2/blueprints/layers/layer_4_world_state/DESIGN.md @@ -19,6 +19,12 @@ Layer 4 currently contains: structured world snapshot. - Optional `RobotBodyStateSpec`: Layer 6 body/local-policy state read by `_Go2StructuredWorldState` when wired. +- V2 memory summaries: normalized `named_objects`, `named_locations`, and + explicit semantic-temporal evidence entries with entity, location, time, + evidence source, and confidence fields. +- V3 snapshot policy: Layer 4 snapshots are explicitly ephemeral read-through + views. Durable writes remain in `SpatialMemory` and `TemporalMemory` until a + concrete snapshot retention requirement exists. ## Runtime Flow @@ -61,6 +67,16 @@ memory, navigation, and odom remain as a fallback. - `temporal` - `fused` - optional `errors` +- V2 data shape: + - `temporal.graph` is read from `TemporalMemorySpec.get_graph_db_stats()` + when available. + - `fused.entries` contains compact evidence entries. Each entry includes + `entity`, `location`, `time`, `evidence_source`, `confidence`, and + `summary`. + - Spatial entries use vector-memory metadata such as `pos_x`, `pos_y`, + `pos_z`, `timestamp`, `frame_id`, and labels when present. + - Temporal entries use state entities, currently-present entities, + `entity_roster`, and graph entities when those sections are available. - Current limits: - Fusion is deterministic metadata packaging. - It does not resolve contradictions between spatial and temporal evidence. @@ -90,11 +106,36 @@ memory, navigation, and odom remain as a fallback. - `memory_state` - `semantic_temporal_map` - optional `errors` +- V2 data shape: + - `memory_state.named_objects` summarizes entities from fused evidence that + are not typed as locations. + - `memory_state.named_locations` summarizes entries that include coordinates + or are typed as locations. + - `memory_state.summary` reports named object count, named location count, + evidence entry count, and contributing evidence sources. +- V3 data shape: + - `snapshot_storage` reports the durability decision for this snapshot API. + The current policy is `ephemeral_read_through` with no separate durable + Layer 4 snapshot backend. - Current limits: - Robot safety and connection state are read from Layer 6 when wired, but they are still descriptive context rather than physical safety guarantees. - The module currently reads navigation state by RPC when available. +### `_Go2StructuredWorldState.get_snapshot_storage_policy(...)` + +- File: `layer_4_world_state/structured_world_state.py` +- Entry point: RPC-only helper; not an MCP skill. +- Purpose: expose the V3 storage decision to callers. Layer 4 snapshots are + normalized read-through views and are not written to a separate durable + store in this stage. +- Return shape: + - `durable`: `False` + - `backend`: `None` + - `policy`: `ephemeral_read_through` + - `reason`: short explanation that durable writes stay in existing memory + modules until snapshot retention requirements are validated. + ## Version Boundaries V1 implemented: @@ -106,18 +147,21 @@ V1 implemented: - Make Layer 3 `ContextProvider` prefer Layer 4 world snapshots while keeping direct-read fallback behavior. -V2 planned: +V2 implemented: - Add named object/location summaries to `memory_state`. - Make semantic-temporal map entries more explicit: entity, location, time, evidence source, and confidence. -V3 planned: +V3 implemented: -- Decide whether stable world snapshots should be written to a durable store - such as `memory2` SQLite, JSONL, or TemporalMemory. -- Add tests that build the full Go2 agentic blueprint and verify the injected - Layer 3 -> Layer 4 RPC path. +- Durable snapshot writes are deferred. Layer 4 snapshots remain ephemeral + read-through state; persistent source data remains in `SpatialMemory` and + `TemporalMemory`. +- Added a focused full-Go2-agentic-blueprint static test that verifies the + injected RPC path from Layer 3 `ContextProvider` to Layer 4 + `_Go2StructuredWorldState`, and from Layer 4 to semantic-temporal memory and + Layer 6 robot-body state. ## Design Rules diff --git a/dimos/robot/unitree/go2/blueprints/layers/layer_4_world_state/semantic_temporal_map.py b/dimos/robot/unitree/go2/blueprints/layers/layer_4_world_state/semantic_temporal_map.py index a44ff0001c..8b5f5b3afe 100644 --- a/dimos/robot/unitree/go2/blueprints/layers/layer_4_world_state/semantic_temporal_map.py +++ b/dimos/robot/unitree/go2/blueprints/layers/layer_4_world_state/semantic_temporal_map.py @@ -113,6 +113,12 @@ def _temporal_section(self, query: str, errors: list[str]) -> dict[str, Any]: logger.warning("Failed to read temporal entity roster", exc_info=True) errors.append(f"temporal_memory.roster: {exc}") + try: + context["graph"] = self._temporal_memory.get_graph_db_stats() + except Exception as exc: + logger.warning("Failed to read temporal graph stats", exc_info=True) + errors.append(f"temporal_memory.graph: {exc}") + return context def _fused_section(self, result: dict[str, Any]) -> dict[str, Any]: @@ -121,11 +127,14 @@ def _fused_section(self, result: dict[str, Any]) -> dict[str, Any]: spatial_matches = spatial.get("matches") or [] temporal_answer = temporal.get("answer") temporal_summary = temporal.get("rolling_summary") + entries = _build_evidence_entries(spatial_matches, temporal) return { "available": bool(spatial.get("available") or temporal.get("available")), "spatial_match_count": len(spatial_matches), "has_temporal_answer": bool(temporal_answer), "has_temporal_summary": bool(temporal_summary), + "entry_count": len(entries), + "entries": entries, } @@ -141,6 +150,190 @@ def _summarize_spatial_match(match: dict[str, Any]) -> dict[str, Any]: return _to_jsonable(summary) +def _build_evidence_entries( + spatial_matches: list[dict[str, Any]], temporal: dict[str, Any] +) -> list[dict[str, Any]]: + entries: list[dict[str, Any]] = [] + + for index, match in enumerate(spatial_matches[:5]): + if not isinstance(match, dict): + continue + entries.append(_spatial_evidence_entry(match, index)) + + for source, entity in _temporal_entities(temporal): + entries.append(_temporal_evidence_entry(entity, source)) + + return [_to_jsonable(entry) for entry in _dedupe_entries(entries)[:10]] + + +def _spatial_evidence_entry(match: dict[str, Any], index: int) -> dict[str, Any]: + metadata = _first_metadata(match) + entity_id = _first_text(metadata, "entity_id", "id", "label", "name") or _first_text( + match, "id", "label", "name" + ) + description = ( + _first_text(match, "text", "description") + or _first_text(metadata, "description", "descriptor", "label", "name") + or entity_id + or f"spatial_match_{index}" + ) + return { + "entity": { + "id": entity_id or f"spatial_match_{index}", + "type": _first_text(metadata, "type", "entity_type") or "spatial_observation", + "description": description, + }, + "location": _location_from(match, metadata), + "time": _time_from(match, metadata), + "evidence_source": "spatial_memory", + "confidence": _confidence_from(match), + "source_index": index, + "summary": description, + } + + +def _temporal_evidence_entry(entity: dict[str, Any], source: str) -> dict[str, Any]: + metadata = _metadata_dict(entity.get("metadata")) + entity_id = _first_text(entity, "entity_id", "id", "name", "label") + description = _first_text(entity, "descriptor", "description", "label", "name") or entity_id + return { + "entity": { + "id": entity_id or description or "unknown_entity", + "type": _first_text(entity, "entity_type", "type") or "unknown", + "description": description or "unknown", + }, + "location": _location_from(entity, metadata), + "time": _time_from(entity, metadata), + "evidence_source": source, + "confidence": _confidence_from(entity), + "summary": description or entity_id or "unknown entity", + } + + +def _temporal_entities(temporal: dict[str, Any]) -> list[tuple[str, dict[str, Any]]]: + entities: list[tuple[str, dict[str, Any]]] = [] + state = temporal.get("state") + if isinstance(state, dict): + for entity in state.get("entities") or state.get("entity_roster") or []: + if isinstance(entity, dict): + entities.append(("temporal_memory.state", entity)) + for entity in state.get("currently_present") or []: + if isinstance(entity, dict): + entities.append(("temporal_memory.currently_present", entity)) + + for entity in temporal.get("entity_roster") or []: + if isinstance(entity, dict): + entities.append(("temporal_memory.entity_roster", entity)) + + graph = temporal.get("graph") + if isinstance(graph, dict): + for entity in graph.get("entities") or []: + if isinstance(entity, dict): + entities.append(("temporal_memory.graph", entity)) + + return entities + + +def _dedupe_entries(entries: list[dict[str, Any]]) -> list[dict[str, Any]]: + deduped: list[dict[str, Any]] = [] + seen: set[tuple[str, str, str]] = set() + for entry in entries: + entity = entry.get("entity") if isinstance(entry.get("entity"), dict) else {} + entity_id = str(entity.get("id") or "") + source = str(entry.get("evidence_source") or "") + location = entry.get("location") + location_key = str(location) if location else "" + key = (entity_id, source, location_key) + if key in seen: + continue + seen.add(key) + deduped.append(entry) + return deduped + + +def _first_metadata(match: dict[str, Any]) -> dict[str, Any]: + return _metadata_dict(match.get("metadata")) + + +def _metadata_dict(value: Any) -> dict[str, Any]: + if isinstance(value, dict): + return value + if isinstance(value, list): + for item in value: + if isinstance(item, dict): + return item + return {} + + +def _location_from(*sources: dict[str, Any]) -> dict[str, Any] | None: + for source in sources: + x = _first_number(source, "world_x", "pos_x", "x") + y = _first_number(source, "world_y", "pos_y", "y") + z = _first_number(source, "world_z", "pos_z", "z") + if x is None or y is None: + continue + location: dict[str, Any] = { + "x": round(x, 3), + "y": round(y, 3), + } + if z is not None: + location["z"] = round(z, 3) + frame_id = _first_text(source, "frame_id", "frame") + if frame_id: + location["frame_id"] = frame_id + label = _first_text(source, "location", "place", "label", "name") + if label: + location["label"] = label + return location + return None + + +def _time_from(*sources: dict[str, Any]) -> dict[str, Any] | None: + for source in sources: + first_seen = _first_number(source, "first_seen_ts", "first_seen", "start_ts") + last_seen = _first_number(source, "last_seen_ts", "last_seen", "end_ts") + timestamp = _first_number(source, "timestamp_s", "timestamp", "time", "ts") + if first_seen is None and last_seen is None and timestamp is None: + continue + result: dict[str, Any] = {} + if timestamp is not None: + result["timestamp"] = round(timestamp, 3) + if first_seen is not None: + result["first_seen_ts"] = round(first_seen, 3) + if last_seen is not None: + result["last_seen_ts"] = round(last_seen, 3) + return result + return None + + +def _confidence_from(source: dict[str, Any]) -> float | None: + explicit = _first_number(source, "confidence", "score", "similarity") + if explicit is not None: + return round(max(0.0, min(1.0, explicit)), 3) + distance = _first_number(source, "distance") + if distance is None: + return None + return round(max(0.0, min(1.0, 1.0 - distance)), 3) + + +def _first_text(source: dict[str, Any], *keys: str) -> str | None: + for key in keys: + value = source.get(key) + if isinstance(value, str) and value.strip(): + return value.strip() + return None + + +def _first_number(source: dict[str, Any], *keys: str) -> float | None: + for key in keys: + value = source.get(key) + if isinstance(value, bool): + continue + if isinstance(value, int | float): + return float(value) + return None + + def _to_jsonable(value: Any, max_string_length: int = 500) -> Any: if value is None or isinstance(value, bool | int | float): return value diff --git a/dimos/robot/unitree/go2/blueprints/layers/layer_4_world_state/structured_world_state.py b/dimos/robot/unitree/go2/blueprints/layers/layer_4_world_state/structured_world_state.py index eeabcc4c0d..40f4589af5 100644 --- a/dimos/robot/unitree/go2/blueprints/layers/layer_4_world_state/structured_world_state.py +++ b/dimos/robot/unitree/go2/blueprints/layers/layer_4_world_state/structured_world_state.py @@ -31,6 +31,9 @@ from dimos.robot.unitree.go2.blueprints.layers.layer_4_world_state.world_state_spec import ( SemanticTemporalMapSpec, ) +from dimos.robot.unitree.go2.blueprints.layers.layer_4_world_state.semantic_temporal_map import ( + _build_evidence_entries, +) from dimos.robot.unitree.go2.blueprints.layers.layer_6_robot_body.robot_body_spec import ( RobotBodyStateSpec, ) @@ -80,6 +83,7 @@ def get_world_snapshot(self, task: str = "", spatial_limit: int = 3) -> dict[str "robot_state": self.get_robot_state(), "memory_state": self._memory_state(task, spatial_limit, semantic_temporal, errors), "semantic_temporal_map": semantic_temporal, + "snapshot_storage": self.get_snapshot_storage_policy(), } if errors: snapshot["errors"] = errors @@ -142,6 +146,20 @@ def get_memory_state(self, task: str = "", spatial_limit: int = 3) -> dict[str, state["errors"] = errors return _to_jsonable(state) + @rpc + def get_snapshot_storage_policy(self) -> dict[str, Any]: + """Return the Layer 4 snapshot durability policy.""" + return { + "durable": False, + "backend": None, + "policy": "ephemeral_read_through", + "reason": ( + "Layer 4 snapshots normalize live source modules. Durable writes stay in " + "SpatialMemory and TemporalMemory until snapshot retention requirements " + "are validated." + ), + } + def _source_status(self, semantic_temporal: dict[str, Any]) -> dict[str, bool]: semantic_sources = semantic_temporal.get("sources", {}) return { @@ -201,16 +219,30 @@ def _memory_state( errors: list[str], ) -> dict[str, Any]: if semantic_temporal.get("spatial") or semantic_temporal.get("temporal"): + spatial = semantic_temporal.get( + "spatial", {"available": False, "matches": []} + ) + temporal = semantic_temporal.get("temporal", {"available": False}) + summaries = _memory_summaries(spatial, temporal, semantic_temporal) return { "query": task, - "spatial": semantic_temporal.get("spatial", {"available": False, "matches": []}), - "temporal": semantic_temporal.get("temporal", {"available": False}), + "spatial": spatial, + "temporal": temporal, + "named_objects": summaries["named_objects"], + "named_locations": summaries["named_locations"], + "summary": summaries["summary"], } + spatial = self._spatial_fallback(task, spatial_limit, errors) + temporal = self._temporal_fallback(errors) + summaries = _memory_summaries(spatial, temporal, semantic_temporal) return { "query": task, - "spatial": self._spatial_fallback(task, spatial_limit, errors), - "temporal": self._temporal_fallback(errors), + "spatial": spatial, + "temporal": temporal, + "named_objects": summaries["named_objects"], + "named_locations": summaries["named_locations"], + "summary": summaries["summary"], } def _spatial_fallback( @@ -279,6 +311,125 @@ def _summarize_spatial_match(match: dict[str, Any]) -> dict[str, Any]: return _to_jsonable(summary) +def _memory_summaries( + spatial: dict[str, Any], + temporal: dict[str, Any], + semantic_temporal: dict[str, Any], +) -> dict[str, Any]: + entries = _evidence_entries(spatial, temporal, semantic_temporal) + named_objects = _named_objects(entries) + named_locations = _named_locations(entries) + return { + "named_objects": named_objects, + "named_locations": named_locations, + "summary": { + "named_object_count": len(named_objects), + "named_location_count": len(named_locations), + "evidence_entry_count": len(entries), + "sources": sorted( + { + str(entry.get("evidence_source")) + for entry in entries + if entry.get("evidence_source") + } + ), + }, + } + + +def _evidence_entries( + spatial: dict[str, Any], + temporal: dict[str, Any], + semantic_temporal: dict[str, Any], +) -> list[dict[str, Any]]: + fused = semantic_temporal.get("fused") + if isinstance(fused, dict) and isinstance(fused.get("entries"), list): + return [entry for entry in fused["entries"] if isinstance(entry, dict)] + + matches = spatial.get("matches") or [] + spatial_matches = [match for match in matches if isinstance(match, dict)] + return _build_evidence_entries(spatial_matches, temporal) + + +def _named_objects(entries: list[dict[str, Any]]) -> list[dict[str, Any]]: + objects: list[dict[str, Any]] = [] + seen: set[str] = set() + for entry in entries: + entity = entry.get("entity") + if not isinstance(entity, dict): + continue + object_type = str(entity.get("type") or "unknown") + if object_type in {"location", "spatial_observation"}: + continue + object_id = str(entity.get("id") or entity.get("description") or "").strip() + if not object_id or object_id in seen: + continue + seen.add(object_id) + objects.append( + _to_jsonable( + { + "id": object_id, + "type": object_type, + "description": entity.get("description"), + "location": entry.get("location"), + "last_seen": _last_seen(entry.get("time")), + "evidence_source": entry.get("evidence_source"), + "confidence": entry.get("confidence"), + } + ) + ) + return objects[:10] + + +def _named_locations(entries: list[dict[str, Any]]) -> list[dict[str, Any]]: + locations: list[dict[str, Any]] = [] + seen: set[str] = set() + for entry in entries: + entity = entry.get("entity") if isinstance(entry.get("entity"), dict) else {} + location = entry.get("location") + entity_type = str(entity.get("type") or "") + if not isinstance(location, dict) and entity_type != "location": + continue + + name = ( + (location or {}).get("label") + or entity.get("description") + or entity.get("id") + or "unknown_location" + ) + key = str(name) + if isinstance(location, dict): + key = f"{key}:{location.get('x')}:{location.get('y')}:{location.get('z')}" + if key in seen: + continue + seen.add(key) + locations.append( + _to_jsonable( + { + "name": name, + "location": location, + "entity_id": entity.get("id"), + "last_seen": _last_seen(entry.get("time")), + "evidence_source": entry.get("evidence_source"), + "confidence": entry.get("confidence"), + } + ) + ) + return locations[:10] + + +def _last_seen(time_value: Any) -> float | None: + if not isinstance(time_value, dict): + return None + for key in ("last_seen_ts", "timestamp", "first_seen_ts"): + value = time_value.get(key) + if isinstance(value, bool): + continue + if isinstance(value, int | float): + return float(value) + return None + + def _section(snapshot: dict[str, Any] | None, key: str) -> dict[str, Any] | None: if snapshot is None: return None diff --git a/dimos/robot/unitree/go2/blueprints/layers/layer_4_world_state/test_world_state.py b/dimos/robot/unitree/go2/blueprints/layers/layer_4_world_state/test_world_state.py index a8d202f055..0bd7bce2a4 100644 --- a/dimos/robot/unitree/go2/blueprints/layers/layer_4_world_state/test_world_state.py +++ b/dimos/robot/unitree/go2/blueprints/layers/layer_4_world_state/test_world_state.py @@ -14,14 +14,31 @@ from typing import Any +import pytest + +from dimos.core.coordination.blueprints import Blueprint, BlueprintAtom, ModuleRef from dimos.msgs.geometry_msgs.PoseStamped import PoseStamped from dimos.navigation.base import NavigationState +from dimos.robot.unitree.go2.blueprints.layers.layer_3_agent_brain.context_provider import ( + _Go2ContextProvider, +) from dimos.robot.unitree.go2.blueprints.layers.layer_4_world_state.semantic_temporal_map import ( _Go2SemanticTemporalMap, ) from dimos.robot.unitree.go2.blueprints.layers.layer_4_world_state.structured_world_state import ( _Go2StructuredWorldState, ) +from dimos.robot.unitree.go2.blueprints.layers.layer_4_world_state.world_state_spec import ( + SemanticTemporalMapSpec, + WorldStateSpec, +) +from dimos.robot.unitree.go2.blueprints.layers.layer_6_robot_body.robot_body_spec import ( + RobotBodyStateSpec, +) +from dimos.robot.unitree.go2.blueprints.layers.layer_6_robot_body.robot_body_state import ( + _Go2RobotBodyState, +) +from dimos.spec.utils import spec_annotation_compliance, spec_structural_compliance def _stop_modules(*modules: object) -> None: @@ -46,16 +63,37 @@ def query(self, question: str) -> str: return f"temporal answer for {question}" def get_state(self) -> dict[str, Any]: - return {"entity_count": 1} + return { + "entity_count": 1, + "entities": [ + { + "id": "cup_1", + "type": "object", + "descriptor": "blue cup", + "confidence": 0.7, + } + ], + } def get_entity_roster(self) -> list[dict[str, Any]]: - return [{"id": "person_1"}] + return [{"id": "person_1", "type": "person", "descriptor": "near the hallway"}] def get_rolling_summary(self) -> str: return "A person was recently near the hallway." def get_graph_db_stats(self) -> dict[str, Any]: - return {"stats": {"entities": 1}} + return { + "stats": {"entities": 1}, + "entities": [ + { + "entity_id": "hallway_1", + "entity_type": "location", + "descriptor": "main hallway", + "last_seen_ts": 12.0, + "metadata": {"world_x": 1.0, "world_y": 2.0, "world_z": 0.0}, + } + ], + } class StubNavigation: @@ -75,17 +113,56 @@ def query_semantic_temporal_map( "sources": {"spatial_memory": True, "temporal_memory": True}, "spatial": { "available": True, - "matches": [{"distance": 0.2, "metadata": [{"label": query}]}], + "matches": [ + { + "distance": 0.2, + "metadata": [ + {"label": query, "pos_x": 1.0, "pos_y": 2.0, "pos_z": 0.0} + ], + } + ], }, "temporal": { "available": True, "rolling_summary": "A person was recently near the hallway.", + "entity_roster": [ + { + "id": "person_1", + "type": "person", + "descriptor": "near the hallway", + } + ], }, "fused": { "available": True, "spatial_match_count": 1, "has_temporal_answer": False, "has_temporal_summary": True, + "entry_count": 2, + "entries": [ + { + "entity": { + "id": query, + "type": "spatial_observation", + "description": query, + }, + "location": {"x": 1.0, "y": 2.0, "z": 0.0, "label": query}, + "time": None, + "evidence_source": "spatial_memory", + "confidence": 0.8, + }, + { + "entity": { + "id": "person_1", + "type": "person", + "description": "near the hallway", + }, + "location": None, + "time": None, + "evidence_source": "temporal_memory.entity_roster", + "confidence": None, + }, + ], }, } @@ -125,6 +202,16 @@ def test_semantic_temporal_map_combines_memory_sources() -> None: assert result["temporal"]["answer"] == "temporal answer for find the hallway" assert result["fused"]["spatial_match_count"] == 1 assert result["fused"]["has_temporal_summary"] is True + assert result["fused"]["entry_count"] == 4 + assert result["fused"]["entries"][0]["entity"]["id"] == "find the hallway" + assert result["fused"]["entries"][0]["location"]["x"] == 1.0 + assert result["fused"]["entries"][0]["evidence_source"] == "spatial_memory" + assert result["fused"]["entries"][0]["confidence"] == 0.8 + assert any( + entry["entity"]["id"] == "hallway_1" + and entry["evidence_source"] == "temporal_memory.graph" + for entry in result["fused"]["entries"] + ) finally: _stop_modules(semantic_map) @@ -148,6 +235,73 @@ def test_structured_world_state_returns_snapshot() -> None: assert snapshot["robot_state"]["sensors"]["odom"]["count"] == 3 assert snapshot["robot_state"]["safety"]["body_pose_available"] is True assert snapshot["memory_state"]["spatial"]["matches"][0]["distance"] == 0.2 + assert snapshot["memory_state"]["named_objects"][0]["id"] == "person_1" + assert snapshot["memory_state"]["named_locations"][0]["name"] == "find the hallway" + assert snapshot["memory_state"]["summary"]["named_object_count"] == 1 + assert snapshot["memory_state"]["summary"]["named_location_count"] == 1 assert snapshot["semantic_temporal_map"]["fused"]["spatial_match_count"] == 1 + assert snapshot["semantic_temporal_map"]["fused"]["entries"][0]["location"]["x"] == 1.0 + assert snapshot["snapshot_storage"]["durable"] is False + assert snapshot["snapshot_storage"]["policy"] == "ephemeral_read_through" finally: _stop_modules(world_state) + + +def test_go2_agentic_blueprint_wires_layer3_to_layer4_rpc_path() -> None: + try: + from dimos.robot.unitree.go2.blueprints.agentic.unitree_go2_agentic import ( + unitree_go2_agentic, + ) + except (ImportError, ModuleNotFoundError, OSError) as exc: + pytest.skip(f"Full Go2 agentic blueprint dependencies are unavailable: {exc}") + + context_atom = _atom_for(unitree_go2_agentic, _Go2ContextProvider) + world_ref = _module_ref(context_atom, "_world_state") + assert world_ref.spec is WorldStateSpec + assert world_ref.optional is True + assert _single_valid_provider(unitree_go2_agentic, context_atom, world_ref) is ( + _Go2StructuredWorldState + ) + + world_atom = _atom_for(unitree_go2_agentic, _Go2StructuredWorldState) + semantic_ref = _module_ref(world_atom, "_semantic_temporal_map") + assert semantic_ref.spec is SemanticTemporalMapSpec + assert semantic_ref.optional is True + assert _single_valid_provider(unitree_go2_agentic, world_atom, semantic_ref) is ( + _Go2SemanticTemporalMap + ) + + body_ref = _module_ref(world_atom, "_robot_body") + assert body_ref.spec is RobotBodyStateSpec + assert body_ref.optional is True + assert _single_valid_provider(unitree_go2_agentic, world_atom, body_ref) is ( + _Go2RobotBodyState + ) + + +def _atom_for(blueprint: Blueprint, module: type[object]) -> BlueprintAtom: + matches = [atom for atom in blueprint.active_blueprints if atom.module is module] + assert len(matches) == 1 + return matches[0] + + +def _module_ref(atom: BlueprintAtom, name: str) -> ModuleRef: + matches = [ref for ref in atom.module_refs if ref.name == name] + assert len(matches) == 1 + return matches[0] + + +def _single_valid_provider( + blueprint: Blueprint, + consumer_atom: BlueprintAtom, + module_ref: ModuleRef, +) -> type[object]: + providers = [ + atom.module + for atom in blueprint.active_blueprints + if atom != consumer_atom + and spec_structural_compliance(atom.module, module_ref.spec) + and spec_annotation_compliance(atom.module, module_ref.spec) + ] + assert len(providers) == 1 + return providers[0] diff --git a/dimos/robot/unitree/go2/blueprints/layers/layer_4_world_state/world_state_spec.py b/dimos/robot/unitree/go2/blueprints/layers/layer_4_world_state/world_state_spec.py index 7e77666e82..c44263ac5e 100644 --- a/dimos/robot/unitree/go2/blueprints/layers/layer_4_world_state/world_state_spec.py +++ b/dimos/robot/unitree/go2/blueprints/layers/layer_4_world_state/world_state_spec.py @@ -28,6 +28,7 @@ def get_world_snapshot(self, task: str = "", spatial_limit: int = 3) -> dict[str def get_robot_state(self) -> dict[str, Any]: ... def get_runtime_state(self) -> dict[str, Any]: ... def get_memory_state(self, task: str = "", spatial_limit: int = 3) -> dict[str, Any]: ... + def get_snapshot_storage_policy(self) -> dict[str, Any]: ... __all__ = ["SemanticTemporalMapSpec", "WorldStateSpec"] diff --git a/dimos/robot/unitree/go2/blueprints/layers/layer_5_skill_interface/DESIGN.md b/dimos/robot/unitree/go2/blueprints/layers/layer_5_skill_interface/DESIGN.md index 59529a9306..9f8ea61176 100644 --- a/dimos/robot/unitree/go2/blueprints/layers/layer_5_skill_interface/DESIGN.md +++ b/dimos/robot/unitree/go2/blueprints/layers/layer_5_skill_interface/DESIGN.md @@ -37,6 +37,10 @@ Layer 3 ContextProvider The registry is read-only. It does not execute robot actions and does not replace the existing MCP skill containers. +Layer 5 V2 compares the static contract registry against an MCP `tools/list` +payload so renamed, missing, or unregistered Go2 skills are detected in focused +tests before the agent relies on stale prompt context. + ## Module Responsibilities `_Go2SkillInterfaceRegistry` @@ -45,6 +49,8 @@ replace the existing MCP skill containers. - Returns all contracts or contracts filtered by domain. - Returns one contract by `skill_name`. - Validates planned arguments against required fields and simple JSON types. +- Compares known contracts against MCP tool names while ignoring Layer 3 and + MCP server internal tools. - Does not call, retry, interrupt, or monitor the skill itself. Existing skill containers @@ -125,6 +131,33 @@ Existing skill containers example, it can confirm that `query` exists, but not that a target location is actually reachable. +### `_Go2SkillInterfaceRegistry.compare_mcp_tools(...)` + +- File: `layer_5_skill_interface/skill_interface_registry.py` +- Entry point: RPC-only helper. +- Purpose: detect drift between the static Layer 5 contracts and the MCP tool + names actually exposed by the full Go2 agentic blueprint. +- Inputs: + - `tools_json`: JSON from MCP `tools/list`, a JSON-RPC result wrapper, a list + of tool-name strings, or a list of tool objects with `name` fields. +- Storage: + - No database or persistent state. +- Algorithm: + - Parse tool names from the payload. + - Compare them with `_SKILL_CONTRACTS`. + - Ignore known non-Layer-5 MCP tools from Layer 3 and `McpServer`, such as + `get_context`, `route_task`, `record_skill_outcome`, and `server_status`. + - Report missing contracts and unexpected MCP tools separately. +- Return shape: + - `valid`, `errors`, `contract_skill_count`, `mcp_tool_count`, + `contract_skill_names`, `mcp_tool_names`, `known_non_layer5_mcp_tools`, + `missing_contracts`, and `unregistered_mcp_tools`. + - `valid` is true only when parsing succeeds and both drift lists are empty. +- Current limits: + - The focused full-blueprint test uses static `@skill` inspection rather than + starting an MCP server process. That catches renamed or newly exposed skill + methods without needing robot hardware. + ## Version Boundaries V1 implemented: @@ -136,10 +169,16 @@ V1 implemented: - Lazy Layer 5 package imports so Layer 3 specs/tests do not pull in the full skill dependency stack. -V2 planned: +V2 implemented: - Compare the static contract registry against the MCP server tool list during tests or startup, so missing/renamed skills are detected automatically. +- Added contracts for MCP-exposed Go2 skills that previously bypassed the + registry: `observe`, `begin_exploration`, `end_exploration`, `start_patrol`, + and `stop_patrol`. + +Remaining V2 work: + - Add more precise argument constraints for Unitree sport commands and perception callbacks. diff --git a/dimos/robot/unitree/go2/blueprints/layers/layer_5_skill_interface/skill_interface_registry.py b/dimos/robot/unitree/go2/blueprints/layers/layer_5_skill_interface/skill_interface_registry.py index 62c3f6f76c..f127d4de5b 100644 --- a/dimos/robot/unitree/go2/blueprints/layers/layer_5_skill_interface/skill_interface_registry.py +++ b/dimos/robot/unitree/go2/blueprints/layers/layer_5_skill_interface/skill_interface_registry.py @@ -65,8 +65,33 @@ def to_dict(self) -> dict[str, Any]: _CONTEXT_PREFLIGHT = ("route_task", "get_context", "predict_skill_outcome") _LIGHT_PREFLIGHT = ("route_task",) +_KNOWN_NON_LAYER5_MCP_TOOLS = frozenset( + { + "agent_send", + "get_context", + "list_modules", + "predict_skill_outcome", + "record_causal_transition", + "record_skill_outcome", + "route_task", + "server_status", + "summarize_causal_patterns", + "summarize_skill_outcomes", + } +) _SKILL_CONTRACTS = ( + _SkillContract( + skill_name="observe", + domain="perception", + module="GO2Connection", + summary="Return the latest robot camera frame for visual world queries.", + requires_context=False, + requires_robot_state=True, + risk_class="low", + recommended_preflight=_LIGHT_PREFLIGHT, + outcome_shape="Image | None", + ), _SkillContract( skill_name="tag_location", domain="memory", @@ -104,6 +129,48 @@ def to_dict(self) -> dict[str, Any]: risk_class="stop", recommended_preflight=(), ), + _SkillContract( + skill_name="begin_exploration", + domain="exploration", + module="WavefrontFrontierExplorer", + summary="Start autonomous frontier exploration of the surrounding area.", + motion_sensitive=True, + requires_context=True, + requires_world_state=True, + requires_robot_state=True, + async_updates=True, + risk_class="high", + recommended_preflight=_CONTEXT_PREFLIGHT, + ), + _SkillContract( + skill_name="end_exploration", + domain="exploration", + module="WavefrontFrontierExplorer", + summary="Stop active frontier exploration and leave the robot at its current pose.", + risk_class="stop", + recommended_preflight=(), + ), + _SkillContract( + skill_name="start_patrol", + domain="navigation", + module="PatrollingModule", + summary="Start autonomous patrolling across known reachable goals.", + motion_sensitive=True, + requires_context=True, + requires_world_state=True, + requires_robot_state=True, + async_updates=True, + risk_class="high", + recommended_preflight=_CONTEXT_PREFLIGHT, + ), + _SkillContract( + skill_name="stop_patrol", + domain="navigation", + module="PatrollingModule", + summary="Stop the active autonomous patrol.", + risk_class="stop", + recommended_preflight=(), + ), _SkillContract( skill_name="follow_person", domain="person_follow", @@ -249,7 +316,7 @@ def get_skill_interface_snapshot(self, domain: str = "") -> dict[str, Any]: ] return { "available": True, - "version": "v1", + "version": "v2", "source": "static_go2_layer5_contracts", "domain_filter": domain, "domains": sorted({contract["domain"] for contract in contracts}), @@ -309,6 +376,34 @@ def validate_skill_request(self, skill_name: str, args_json: str = "{}") -> dict "contract": contract.to_dict(), } + @rpc + def compare_mcp_tools(self, tools_json: str) -> dict[str, Any]: + """Compare Layer 5 contracts with an MCP tools/list payload.""" + tool_names, parse_error = _parse_mcp_tool_names(tools_json) + contract_names = {contract.skill_name for contract in _SKILL_CONTRACTS} + errors: list[str] = [] + if parse_error: + errors.append(parse_error) + + mcp_names = set(tool_names) + known_internal = sorted(mcp_names & _KNOWN_NON_LAYER5_MCP_TOOLS) + missing_contracts = sorted(contract_names - mcp_names) if tool_names else [] + unregistered_mcp_tools = sorted( + mcp_names - contract_names - _KNOWN_NON_LAYER5_MCP_TOOLS + ) + + return { + "valid": not errors and not missing_contracts and not unregistered_mcp_tools, + "errors": errors, + "contract_skill_count": len(contract_names), + "mcp_tool_count": len(mcp_names), + "contract_skill_names": sorted(contract_names), + "mcp_tool_names": sorted(mcp_names), + "known_non_layer5_mcp_tools": known_internal, + "missing_contracts": missing_contracts, + "unregistered_mcp_tools": unregistered_mcp_tools, + } + def _contract_by_name(skill_name: str) -> _SkillContract | None: for contract in _SKILL_CONTRACTS: @@ -329,6 +424,38 @@ def _parse_args(args_json: str) -> tuple[dict[str, Any] | None, str | None]: return parsed, None +def _parse_mcp_tool_names(tools_json: str) -> tuple[list[str], str | None]: + if not tools_json.strip(): + return [], "tools_json is required" + try: + parsed = json.loads(tools_json) + except json.JSONDecodeError as exc: + return [], f"tools_json is invalid JSON: {exc.msg}" + + tools_payload: Any = parsed + if isinstance(parsed, dict): + if isinstance(parsed.get("result"), dict): + tools_payload = parsed["result"].get("tools", []) + elif "tools" in parsed: + tools_payload = parsed["tools"] + elif "skills" in parsed: + tools_payload = parsed["skills"] + + names: list[str] = [] + if isinstance(tools_payload, list): + for item in tools_payload: + if isinstance(item, str): + names.append(item) + elif isinstance(item, dict) and isinstance(item.get("name"), str): + names.append(item["name"]) + elif isinstance(item, dict) and isinstance(item.get("func_name"), str): + names.append(item["func_name"]) + else: + return [], "tools_json must contain a list of tool names or tool objects" + + return sorted(set(names)), None + + def _is_missing(value: Any) -> bool: if value is None: return True diff --git a/dimos/robot/unitree/go2/blueprints/layers/layer_5_skill_interface/skill_interface_spec.py b/dimos/robot/unitree/go2/blueprints/layers/layer_5_skill_interface/skill_interface_spec.py index 02fcb39bf9..fdc5bd0d8c 100644 --- a/dimos/robot/unitree/go2/blueprints/layers/layer_5_skill_interface/skill_interface_spec.py +++ b/dimos/robot/unitree/go2/blueprints/layers/layer_5_skill_interface/skill_interface_spec.py @@ -21,6 +21,7 @@ class SkillInterfaceSpec(Spec, Protocol): def get_skill_interface_snapshot(self, domain: str = "") -> dict[str, Any]: ... def get_skill_contract(self, skill_name: str) -> dict[str, Any] | None: ... def validate_skill_request(self, skill_name: str, args_json: str = "{}") -> dict[str, Any]: ... + def compare_mcp_tools(self, tools_json: str) -> dict[str, Any]: ... __all__ = ["SkillInterfaceSpec"] diff --git a/dimos/robot/unitree/go2/blueprints/layers/layer_5_skill_interface/test_skill_interface_registry.py b/dimos/robot/unitree/go2/blueprints/layers/layer_5_skill_interface/test_skill_interface_registry.py index 5328cea261..0315ba9d51 100644 --- a/dimos/robot/unitree/go2/blueprints/layers/layer_5_skill_interface/test_skill_interface_registry.py +++ b/dimos/robot/unitree/go2/blueprints/layers/layer_5_skill_interface/test_skill_interface_registry.py @@ -12,6 +12,12 @@ # See the License for the specific language governing permissions and # limitations under the License. +import json +from typing import Any + +import pytest + +from dimos.core.coordination.blueprints import Blueprint from dimos.robot.unitree.go2.blueprints.layers.layer_5_skill_interface.skill_interface_registry import ( _Go2SkillInterfaceRegistry, ) @@ -31,9 +37,10 @@ def test_snapshot_lists_go2_skill_contracts() -> None: skill_names = {skill["skill_name"] for skill in snapshot["skills"]} assert snapshot["available"] is True - assert snapshot["version"] == "v1" + assert snapshot["version"] == "v2" assert "navigation" in snapshot["domains"] assert "robot_motion" in snapshot["domains"] + assert "observe" in skill_names assert "navigate_with_text" in skill_names assert "follow_person" in skill_names assert "speak" in skill_names @@ -41,6 +48,76 @@ def test_snapshot_lists_go2_skill_contracts() -> None: _stop_modules(registry) +def test_compare_mcp_tools_accepts_matching_mcp_tool_payload() -> None: + registry = _Go2SkillInterfaceRegistry() + try: + snapshot = registry.get_skill_interface_snapshot() + tool_names = [skill["skill_name"] for skill in snapshot["skills"]] + payload = { + "result": { + "tools": [{"name": name, "inputSchema": {"type": "object"}} for name in tool_names] + + [{"name": "get_context"}, {"name": "route_task"}, {"name": "server_status"}] + } + } + + result = registry.compare_mcp_tools(json.dumps(payload)) + + assert result["valid"] is True + assert result["missing_contracts"] == [] + assert result["unregistered_mcp_tools"] == [] + assert "get_context" in result["known_non_layer5_mcp_tools"] + finally: + _stop_modules(registry) + + +def test_compare_mcp_tools_reports_missing_and_unregistered_tools() -> None: + registry = _Go2SkillInterfaceRegistry() + try: + payload = {"tools": [{"name": "observe"}, {"name": "unexpected_tool"}]} + + result = registry.compare_mcp_tools(json.dumps(payload)) + + assert result["valid"] is False + assert "navigate_with_text" in result["missing_contracts"] + assert result["unregistered_mcp_tools"] == ["unexpected_tool"] + finally: + _stop_modules(registry) + + +def test_go2_agentic_mcp_tools_match_layer5_contracts() -> None: + try: + from dimos.robot.unitree.go2.blueprints.agentic.unitree_go2_agentic import ( + unitree_go2_agentic, + ) + except (ImportError, ModuleNotFoundError, OSError) as exc: + pytest.skip(f"Full Go2 agentic blueprint dependencies are unavailable: {exc}") + + registry = _Go2SkillInterfaceRegistry() + try: + mcp_tool_names = _static_mcp_tool_names(unitree_go2_agentic) + payload = {"tools": [{"name": name} for name in sorted(mcp_tool_names)]} + + result = registry.compare_mcp_tools(json.dumps(payload)) + + assert result["valid"] is True + assert result["missing_contracts"] == [] + assert result["unregistered_mcp_tools"] == [] + assert "observe" in result["mcp_tool_names"] + assert "server_status" in result["known_non_layer5_mcp_tools"] + finally: + _stop_modules(registry) + + +def _static_mcp_tool_names(blueprint: Blueprint) -> set[str]: + tool_names: set[str] = set() + for atom in blueprint.active_blueprints: + for name in dir(atom.module): + attr: Any = getattr(atom.module, name) + if callable(attr) and hasattr(attr, "__skill__"): + tool_names.add(name) + return tool_names + + def test_snapshot_can_filter_by_domain() -> None: registry = _Go2SkillInterfaceRegistry() try: From 0494faceaff8fe3f8e7a1253bc8e17b683a284ce Mon Sep 17 00:00:00 2001 From: z1050297972-spec Date: Sun, 7 Jun 2026 17:57:19 +0800 Subject: [PATCH 09/36] use ollama for g1 agent --- dimos/agents/mcp/mcp_client.py | 2 +- .../security_demo/security_module.py | 35 ++++++++++++++++--- .../g1/blueprints/agentic/_agentic_skills.py | 2 +- dimos/robot/unitree/go2/PROJECT_CHANGELOG.md | 18 ++++++++++ 4 files changed, 51 insertions(+), 6 deletions(-) diff --git a/dimos/agents/mcp/mcp_client.py b/dimos/agents/mcp/mcp_client.py index 76394c6509..ffa2e9ddfb 100644 --- a/dimos/agents/mcp/mcp_client.py +++ b/dimos/agents/mcp/mcp_client.py @@ -74,7 +74,7 @@ class McpClientConfig(ModuleConfig): system_prompt: str | None = SYSTEM_PROMPT - model: str = "gpt-4o" + model: str = "ollama:qwen3.6:latest" model_fixture: str | None = None mcp_server_url: str = "http://localhost:9990/mcp" diff --git a/dimos/experimental/security_demo/security_module.py b/dimos/experimental/security_demo/security_module.py index 9569227805..eb231583ca 100644 --- a/dimos/experimental/security_demo/security_module.py +++ b/dimos/experimental/security_demo/security_module.py @@ -149,6 +149,7 @@ class SecurityModule(Module): _planner_spec: ReplanningAStarPlannerSpec _speak_skill: SpeakSkillSpec + _tracker: EdgeTAMProcessor | None def __init__(self, **kwargs: Any) -> None: super().__init__(**kwargs) @@ -156,7 +157,7 @@ def __init__(self, **kwargs: Any) -> None: self._router: PatrolRouter = _create_router(self.config.g) self._visual_servo = _create_visual_servo(self.config, self.config.g) self._detector = YoloPersonDetector() - self._tracker = EdgeTAMProcessor() + self._tracker = None self._depth_estimator = DepthEstimator(self.depth_image.publish) @@ -185,7 +186,8 @@ def stop(self) -> None: self._stop_security_patrol_internal() self._depth_estimator.stop() self._detector.stop() - self._tracker.stop() + if self._tracker is not None: + self._tracker.stop() super().stop() @skill @@ -198,6 +200,10 @@ def start_security_patrol(self) -> str: if self._main_thread is not None and self._main_thread.is_alive(): return "Security patrol is already running. Use `stop_security_patrol` to stop." + tracker_error = self._ensure_tracker_available() + if tracker_error is not None: + return tracker_error + if not self._depth_started: self._depth_estimator.start() self._depth_started = True @@ -222,6 +228,16 @@ def start_security_patrol(self) -> str: "persons automatically. Use `stop_security_patrol` to stop." ) + def _ensure_tracker_available(self) -> str | None: + if self._tracker is not None: + return None + try: + self._tracker = EdgeTAMProcessor() + except RuntimeError as exc: + logger.warning("Security patrol unavailable", error=str(exc)) + return f"Security patrol unavailable: {exc}" + return None + @skill def stop_security_patrol(self) -> str: """Stop the security patrol behavior entirely.""" @@ -310,7 +326,12 @@ def _patrol_step(self) -> None: # Init EdgeTAM with YOLO bbox for continuous tracking box = np.array(list(best.bbox), dtype=np.float32) - self._tracker.init_track(image=image, box=box, obj_id=1) + tracker = self._tracker + if tracker is None: + logger.warning("Security tracker unavailable, stopping security patrol") + self._stop_security_patrol_internal() + return + tracker.init_track(image=image, box=box, obj_id=1) self._cancel_current_goal() self._has_active_goal = False @@ -326,7 +347,13 @@ def _follow_step(self) -> None: self._stop_event.wait(timeout=_ANTI_BUSY_LOOP_TIMEOUT) return - detections = self._tracker.process_image(latest_image) + tracker = self._tracker + if tracker is None: + logger.warning("Security tracker unavailable, resuming patrol") + self._transition_to("PATROLLING") + return + + detections = tracker.process_image(latest_image) if len(detections) == 0: self.cmd_vel.publish(Twist.zero()) diff --git a/dimos/robot/unitree/g1/blueprints/agentic/_agentic_skills.py b/dimos/robot/unitree/g1/blueprints/agentic/_agentic_skills.py index 557554e061..53ecb464b3 100644 --- a/dimos/robot/unitree/g1/blueprints/agentic/_agentic_skills.py +++ b/dimos/robot/unitree/g1/blueprints/agentic/_agentic_skills.py @@ -25,7 +25,7 @@ _agentic_skills = autoconnect( McpServer.blueprint(), - McpClient.blueprint(system_prompt=G1_SYSTEM_PROMPT), + McpClient.blueprint(system_prompt=G1_SYSTEM_PROMPT,model="ollama:qwen3.6:latest"), NavigationSkillContainer.blueprint(), SpeakSkill.blueprint(), UnitreeG1SkillContainer.blueprint(), diff --git a/dimos/robot/unitree/go2/PROJECT_CHANGELOG.md b/dimos/robot/unitree/go2/PROJECT_CHANGELOG.md index 28b0767b34..a5382aca50 100644 --- a/dimos/robot/unitree/go2/PROJECT_CHANGELOG.md +++ b/dimos/robot/unitree/go2/PROJECT_CHANGELOG.md @@ -6,6 +6,24 @@ work so future contributors can understand why the fork differs from upstream. Entries are listed in reverse chronological order. +## 2026-05-28 + +- Branch: `refactor/go2-architecture-layers` +- Summary: Made the security patrol tracker initialize lazily so the Go2 + agentic blueprint can deploy in CPU-only MuJoCo/WSL environments when + security patrol is not used. Calling `start_security_patrol` still reports + the EdgeTAM CUDA requirement if no compatible GPU is available. +- Files/modules: + - `dimos/experimental/security_demo/security_module.py` + - `dimos/robot/unitree/go2/PROJECT_CHANGELOG.md` +- Validation: + - Ran `python -m py_compile dimos/experimental/security_demo/security_module.py`. + - Instantiated `SecurityModule(camera_info=CameraInfo())` in UbuntuDev using + the existing WSL dependency environment; construction completed on CPU. +- Open items: + - Add a CPU-capable tracker fallback if security patrol needs to run without + CUDA. + ## 2026-05-22 - Branch: `refactor/go2-architecture-layers` From 65444534fb27513fe3cd01c1c8b58b572c921cc8 Mon Sep 17 00:00:00 2001 From: ZhangZheMing <1050297972@qq.com> Date: Sat, 4 Jul 2026 22:59:29 +0800 Subject: [PATCH 10/36] Improve MCP and coordinator runtime plumbing --- dimos/agents/mcp/mcp_client.py | 207 ++++++++++++++---- dimos/agents/mcp/mcp_server.py | 62 ++++-- dimos/agents/mcp/test_mcp_client_unit.py | 154 ++++++++++++- dimos/agents/mcp/test_mcp_server.py | 56 ++++- dimos/core/coordination/module_coordinator.py | 20 ++ .../coordination/test_module_coordinator.py | 71 ++++++ dimos/core/coordination/test_worker.py | 33 +++ .../coordination/worker_manager_python.py | 25 ++- dimos/e2e_tests/dimos_cli_call.py | 7 +- dimos/e2e_tests/test_dimos_cli_call.py | 29 +++ .../service/system_configurator/base.py | 4 + .../service/test_system_configurator.py | 6 + 12 files changed, 609 insertions(+), 65 deletions(-) create mode 100644 dimos/e2e_tests/test_dimos_cli_call.py diff --git a/dimos/agents/mcp/mcp_client.py b/dimos/agents/mcp/mcp_client.py index ffa2e9ddfb..7e1eb72f7c 100644 --- a/dimos/agents/mcp/mcp_client.py +++ b/dimos/agents/mcp/mcp_client.py @@ -12,8 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -import json from collections.abc import Callable +import json from queue import Empty, Queue from threading import Event, RLock, Thread import time @@ -21,7 +21,6 @@ import uuid import httpx -from langchain.agents import create_agent from langchain_core.messages import HumanMessage from langchain_core.messages.base import BaseMessage from langchain_core.tools import StructuredTool @@ -33,8 +32,8 @@ from dimos.agents.utils import pretty_print_langchain_message from dimos.constants import DEFAULT_THREAD_JOIN_TIMEOUT from dimos.core.core import rpc -from dimos.core.module import Module, ModuleConfig -from dimos.core.rpc_client import RPCClient +from dimos.core.module import Module, ModuleConfig, SkillInfo +from dimos.core.rpc_client import RpcCall, RPCClient from dimos.core.stream import In, Out from dimos.utils.logging_config import setup_logger from dimos.utils.sequential_ids import SequentialIds @@ -70,13 +69,14 @@ "wait": "utility", "current_time": "utility", } +_AGENT_INIT_START_DELAY_SECONDS = 0.25 class McpClientConfig(ModuleConfig): system_prompt: str | None = SYSTEM_PROMPT model: str = "ollama:qwen3.6:latest" model_fixture: str | None = None - mcp_server_url: str = "http://localhost:9990/mcp" + mcp_server_url: str = "http://127.0.0.1:9990/mcp" class McpClient(Module): @@ -86,11 +86,14 @@ class McpClient(Module): agent_idle: Out[bool] _lock: RLock - _state_graph: CompiledStateGraph[Any, Any, Any, Any] | None + _state_graph: "CompiledStateGraph[Any, Any, Any, Any] | None" _message_queue: Queue[BaseMessage] _tool_registry: dict[str, dict[str, Any]] + _direct_tool_payload: dict[str, Any] | None + _direct_rpc_calls: dict[str, Callable[..., Any]] _history: list[BaseMessage] _thread: Thread + _agent_init_thread: Thread | None _stop_event: Event _http_client: httpx.Client _seq_ids: SequentialIds @@ -102,21 +105,29 @@ def __init__(self, **kwargs: Any) -> None: self._state_graph = None self._message_queue = Queue() self._tool_registry = {} + self._direct_tool_payload = None + self._direct_rpc_calls = {} self._history = [] self._thread = Thread( target=self._thread_loop, name=f"{self.__class__.__name__}-thread", daemon=True, ) + self._agent_init_thread = None self._stop_event = Event() - self._http_client = httpx.Client(timeout=120.0) + self._http_client = httpx.Client(timeout=120.0, trust_env=False) self._seq_ids = SequentialIds() self._tool_stream_cleanup = None def __reduce__(self) -> Any: return (self.__class__, (), {}) - def _mcp_request(self, method: str, params: dict[str, Any] | None = None) -> dict[str, Any]: + def _mcp_request( + self, + method: str, + params: dict[str, Any] | None = None, + timeout: float | None = None, + ) -> dict[str, Any]: body: dict[str, Any] = { "jsonrpc": "2.0", "id": self._seq_ids.next(), @@ -125,7 +136,10 @@ def _mcp_request(self, method: str, params: dict[str, Any] | None = None) -> dic if params is not None: body["params"] = params - resp = self._http_client.post(self.config.mcp_server_url, json=body) + request_kwargs: dict[str, Any] = {"json": body} + if timeout is not None: + request_kwargs["timeout"] = timeout + resp = self._http_client.post(self.config.mcp_server_url, **request_kwargs) resp.raise_for_status() data = resp.json() @@ -140,14 +154,17 @@ def _mcp_tool_call( ) -> dict[str, Any]: progress_token = str(uuid.uuid4()) try: - result = self._mcp_request( - "tools/call", - { - "name": name, - "arguments": arguments, - "_meta": {"progressToken": progress_token}, - }, - ) + if name in self._direct_rpc_calls: + result = self._direct_tool_call(name, arguments, progress_token) + else: + result = self._mcp_request( + "tools/call", + { + "name": name, + "arguments": arguments, + "_meta": {"progressToken": progress_token}, + }, + ) except Exception as exc: if record_outcome: self._record_tool_exception(name, exc) @@ -186,6 +203,23 @@ def _record_tool_payload(self, payload: dict[str, Any]) -> None: except Exception: logger.warning("Failed to record MCP tool outcome", tool=payload["skill_name"]) + def _direct_tool_call( + self, name: str, arguments: dict[str, Any], progress_token: str + ) -> dict[str, Any]: + rpc_call = self._direct_rpc_calls[name] + call_kwargs = dict(arguments) + call_kwargs["_mcp_context"] = {"progress_token": progress_token} + + try: + result = rpc_call(**call_kwargs) + except Exception as e: + logger.exception("MCP direct tool error", tool=name) + return {"content": [{"type": "text", "text": f"Error running tool '{name}': {e}"}]} + + if hasattr(result, "agent_encode"): + return {"content": result.agent_encode()} + return {"content": [{"type": "text", "text": str(result)}]} + def _on_tool_stream_message(self, msg: dict[str, Any]) -> None: method = msg.get("method") params = msg.get("params") or {} @@ -202,11 +236,13 @@ def _on_tool_stream_message(self, msg: dict[str, Any]) -> None: self._message_queue.put(HumanMessage(content=f"[tool:{tool_name}] {text}")) def _fetch_tools(self, timeout: float = 60.0, interval: float = 1.0) -> list[StructuredTool]: - result = self._try_fetch_tools(timeout=timeout, interval=interval) + result = self._direct_tool_payload if result is None: - raise RuntimeError( - f"Failed to fetch tools from MCP server {self.config.mcp_server_url}" - ) + result = self._try_fetch_tools(timeout=timeout, interval=interval) + if result is None: + raise RuntimeError( + f"Failed to fetch tools from MCP server {self.config.mcp_server_url}" + ) raw_tools = result.get("tools", []) self._tool_registry = {t["name"]: t for t in raw_tools} @@ -225,14 +261,14 @@ def _try_fetch_tools(self, timeout: float, interval: float) -> dict[str, Any] | while True: try: - self._mcp_request("initialize") + self._mcp_request("initialize", timeout=min(max(interval, 0.1), 5.0)) break - except (httpx.ConnectError, httpx.RemoteProtocolError): + except (httpx.ConnectError, httpx.RemoteProtocolError, httpx.TimeoutException): if time.monotonic() >= deadline: return None time.sleep(interval) - return self._mcp_request("tools/list") + return self._mcp_request("tools/list", timeout=5.0) def _mcp_tool_to_langchain(self, mcp_tool: dict[str, Any]) -> StructuredTool: name = mcp_tool["name"] @@ -281,22 +317,102 @@ def _on_human_input(string: str) -> None: @rpc def on_system_modules(self, _modules: list[RPCClient]) -> None: - tools = self._fetch_tools() - - model: str | Any = self.config.model - if self.config.model_fixture is not None: - from dimos.agents.testing import MockModel + with self._lock: + if self._state_graph is not None: + if not self._thread.is_alive(): + self._thread.start() + return + if self._agent_init_thread is not None and self._agent_init_thread.is_alive(): + return + modules = list(_modules) + self._agent_init_thread = Thread( + target=self._delayed_register_and_initialize_agent_graph, + args=(modules,), + name=f"{self.__class__.__name__}-init-thread", + daemon=True, + ) + self._agent_init_thread.start() - model = MockModel(json_path=self.config.model_fixture) + def _register_direct_tools(self, modules: list[RPCClient]) -> None: + from dimos.agents.mcp.mcp_server import _skill_infos_from_class - with self._lock: - self._state_graph = create_agent( - model=model, - tools=tools, - system_prompt=self.config.system_prompt, + skills: list[SkillInfo] = [] + for module in modules: + if getattr(module, "remote_name", None) == self.__class__.__name__: + continue + if actor_class := getattr(module, "actor_class", None): + skills.extend(_skill_infos_from_class(actor_class)) + continue + try: + skills.extend(module.get_skills() or []) + except Exception: + logger.debug( + "Skipping direct MCP tool registration for module.", + module=getattr(module, "remote_name", None), + ) + + rpc_client = self.rpc + if not skills or rpc_client is None: + self._direct_tool_payload = None + self._direct_rpc_calls = {} + return + self._direct_tool_payload = _tools_payload_from_skill_infos(skills) + self._direct_rpc_calls = { + skill.func_name: RpcCall( + None, + rpc_client, + skill.func_name, + skill.class_name, + [], ) - if not self._thread.is_alive(): - self._thread.start() + for skill in skills + } + + def _delayed_register_and_initialize_agent_graph(self, modules: list[RPCClient]) -> None: + # Let the on_system_modules RPC response flush before this thread + # starts direct tool registration, HTTP polling, and LangGraph setup. + if self._stop_event.wait(_AGENT_INIT_START_DELAY_SECONDS): + return + try: + self._register_direct_tools(modules) + except Exception: + logger.exception("MCP direct tool registration failed; falling back to HTTP tools.") + self._initialize_agent_graph() + + def _initialize_agent_graph(self) -> None: + from langchain.agents import create_agent + + retry_seconds = 2.0 + while not self._stop_event.is_set(): + try: + tools = self._fetch_tools() + + model: str | Any = self.config.model + if self.config.model_fixture is not None: + from dimos.agents.testing import MockModel + + model = MockModel(json_path=self.config.model_fixture) + + state_graph = create_agent( + model=model, + tools=tools, + system_prompt=self.config.system_prompt, + ) + except Exception: + logger.exception( + "MCP client agent initialization failed; retrying.", + retry_seconds=retry_seconds, + ) + self._stop_event.wait(retry_seconds) + continue + + with self._lock: + if self._stop_event.is_set(): + return + self._state_graph = state_graph + if not self._thread.is_alive(): + self._thread.start() + return @rpc def stop(self) -> None: @@ -306,6 +422,8 @@ def stop(self) -> None: self._tool_stream_cleanup() self._tool_stream_cleanup = None self._stop_event.set() + if self._agent_init_thread is not None and self._agent_init_thread.is_alive(): + self._agent_init_thread.join(timeout=DEFAULT_THREAD_JOIN_TIMEOUT) if self._thread.is_alive(): self._thread.join(timeout=DEFAULT_THREAD_JOIN_TIMEOUT) self._http_client.close() @@ -387,7 +505,7 @@ def _thread_loop(self) -> None: self._process_message(self._state_graph, message) def _process_message( - self, state_graph: CompiledStateGraph[Any, Any, Any, Any], message: BaseMessage + self, state_graph: "CompiledStateGraph[Any, Any, Any, Any]", message: BaseMessage ) -> None: self.agent_idle.publish(False) self._history.append(message) @@ -421,6 +539,19 @@ def _append_image_to_history( ) +def _tools_payload_from_skill_infos(skills: list[SkillInfo]) -> dict[str, Any]: + tools = [] + for skill in skills: + schema = json.loads(skill.args_schema) + description = schema.pop("description", None) + schema.pop("title", None) + tool: dict[str, Any] = {"name": skill.func_name, "inputSchema": schema} + if description: + tool["description"] = description + tools.append(tool) + return {"tools": tools} + + def _outcome_payload_from_result(name: str, result: dict[str, Any]) -> dict[str, Any]: text = _content_text(result) structured = _parse_skill_result_text(text) diff --git a/dimos/agents/mcp/mcp_server.py b/dimos/agents/mcp/mcp_server.py index dbd31f8d87..05583fb344 100644 --- a/dimos/agents/mcp/mcp_server.py +++ b/dimos/agents/mcp/mcp_server.py @@ -15,11 +15,11 @@ import asyncio from collections.abc import AsyncGenerator, Callable -import concurrent.futures import json import os +import threading import time -from typing import TYPE_CHECKING, Any +from typing import Any from fastapi import FastAPI from fastapi.middleware.cors import CORSMiddleware @@ -31,13 +31,10 @@ from dimos.agents.annotation import skill from dimos.agents.mcp import tool_stream from dimos.core.core import rpc -from dimos.core.module import Module +from dimos.core.module import Module, SkillInfo from dimos.core.rpc_client import RpcCall, RPCClient from dimos.utils.logging_config import setup_logger -if TYPE_CHECKING: - from dimos.core.module import SkillInfo - logger = setup_logger() @@ -94,6 +91,27 @@ def _handle_tools_list(req_id: Any, skills: list[SkillInfo]) -> dict[str, Any]: return _jsonrpc_result(req_id, {"tools": tools}) +def _skill_infos_from_class(module_class: type[Any]) -> list[SkillInfo]: + """Collect declared skills without making an RPC call to a worker.""" + from langchain_core.tools import tool + + skills: list[SkillInfo] = [] + dummy_self = object() + for name in dir(module_class): + attr = getattr(module_class, name, None) + if callable(attr) and hasattr(attr, "__skill__"): + bound_attr = attr.__get__(dummy_self, module_class) + schema = json.dumps(tool(bound_attr).args_schema.model_json_schema()) + skills.append( + SkillInfo( + class_name=module_class.__name__, + func_name=name, + args_schema=schema, + ) + ) + return skills + + async def _handle_tools_call( req_id: Any, params: dict[str, Any], rpc_calls: dict[str, Any] ) -> dict[str, Any]: @@ -244,7 +262,7 @@ async def event_generator() -> AsyncGenerator[str, None]: class McpServer(Module): _uvicorn_server: uvicorn.Server | None = None - _serve_future: concurrent.futures.Future[None] | None = None + _uvicorn_server_thread: threading.Thread | None = None _tool_stream_cleanup: Callable[[], None] | None = None @rpc @@ -268,26 +286,37 @@ def stop(self) -> None: if self._uvicorn_server: self._uvicorn_server.should_exit = True - loop = self._loop - if loop is not None and self._serve_future is not None: - self._serve_future.result(timeout=5.0) + if self._uvicorn_server_thread is not None: + self._uvicorn_server_thread.join(timeout=5.0) self._uvicorn_server = None - self._serve_future = None + self._uvicorn_server_thread = None super().stop() @rpc def on_system_modules(self, modules: list[RPCClient]) -> None: # TODO: this is a bit hacky, also not thread-safe assert self.rpc is not None - app.state.skills = [ - skill_info for module in modules for skill_info in (module.get_skills() or []) - ] + skills: list[SkillInfo] = [] + for module in modules: + if getattr(module, "remote_name", None) == self.__class__.__name__: + skills.extend(self.get_skills()) + elif actor_class := getattr(module, "actor_class", None): + skills.extend(_skill_infos_from_class(actor_class)) + else: + skills.extend(module.get_skills() or []) + + app.state.skills = skills app.state.rpc_calls = { skill_info.func_name: RpcCall( None, self.rpc, skill_info.func_name, skill_info.class_name, [] ) for skill_info in app.state.skills } + logger.info( + "Registered MCP tools.", + n_tools=len(app.state.skills), + tools=[skill.func_name for skill in app.state.skills], + ) @skill def server_status(self) -> str: @@ -339,6 +368,5 @@ def _start_server(self, port: int | None = None) -> None: config = uvicorn.Config(app, host=_host, port=_port, log_level="warning", access_log=False) server = uvicorn.Server(config) self._uvicorn_server = server - loop = self._loop - assert loop is not None - self._serve_future = asyncio.run_coroutine_threadsafe(server.serve(), loop) + self._uvicorn_server_thread = threading.Thread(target=server.run, daemon=True) + self._uvicorn_server_thread.start() diff --git a/dimos/agents/mcp/test_mcp_client_unit.py b/dimos/agents/mcp/test_mcp_client_unit.py index a1b8d7577a..d513f235a5 100644 --- a/dimos/agents/mcp/test_mcp_client_unit.py +++ b/dimos/agents/mcp/test_mcp_client_unit.py @@ -15,13 +15,18 @@ import json from queue import Empty, Queue +from threading import Event +import time from unittest.mock import MagicMock, patch +import httpx from langchain_core.messages import HumanMessage from langchain_core.messages.base import BaseMessage import pytest -from dimos.agents.mcp.mcp_client import McpClient +from dimos.agents.annotation import skill +from dimos.agents.mcp.mcp_client import McpClient, McpClientConfig +from dimos.core.module import SkillInfo from dimos.utils.sequential_ids import SequentialIds @@ -105,11 +110,81 @@ def mcp_client() -> McpClient: client._http_client = mock_http client._seq_ids = SequentialIds() + client._tool_registry = {} + client._direct_tool_payload = None + client._direct_rpc_calls = {} client.config = MagicMock() client.config.mcp_server_url = "http://localhost:9990/mcp" return client +def test_mcp_client_does_not_inherit_proxy_environment() -> None: + client: McpClient | None = None + with patch("dimos.agents.mcp.mcp_client.httpx.Client") as client_factory: + try: + client = McpClient() + finally: + if client is not None: + client.stop() + + client_factory.assert_called_once_with(timeout=120.0, trust_env=False) + + +def test_default_mcp_server_url_uses_ipv4_loopback() -> None: + assert McpClientConfig().mcp_server_url == "http://127.0.0.1:9990/mcp" + + +def test_on_system_modules_initializes_agent_without_blocking_startup() -> None: + client = McpClient() + fetch_started = Event() + + def slow_fetch_tools() -> list[object]: + fetch_started.set() + time.sleep(0.25) + return [] + + def fake_create_agent(**kwargs: object) -> MagicMock: + return MagicMock() + + client._fetch_tools = slow_fetch_tools # type: ignore[method-assign] + + try: + with patch("langchain.agents.create_agent", side_effect=fake_create_agent): + started_at = time.monotonic() + client.on_system_modules([]) + elapsed = time.monotonic() - started_at + + assert elapsed < 0.1 + assert fetch_started.wait(1.0) + finally: + client.stop() + + +def test_on_system_modules_defers_direct_tool_registration_outside_rpc_path() -> None: + client = McpClient() + registration_started = Event() + release_registration = Event() + + def slow_register_tools(modules: list[object]) -> None: + registration_started.set() + release_registration.wait(1.0) + + client._register_direct_tools = slow_register_tools # type: ignore[method-assign] + client._initialize_agent_graph = MagicMock() # type: ignore[method-assign] + + try: + with patch("dimos.agents.mcp.mcp_client._AGENT_INIT_START_DELAY_SECONDS", 0.0): + started_at = time.monotonic() + client.on_system_modules([MagicMock()]) + elapsed = time.monotonic() - started_at + + assert elapsed < 0.1 + assert registration_started.wait(1.0) + finally: + release_registration.set() + client.stop() + + def test_fetch_tools_from_mcp_server(mcp_client: McpClient) -> None: tools = mcp_client._fetch_tools() @@ -118,6 +193,83 @@ def test_fetch_tools_from_mcp_server(mcp_client: McpClient) -> None: assert tools[1].name == "greet" +def test_try_fetch_tools_retries_transient_initialize_timeout(mcp_client: McpClient) -> None: + initialize_attempts = 0 + + def flaky_request( + method: str, + params: dict[str, object] | None = None, + timeout: float | None = None, + ) -> dict[str, object]: + nonlocal initialize_attempts + if method == "initialize": + initialize_attempts += 1 + if initialize_attempts == 1: + raise httpx.TimeoutException("server not accepting requests yet") + return {"protocolVersion": "2024-11-05"} + if method == "tools/list": + return {"tools": []} + raise AssertionError(f"Unexpected MCP request: {method}") + + mcp_client._mcp_request = flaky_request # type: ignore[method-assign] + + result = mcp_client._try_fetch_tools(timeout=1.0, interval=0.01) + + assert result == {"tools": []} + assert initialize_attempts == 2 + + +def test_register_direct_tools_from_system_modules(mcp_client: McpClient) -> None: + class DirectSkills: + @skill + def ping(self, name: str) -> str: + """Ping by name.""" + return name + + class DirectProxy: + remote_name = "DirectSkills" + actor_class = DirectSkills + + def get_skills(self) -> list[SkillInfo]: + raise AssertionError("direct registration should prefer actor_class") + + mcp_client.rpc = MagicMock() + + mcp_client._register_direct_tools([DirectProxy()]) + + assert mcp_client._direct_tool_payload is not None + assert [tool["name"] for tool in mcp_client._direct_tool_payload["tools"]] == ["ping"] + assert set(mcp_client._direct_rpc_calls) == {"ping"} + + +def test_fetch_tools_uses_direct_registry_without_http(mcp_client: McpClient) -> None: + direct_call = MagicMock(return_value="pong") + mcp_client._direct_tool_payload = { + "tools": [ + { + "name": "ping", + "description": "Ping by name.", + "inputSchema": { + "type": "object", + "properties": {"name": {"type": "string"}}, + "required": ["name"], + }, + } + ] + } + mcp_client._direct_rpc_calls = {"ping": direct_call} + mcp_client._mcp_request = MagicMock(side_effect=AssertionError("HTTP should not be used")) + + tools = mcp_client._fetch_tools() + + assert [tool.name for tool in tools] == ["ping"] + assert tools[0].func(name="agent") == "pong" + direct_call.assert_called_once_with( + name="agent", + _mcp_context={"progress_token": direct_call.call_args.kwargs["_mcp_context"]["progress_token"]}, + ) + + def test_tool_invocation_via_mcp(mcp_client: McpClient) -> None: tools = mcp_client._fetch_tools() add_tool = next(t for t in tools if t.name == "add") diff --git a/dimos/agents/mcp/test_mcp_server.py b/dimos/agents/mcp/test_mcp_server.py index fd514b0643..c8cd0f70d3 100644 --- a/dimos/agents/mcp/test_mcp_server.py +++ b/dimos/agents/mcp/test_mcp_server.py @@ -16,10 +16,11 @@ import asyncio import json -from unittest.mock import MagicMock +from unittest.mock import MagicMock, patch -from dimos.agents.mcp.mcp_server import handle_request +from dimos.agents.mcp.mcp_server import McpServer, app, handle_request from dimos.core.module import SkillInfo +from dimos.core.tests.stress_test_module import StressTestModule def _make_rpc_calls( @@ -161,3 +162,54 @@ def test_mcp_module_initialize_and_unknown() -> None: response = asyncio.run(handle_request({"method": "unknown/method", "id": 2}, [], {})) assert response["error"]["code"] == -32601 + + +def test_mcp_server_starts_uvicorn_in_dedicated_thread_without_module_loop() -> None: + server = McpServer.__new__(McpServer) + server._loop = None + server._uvicorn_server = None + server._uvicorn_server_thread = None + uvicorn_server = MagicMock() + thread = MagicMock() + + with ( + patch("dimos.agents.mcp.mcp_server.uvicorn.Server", return_value=uvicorn_server), + patch("dimos.agents.mcp.mcp_server.threading.Thread", return_value=thread) as thread_cls, + ): + server._start_server(port=0) + + assert server._uvicorn_server is uvicorn_server + assert server._uvicorn_server_thread is thread + thread_cls.assert_called_once() + assert thread_cls.call_args.kwargs["daemon"] is True + thread.start.assert_called_once_with() + + +def test_mcp_server_uses_local_skills_for_its_own_proxy() -> None: + class SelfProxy: + remote_name = "McpServer" + + def get_skills(self) -> list[SkillInfo]: + raise AssertionError("self proxy should not be queried over RPC") + + class OtherProxy: + remote_name = "StressTestModule" + actor_class = StressTestModule + + def get_skills(self) -> list[SkillInfo]: + raise AssertionError("remote get_skills should not be used") + + server_skill = SkillInfo( + class_name="McpServer", + func_name="server_status", + args_schema=json.dumps({"type": "object", "properties": {}}), + ) + server = McpServer.__new__(McpServer) + server.rpc = MagicMock() + server.get_skills = MagicMock(return_value=[server_skill]) + + server.on_system_modules([SelfProxy(), OtherProxy()]) + + tool_names = {skill.func_name for skill in app.state.skills} + assert {"server_status", "echo", "ping", "slow", "info"} <= tool_names + server.get_skills.assert_called_once_with() diff --git a/dimos/core/coordination/module_coordinator.py b/dimos/core/coordination/module_coordinator.py index 9ac46e3c20..845cd43b59 100644 --- a/dimos/core/coordination/module_coordinator.py +++ b/dimos/core/coordination/module_coordinator.py @@ -42,6 +42,8 @@ logger = setup_logger() +_SYSTEM_MODULES_NOWAIT_REMOTE_NAMES = frozenset({"McpClient"}) + class ModuleCoordinator(Resource): _managers: dict[str, WorkerManager] @@ -212,6 +214,8 @@ def get_instance(self, module: type[ModuleBase]) -> ModuleProxy: def _send_on_system_modules(self) -> None: modules = list(self._deployed_modules.values()) for module in modules: + if _notify_remote_rpc_nowait(module, "on_system_modules", modules): + continue if hasattr(module, "on_system_modules"): module.on_system_modules(modules) @@ -665,6 +669,22 @@ def _ref_msg(module_name: str, ref: object, spec_name: str, detail: str) -> str: ) +def _notify_remote_rpc_nowait(module: Any, rpc_name: str, modules: list[Any]) -> bool: + rpcs = getattr(module, "rpcs", ()) + if rpc_name not in rpcs: + return False + + rpc_client = getattr(module, "rpc", None) + remote_name = getattr(module, "remote_name", None) + if rpc_client is None or remote_name is None: + return False + if remote_name not in _SYSTEM_MODULES_NOWAIT_REMOTE_NAMES: + return False + + rpc_client.call_nowait(f"{remote_name}/{rpc_name}", ([modules], {})) + return True + + def _resolve_single_ref( bp: Any, module_ref: Any, diff --git a/dimos/core/coordination/test_module_coordinator.py b/dimos/core/coordination/test_module_coordinator.py index 9215c16a0d..f2252ba947 100644 --- a/dimos/core/coordination/test_module_coordinator.py +++ b/dimos/core/coordination/test_module_coordinator.py @@ -85,6 +85,24 @@ class ModuleC(Module): data3: In[Data3] +class ModuleWithSystemNotification(Module): + @rpc + def on_system_modules(self, _modules: list[object]) -> None: + pass + + +class McpClient(Module): + @rpc + def on_system_modules(self, _modules: list[object]) -> None: + pass + + +class McpServer(Module): + @rpc + def on_system_modules(self, _modules: list[object]) -> None: + pass + + class SourceModule(Module): color_image: Out[Data1] @@ -193,6 +211,59 @@ def test_build_happy_path() -> None: coordinator.stop() +def test_mcp_client_system_module_notification_is_sent_without_waiting_for_response() -> None: + calls: list[tuple[str, tuple[list[object], dict[str, object]]]] = [] + + class Rpc: + def call_nowait(self, name: str, arguments: tuple[list[object], dict[str, object]]) -> None: + calls.append((name, arguments)) + + class RemoteProxy: + remote_name = "McpClient" + rpcs = {"on_system_modules"} + rpc = Rpc() + + def on_system_modules(self, _modules: list[object]) -> None: + raise AssertionError("McpClient initialization must not block coordinator startup") + + coordinator = ModuleCoordinator() + proxy = RemoteProxy() + coordinator._deployed_modules = {McpClient: proxy} # type: ignore[dict-item] + + coordinator._send_on_system_modules() + + assert calls == [ + ( + "McpClient/on_system_modules", + ([[proxy]], {}), + ) + ] + + +def test_mcp_server_system_module_notification_waits_for_tool_registration() -> None: + calls: list[list[object]] = [] + + class Rpc: + def call_nowait(self, _name: str, _arguments: tuple[list[object], dict[str, object]]) -> None: + raise AssertionError("McpServer must register tools before startup continues") + + class RemoteProxy: + remote_name = "McpServer" + rpcs = {"on_system_modules"} + rpc = Rpc() + + def on_system_modules(self, modules: list[object]) -> None: + calls.append(modules) + + coordinator = ModuleCoordinator() + proxy = RemoteProxy() + coordinator._deployed_modules = {McpServer: proxy} # type: ignore[dict-item] + + coordinator._send_on_system_modules() + + assert calls == [[proxy]] + + def test_name_conflicts_are_reported() -> None: class ModuleA(Module): shared_data: Out[Data1] diff --git a/dimos/core/coordination/test_worker.py b/dimos/core/coordination/test_worker.py index c606dcda73..dfa35b5acf 100644 --- a/dimos/core/coordination/test_worker.py +++ b/dimos/core/coordination/test_worker.py @@ -189,6 +189,39 @@ def test_worker_manager_parallel_deployment(create_worker_manager): module3.stop() +def test_deploy_parallel_errors_identify_module_and_worker(): + class FailingWorker: + dedicated = False + module_count = 0 + worker_id = 42 + + def reserve_slot(self): + pass + + def deploy_module(self, module_class, global_config, kwargs): + raise EOFError("worker pipe closed") + + def shutdown(self): + pass + + manager = WorkerManagerPython(g=GlobalConfig(n_workers=1)) + manager._started = True + manager._workers = [FailingWorker()] # type: ignore[list-item] + + with pytest.raises(ExceptionGroup) as exc_info: + manager.deploy_parallel([(SimpleModule, global_config, {})], {}) + + errors = exc_info.value.exceptions + assert len(errors) == 1 + error = errors[0] + assert isinstance(error, RuntimeError) + assert "SimpleModule" in str(error) + assert "worker 42" in str(error) + assert "EOFError" in str(error) + assert "worker pipe closed" in str(error) + assert isinstance(error.__cause__, EOFError) + + @pytest.mark.skipif_macos_bug def test_collect_stats(create_worker_manager): from dimos.core.resource_monitor.monitor import StatsMonitor diff --git a/dimos/core/coordination/worker_manager_python.py b/dimos/core/coordination/worker_manager_python.py index 4963db2dac..537b28514f 100644 --- a/dimos/core/coordination/worker_manager_python.py +++ b/dimos/core/coordination/worker_manager_python.py @@ -168,9 +168,28 @@ def deploy_parallel( def _deploy(item: tuple[PythonWorker, ModuleSpec]) -> ModuleProxyProtocol: worker, (module_class, global_config, kwargs) = item - return RPCClient( - worker.deploy_module(module_class, global_config, kwargs), module_class - ) + module_name = module_class.__name__ + worker_id = worker.worker_id + try: + logger.info("Deploying module.", module=module_name, worker_id=worker_id) + return RPCClient( + worker.deploy_module(module_class, global_config, kwargs), module_class + ) + except Exception as exc: + error_type = type(exc).__name__ + error_repr = repr(exc) + logger.error( + "Failed to deploy module.", + module=module_name, + worker_id=worker_id, + error_type=error_type, + error_repr=error_repr, + exc_info=True, + ) + raise RuntimeError( + f"Failed to deploy {module_name} on worker {worker_id}: " + f"{error_type}: {error_repr}" + ) from exc try: return safe_thread_map(assignments, _deploy) diff --git a/dimos/e2e_tests/dimos_cli_call.py b/dimos/e2e_tests/dimos_cli_call.py index 83f7c5566f..59aea87fc4 100644 --- a/dimos/e2e_tests/dimos_cli_call.py +++ b/dimos/e2e_tests/dimos_cli_call.py @@ -36,9 +36,8 @@ def start(self) -> None: args = ["run", *args] # If a port was supplied, override `global_config.mcp_port` (used by - # `McpServer.start` to bind) and `McpClient.mcp_server_url` (which - # defaults to a hard-coded `http://localhost:9990/mcp`) so server - # and client agree on the same port. + # `McpServer.start` to bind) and `McpClient.mcp_server_url` so server + # and client agree on the same loopback interface and port. # # The McpClient URL goes through an env var rather than a `-o` # blueprint override: `load_config_args` silently skips env-var @@ -49,7 +48,7 @@ def start(self) -> None: env = os.environ.copy() if self.mcp_port is not None: global_overrides += ["--mcp-port", str(self.mcp_port)] - env["MCPCLIENT__MCP_SERVER_URL"] = f"http://localhost:{self.mcp_port}/mcp" + env["MCPCLIENT__MCP_SERVER_URL"] = f"http://127.0.0.1:{self.mcp_port}/mcp" self.process = subprocess.Popen( [ diff --git a/dimos/e2e_tests/test_dimos_cli_call.py b/dimos/e2e_tests/test_dimos_cli_call.py new file mode 100644 index 0000000000..f5f9a7dadb --- /dev/null +++ b/dimos/e2e_tests/test_dimos_cli_call.py @@ -0,0 +1,29 @@ +# Copyright 2025-2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from unittest.mock import patch + +from dimos.e2e_tests.dimos_cli_call import DimosCliCall + + +def test_start_aligns_mcp_client_url_with_ipv4_loopback_for_custom_port() -> None: + call = DimosCliCall() + call.demo_args = ["coordinator-mock"] + call.mcp_port = 12345 + + with patch("dimos.e2e_tests.dimos_cli_call.subprocess.Popen") as popen: + call.start() + + kwargs = popen.call_args.kwargs + assert kwargs["env"]["MCPCLIENT__MCP_SERVER_URL"] == "http://127.0.0.1:12345/mcp" diff --git a/dimos/protocol/service/system_configurator/base.py b/dimos/protocol/service/system_configurator/base.py index 8c9e2b47e7..fd35c5e5b3 100644 --- a/dimos/protocol/service/system_configurator/base.py +++ b/dimos/protocol/service/system_configurator/base.py @@ -77,6 +77,10 @@ def fix(self) -> None: def configure_system(checks: list[SystemConfigurator], check_only: bool = False) -> None: + if os.environ.get("DIMOS_SKIP_SYSTEM_CONFIG"): + logger.warning("DIMOS_SKIP_SYSTEM_CONFIG is set: skipping system configuration.") + return + # Skip in test runs — we'd otherwise prompt for sudo from a non- # interactive subprocess and kill the worker. The self-hosted CI # runner has its host config baked in via the container image, so diff --git a/dimos/protocol/service/test_system_configurator.py b/dimos/protocol/service/test_system_configurator.py index dac1bdabbf..98131f1a29 100644 --- a/dimos/protocol/service/test_system_configurator.py +++ b/dimos/protocol/service/test_system_configurator.py @@ -138,6 +138,12 @@ def test_skips_in_pytest_environment(self, mocker) -> None: configure_system([mock_check]) assert not mock_check.fix_called + def test_skips_when_explicitly_disabled(self, mocker) -> None: + mocker.patch.dict(os.environ, {"DIMOS_SKIP_SYSTEM_CONFIG": "1"}) + mock_check = MockConfigurator(passes=False, is_critical=True) + configure_system([mock_check]) + assert not mock_check.fix_called + def test_does_nothing_when_all_checks_pass(self) -> None: mock_check = MockConfigurator(passes=True) configure_system([mock_check]) From 8f2c32a45c9a18cedbc080cda1a3ad21be7d5cab Mon Sep 17 00:00:00 2001 From: ZhangZheMing <1050297972@qq.com> Date: Sat, 4 Jul 2026 22:59:52 +0800 Subject: [PATCH 11/36] Harden Go2 runtime adapters --- dimos/agents/skills/speak_skill.py | 19 +- dimos/agents/skills/test_speak_skill.py | 35 +++ .../security_demo/security_module.py | 44 ++- .../security_demo/test_security_module.py | 25 ++ dimos/perception/spatial_perception.py | 17 +- .../test_spatial_perception_config.py | 32 +++ dimos/robot/unitree/go2/connection.py | 3 + dimos/robot/unitree/go2/test_connection.py | 26 ++ dimos/robot/unitree/mujoco_connection.py | 40 ++- dimos/simulation/mujoco/mujoco_process.py | 272 +++++++++++------- .../simulation/mujoco/test_mujoco_process.py | 31 ++ 11 files changed, 419 insertions(+), 125 deletions(-) create mode 100644 dimos/agents/skills/test_speak_skill.py create mode 100644 dimos/perception/test_spatial_perception_config.py create mode 100644 dimos/robot/unitree/go2/test_connection.py create mode 100644 dimos/simulation/mujoco/test_mujoco_process.py diff --git a/dimos/agents/skills/speak_skill.py b/dimos/agents/skills/speak_skill.py index b46de157c4..6aaae644bd 100644 --- a/dimos/agents/skills/speak_skill.py +++ b/dimos/agents/skills/speak_skill.py @@ -38,9 +38,22 @@ class SpeakSkill(Module): @rpc def start(self) -> None: super().start() - self._tts_node = OpenAITTSNode(speed=1.2, voice=Voice.ONYX) - self._audio_output = SounddeviceAudioOutput(sample_rate=24000) - self._audio_output.consume_audio(self._tts_node.emit_audio()) + try: + self._tts_node = OpenAITTSNode(speed=1.2, voice=Voice.ONYX) + self._audio_output = SounddeviceAudioOutput(sample_rate=24000) + self._audio_output.consume_audio(self._tts_node.emit_audio()) + except Exception as exc: + logger.warning( + "SpeakSkill audio unavailable; speech is disabled.", + error=str(exc), + exc_info=True, + ) + if self._audio_output: + self._audio_output.stop() + if self._tts_node: + self._tts_node.dispose() + self._audio_output = None + self._tts_node = None @rpc def stop(self) -> None: diff --git a/dimos/agents/skills/test_speak_skill.py b/dimos/agents/skills/test_speak_skill.py new file mode 100644 index 0000000000..96a41d2636 --- /dev/null +++ b/dimos/agents/skills/test_speak_skill.py @@ -0,0 +1,35 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import annotations + +from dimos.agents.skills.speak_skill import SpeakSkill + + +def test_start_disables_tts_when_audio_initialization_fails(mocker): + mocker.patch( + "dimos.agents.skills.speak_skill.OpenAITTSNode", + side_effect=RuntimeError("missing api key"), + ) + audio_output = mocker.patch("dimos.agents.skills.speak_skill.SounddeviceAudioOutput") + + skill = SpeakSkill() + try: + skill.start() + + assert skill._tts_node is None + audio_output.assert_not_called() + assert skill.speak("hello") == "Error: TTS not initialized" + finally: + skill.stop() diff --git a/dimos/experimental/security_demo/security_module.py b/dimos/experimental/security_demo/security_module.py index eb231583ca..0007b89ad3 100644 --- a/dimos/experimental/security_demo/security_module.py +++ b/dimos/experimental/security_demo/security_module.py @@ -149,6 +149,8 @@ class SecurityModule(Module): _planner_spec: ReplanningAStarPlannerSpec _speak_skill: SpeakSkillSpec + _detector: YoloPersonDetector | None + _depth_estimator: DepthEstimator | None _tracker: EdgeTAMProcessor | None def __init__(self, **kwargs: Any) -> None: @@ -156,10 +158,9 @@ def __init__(self, **kwargs: Any) -> None: self._router: PatrolRouter = _create_router(self.config.g) self._visual_servo = _create_visual_servo(self.config, self.config.g) - self._detector = YoloPersonDetector() + self._detector = None self._tracker = None - - self._depth_estimator = DepthEstimator(self.depth_image.publish) + self._depth_estimator = None self._lock = threading.RLock() self._stop_event = threading.Event() @@ -184,10 +185,16 @@ def start(self) -> None: @rpc def stop(self) -> None: self._stop_security_patrol_internal() - self._depth_estimator.stop() - self._detector.stop() + if self._depth_estimator is not None: + self._depth_estimator.stop() + self._depth_estimator = None + self._depth_started = False + if self._detector is not None: + self._detector.stop() + self._detector = None if self._tracker is not None: self._tracker.stop() + self._tracker = None super().stop() @skill @@ -200,12 +207,19 @@ def start_security_patrol(self) -> str: if self._main_thread is not None and self._main_thread.is_alive(): return "Security patrol is already running. Use `stop_security_patrol` to stop." + try: + self._ensure_detector_available() + depth_estimator = self._ensure_depth_estimator_available() + except Exception as exc: + logger.warning("Security patrol unavailable", error=str(exc), exc_info=True) + return f"Security patrol unavailable: {exc}" + tracker_error = self._ensure_tracker_available() if tracker_error is not None: return tracker_error if not self._depth_started: - self._depth_estimator.start() + depth_estimator.start() self._depth_started = True self._router.reset() @@ -228,6 +242,16 @@ def start_security_patrol(self) -> str: "persons automatically. Use `stop_security_patrol` to stop." ) + def _ensure_detector_available(self) -> YoloPersonDetector: + if self._detector is None: + self._detector = YoloPersonDetector() + return self._detector + + def _ensure_depth_estimator_available(self) -> DepthEstimator: + if self._depth_estimator is None: + self._depth_estimator = DepthEstimator(self.depth_image.publish) + return self._depth_estimator + def _ensure_tracker_available(self) -> str | None: if self._tracker is not None: return None @@ -255,7 +279,8 @@ def _on_goal_reached(self, _msg: Bool) -> None: def _on_color_image(self, image: Image) -> None: with self._lock: self._latest_image = image - self._depth_estimator.submit(image) + if self._depth_started and self._depth_estimator is not None: + self._depth_estimator.submit(image) def _main_loop(self) -> None: self._transition_to("PATROLLING") @@ -375,7 +400,8 @@ def _follow_step(self) -> None: overlay[mask_bool] = cv2.addWeighted(overlay[mask_bool], 0.6, green[mask_bool], 0.4, 0) # Run pose estimation on the tracked frame and draw skeleton - pose_detections = self._detector.process_image(latest_image) + detector = self._ensure_detector_available() + pose_detections = detector.process_image(latest_image) persons = [ d for d in pose_detections.detections @@ -392,7 +418,7 @@ def _follow_step(self) -> None: def _find_best_person(self, image: Image) -> Detection2DBBox | None: """Run YOLO and return the largest person detection, or None.""" - all_detections = self._detector.process_image(image) + all_detections = self._ensure_detector_available().process_image(image) persons = [d for d in all_detections.detections if d.name == "person"] if not persons: return None diff --git a/dimos/experimental/security_demo/test_security_module.py b/dimos/experimental/security_demo/test_security_module.py index 6d994440f2..6d78e1bbeb 100644 --- a/dimos/experimental/security_demo/test_security_module.py +++ b/dimos/experimental/security_demo/test_security_module.py @@ -18,11 +18,36 @@ import pytest +from dimos.experimental.security_demo.security_module import SecurityModule from dimos.msgs.geometry_msgs.PoseStamped import PoseStamped from dimos.msgs.geometry_msgs.Twist import Twist +from dimos.msgs.sensor_msgs.CameraInfo import CameraInfo from dimos.perception.detection.type.detection2d.imageDetections2D import ImageDetections2D +def test_security_module_initialization_defers_heavy_models(mocker): + yolo_detector = mocker.patch( + "dimos.experimental.security_demo.security_module.YoloPersonDetector" + ) + depth_estimator = mocker.patch( + "dimos.experimental.security_demo.security_module.DepthEstimator" + ) + mocker.patch("dimos.experimental.security_demo.security_module._create_router") + mocker.patch("dimos.experimental.security_demo.security_module._create_visual_servo") + + module = SecurityModule(camera_info=CameraInfo()) + module._planner_spec = mocker.MagicMock() + module._speak_skill = mocker.MagicMock() + + try: + yolo_detector.assert_not_called() + depth_estimator.assert_not_called() + assert module._detector is None + assert module._depth_estimator is None + finally: + module.stop() + + @pytest.mark.self_hosted def test_find_best_person_detects_person(security_module, yolo_detector, person_image): security_module._detector = yolo_detector diff --git a/dimos/perception/spatial_perception.py b/dimos/perception/spatial_perception.py index 1265653d66..b2cf702cd5 100644 --- a/dimos/perception/spatial_perception.py +++ b/dimos/perception/spatial_perception.py @@ -18,6 +18,7 @@ from datetime import datetime import os +from pathlib import Path import time from typing import TYPE_CHECKING, Any import uuid @@ -30,7 +31,7 @@ from dimos.agents_deprecated.memory.image_embedding import ImageEmbeddingProvider from dimos.agents_deprecated.memory.spatial_vector_db import SpatialVectorDB from dimos.agents_deprecated.memory.visual_memory import VisualMemory -from dimos.constants import DIMOS_PROJECT_ROOT +from dimos.constants import STATE_DIR from dimos.core.coordination.module_coordinator import ModuleCoordinator from dimos.core.core import rpc from dimos.core.module import Module, ModuleConfig @@ -43,9 +44,17 @@ if TYPE_CHECKING: from dimos.msgs.geometry_msgs.Vector3 import Vector3 -_OUTPUT_DIR = DIMOS_PROJECT_ROOT / "assets" / "output" -_MEMORY_DIR = _OUTPUT_DIR / "memory" -_SPATIAL_MEMORY_DIR = _MEMORY_DIR / "spatial_memory" +_SPATIAL_MEMORY_DIR_ENV = "DIMOS_SPATIAL_MEMORY_DIR" + + +def _default_spatial_memory_dir() -> Path: + configured = os.environ.get(_SPATIAL_MEMORY_DIR_ENV) + if configured: + return Path(configured).expanduser() + return STATE_DIR / "spatial_memory" + + +_SPATIAL_MEMORY_DIR = _default_spatial_memory_dir() _DB_PATH = _SPATIAL_MEMORY_DIR / "chromadb_data" _VISUAL_MEMORY_PATH = _SPATIAL_MEMORY_DIR / "visual_memory.pkl" diff --git a/dimos/perception/test_spatial_perception_config.py b/dimos/perception/test_spatial_perception_config.py new file mode 100644 index 0000000000..850e11c1d4 --- /dev/null +++ b/dimos/perception/test_spatial_perception_config.py @@ -0,0 +1,32 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from dimos.constants import STATE_DIR +from dimos.perception.spatial_perception import ( + SpatialConfig, + _default_spatial_memory_dir, +) + + +def test_spatial_memory_defaults_to_state_dir(monkeypatch) -> None: + monkeypatch.delenv("DIMOS_SPATIAL_MEMORY_DIR", raising=False) + + assert _default_spatial_memory_dir() == STATE_DIR / "spatial_memory" + assert SpatialConfig().db_path == str(STATE_DIR / "spatial_memory" / "chromadb_data") + + +def test_spatial_memory_dir_honors_env(monkeypatch, tmp_path) -> None: + monkeypatch.setenv("DIMOS_SPATIAL_MEMORY_DIR", str(tmp_path)) + + assert _default_spatial_memory_dir() == tmp_path diff --git a/dimos/robot/unitree/go2/connection.py b/dimos/robot/unitree/go2/connection.py index e91a378fe0..8463b3ff48 100644 --- a/dimos/robot/unitree/go2/connection.py +++ b/dimos/robot/unitree/go2/connection.py @@ -150,6 +150,9 @@ def connect(self) -> None: def start(self) -> None: pass + def stop(self) -> None: + pass + def standup(self) -> bool: return True diff --git a/dimos/robot/unitree/go2/test_connection.py b/dimos/robot/unitree/go2/test_connection.py new file mode 100644 index 0000000000..9b0706c941 --- /dev/null +++ b/dimos/robot/unitree/go2/test_connection.py @@ -0,0 +1,26 @@ +from io import BytesIO +import sys + +from dimos.robot.unitree.go2.connection import ReplayConnection +from dimos.robot.unitree.mujoco_connection import ( + _mujoco_subprocess_executable, + _read_process_stderr, +) + + +def test_replay_connection_stop_is_noop() -> None: + connection = ReplayConnection() + + connection.stop() + + +def test_read_process_stderr_decodes_available_output() -> None: + class Process: + stderr = BytesIO(b"first line\nsecond line\n") + + assert _read_process_stderr(Process()) == "first line\nsecond line" + + +def test_headless_mujoco_uses_current_python_on_macos() -> None: + assert _mujoco_subprocess_executable(headless=True, platform="darwin") == sys.executable + assert _mujoco_subprocess_executable(headless=False, platform="darwin") == "mjpython" diff --git a/dimos/robot/unitree/mujoco_connection.py b/dimos/robot/unitree/mujoco_connection.py index 4c455899e8..fc8449c0da 100644 --- a/dimos/robot/unitree/mujoco_connection.py +++ b/dimos/robot/unitree/mujoco_connection.py @@ -63,6 +63,35 @@ logger = setup_logger() T = TypeVar("T") +_MUJOCO_HEADLESS_ENV = "DIMOS_MUJOCO_HEADLESS" + + +def _mujoco_headless_enabled() -> bool: + return os.environ.get(_MUJOCO_HEADLESS_ENV, "").strip().lower() in { + "1", + "true", + "yes", + "on", + } + + +def _mujoco_subprocess_executable(headless: bool, platform: str = sys.platform) -> str: + if platform == "darwin" and not headless: + return "mjpython" + return sys.executable + + +def _read_process_stderr(process: subprocess.Popen[bytes] | Any) -> str: + stderr = getattr(process, "stderr", None) + if stderr is None: + return "" + try: + data = stderr.read() + except Exception: + return "" + if isinstance(data, bytes): + return data.decode(errors="replace").strip() + return str(data).strip() class MujocoConnection: @@ -115,9 +144,10 @@ def start(self) -> None: # It needs libpython on the dylib search path; uv-installed Pythons # use @rpath which doesn't always resolve inside venvs, so we # point DYLD_LIBRARY_PATH at the real libpython directory. - executable = sys.executable if sys.platform != "darwin" else "mjpython" + headless = _mujoco_headless_enabled() + executable = _mujoco_subprocess_executable(headless=headless) env = os.environ.copy() - if sys.platform == "darwin": + if sys.platform == "darwin" and not headless: # on some systems mujoco looks in the wrong place for shared libraries. So we force it look in the right place libdir = Path(sysconfig.get_config_var("LIBDIR") or "") if libdir.is_dir(): @@ -141,8 +171,12 @@ def start(self) -> None: while time.time() - start_time < ready_timeout: if self.process.poll() is not None: exit_code = self.process.returncode + stderr = _read_process_stderr(self.process) self.stop() - raise RuntimeError(f"MuJoCo process failed to start (exit code {exit_code})") + detail = f"MuJoCo process failed to start (exit code {exit_code})" + if stderr: + detail = f"{detail}: {stderr[-4000:]}" + raise RuntimeError(detail) if self.shm_data.is_ready(): logger.info("MuJoCo process started successfully") # Register atexit handler to ensure subprocess is cleaned up diff --git a/dimos/simulation/mujoco/mujoco_process.py b/dimos/simulation/mujoco/mujoco_process.py index 8ed4f20da2..0270fafc52 100755 --- a/dimos/simulation/mujoco/mujoco_process.py +++ b/dimos/simulation/mujoco/mujoco_process.py @@ -16,6 +16,7 @@ import base64 import json +import os import pickle import signal import sys @@ -23,7 +24,6 @@ from typing import Any import mujoco -from mujoco import viewer import numpy as np from numpy.typing import NDArray import open3d as o3d # type: ignore[import-untyped] @@ -46,6 +46,17 @@ logger = setup_logger() +_MUJOCO_HEADLESS_ENV = "DIMOS_MUJOCO_HEADLESS" + + +def _mujoco_headless_enabled() -> bool: + return os.environ.get(_MUJOCO_HEADLESS_ENV, "").strip().lower() in { + "1", + "true", + "yes", + "on", + } + class MockController: """Controller that reads commands from shared memory.""" @@ -106,126 +117,175 @@ def _run_simulation(config: GlobalConfig, shm: ShmReader) -> None: model, mujoco.mjtObj.mjOBJ_CAMERA, "lidar_right_camera" ) - shm.signal_ready() - - with viewer.launch_passive(model, data, show_left_ui=False, show_right_ui=False) as m_viewer: - camera_size = (VIDEO_WIDTH, VIDEO_HEIGHT) - - # Create renderers - rgb_renderer = mujoco.Renderer(model, height=camera_size[1], width=camera_size[0]) - depth_renderer = mujoco.Renderer(model, height=camera_size[1], width=camera_size[0]) - depth_renderer.enable_depth_rendering() - - depth_left_renderer = mujoco.Renderer(model, height=camera_size[1], width=camera_size[0]) - depth_left_renderer.enable_depth_rendering() - - depth_right_renderer = mujoco.Renderer(model, height=camera_size[1], width=camera_size[0]) - depth_right_renderer.enable_depth_rendering() - - scene_option = mujoco.MjvOption() + try: + shm.signal_ready() + + if _mujoco_headless_enabled(): + logger.info("Running MuJoCo in headless mode") + _run_simulation_loop( + config, + shm, + model, + data, + camera_id, + lidar_camera_id, + lidar_left_camera_id, + lidar_right_camera_id, + person_position_controller, + m_viewer=None, + ) + return + + from mujoco import viewer + + with viewer.launch_passive( + model, data, show_left_ui=False, show_right_ui=False + ) as m_viewer: + _run_simulation_loop( + config, + shm, + model, + data, + camera_id, + lidar_camera_id, + lidar_left_camera_id, + lidar_right_camera_id, + person_position_controller, + m_viewer=m_viewer, + ) + finally: + person_position_controller.stop() - # Timing control - last_video_time = 0.0 - last_lidar_time = 0.0 - video_interval = 1.0 / VIDEO_FPS - lidar_interval = 1.0 / LIDAR_FPS +def _run_simulation_loop( + config: GlobalConfig, + shm: ShmReader, + model: mujoco.MjModel, + data: mujoco.MjData, + camera_id: int, + lidar_camera_id: int, + lidar_left_camera_id: int, + lidar_right_camera_id: int, + person_position_controller: PersonPositionController, + m_viewer: Any | None, +) -> None: + camera_size = (VIDEO_WIDTH, VIDEO_HEIGHT) + + # Create renderers + rgb_renderer = mujoco.Renderer(model, height=camera_size[1], width=camera_size[0]) + depth_renderer = mujoco.Renderer(model, height=camera_size[1], width=camera_size[0]) + depth_renderer.enable_depth_rendering() + + depth_left_renderer = mujoco.Renderer(model, height=camera_size[1], width=camera_size[0]) + depth_left_renderer.enable_depth_rendering() + + depth_right_renderer = mujoco.Renderer(model, height=camera_size[1], width=camera_size[0]) + depth_right_renderer.enable_depth_rendering() + + scene_option = mujoco.MjvOption() + + # Timing control + last_video_time = 0.0 + last_lidar_time = 0.0 + video_interval = 1.0 / VIDEO_FPS + lidar_interval = 1.0 / LIDAR_FPS + + if m_viewer is not None: m_viewer.cam.lookat = config.mujoco_camera_position_float[0:3] m_viewer.cam.distance = config.mujoco_camera_position_float[3] m_viewer.cam.azimuth = config.mujoco_camera_position_float[4] m_viewer.cam.elevation = config.mujoco_camera_position_float[5] - while m_viewer.is_running() and not shm.should_stop(): - step_start = time.time() + while (m_viewer is None or m_viewer.is_running()) and not shm.should_stop(): + step_start = time.time() - # Step simulation - for _ in range(config.mujoco_steps_per_frame): - mujoco.mj_step(model, data) + # Step simulation + for _ in range(config.mujoco_steps_per_frame): + mujoco.mj_step(model, data) - person_position_controller.tick(data) + person_position_controller.tick(data) + if m_viewer is not None: m_viewer.sync() - # Always update odometry - pos = data.qpos[0:3].copy() - quat = data.qpos[3:7].copy() # (w, x, y, z) - shm.write_odom(pos, quat, time.time()) - - current_time = time.time() - - # Video rendering - if current_time - last_video_time >= video_interval: - rgb_renderer.update_scene(data, camera=camera_id, scene_option=scene_option) - pixels = rgb_renderer.render() - shm.write_video(pixels) - last_video_time = current_time - - # Lidar/depth rendering - if current_time - last_lidar_time >= lidar_interval: - # Render all depth cameras - depth_renderer.update_scene(data, camera=lidar_camera_id, scene_option=scene_option) - depth_front = depth_renderer.render() - - depth_left_renderer.update_scene( - data, camera=lidar_left_camera_id, scene_option=scene_option + # Always update odometry + pos = data.qpos[0:3].copy() + quat = data.qpos[3:7].copy() # (w, x, y, z) + shm.write_odom(pos, quat, time.time()) + + current_time = time.time() + + # Video rendering + if current_time - last_video_time >= video_interval: + rgb_renderer.update_scene(data, camera=camera_id, scene_option=scene_option) + pixels = rgb_renderer.render() + shm.write_video(pixels) + last_video_time = current_time + + # Lidar/depth rendering + if current_time - last_lidar_time >= lidar_interval: + # Render all depth cameras + depth_renderer.update_scene(data, camera=lidar_camera_id, scene_option=scene_option) + depth_front = depth_renderer.render() + + depth_left_renderer.update_scene( + data, camera=lidar_left_camera_id, scene_option=scene_option + ) + depth_left = depth_left_renderer.render() + + depth_right_renderer.update_scene( + data, camera=lidar_right_camera_id, scene_option=scene_option + ) + depth_right = depth_right_renderer.render() + + shm.write_depth(depth_front, depth_left, depth_right) + + # Process depth images into lidar message + all_points = [] + cameras_data = [ + ( + depth_front, + data.cam_xpos[lidar_camera_id], + data.cam_xmat[lidar_camera_id].reshape(3, 3), + ), + ( + depth_left, + data.cam_xpos[lidar_left_camera_id], + data.cam_xmat[lidar_left_camera_id].reshape(3, 3), + ), + ( + depth_right, + data.cam_xpos[lidar_right_camera_id], + data.cam_xmat[lidar_right_camera_id].reshape(3, 3), + ), + ] + + for depth_image, camera_pos, camera_mat in cameras_data: + points = depth_image_to_point_cloud( + depth_image, camera_pos, camera_mat, fov_degrees=DEPTH_CAMERA_FOV ) - depth_left = depth_left_renderer.render() - - depth_right_renderer.update_scene( - data, camera=lidar_right_camera_id, scene_option=scene_option + if points.size > 0: + all_points.append(points) + + if all_points: + combined_points = np.vstack(all_points) + pcd = o3d.geometry.PointCloud() + pcd.points = o3d.utility.Vector3dVector(combined_points) + pcd = pcd.voxel_down_sample(voxel_size=LIDAR_RESOLUTION) + + lidar_msg = PointCloud2( + pointcloud=pcd, + ts=time.time(), + frame_id="world", ) - depth_right = depth_right_renderer.render() - - shm.write_depth(depth_front, depth_left, depth_right) - - # Process depth images into lidar message - all_points = [] - cameras_data = [ - ( - depth_front, - data.cam_xpos[lidar_camera_id], - data.cam_xmat[lidar_camera_id].reshape(3, 3), - ), - ( - depth_left, - data.cam_xpos[lidar_left_camera_id], - data.cam_xmat[lidar_left_camera_id].reshape(3, 3), - ), - ( - depth_right, - data.cam_xpos[lidar_right_camera_id], - data.cam_xmat[lidar_right_camera_id].reshape(3, 3), - ), - ] - - for depth_image, camera_pos, camera_mat in cameras_data: - points = depth_image_to_point_cloud( - depth_image, camera_pos, camera_mat, fov_degrees=DEPTH_CAMERA_FOV - ) - if points.size > 0: - all_points.append(points) - - if all_points: - combined_points = np.vstack(all_points) - pcd = o3d.geometry.PointCloud() - pcd.points = o3d.utility.Vector3dVector(combined_points) - pcd = pcd.voxel_down_sample(voxel_size=LIDAR_RESOLUTION) - - lidar_msg = PointCloud2( - pointcloud=pcd, - ts=time.time(), - frame_id="world", - ) - shm.write_lidar(lidar_msg) - - last_lidar_time = current_time - - # Control simulation speed - time_until_next_step = model.opt.timestep - (time.time() - step_start) - if time_until_next_step > 0: - time.sleep(time_until_next_step) + shm.write_lidar(lidar_msg) - person_position_controller.stop() + last_lidar_time = current_time + + # Control simulation speed + time_until_next_step = model.opt.timestep - (time.time() - step_start) + if time_until_next_step > 0: + time.sleep(time_until_next_step) if __name__ == "__main__": diff --git a/dimos/simulation/mujoco/test_mujoco_process.py b/dimos/simulation/mujoco/test_mujoco_process.py new file mode 100644 index 0000000000..de6271fba4 --- /dev/null +++ b/dimos/simulation/mujoco/test_mujoco_process.py @@ -0,0 +1,31 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import annotations + +from dimos.simulation.mujoco.mujoco_process import _mujoco_headless_enabled + + +def test_mujoco_headless_enabled_from_env(monkeypatch): + monkeypatch.delenv("DIMOS_MUJOCO_HEADLESS", raising=False) + assert _mujoco_headless_enabled() is False + + monkeypatch.setenv("DIMOS_MUJOCO_HEADLESS", "1") + assert _mujoco_headless_enabled() is True + + monkeypatch.setenv("DIMOS_MUJOCO_HEADLESS", "true") + assert _mujoco_headless_enabled() is True + + monkeypatch.setenv("DIMOS_MUJOCO_HEADLESS", "0") + assert _mujoco_headless_enabled() is False From d830e7e90319fb6951280a8ed88800305dc97f3c Mon Sep 17 00:00:00 2001 From: ZhangZheMing <1050297972@qq.com> Date: Sat, 4 Jul 2026 23:00:09 +0800 Subject: [PATCH 12/36] Add Go2 world model dashboard contracts --- .../causal_effect_estimator.py | 229 +++++ .../layer_3_agent_brain/causal_world_model.py | 949 +++++++++++++++++- .../layer_3_agent_brain/intervention_log.py | 274 +++++ .../online_transition_model.py | 198 ++++ .../skill_outcome_predictor.py | 64 +- .../structural_causal_model.py | 225 +++++ .../test_causal_world_model.py | 402 ++++++++ .../test_skill_outcomes.py | 4 +- .../test_world_model_contract.py | 73 ++ .../world_model_contract.py | 98 ++ .../layers/layer_4_world_state/DESIGN.md | 29 + .../layers/layer_4_world_state/__init__.py | 53 +- dimos/web/templates/rerun_dashboard.html | 30 +- .../test_websocket_vis_module.py | 71 ++ .../web/websocket_vis/websocket_vis_module.py | 240 ++++- dimos/web/websocket_vis_spec.py | 4 +- 16 files changed, 2907 insertions(+), 36 deletions(-) create mode 100644 dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/causal_effect_estimator.py create mode 100644 dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/intervention_log.py create mode 100644 dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/online_transition_model.py create mode 100644 dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/structural_causal_model.py create mode 100644 dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/test_world_model_contract.py create mode 100644 dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/world_model_contract.py create mode 100644 dimos/web/websocket_vis/test_websocket_vis_module.py diff --git a/dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/causal_effect_estimator.py b/dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/causal_effect_estimator.py new file mode 100644 index 0000000000..da532f007d --- /dev/null +++ b/dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/causal_effect_estimator.py @@ -0,0 +1,229 @@ +#!/usr/bin/env python3 +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import annotations + +from dataclasses import dataclass +from typing import Any + + +@dataclass(frozen=True) +class _CausalObservation: + features: set[str] + outcome_success: bool + skill_name: str + inferred_cause: str + + +class CausalEffectEstimator: + """Observational feature-effect estimator for symbolic world transitions.""" + + method = "observational_feature_effect" + + def __init__(self, max_observations: int = 500) -> None: + self._max_observations = max_observations + self._observations: list[_CausalObservation] = [] + + def update( + self, + snapshot: dict[str, Any], + action: dict[str, Any], + outcome_success: bool | None, + inferred_cause: str, + ) -> None: + if outcome_success is None: + return + self._observations.append( + _CausalObservation( + features=_causal_features(snapshot, action), + outcome_success=outcome_success, + skill_name=str(action.get("skill_name") or ""), + inferred_cause=inferred_cause, + ) + ) + if len(self._observations) > self._max_observations: + self._observations = self._observations[-self._max_observations :] + + def estimate(self, snapshot: dict[str, Any], action: dict[str, Any]) -> dict[str, Any]: + active = _causal_features(snapshot, action) + skill_name = str(action.get("skill_name") or "") + observations = [ + observation + for observation in self._observations + if not skill_name or observation.skill_name == skill_name + ] + if not observations: + return self._empty(active) + + effects: list[dict[str, Any]] = [] + for feature in sorted(active): + treated = [obs for obs in observations if feature in obs.features] + control = [obs for obs in observations if feature not in obs.features] + if not treated or not control: + continue + treated_rate = _smoothed_success_rate(treated) + control_rate = _smoothed_success_rate(control) + effect = treated_rate - control_rate + effects.append( + { + "feature": feature, + "effect_on_success": round(effect, 3), + "treated_success_rate": round(treated_rate, 3), + "control_success_rate": round(control_rate, 3), + "treated_count": len(treated), + "control_count": len(control), + "confidence": _effect_confidence(len(treated), len(control), effect), + } + ) + + effects.sort(key=lambda item: float(item["effect_on_success"])) + risk_factors = [effect for effect in effects if float(effect["effect_on_success"]) < -0.1] + supports = [ + effect for effect in reversed(effects) if float(effect["effect_on_success"]) > 0.1 + ] + return { + "method": self.method, + "assumption": "observational estimate; unobserved confounders are not controlled", + "sample_count": len(observations), + "active_features": sorted(active), + "risk_factors": risk_factors[:5], + "supporting_factors": supports[:5], + "interventions": _interventions(risk_factors[:5]), + } + + def snapshot(self) -> dict[str, Any]: + return { + "method": self.method, + "sample_count": len(self._observations), + "max_observations": self._max_observations, + } + + def to_dict(self) -> dict[str, Any]: + return { + "method": self.method, + "max_observations": self._max_observations, + "observations": [ + { + "features": sorted(observation.features), + "outcome_success": observation.outcome_success, + "skill_name": observation.skill_name, + "inferred_cause": observation.inferred_cause, + } + for observation in self._observations + ], + } + + def load_dict(self, state: dict[str, Any]) -> None: + self._max_observations = max( + 1, + int(state.get("max_observations") or self._max_observations), + ) + raw_observations = ( + state.get("observations") if isinstance(state.get("observations"), list) else [] + ) + observations: list[_CausalObservation] = [] + for item in raw_observations[-self._max_observations :]: + if not isinstance(item, dict): + continue + features = item.get("features") if isinstance(item.get("features"), list) else [] + outcome_success = item.get("outcome_success") + if not isinstance(outcome_success, bool): + continue + observations.append( + _CausalObservation( + features={str(feature) for feature in features}, + outcome_success=outcome_success, + skill_name=str(item.get("skill_name") or ""), + inferred_cause=str(item.get("inferred_cause") or ""), + ) + ) + self._observations = observations + + def _empty(self, active: set[str]) -> dict[str, Any]: + return { + "method": self.method, + "assumption": "observational estimate; unobserved confounders are not controlled", + "sample_count": 0, + "active_features": sorted(active), + "risk_factors": [], + "supporting_factors": [], + "interventions": [], + } + + +def _causal_features(snapshot: dict[str, Any], action: dict[str, Any]) -> set[str]: + features: set[str] = set() + skill_name = str(action.get("skill_name") or "") + domain = str(action.get("domain") or "") + args = action.get("args") if isinstance(action.get("args"), dict) else {} + if skill_name: + features.add(f"skill:{skill_name}") + if domain: + features.add(f"domain:{domain}") + + robot_state = ( + snapshot.get("robot_state") if isinstance(snapshot.get("robot_state"), dict) else {} + ) + memory_state = ( + snapshot.get("memory_state") if isinstance(snapshot.get("memory_state"), dict) else {} + ) + spatial = memory_state.get("spatial") if isinstance(memory_state.get("spatial"), dict) else {} + temporal = ( + memory_state.get("temporal") if isinstance(memory_state.get("temporal"), dict) else {} + ) + + features.add(f"has_odom:{bool(robot_state.get('odom'))}") + if "available" in spatial: + features.add(f"spatial_available:{bool(spatial.get('available'))}") + matches = spatial.get("matches") if isinstance(spatial.get("matches"), list) else [] + features.add(f"spatial_has_matches:{bool(matches)}") + if "available" in temporal: + features.add(f"temporal_available:{bool(temporal.get('available'))}") + + for key, value in args.items(): + features.add(f"arg_present:{key}:{bool(str(value).strip())}") + return features + + +def _smoothed_success_rate(observations: list[_CausalObservation]) -> float: + successes = sum(1 for observation in observations if observation.outcome_success) + return (successes + 1.0) / (len(observations) + 2.0) + + +def _effect_confidence(treated_count: int, control_count: int, effect: float) -> str: + support = min(treated_count, control_count) + if support >= 5 and abs(effect) >= 0.25: + return "high" + if support >= 2 and abs(effect) >= 0.15: + return "medium" + return "low" + + +def _interventions(risk_factors: list[dict[str, Any]]) -> list[str]: + suggestions: list[str] = [] + for factor in risk_factors: + feature = str(factor.get("feature") or "") + if feature == "spatial_has_matches:False": + suggestions.append("Use perception or tag the target before semantic navigation.") + elif feature == "spatial_available:False": + suggestions.append("Wait for spatial memory or use direct perception.") + elif feature == "has_odom:False": + suggestions.append("Wait for odometry before motion-sensitive skills.") + elif feature.startswith("arg_present:") and feature.endswith(":False"): + suggestions.append("Fill the missing action argument before execution.") + return suggestions + + +__all__ = ["CausalEffectEstimator"] diff --git a/dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/causal_world_model.py b/dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/causal_world_model.py index 77149dbe65..38465b7950 100644 --- a/dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/causal_world_model.py +++ b/dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/causal_world_model.py @@ -18,6 +18,8 @@ from collections import Counter, deque from dataclasses import dataclass import json +import os +from pathlib import Path import time from typing import Any, Protocol @@ -25,14 +27,35 @@ from dimos.agents.skill_result import SkillResult from dimos.core.core import rpc from dimos.core.module import Module +from dimos.robot.unitree.go2.blueprints.layers.layer_3_agent_brain.causal_effect_estimator import ( + CausalEffectEstimator, +) +from dimos.robot.unitree.go2.blueprints.layers.layer_3_agent_brain.intervention_log import ( + InterventionLog, +) +from dimos.robot.unitree.go2.blueprints.layers.layer_3_agent_brain.online_transition_model import ( + OnlineTransitionOutcomeModel, +) from dimos.robot.unitree.go2.blueprints.layers.layer_3_agent_brain.skill_outcome_store import ( SkillOutcomeStoreSpec, ) +from dimos.robot.unitree.go2.blueprints.layers.layer_3_agent_brain.structural_causal_model import ( + StructuralCausalModel, +) +from dimos.robot.unitree.go2.blueprints.layers.layer_3_agent_brain.world_model_contract import ( + WORLD_MODEL_DASHBOARD_SCHEMA, + WORLD_MODEL_PREDICTION_SCHEMA, + provider_contract, +) from dimos.spec.utils import Spec +from dimos.web.websocket_vis_spec import WebsocketVisSpec + +_STATE_SCHEMA = "go2_causal_world_model_state.v1" +_STATE_PATH_ENV = "DIMOS_GO2_WORLD_MODEL_STATE" class CausalWorldModelSpec(Spec, Protocol): - """RPC surface for Layer 3 modules that need recent causal history.""" + """RPC surface for Layer 3 modules that need world-model predictions.""" def get_recent_transitions( self, @@ -42,23 +65,56 @@ def get_recent_transitions( cause: str = "", ) -> list[dict[str, Any]]: ... + def predict_next_state( + self, + snapshot_json: str = "", + action_json: str = "", + goal: str = "", + horizon: int = 1, + ) -> dict[str, Any]: ... + + def score_action( + self, + snapshot_json: str = "", + action_json: str = "", + goal: str = "", + ) -> dict[str, Any]: ... + + def get_intervention_log( + self, + limit: int = 10, + target_variable: str = "", + intervention_name: str = "", + ) -> list[dict[str, Any]]: ... + + def save_world_model_state(self, path: str = "") -> SkillResult: ... + + def load_world_model_state(self, path: str = "") -> SkillResult: ... + + def get_model_state(self) -> dict[str, Any]: ... + + def get_provider_contract(self) -> dict[str, Any]: ... + @dataclass(frozen=True) -class _CausalTransition: - """One observed action transition with inferred cause and recovery.""" +class _WorldTransition: + """One observed world-state transition with inferred failure cause.""" timestamp: float task: str domain: str skill_name: str args: dict[str, Any] + before_state: dict[str, Any] before_context: str prediction_risk: str prediction_reasons: list[str] outcome_success: bool | None outcome_error_code: str outcome_message: str + after_state: dict[str, Any] after_context: str + state_delta: dict[str, Any] inferred_cause: str recovery: str confidence: str @@ -70,13 +126,16 @@ def to_dict(self) -> dict[str, Any]: "domain": self.domain, "skill_name": self.skill_name, "args": self.args, + "before_state": _bounded_jsonable(self.before_state), "before_context": self.before_context, "prediction_risk": self.prediction_risk, "prediction_reasons": self.prediction_reasons, "outcome_success": self.outcome_success, "outcome_error_code": self.outcome_error_code, "outcome_message": self.outcome_message, + "after_state": _bounded_jsonable(self.after_state), "after_context": self.after_context, + "state_delta": _bounded_jsonable(self.state_delta), "inferred_cause": self.inferred_cause, "recovery": self.recovery, "confidence": self.confidence, @@ -84,14 +143,33 @@ def to_dict(self) -> dict[str, Any]: class _Go2CausalWorldModel(Module): - """In-memory Layer 3 causal transition recorder for Go2.""" + """In-memory Layer 3 predictive world model for Go2. + + The model consumes Layer 4 JSON world snapshots plus candidate skill actions. + It does not learn continuous dynamics yet; it provides an agent-friendly + transition memory, risk estimate, predicted symbolic state delta, and action + score so the coding/LLM agent can run a physical-autoresearch style loop. + """ _skill_outcomes: SkillOutcomeStoreSpec | None = None + _websocket_vis: WebsocketVisSpec | None = None _max_transitions = 200 def __init__(self, **kwargs: Any) -> None: super().__init__(**kwargs) - self._transitions: deque[_CausalTransition] = deque(maxlen=self._max_transitions) + self._transitions: deque[_WorldTransition] = deque(maxlen=self._max_transitions) + self._outcome_model = OnlineTransitionOutcomeModel() + self._causal_estimator = CausalEffectEstimator() + self._intervention_log = InterventionLog() + self._scm = StructuralCausalModel() + self._persistence_path = _configured_persistence_path() + if self._persistence_path is not None: + self._load_world_model_state_from_path(self._persistence_path, missing_ok=True) + + @rpc + def start(self) -> None: + super().start() + self._publish_dashboard_state(event="world_model_start") @skill def record_causal_transition( @@ -104,18 +182,23 @@ def record_causal_transition( prediction_json: str = "", outcome_json: str = "", domain: str = "", + before_state_json: str = "", + after_state_json: str = "", ) -> SkillResult: - """Record one before/action/result/after causal transition. + """Record one before/action/result/after world-state transition. Use this after an important physical or recovery-sensitive tool call. - The tool stores a compact event, infers a coarse cause, and returns a - recovery suggestion. It does not execute or retry robot actions. + The tool stores compact JSON state snapshots, infers a coarse cause, + and returns a recovery suggestion. It does not execute or retry robot + actions. Args: task: User goal or agent subtask that led to the action. skill_name: Tool or skill that was executed. args_json: Optional JSON object string with the executed arguments. + before_state_json: Optional Layer 4 world snapshot before action. before_context: Context summary captured before the action. + after_state_json: Optional Layer 4 world snapshot after action. after_context: Context summary captured after the action. prediction_json: Optional JSON object from predict_skill_outcome. outcome_json: Optional JSON object from the tool result or outcome @@ -138,6 +221,14 @@ def record_causal_transition( if isinstance(parsed_args, str): return SkillResult.fail("INVALID_INPUT", parsed_args) + before_state = _parse_json_object(before_state_json, "before_state_json") + if isinstance(before_state, str): + return SkillResult.fail("INVALID_INPUT", before_state) + + after_state = _parse_json_object(after_state_json, "after_state_json") + if isinstance(after_state, str): + return SkillResult.fail("INVALID_INPUT", after_state) + prediction = _parse_json_object(prediction_json, "prediction_json") if isinstance(prediction, str): return SkillResult.fail("INVALID_INPUT", prediction) @@ -156,6 +247,11 @@ def record_causal_transition( outcome_message = str(outcome_view.get("message") or "") prediction_risk = str(prediction_view.get("risk") or "unknown") resolved_domain = domain or str(prediction_view.get("domain") or _infer_domain(skill_name)) + action = { + "skill_name": skill_name, + "domain": resolved_domain, + "args": parsed_args, + } cause, recovery, confidence = _infer_cause( skill_name=skill_name, @@ -166,31 +262,74 @@ def record_causal_transition( outcome_message=outcome_message, outcome_error_code=outcome_error_code, ) + state_delta = _diff_world_state(before_state, after_state) + self._outcome_model.update(before_state, action, outcome_success) + self._causal_estimator.update(before_state, action, outcome_success, cause) - transition = _CausalTransition( + transition = _WorldTransition( timestamp=time.time(), task=task, domain=resolved_domain, skill_name=skill_name, args=parsed_args, + before_state=before_state, before_context=before_context[:500], prediction_risk=prediction_risk, prediction_reasons=prediction_reasons, outcome_success=outcome_success, outcome_error_code=outcome_error_code, outcome_message=outcome_message[:500], + after_state=after_state, after_context=after_context[:500], + state_delta=state_delta, inferred_cause=cause, recovery=recovery, confidence=confidence, ) self._transitions.append(transition) + autosave_error = self._autosave_if_configured() + dashboard_error = self._publish_dashboard_state(event="record_causal_transition") return SkillResult.ok( f"Recorded causal transition for {skill_name}: {cause}", transition=transition.to_dict(), total_transitions=len(self._transitions), + autosave_error=autosave_error, + dashboard_error=dashboard_error, + ) + + @skill + def predict_world_transition( + self, + snapshot_json: str = "", + action_json: str = "", + goal: str = "", + horizon: int = 1, + ) -> SkillResult: + """Predict symbolic state delta, risk, and failure modes for an action. + + Args: + snapshot_json: Layer 4 world snapshot JSON object. + action_json: Candidate action JSON object. Expected keys are + skill_name and args. + goal: Optional user goal for scoring. + horizon: Number of symbolic action steps to roll out. + """ + prediction = self.predict_next_state( + snapshot_json=snapshot_json, + action_json=action_json, + goal=goal, + horizon=horizon, ) + if prediction.get("error"): + return SkillResult.fail("INVALID_INPUT", str(prediction["error"])) + + model = prediction["prediction"] + message = ( + f"Predicted world risk={model['risk']}, " + f"score={model['score']}, confidence={model['confidence']}" + ) + return SkillResult.ok(message, **prediction) @skill def summarize_causal_patterns( @@ -226,6 +365,113 @@ def summarize_causal_patterns( ) return SkillResult.ok(message, patterns=patterns, transitions=transitions) + @skill + def record_intervention( + self, + task: str, + intervention_name: str, + target_variable: str, + before_value_json: str = "", + after_value_json: str = "", + action_json: str = "", + snapshot_before_json: str = "", + snapshot_after_json: str = "", + outcome_json: str = "", + causal_hypothesis: str = "", + ) -> SkillResult: + """Record an explicit intervention and its observed result. + + Use this when the agent intentionally changes one world variable, such + as adding a spatial tag, waiting for odometry, or running perception, + then observes whether the downstream task improved. + """ + task = task.strip() + intervention_name = intervention_name.strip() + target_variable = target_variable.strip() + causal_hypothesis = causal_hypothesis.strip() + if not task: + return SkillResult.fail("INVALID_INPUT", "task is required") + if not intervention_name: + return SkillResult.fail("INVALID_INPUT", "intervention_name is required") + if not target_variable: + return SkillResult.fail("INVALID_INPUT", "target_variable is required") + + before_value = _parse_json_value(before_value_json, "before_value_json") + if isinstance(before_value, _JsonParseError): + return SkillResult.fail("INVALID_INPUT", before_value.message) + after_value = _parse_json_value(after_value_json, "after_value_json") + if isinstance(after_value, _JsonParseError): + return SkillResult.fail("INVALID_INPUT", after_value.message) + + action = _parse_json_object(action_json, "action_json") + if isinstance(action, str): + return SkillResult.fail("INVALID_INPUT", action) + snapshot_before = _parse_json_object(snapshot_before_json, "snapshot_before_json") + if isinstance(snapshot_before, str): + return SkillResult.fail("INVALID_INPUT", snapshot_before) + snapshot_after = _parse_json_object(snapshot_after_json, "snapshot_after_json") + if isinstance(snapshot_after, str): + return SkillResult.fail("INVALID_INPUT", snapshot_after) + outcome = _parse_json_object(outcome_json, "outcome_json") + if isinstance(outcome, str): + return SkillResult.fail("INVALID_INPUT", outcome) + + outcome_view = _metadata_view(outcome) + record = self._intervention_log.record( + task=task, + intervention_name=intervention_name, + target_variable=target_variable, + before_value=before_value, + after_value=after_value, + action=_normalize_action(action), + snapshot_before=snapshot_before, + snapshot_after=snapshot_after, + outcome_success=_optional_bool(outcome_view.get("success")), + outcome_message=str(outcome_view.get("message") or ""), + causal_hypothesis=causal_hypothesis, + ) + autosave_error = self._autosave_if_configured() + dashboard_error = self._publish_dashboard_state(event="record_intervention") + return SkillResult.ok( + f"Recorded intervention {intervention_name} on {target_variable}", + intervention=record, + total_interventions=self._intervention_log.snapshot()["record_count"], + autosave_error=autosave_error, + dashboard_error=dashboard_error, + ) + + @skill + def save_world_model_state(self, path: str = "") -> SkillResult: + """Persist transitions, interventions, and learned model weights to JSON.""" + state_path = self._resolve_state_path(path) + if state_path is None: + return SkillResult.fail( + "INVALID_INPUT", + f"path is required unless {_STATE_PATH_ENV} is set", + ) + try: + summary = self._save_world_model_state_to_path(state_path) + except OSError as exc: + return SkillResult.fail("IO_ERROR", str(exc)) + summary["dashboard_error"] = self._publish_dashboard_state(event="save_world_model_state") + return SkillResult.ok(f"Saved world model state to {state_path}", **summary) + + @skill + def load_world_model_state(self, path: str = "") -> SkillResult: + """Load transitions, interventions, and learned model weights from JSON.""" + state_path = self._resolve_state_path(path) + if state_path is None: + return SkillResult.fail( + "INVALID_INPUT", + f"path is required unless {_STATE_PATH_ENV} is set", + ) + try: + summary = self._load_world_model_state_from_path(state_path) + except (OSError, ValueError, json.JSONDecodeError) as exc: + return SkillResult.fail("IO_ERROR", str(exc)) + summary["dashboard_error"] = self._publish_dashboard_state(event="load_world_model_state") + return SkillResult.ok(f"Loaded world model state from {state_path}", **summary) + @rpc def get_recent_transitions( self, @@ -249,6 +495,291 @@ def get_recent_transitions( ] return [transition.to_dict() for transition in transitions[:limit]] + @rpc + def get_intervention_log( + self, + limit: int = 10, + target_variable: str = "", + intervention_name: str = "", + ) -> list[dict[str, Any]]: + """Return newest recorded interventions first, optionally filtered.""" + return self._intervention_log.recent( + limit=limit, + target_variable=target_variable, + intervention_name=intervention_name, + ) + + @rpc + def predict_next_state( + self, + snapshot_json: str = "", + action_json: str = "", + goal: str = "", + horizon: int = 1, + ) -> dict[str, Any]: + """Predict a symbolic next-state delta for a candidate action.""" + snapshot = _parse_json_object(snapshot_json, "snapshot_json") + if isinstance(snapshot, str): + return {"error": snapshot} + + action = _parse_json_object(action_json, "action_json") + if isinstance(action, str): + return {"error": action} + + normalized_action = _normalize_action(action) + skill_name = normalized_action["skill_name"] + domain = _infer_domain(skill_name) + horizon = max(1, min(horizon, 10)) + + recent = self.get_recent_transitions(limit=20, skill_name=skill_name, domain=domain) + failure_modes = _failure_modes(recent) + reasons = _world_model_risk_reasons(snapshot, normalized_action, failure_modes) + rule_risk = _risk_level(reasons) + model_prediction = self._outcome_model.predict(snapshot, normalized_action) + causal_attribution = self._causal_estimator.estimate(snapshot, normalized_action) + structural_causal_model = self._scm.explain( + snapshot, + normalized_action, + causal_attribution, + ) + intervention_evidence = self._intervention_log.evidence_for( + causal_attribution, + structural_causal_model, + ) + reasons.extend(_causal_risk_reasons(causal_attribution)) + rule_risk = _risk_level(reasons) + risk = _combined_risk(rule_risk, str(model_prediction["risk"])) + score = _combined_score( + rule_score=_action_score(rule_risk, reasons, failure_modes), + model_score=float(model_prediction["score"]), + has_model_samples=int(model_prediction["sample_count"]) > 0, + risk=risk, + ) + predicted_delta = _predict_symbolic_delta( + snapshot=snapshot, + action=normalized_action, + risk=risk, + horizon=horizon, + ) + confidence = _prediction_confidence(snapshot, normalized_action, failure_modes) + evidence = _prediction_evidence(snapshot, normalized_action, recent, reasons) + + result = { + "schema": WORLD_MODEL_PREDICTION_SCHEMA, + "goal": goal.strip(), + "horizon": horizon, + "action": normalized_action, + "snapshot_summary": _snapshot_summary(snapshot), + "prediction": { + "risk": risk, + "predicted_success": risk != "high", + "score": score, + "confidence": _combined_confidence(confidence, model_prediction), + "predicted_state_delta": predicted_delta, + "failure_modes": failure_modes, + "reasons": reasons, + "evidence": evidence, + "model": model_prediction, + "causal_attribution": causal_attribution, + "structural_causal_model": structural_causal_model, + "intervention_evidence": intervention_evidence, + }, + } + dashboard_error = self._publish_dashboard_state( + prediction=result, + event="predict_next_state", + ) + if dashboard_error: + result["dashboard_error"] = dashboard_error + return result + + @rpc + def score_action( + self, + snapshot_json: str = "", + action_json: str = "", + goal: str = "", + ) -> dict[str, Any]: + """Return a compact action score derived from predict_next_state.""" + prediction = self.predict_next_state( + snapshot_json=snapshot_json, + action_json=action_json, + goal=goal, + ) + if prediction.get("error"): + return prediction + model = prediction["prediction"] + return { + "goal": prediction["goal"], + "action": prediction["action"], + "score": model["score"], + "risk": model["risk"], + "confidence": model["confidence"], + "predicted_success": model["predicted_success"], + "failure_modes": model["failure_modes"], + "reasons": model["reasons"], + "model": model["model"], + "causal_attribution": model["causal_attribution"], + "structural_causal_model": model["structural_causal_model"], + "intervention_evidence": model["intervention_evidence"], + } + + @rpc + def get_provider_contract(self) -> dict[str, Any]: + """Return the explicit world-model provider contract for this module.""" + return provider_contract( + name="go2_causal_world_model", + model_type="symbolic_causal_online_transition", + capabilities=[ + "predict_next_state", + "score_action", + "record_causal_transition", + "record_intervention", + "get_model_state", + ], + output_schemas={ + "prediction": WORLD_MODEL_PREDICTION_SCHEMA, + "dashboard": WORLD_MODEL_DASHBOARD_SCHEMA, + }, + ) + + @rpc + def get_model_state(self) -> dict[str, Any]: + """Return compact metadata for the learned local transition model.""" + state = self._outcome_model.snapshot() + state["available"] = True + state["provider_contract"] = self.get_provider_contract() + state["causal_estimator"] = self._causal_estimator.snapshot() + state["intervention_log"] = self._intervention_log.snapshot() + state["structural_causal_model"] = self._scm.snapshot() + state["persistence"] = { + "schema": _STATE_SCHEMA, + "path": str(self._persistence_path) if self._persistence_path else "", + "autosave": self._persistence_path is not None, + } + return state + + def _resolve_state_path(self, path: str) -> Path | None: + path = path.strip() + if path: + return Path(path).expanduser() + return self._persistence_path + + def _autosave_if_configured(self) -> str: + if self._persistence_path is None: + return "" + try: + self._save_world_model_state_to_path(self._persistence_path) + except OSError as exc: + return str(exc) + return "" + + def _save_world_model_state_to_path(self, path: Path) -> dict[str, Any]: + path.parent.mkdir(parents=True, exist_ok=True) + payload = self._state_payload() + path.write_text(json.dumps(payload, indent=2, sort_keys=True), encoding="utf-8") + return _state_summary(payload, str(path)) + + def _load_world_model_state_from_path( + self, + path: Path, + missing_ok: bool = False, + ) -> dict[str, Any]: + if not path.exists(): + if missing_ok: + return { + "schema": _STATE_SCHEMA, + "path": str(path), + "loaded": False, + "reason": "state file does not exist", + } + raise FileNotFoundError(path) + + payload = json.loads(path.read_text(encoding="utf-8")) + if not isinstance(payload, dict): + raise ValueError("world model state must be a JSON object") + if payload.get("schema") != _STATE_SCHEMA: + raise ValueError(f"unsupported world model state schema: {payload.get('schema')}") + + self._transitions.clear() + raw_transitions = ( + payload.get("transitions") if isinstance(payload.get("transitions"), list) else [] + ) + for item in raw_transitions[-self._max_transitions :]: + if isinstance(item, dict): + self._transitions.append(_transition_from_dict(item)) + + outcome_model = ( + payload.get("outcome_model") if isinstance(payload.get("outcome_model"), dict) else {} + ) + self._outcome_model.load_dict(outcome_model) + causal_estimator = ( + payload.get("causal_estimator") + if isinstance(payload.get("causal_estimator"), dict) + else {} + ) + self._causal_estimator.load_dict(causal_estimator) + intervention_log = ( + payload.get("intervention_log") + if isinstance(payload.get("intervention_log"), dict) + else {} + ) + self._intervention_log.load_dict(intervention_log) + return _state_summary(payload, str(path)) | {"loaded": True} + + def _state_payload(self) -> dict[str, Any]: + return { + "schema": _STATE_SCHEMA, + "saved_at": round(time.time(), 3), + "transitions": [transition.to_dict() for transition in self._transitions], + "outcome_model": self._outcome_model.to_dict(), + "causal_estimator": self._causal_estimator.to_dict(), + "intervention_log": self._intervention_log.to_dict(), + "structural_causal_model": self._scm.snapshot(), + } + + def _publish_dashboard_state( + self, + prediction: dict[str, Any] | None = None, + event: str = "", + ) -> str: + if self._websocket_vis is None: + return "" + try: + self._websocket_vis.set_world_model_state( + self._dashboard_state(prediction=prediction, event=event) + ) + except Exception as exc: + return str(exc) + return "" + + def _dashboard_state( + self, + prediction: dict[str, Any] | None = None, + event: str = "", + ) -> dict[str, Any]: + dashboard_state: dict[str, Any] = { + "schema": WORLD_MODEL_DASHBOARD_SCHEMA, + "available": True, + "event": event, + "updated_at": round(time.time(), 3), + "model_state": self.get_model_state(), + "recent_transitions": self.get_recent_transitions(limit=5), + "recent_interventions": self.get_intervention_log(limit=5), + "prediction": {}, + "action_role": "none", + "action_label": "No action", + } + if prediction is not None: + dashboard_state["goal"] = prediction.get("goal") or "" + dashboard_state["horizon"] = prediction.get("horizon") + dashboard_state["action"] = prediction.get("action") or {} + dashboard_state["action_role"] = "predicted_future_action" + dashboard_state["action_label"] = "Predicted future action" + dashboard_state["snapshot_summary"] = prediction.get("snapshot_summary") or {} + dashboard_state["prediction"] = prediction.get("prediction") or {} + return _bounded_jsonable(dashboard_state) + def _latest_outcome(self, skill_name: str) -> dict[str, Any]: if self._skill_outcomes is None: return {} @@ -256,6 +787,390 @@ def _latest_outcome(self, skill_name: str) -> dict[str, Any]: return outcomes[0] if outcomes else {} +def _configured_persistence_path() -> Path | None: + raw_path = os.environ.get(_STATE_PATH_ENV, "").strip() + if not raw_path: + return None + return Path(raw_path).expanduser() + + +def _state_summary(payload: dict[str, Any], path: str) -> dict[str, Any]: + transitions = payload.get("transitions") if isinstance(payload.get("transitions"), list) else [] + outcome_model = ( + payload.get("outcome_model") if isinstance(payload.get("outcome_model"), dict) else {} + ) + intervention_log = ( + payload.get("intervention_log") if isinstance(payload.get("intervention_log"), dict) else {} + ) + interventions = ( + intervention_log.get("records") if isinstance(intervention_log.get("records"), list) else [] + ) + return { + "schema": str(payload.get("schema") or ""), + "path": path, + "transition_count": len(transitions), + "intervention_count": len(interventions), + "sample_count": int(outcome_model.get("sample_count") or 0), + "saved_at": payload.get("saved_at"), + } + + +def _transition_from_dict(value: dict[str, Any]) -> _WorldTransition: + args = value.get("args") if isinstance(value.get("args"), dict) else {} + before_state = value.get("before_state") if isinstance(value.get("before_state"), dict) else {} + after_state = value.get("after_state") if isinstance(value.get("after_state"), dict) else {} + state_delta = value.get("state_delta") if isinstance(value.get("state_delta"), dict) else {} + prediction_reasons = ( + value.get("prediction_reasons") if isinstance(value.get("prediction_reasons"), list) else [] + ) + outcome_success = value.get("outcome_success") + if not isinstance(outcome_success, bool): + outcome_success = None + return _WorldTransition( + timestamp=float(value.get("timestamp") or time.time()), + task=str(value.get("task") or ""), + domain=str(value.get("domain") or ""), + skill_name=str(value.get("skill_name") or ""), + args=args, + before_state=before_state, + before_context=str(value.get("before_context") or ""), + prediction_risk=str(value.get("prediction_risk") or "unknown"), + prediction_reasons=[str(reason) for reason in prediction_reasons], + outcome_success=outcome_success, + outcome_error_code=str(value.get("outcome_error_code") or ""), + outcome_message=str(value.get("outcome_message") or ""), + after_state=after_state, + after_context=str(value.get("after_context") or ""), + state_delta=state_delta, + inferred_cause=str(value.get("inferred_cause") or "unknown_transition"), + recovery=str(value.get("recovery") or ""), + confidence=str(value.get("confidence") or "low"), + ) + + +def _normalize_action(action: dict[str, Any]) -> dict[str, Any]: + skill_name = str( + action.get("skill_name") + or action.get("tool_name") + or action.get("action") + or action.get("name") + or "" + ).strip() + args = action.get("args") or action.get("arguments") or {} + if not isinstance(args, dict): + args = {"value": args} + return { + "skill_name": skill_name, + "domain": str(action.get("domain") or _infer_domain(skill_name)), + "args": args, + } + + +def _failure_modes(transitions: list[dict[str, Any]]) -> list[dict[str, Any]]: + counts: Counter[str] = Counter() + examples: dict[str, dict[str, Any]] = {} + for transition in transitions: + if transition.get("outcome_success") is not False: + continue + cause = str(transition.get("inferred_cause") or "unknown_failure") + if cause == "success_no_failure": + continue + counts[cause] += 1 + examples.setdefault(cause, transition) + + modes: list[dict[str, Any]] = [] + for cause, count in counts.most_common(): + example = examples[cause] + modes.append( + { + "cause": cause, + "count": count, + "recovery": str(example.get("recovery") or ""), + "latest_message": str(example.get("outcome_message") or ""), + "confidence": "high" if count >= 2 else "medium", + } + ) + return modes + + +def _world_model_risk_reasons( + snapshot: dict[str, Any], + action: dict[str, Any], + failure_modes: list[dict[str, Any]], +) -> list[str]: + reasons: list[str] = [] + skill_name = str(action.get("skill_name") or "") + name = skill_name.casefold() + args = action.get("args") if isinstance(action.get("args"), dict) else {} + + for mode in failure_modes: + cause = str(mode.get("cause") or "unknown_failure") + count = int(mode.get("count") or 0) + if count >= 2: + reasons.append(f"same skill repeatedly failed from causal cause: {cause}") + else: + reasons.append(f"same skill has a recent causal failure: {cause}") + + robot_state = ( + snapshot.get("robot_state") if isinstance(snapshot.get("robot_state"), dict) else {} + ) + memory_state = ( + snapshot.get("memory_state") if isinstance(snapshot.get("memory_state"), dict) else {} + ) + spatial = memory_state.get("spatial") if isinstance(memory_state.get("spatial"), dict) else {} + temporal = ( + memory_state.get("temporal") if isinstance(memory_state.get("temporal"), dict) else {} + ) + + if name in {"navigate_with_text", "relative_move", "follow_person"}: + if not robot_state.get("odom"): + reasons.append("robot pose or odometry may be unavailable") + + if name == "navigate_with_text": + query = str(args.get("query") or "").strip() + if not query: + reasons.append("navigation query is missing") + if spatial.get("available") is False: + reasons.append("spatial memory is unavailable for semantic navigation") + elif spatial.get("available") is True and not spatial.get("matches"): + reasons.append("spatial memory has no relevant matches") + + if name == "follow_person": + query = str(args.get("query") or "").strip() + if not query: + reasons.append("person-follow query is missing") + if spatial.get("available") is False and temporal.get("available") is False: + reasons.append("no memory source is available to help identify the target") + + if name == "look_out_for" and not args.get("description_of_things"): + reasons.append("look_out_for descriptions are missing") + + return reasons + + +def _predict_symbolic_delta( + snapshot: dict[str, Any], + action: dict[str, Any], + risk: str, + horizon: int, +) -> dict[str, Any]: + skill_name = str(action.get("skill_name") or "").casefold() + args = action.get("args") if isinstance(action.get("args"), dict) else {} + blocked = risk == "high" + + if skill_name == "navigate_with_text": + return { + "navigation.state": "blocked_before_navigation" if blocked else "following_path", + "navigation.target_query": str(args.get("query") or ""), + "navigation.horizon_steps": horizon, + "memory_state.spatial.required_match": "missing_or_untrusted" + if blocked + else "used_for_target", + } + + if skill_name == "relative_move": + return { + "robot_state.odom": "expected_motion_delta" if not blocked else "unchanged", + "robot_state.motion_command": _bounded_jsonable(args), + "safety.constraint_status": "must_pass_before_motion", + } + + if skill_name == "follow_person": + return { + "navigation.state": "blocked_before_follow" if blocked else "following_person", + "tracking.target_query": str(args.get("query") or ""), + "memory_state.temporal.expected_update": "target_track_observation", + } + + if skill_name == "look_out_for": + return { + "perception.query": _bounded_jsonable(args.get("description_of_things")), + "memory_state.temporal.expected_update": "visual_observation", + "semantic_temporal_map.expected_evidence": "perception", + } + + if skill_name.startswith("stop_"): + return { + "navigation.state": "stopping_or_idle", + "control.mode": "safe_stop_requested", + } + + return { + "world_state": "unknown_symbolic_delta", + "action": str(action.get("skill_name") or ""), + } + + +def _prediction_confidence( + snapshot: dict[str, Any], + action: dict[str, Any], + failure_modes: list[dict[str, Any]], +) -> str: + has_snapshot = bool(snapshot) + has_action = bool(action.get("skill_name")) + repeated_failures = any(int(mode.get("count") or 0) >= 2 for mode in failure_modes) + if has_snapshot and has_action and not failure_modes: + return "high" + if has_action and (has_snapshot or repeated_failures): + return "medium" + return "low" + + +def _prediction_evidence( + snapshot: dict[str, Any], + action: dict[str, Any], + recent: list[dict[str, Any]], + reasons: list[str], +) -> dict[str, Any]: + sources = snapshot.get("sources") if isinstance(snapshot.get("sources"), dict) else {} + return { + "snapshot_sources": _bounded_jsonable(sources), + "skill_name": action.get("skill_name") or "", + "recent_transition_count": len(recent), + "risk_reason_count": len(reasons), + "has_robot_state": isinstance(snapshot.get("robot_state"), dict), + "has_memory_state": isinstance(snapshot.get("memory_state"), dict), + } + + +def _action_score( + risk: str, + reasons: list[str], + failure_modes: list[dict[str, Any]], +) -> float: + score_by_risk = {"low": 0.85, "medium": 0.55, "high": 0.25} + score = score_by_risk.get(risk, 0.4) + score -= min(0.2, 0.04 * len(reasons)) + score -= min(0.2, 0.05 * sum(int(mode.get("count") or 0) for mode in failure_modes)) + return round(max(0.0, min(1.0, score)), 3) + + +def _combined_score( + rule_score: float, + model_score: float, + has_model_samples: bool, + risk: str, +) -> float: + if not has_model_samples: + return rule_score + score = 0.55 * rule_score + 0.45 * model_score + if risk == "high": + score = min(score, 0.45) + return round(max(0.0, min(1.0, score)), 3) + + +def _combined_risk(rule_risk: str, model_risk: str) -> str: + severity = {"low": 0, "medium": 1, "high": 2} + if severity.get(model_risk, 0) > severity.get(rule_risk, 0): + return model_risk + return rule_risk + + +def _combined_confidence(rule_confidence: str, model_prediction: dict[str, Any]) -> str: + model_confidence = str(model_prediction.get("confidence") or "low") + if int(model_prediction.get("sample_count") or 0) == 0: + return rule_confidence + severity = {"low": 0, "medium": 1, "high": 2} + return ( + model_confidence + if severity.get(model_confidence, 0) > severity.get(rule_confidence, 0) + else rule_confidence + ) + + +def _causal_risk_reasons(causal_attribution: dict[str, Any]) -> list[str]: + reasons: list[str] = [] + for factor in causal_attribution.get("risk_factors") or []: + feature = str(factor.get("feature") or "") + effect = factor.get("effect_on_success") + confidence = str(factor.get("confidence") or "low") + reasons.append(f"causal attribution: {feature} changes success by {effect} ({confidence})") + return reasons + + +def _risk_level(reasons: list[str]) -> str: + high_markers = ( + "failed repeatedly", + "repeatedly failed from causal cause", + "navigation query is missing", + "person-follow query is missing", + "descriptions are missing", + ) + if any(any(marker in reason for marker in high_markers) for reason in reasons): + return "high" + if reasons: + return "medium" + return "low" + + +def _snapshot_summary(snapshot: dict[str, Any]) -> dict[str, Any]: + robot_state = ( + snapshot.get("robot_state") if isinstance(snapshot.get("robot_state"), dict) else {} + ) + memory_state = ( + snapshot.get("memory_state") if isinstance(snapshot.get("memory_state"), dict) else {} + ) + spatial = memory_state.get("spatial") if isinstance(memory_state.get("spatial"), dict) else {} + temporal = ( + memory_state.get("temporal") if isinstance(memory_state.get("temporal"), dict) else {} + ) + return { + "sources": _bounded_jsonable(snapshot.get("sources") or {}), + "has_robot_state": bool(robot_state), + "navigation": _bounded_jsonable(robot_state.get("navigation") or {}), + "spatial_available": spatial.get("available"), + "spatial_match_count": len(spatial.get("matches") or []), + "temporal_available": temporal.get("available"), + "semantic_temporal_available": bool(snapshot.get("semantic_temporal_map")), + } + + +def _diff_world_state(before: dict[str, Any], after: dict[str, Any]) -> dict[str, Any]: + if not before or not after: + return {} + delta: dict[str, Any] = {} + for path in ( + ("robot_state", "navigation", "state"), + ("robot_state", "navigation", "goal_reached"), + ("robot_state", "odom"), + ("memory_state", "spatial", "available"), + ("memory_state", "temporal", "rolling_summary"), + ("semantic_temporal_map", "fused", "entry_count"), + ): + before_value = _get_path(before, path) + after_value = _get_path(after, path) + if before_value != after_value: + delta[".".join(path)] = { + "before": _bounded_jsonable(before_value), + "after": _bounded_jsonable(after_value), + } + return delta + + +def _get_path(value: dict[str, Any], path: tuple[str, ...]) -> Any: + current: Any = value + for key in path: + if not isinstance(current, dict) or key not in current: + return None + current = current[key] + return current + + +@dataclass(frozen=True) +class _JsonParseError: + message: str + + +def _parse_json_value(value: str, field_name: str) -> Any | _JsonParseError: + value = value.strip() + if not value: + return None + try: + return json.loads(value) + except json.JSONDecodeError as exc: + return _JsonParseError(f"{field_name} must be valid JSON: {exc}") + + def _parse_json_object(value: str, field_name: str) -> dict[str, Any] | str: value = value.strip() if not value: @@ -269,6 +1184,22 @@ def _parse_json_object(value: str, field_name: str) -> dict[str, Any] | str: return parsed +def _bounded_jsonable(value: Any, max_string_length: int = 500) -> Any: + if value is None or isinstance(value, bool | int | float): + return value + if isinstance(value, str): + return value[:max_string_length] + if isinstance(value, dict): + return { + str(key): _bounded_jsonable(item, max_string_length) + for key, item in value.items() + if not str(key).startswith("_") + } + if isinstance(value, list | tuple): + return [_bounded_jsonable(item, max_string_length) for item in value[:10]] + return str(value)[:max_string_length] + + def _metadata_view(value: dict[str, Any]) -> dict[str, Any]: metadata = value.get("metadata") if isinstance(metadata, dict): diff --git a/dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/intervention_log.py b/dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/intervention_log.py new file mode 100644 index 0000000000..1e091fec3c --- /dev/null +++ b/dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/intervention_log.py @@ -0,0 +1,274 @@ +#!/usr/bin/env python3 +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import annotations + +from collections import Counter, deque +from dataclasses import dataclass +import time +from typing import Any + + +@dataclass(frozen=True) +class InterventionRecord: + """One explicit do-operation and the observed result.""" + + timestamp: float + task: str + intervention_name: str + target_variable: str + before_value: Any + after_value: Any + action: dict[str, Any] + snapshot_before: dict[str, Any] + snapshot_after: dict[str, Any] + outcome_success: bool | None + outcome_message: str + causal_hypothesis: str + + def to_dict(self) -> dict[str, Any]: + return { + "timestamp": round(self.timestamp, 3), + "task": self.task, + "intervention_name": self.intervention_name, + "target_variable": self.target_variable, + "before_value": _bounded_jsonable(self.before_value), + "after_value": _bounded_jsonable(self.after_value), + "action": _bounded_jsonable(self.action), + "snapshot_before": _bounded_jsonable(self.snapshot_before), + "snapshot_after": _bounded_jsonable(self.snapshot_after), + "outcome_success": self.outcome_success, + "outcome_message": self.outcome_message, + "causal_hypothesis": self.causal_hypothesis, + } + + +class InterventionLog: + """Bounded memory of agent interventions for causal reuse.""" + + def __init__(self, max_records: int = 200) -> None: + self._records: deque[InterventionRecord] = deque(maxlen=max_records) + + def record( + self, + *, + task: str, + intervention_name: str, + target_variable: str, + before_value: Any, + after_value: Any, + action: dict[str, Any], + snapshot_before: dict[str, Any], + snapshot_after: dict[str, Any], + outcome_success: bool | None, + outcome_message: str, + causal_hypothesis: str, + ) -> dict[str, Any]: + record = InterventionRecord( + timestamp=time.time(), + task=task, + intervention_name=intervention_name, + target_variable=target_variable, + before_value=before_value, + after_value=after_value, + action=action, + snapshot_before=snapshot_before, + snapshot_after=snapshot_after, + outcome_success=outcome_success, + outcome_message=outcome_message[:500], + causal_hypothesis=causal_hypothesis[:500], + ) + self._records.append(record) + return record.to_dict() + + def recent( + self, + *, + limit: int = 10, + target_variable: str = "", + intervention_name: str = "", + ) -> list[dict[str, Any]]: + limit = max(0, min(limit, self._records.maxlen or limit)) + target_variable = target_variable.strip() + intervention_name = intervention_name.strip() + records = [ + record + for record in reversed(self._records) + if (not target_variable or record.target_variable == target_variable) + and (not intervention_name or record.intervention_name == intervention_name) + ] + return [record.to_dict() for record in records[:limit]] + + def evidence_for( + self, + causal_attribution: dict[str, Any], + structural_causal_model: dict[str, Any], + ) -> dict[str, Any]: + risk_features = [ + str(factor.get("feature") or "") + for factor in causal_attribution.get("risk_factors") or [] + ] + variables = structural_causal_model.get("variables") + if isinstance(variables, dict): + risk_features.extend(_risk_features_from_variables(variables)) + + target_variables = [_feature_variable(feature) for feature in risk_features] + target_variables = [variable for variable in target_variables if variable] + suggestions = self._suggestions(target_variables, risk_features) + return { + "type": "intervention_log", + "record_count": len(self._records), + "risk_features": sorted({feature for feature in risk_features if feature}), + "matched_count": len(suggestions), + "suggestions": suggestions[:5], + } + + def snapshot(self) -> dict[str, Any]: + successes = sum(1 for record in self._records if record.outcome_success is True) + failures = sum(1 for record in self._records if record.outcome_success is False) + target_counts = Counter(record.target_variable for record in self._records) + intervention_counts = Counter(record.intervention_name for record in self._records) + return { + "type": "intervention_log", + "record_count": len(self._records), + "positive_count": successes, + "negative_count": failures, + "target_variables": dict(target_counts), + "intervention_names": dict(intervention_counts), + } + + def to_dict(self) -> dict[str, Any]: + return { + "type": "intervention_log", + "max_records": self._records.maxlen, + "records": [record.to_dict() for record in self._records], + } + + def load_dict(self, state: dict[str, Any]) -> None: + raw_records = state.get("records") if isinstance(state.get("records"), list) else [] + max_records = self._records.maxlen or len(raw_records) + self._records.clear() + for item in raw_records[-max_records:]: + if not isinstance(item, dict): + continue + action = item.get("action") if isinstance(item.get("action"), dict) else {} + snapshot_before = ( + item.get("snapshot_before") if isinstance(item.get("snapshot_before"), dict) else {} + ) + snapshot_after = ( + item.get("snapshot_after") if isinstance(item.get("snapshot_after"), dict) else {} + ) + outcome_success = item.get("outcome_success") + if not isinstance(outcome_success, bool): + outcome_success = None + self._records.append( + InterventionRecord( + timestamp=float(item.get("timestamp") or time.time()), + task=str(item.get("task") or ""), + intervention_name=str(item.get("intervention_name") or ""), + target_variable=str(item.get("target_variable") or ""), + before_value=item.get("before_value"), + after_value=item.get("after_value"), + action=action, + snapshot_before=snapshot_before, + snapshot_after=snapshot_after, + outcome_success=outcome_success, + outcome_message=str(item.get("outcome_message") or ""), + causal_hypothesis=str(item.get("causal_hypothesis") or ""), + ) + ) + + def _suggestions( + self, + target_variables: list[str], + risk_features: list[str], + ) -> list[dict[str, Any]]: + variables = set(target_variables) + grouped: dict[tuple[str, str], list[InterventionRecord]] = {} + for record in self._records: + if record.target_variable not in variables: + continue + grouped.setdefault((record.target_variable, record.intervention_name), []).append( + record + ) + + suggestions: list[dict[str, Any]] = [] + for (target_variable, intervention_name), records in grouped.items(): + successful = [record for record in records if record.outcome_success is True] + if not successful: + continue + latest = successful[-1] + suggestions.append( + { + "intervention_name": intervention_name, + "target_variable": target_variable, + "before_value": _bounded_jsonable(latest.before_value), + "after_value": _bounded_jsonable(latest.after_value), + "success_count": len(successful), + "total_count": len(records), + "success_rate": round(len(successful) / len(records), 3), + "latest_message": latest.outcome_message, + "causal_hypothesis": latest.causal_hypothesis, + "matched_risk_features": [ + feature + for feature in risk_features + if _feature_variable(feature) == target_variable + ], + } + ) + + suggestions.sort( + key=lambda item: (float(item["success_rate"]), int(item["success_count"])), + reverse=True, + ) + return suggestions + + +def _feature_variable(feature: str) -> str: + if not feature: + return "" + return feature.split(":", 1)[0] + + +def _risk_features_from_variables(variables: dict[str, Any]) -> list[str]: + risk_features: list[str] = [] + if variables.get("spatial_has_matches") is False: + risk_features.append("spatial_has_matches:False") + if variables.get("spatial_available") is False: + risk_features.append("spatial_available:False") + if variables.get("odom_ready") is False: + risk_features.append("has_odom:False") + if variables.get("target_query_present") is False: + risk_features.append("arg_present:query:False") + return risk_features + + +def _bounded_jsonable(value: Any, max_string_length: int = 500) -> Any: + if value is None or isinstance(value, bool | int | float): + return value + if isinstance(value, str): + return value[:max_string_length] + if isinstance(value, dict): + return { + str(key): _bounded_jsonable(item, max_string_length) + for key, item in value.items() + if not str(key).startswith("_") + } + if isinstance(value, list | tuple): + return [_bounded_jsonable(item, max_string_length) for item in value[:10]] + return str(value)[:max_string_length] + + +__all__ = ["InterventionLog", "InterventionRecord"] diff --git a/dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/online_transition_model.py b/dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/online_transition_model.py new file mode 100644 index 0000000000..282cf09b30 --- /dev/null +++ b/dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/online_transition_model.py @@ -0,0 +1,198 @@ +#!/usr/bin/env python3 +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import annotations + +from collections import Counter +import math +from typing import Any + + +class OnlineTransitionOutcomeModel: + """Small online model for predicting action outcome from world-state features.""" + + model_type = "online_transition_outcome" + + def __init__(self, learning_rate: float = 0.35, l2: float = 0.001) -> None: + self._learning_rate = learning_rate + self._l2 = l2 + self._weights: dict[str, float] = {} + self._sample_count = 0 + self._positive_count = 0 + self._negative_count = 0 + self._skill_counts: Counter[str] = Counter() + + def update( + self, + snapshot: dict[str, Any], + action: dict[str, Any], + outcome_success: bool | None, + ) -> None: + if outcome_success is None: + return + features = self._features(snapshot, action) + label = 1.0 if outcome_success else 0.0 + probability = self._probability(features) + error = label - probability + + for name, value in features.items(): + current = self._weights.get(name, 0.0) + self._weights[name] = current + self._learning_rate * ( + error * value - self._l2 * current + ) + + self._sample_count += 1 + if outcome_success: + self._positive_count += 1 + else: + self._negative_count += 1 + skill_name = str(action.get("skill_name") or "") + if skill_name: + self._skill_counts[skill_name] += 1 + + def predict(self, snapshot: dict[str, Any], action: dict[str, Any]) -> dict[str, Any]: + features = self._features(snapshot, action) + probability = self._probability(features) + risk = self._risk(probability) + return { + "type": self.model_type, + "sample_count": self._sample_count, + "positive_count": self._positive_count, + "negative_count": self._negative_count, + "success_probability": round(probability, 3), + "score": round(probability, 3), + "risk": risk, + "confidence": self._confidence(action), + "top_features": self._top_features(features), + } + + def snapshot(self) -> dict[str, Any]: + return { + "type": self.model_type, + "sample_count": self._sample_count, + "positive_count": self._positive_count, + "negative_count": self._negative_count, + "skill_counts": dict(self._skill_counts), + "weight_count": len(self._weights), + } + + def to_dict(self) -> dict[str, Any]: + return { + "type": self.model_type, + "learning_rate": self._learning_rate, + "l2": self._l2, + "weights": dict(self._weights), + "sample_count": self._sample_count, + "positive_count": self._positive_count, + "negative_count": self._negative_count, + "skill_counts": dict(self._skill_counts), + } + + def load_dict(self, state: dict[str, Any]) -> None: + weights = state.get("weights") if isinstance(state.get("weights"), dict) else {} + skill_counts = ( + state.get("skill_counts") if isinstance(state.get("skill_counts"), dict) else {} + ) + self._learning_rate = float(state.get("learning_rate") or self._learning_rate) + self._l2 = float(state.get("l2") or self._l2) + self._weights = {str(key): float(value) for key, value in weights.items()} + self._sample_count = max(0, int(state.get("sample_count") or 0)) + self._positive_count = max(0, int(state.get("positive_count") or 0)) + self._negative_count = max(0, int(state.get("negative_count") or 0)) + self._skill_counts = Counter( + {str(key): max(0, int(value)) for key, value in skill_counts.items()} + ) + + def _probability(self, features: dict[str, float]) -> float: + logit = sum(self._weights.get(name, 0.0) * value for name, value in features.items()) + return 1.0 / (1.0 + math.exp(-max(-20.0, min(20.0, logit)))) + + def _confidence(self, action: dict[str, Any]) -> str: + skill_count = self._skill_counts.get(str(action.get("skill_name") or ""), 0) + if self._sample_count >= 10 and skill_count >= 4: + return "high" + if self._sample_count >= 2 and skill_count >= 1: + return "medium" + return "low" + + def _top_features(self, features: dict[str, float]) -> list[dict[str, Any]]: + contributions = [ + { + "feature": name, + "value": round(value, 3), + "weight": round(self._weights.get(name, 0.0), 3), + "contribution": round(self._weights.get(name, 0.0) * value, 3), + } + for name, value in features.items() + if self._weights.get(name, 0.0) + ] + contributions.sort(key=lambda item: abs(float(item["contribution"])), reverse=True) + return contributions[:8] + + def _features(self, snapshot: dict[str, Any], action: dict[str, Any]) -> dict[str, float]: + features: dict[str, float] = {"bias": 1.0} + skill_name = str(action.get("skill_name") or "") + domain = str(action.get("domain") or "") + args = action.get("args") if isinstance(action.get("args"), dict) else {} + if skill_name: + features[f"skill:{skill_name}"] = 1.0 + if domain: + features[f"domain:{domain}"] = 1.0 + + robot_state = ( + snapshot.get("robot_state") if isinstance(snapshot.get("robot_state"), dict) else {} + ) + memory_state = ( + snapshot.get("memory_state") if isinstance(snapshot.get("memory_state"), dict) else {} + ) + spatial = ( + memory_state.get("spatial") if isinstance(memory_state.get("spatial"), dict) else {} + ) + temporal = ( + memory_state.get("temporal") if isinstance(memory_state.get("temporal"), dict) else {} + ) + navigation = ( + robot_state.get("navigation") if isinstance(robot_state.get("navigation"), dict) else {} + ) + + features["has_odom"] = 1.0 if robot_state.get("odom") else 0.0 + nav_state = str(navigation.get("state") or "") + if nav_state: + features[f"navigation_state:{nav_state}"] = 1.0 + if "available" in spatial: + features[f"spatial_available:{bool(spatial.get('available'))}"] = 1.0 + spatial_matches = spatial.get("matches") if isinstance(spatial.get("matches"), list) else [] + features["spatial_match_count"] = min(5.0, math.log1p(len(spatial_matches))) + if "available" in temporal: + features[f"temporal_available:{bool(temporal.get('available'))}"] = 1.0 + + for key, value in args.items(): + features[f"arg_present:{key}"] = 1.0 if str(value).strip() else 0.0 + if key in {"query", "target", "description"} and str(value).strip(): + normalized = str(value).strip().casefold()[:64] + features[f"arg_value:{key}:{normalized}"] = 1.0 + + return features + + @staticmethod + def _risk(probability: float) -> str: + if probability < 0.35: + return "high" + if probability < 0.65: + return "medium" + return "low" + + +__all__ = ["OnlineTransitionOutcomeModel"] diff --git a/dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/skill_outcome_predictor.py b/dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/skill_outcome_predictor.py index 8bc6d30b61..3c8b543c28 100644 --- a/dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/skill_outcome_predictor.py +++ b/dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/skill_outcome_predictor.py @@ -84,8 +84,13 @@ def predict_skill_outcome( # 5. causal_recent: same-skill transitions from CausalWorldModel. recent = self._recent_outcomes(skill_name) causal_recent = self._recent_causal_transitions(skill_name) - reasons = _risk_reasons(skill_name, parsed_args, context, recent, causal_recent) + world_prediction = self._world_model_prediction(skill_name, parsed_args, context) + reasons = _risk_reasons( + skill_name, parsed_args, context, recent, causal_recent, world_prediction + ) risk = _risk_level(reasons) + if world_prediction.get("risk") == "high": + risk = "high" predicted_success = risk != "high" suggestions = _recovery_suggestions(skill_name, reasons) @@ -98,6 +103,7 @@ def predict_skill_outcome( "recovery_suggestions": suggestions, "recent_outcomes": recent, "recent_causal_transitions": causal_recent, + "world_model_prediction": world_prediction, "outcome_store_available": self._skill_outcomes is not None, "causal_world_model_available": self._causal_world_model is not None, } @@ -119,6 +125,28 @@ def _recent_causal_transitions(self, skill_name: str) -> list[dict[str, Any]]: return [] return self._causal_world_model.get_recent_transitions(limit=5, skill_name=skill_name) + def _world_model_prediction( + self, skill_name: str, args: dict[str, Any], context: str + ) -> dict[str, Any]: + """Ask the world model for an action score when that RPC is available.""" + if self._causal_world_model is None: + return {"available": False} + score_action = getattr(self._causal_world_model, "score_action", None) + if score_action is None: + return {"available": False} + snapshot_json = _extract_snapshot_json(context) + action_json = json.dumps({"skill_name": skill_name, "args": args}) + try: + prediction = score_action( + snapshot_json=snapshot_json, + action_json=action_json, + goal="", + ) + except Exception as exc: + return {"available": True, "error": str(exc)} + prediction["available"] = True + return prediction + def _parse_args_json(args_json: str) -> dict[str, Any] | str: """Parse the planned skill arguments. @@ -144,6 +172,7 @@ def _risk_reasons( context: str, recent: list[dict[str, Any]], causal_recent: list[dict[str, Any]], + world_prediction: dict[str, Any] | None = None, ) -> list[str]: """Collect human-readable risk reasons. @@ -179,6 +208,15 @@ def _risk_reasons( cause = causal_failures[0].get("inferred_cause") or "unknown_failure" reasons.append(f"same skill has a recent causal failure: {cause}") + if world_prediction and world_prediction.get("available"): + risk = str(world_prediction.get("risk") or "") + if risk and risk != "low": + reasons.append(f"world model predicted {risk} risk") + for mode in world_prediction.get("failure_modes") or []: + cause = str(mode.get("cause") or "") + if cause: + reasons.append(f"world model failure mode: {cause}") + if name in {"navigate_with_text", "relative_move", "follow_person"}: # ContextProvider formats missing odom as "Robot pose: unavailable". # Some structured summaries may instead include an odom=false-style @@ -240,6 +278,7 @@ def _risk_level(reasons: list[str]) -> str: "navigation query is missing", "person-follow query is missing", "descriptions are missing", + "world model predicted high risk", ) if any(any(marker in reason for marker in high_markers) for reason in reasons): return "high" @@ -248,6 +287,29 @@ def _risk_level(reasons: list[str]) -> str: return "low" +def _extract_snapshot_json(context: str) -> str: + """Best-effort extraction for callers that pass ContextProvider text/JSON.""" + context = context.strip() + if not context: + return "{}" + try: + parsed = json.loads(context) + except json.JSONDecodeError: + return "{}" + if not isinstance(parsed, dict): + return "{}" + metadata = parsed.get("metadata") if isinstance(parsed.get("metadata"), dict) else parsed + world_state = ( + metadata.get("world_state") if isinstance(metadata.get("world_state"), dict) else {} + ) + snapshot = world_state.get("snapshot") if isinstance(world_state.get("snapshot"), dict) else {} + if snapshot: + return json.dumps(snapshot) + if "robot_state" in metadata or "memory_state" in metadata: + return json.dumps(metadata) + return "{}" + + def _repeated_causal_cause(transitions: list[dict[str, Any]]) -> str: counts: dict[str, int] = {} for transition in transitions: diff --git a/dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/structural_causal_model.py b/dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/structural_causal_model.py new file mode 100644 index 0000000000..dcd5bff2fc --- /dev/null +++ b/dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/structural_causal_model.py @@ -0,0 +1,225 @@ +#!/usr/bin/env python3 +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import annotations + +from typing import Any + + +class StructuralCausalModel: + """Hand-authored symbolic SCM for Go2 Layer 3 action decisions.""" + + model_type = "structural_causal_model" + + def __init__(self) -> None: + self._edges = [ + ["target_query_present", "target_resolvability"], + ["spatial_available", "spatial_has_matches"], + ["spatial_has_matches", "target_resolvability"], + ["visual_detection", "target_resolvability"], + ["odom_ready", "motion_safety"], + ["obstacle_clearance", "motion_safety"], + ["target_resolvability", "navigation_success"], + ["motion_safety", "navigation_success"], + ] + + def explain( + self, + snapshot: dict[str, Any], + action: dict[str, Any], + causal_attribution: dict[str, Any] | None = None, + ) -> dict[str, Any]: + variables = self.evaluate(snapshot, action) + risk_features = _risk_features(causal_attribution or {}) + return { + "type": self.model_type, + "assumption": ( + "symbolic SCM; graph structure is hand-authored and updated from " + "observed/intervened outcomes" + ), + "variables": variables, + "edges": self.edges(), + "counterfactuals": self.counterfactuals(variables, action, risk_features), + } + + def evaluate(self, snapshot: dict[str, Any], action: dict[str, Any]) -> dict[str, Any]: + skill_name = str(action.get("skill_name") or "") + args = action.get("args") if isinstance(action.get("args"), dict) else {} + robot_state = ( + snapshot.get("robot_state") if isinstance(snapshot.get("robot_state"), dict) else {} + ) + memory_state = ( + snapshot.get("memory_state") if isinstance(snapshot.get("memory_state"), dict) else {} + ) + spatial = ( + memory_state.get("spatial") if isinstance(memory_state.get("spatial"), dict) else {} + ) + semantic_temporal_map = ( + snapshot.get("semantic_temporal_map") + if isinstance(snapshot.get("semantic_temporal_map"), dict) + else {} + ) + safety = robot_state.get("safety") if isinstance(robot_state.get("safety"), dict) else {} + + target_query_present = bool(str(args.get("query") or args.get("target") or "").strip()) + spatial_available = bool(spatial.get("available")) + matches = spatial.get("matches") if isinstance(spatial.get("matches"), list) else [] + spatial_has_matches = bool(matches) + visual_detection = _has_visual_detection(semantic_temporal_map) + odom_ready = bool(robot_state.get("odom")) + obstacle_clearance = safety.get("obstacle_clearance") is not False + target_resolvability = bool( + target_query_present and (spatial_has_matches or visual_detection) + ) + motion_safety = bool(odom_ready and obstacle_clearance) + + is_navigation = skill_name == "navigate_with_text" + navigation_success = ( + bool(target_resolvability and motion_safety) if is_navigation else motion_safety + ) + return { + "skill_name": skill_name, + "target_query_present": target_query_present, + "spatial_available": spatial_available, + "spatial_has_matches": spatial_has_matches, + "visual_detection": visual_detection, + "target_resolvability": target_resolvability, + "odom_ready": odom_ready, + "obstacle_clearance": obstacle_clearance, + "motion_safety": motion_safety, + "navigation_success": navigation_success, + } + + def counterfactuals( + self, + variables: dict[str, Any], + action: dict[str, Any], + risk_features: list[str], + ) -> list[dict[str, Any]]: + counterfactuals: list[dict[str, Any]] = [] + skill_name = str(action.get("skill_name") or "") + + if skill_name == "navigate_with_text" and ( + variables.get("spatial_has_matches") is False + or "spatial_has_matches:False" in risk_features + ): + counterfactuals.append( + { + "intervention": "do(spatial_has_matches=True)", + "target_variable": "spatial_has_matches", + "current_value": variables.get("spatial_has_matches"), + "counterfactual_value": True, + "expected_changes": { + "spatial_has_matches": True, + "target_resolvability": bool(variables.get("target_query_present")), + "navigation_success": bool( + variables.get("target_query_present") and variables.get("motion_safety") + ), + }, + "recommended_action": ( + "Use look_out_for or tag the target before semantic navigation." + ), + } + ) + + if variables.get("visual_detection") is False and skill_name == "navigate_with_text": + counterfactuals.append( + { + "intervention": "do(visual_detection=True)", + "target_variable": "visual_detection", + "current_value": False, + "counterfactual_value": True, + "expected_changes": { + "visual_detection": True, + "target_resolvability": bool(variables.get("target_query_present")), + }, + "recommended_action": "Run a perception query for the target.", + } + ) + + if variables.get("odom_ready") is False or "has_odom:False" in risk_features: + counterfactuals.append( + { + "intervention": "do(odom_ready=True)", + "target_variable": "odom_ready", + "current_value": variables.get("odom_ready"), + "counterfactual_value": True, + "expected_changes": { + "motion_safety": bool(variables.get("obstacle_clearance")), + "navigation_success": bool( + variables.get("target_resolvability") + and variables.get("obstacle_clearance") + ), + }, + "recommended_action": "Wait for odometry before issuing motion.", + } + ) + + if variables.get("target_query_present") is False: + counterfactuals.append( + { + "intervention": "do(target_query_present=True)", + "target_variable": "target_query_present", + "current_value": False, + "counterfactual_value": True, + "expected_changes": { + "target_query_present": True, + "target_resolvability": bool( + variables.get("spatial_has_matches") + or variables.get("visual_detection") + ), + }, + "recommended_action": "Fill the missing navigation target argument.", + } + ) + + return counterfactuals + + def snapshot(self) -> dict[str, Any]: + return { + "type": self.model_type, + "edges": self.edges(), + "node_count": len({node for edge in self._edges for node in edge}), + "edge_count": len(self._edges), + } + + def edges(self) -> list[list[str]]: + return [list(edge) for edge in self._edges] + + +def _has_visual_detection(semantic_temporal_map: dict[str, Any]) -> bool: + fused = ( + semantic_temporal_map.get("fused") + if isinstance(semantic_temporal_map.get("fused"), dict) + else {} + ) + if fused.get("entry_count") or fused.get("spatial_match_count"): + return True + for key in ("detections", "observations", "entries"): + value = semantic_temporal_map.get(key) + if isinstance(value, list) and value: + return True + return False + + +def _risk_features(causal_attribution: dict[str, Any]) -> list[str]: + return [ + str(factor.get("feature") or "") + for factor in causal_attribution.get("risk_factors") or [] + if factor.get("feature") + ] + + +__all__ = ["StructuralCausalModel"] diff --git a/dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/test_causal_world_model.py b/dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/test_causal_world_model.py index 8154e97f7e..4955da64ca 100644 --- a/dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/test_causal_world_model.py +++ b/dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/test_causal_world_model.py @@ -21,6 +21,12 @@ from dimos.robot.unitree.go2.blueprints.layers.layer_3_agent_brain.skill_outcome_store import ( _Go2SkillOutcomeStore, ) +from dimos.robot.unitree.go2.blueprints.layers.layer_3_agent_brain.world_model_contract import ( + WORLD_MODEL_DASHBOARD_SCHEMA, + WORLD_MODEL_PREDICTION_SCHEMA, + WORLD_MODEL_PROVIDER_SCHEMA, + validate_world_model_prediction, +) def _stop_modules(*modules: object) -> None: @@ -30,6 +36,15 @@ def _stop_modules(*modules: object) -> None: stop() +class StubWebsocketVis: + def __init__(self) -> None: + self.states: list[dict] = [] + + def set_world_model_state(self, state: dict) -> dict: + self.states.append(state) + return {"available": True} + + def test_record_causal_transition_infers_semantic_map_failure() -> None: model = _Go2CausalWorldModel() try: @@ -115,6 +130,29 @@ def test_record_causal_transition_rejects_invalid_json() -> None: _stop_modules(model) +def test_record_causal_transition_preserves_legacy_positional_arguments() -> None: + model = _Go2CausalWorldModel() + try: + result = model.record_causal_transition( + "go to the kitchen", + "navigate_with_text", + '{"query": "kitchen"}', + "Spatial memory: available, no relevant matches", + "Navigation returned no matching location", + '{"risk": "medium", "failure_reasons": []}', + '{"success": false, "message": "No matching location found"}', + "navigation", + ) + + assert result.success is True + transition = model.get_recent_transitions(limit=1)[0] + assert transition["before_context"] == "Spatial memory: available, no relevant matches" + assert transition["after_context"] == "Navigation returned no matching location" + assert transition["before_state"] == {} + finally: + _stop_modules(model) + + def test_predictor_uses_repeated_causal_failures() -> None: model = _Go2CausalWorldModel() predictor = _Go2SkillOutcomePredictor() @@ -141,3 +179,367 @@ def test_predictor_uses_repeated_causal_failures() -> None: assert any("causal cause" in reason for reason in result.metadata["failure_reasons"]) finally: _stop_modules(predictor, model) + + +def test_predict_next_state_uses_world_snapshot_and_recent_failures() -> None: + model = _Go2CausalWorldModel() + try: + for target in ("kitchen", "office"): + model.record_causal_transition( + task=f"go to the {target}", + skill_name="navigate_with_text", + args_json=f'{{"query": "{target}"}}', + before_state_json=( + '{"memory_state": {"spatial": {"available": true, "matches": []}}}' + ), + after_state_json=( + '{"memory_state": {"spatial": {"available": true, "matches": []}}}' + ), + outcome_json='{"success": false, "message": "No matching location found"}', + ) + + prediction = model.predict_next_state( + snapshot_json=( + '{"robot_state": {"navigation": {"state": "idle"}}, ' + '"memory_state": {"spatial": {"available": true, "matches": []}}}' + ), + action_json='{"skill_name": "navigate_with_text", "args": {"query": "kitchen"}}', + goal="go to the kitchen", + ) + + assert prediction["action"]["skill_name"] == "navigate_with_text" + assert prediction["schema"] == WORLD_MODEL_PREDICTION_SCHEMA + assert validate_world_model_prediction(prediction) == [] + assert prediction["prediction"]["risk"] == "high" + assert prediction["prediction"]["confidence"] == "medium" + assert "navigation.state" in prediction["prediction"]["predicted_state_delta"] + assert any( + failure["cause"] == "semantic_map_missing_target" + for failure in prediction["prediction"]["failure_modes"] + ) + assert prediction["prediction"]["score"] < 0.5 + finally: + _stop_modules(model) + + +def test_world_model_provider_contract_is_explicit() -> None: + model = _Go2CausalWorldModel() + try: + contract = model.get_provider_contract() + + assert contract["schema"] == WORLD_MODEL_PROVIDER_SCHEMA + assert contract["name"] == "go2_causal_world_model" + assert contract["model_type"] == "symbolic_causal_online_transition" + assert "predict_next_state" in contract["capabilities"] + assert contract["output_schemas"]["prediction"] == WORLD_MODEL_PREDICTION_SCHEMA + assert model.get_model_state()["provider_contract"] == contract + finally: + _stop_modules(model) + + +def test_predict_next_state_uses_online_transition_model() -> None: + model = _Go2CausalWorldModel() + snapshot_json = ( + '{"robot_state": {"odom": {"position": {"x": 1.0, "y": 2.0}}, ' + '"navigation": {"state": "idle"}}, ' + '"memory_state": {"spatial": {"available": true, "matches": [{"id": "kitchen"}]}, ' + '"temporal": {"available": true}}}' + ) + try: + for _ in range(2): + model.record_causal_transition( + task="go to the kitchen", + skill_name="navigate_with_text", + args_json='{"query": "kitchen"}', + before_state_json=snapshot_json, + after_state_json=( + '{"robot_state": {"navigation": {"state": "goal_reached"}}, ' + '"memory_state": {"spatial": {"available": true, "matches": [{"id": "kitchen"}]}}}' + ), + outcome_json='{"success": true, "message": "goal reached"}', + ) + + prediction = model.predict_next_state( + snapshot_json=snapshot_json, + action_json='{"skill_name": "navigate_with_text", "args": {"query": "kitchen"}}', + goal="go to the kitchen", + ) + + learned_model = prediction["prediction"]["model"] + assert learned_model["type"] == "online_transition_outcome" + assert learned_model["sample_count"] == 2 + assert learned_model["success_probability"] > 0.5 + assert prediction["prediction"]["score"] > 0.5 + finally: + _stop_modules(model) + + +def test_predict_next_state_reports_causal_attribution_for_missing_spatial_evidence() -> None: + model = _Go2CausalWorldModel() + missing_snapshot_json = ( + '{"robot_state": {"odom": {"position": {"x": 1.0, "y": 2.0}}, ' + '"navigation": {"state": "idle"}}, ' + '"memory_state": {"spatial": {"available": true, "matches": []}, ' + '"temporal": {"available": true}}}' + ) + matched_snapshot_json = ( + '{"robot_state": {"odom": {"position": {"x": 1.0, "y": 2.0}}, ' + '"navigation": {"state": "idle"}}, ' + '"memory_state": {"spatial": {"available": true, "matches": [{"id": "kitchen"}]}, ' + '"temporal": {"available": true}}}' + ) + try: + for _ in range(3): + model.record_causal_transition( + task="go to the kitchen", + skill_name="navigate_with_text", + args_json='{"query": "kitchen"}', + before_state_json=missing_snapshot_json, + after_state_json=missing_snapshot_json, + outcome_json='{"success": false, "message": "No matching location found"}', + ) + model.record_causal_transition( + task="go to the kitchen", + skill_name="navigate_with_text", + args_json='{"query": "kitchen"}', + before_state_json=matched_snapshot_json, + after_state_json=( + '{"robot_state": {"navigation": {"state": "goal_reached"}}, ' + '"memory_state": {"spatial": {"available": true, ' + '"matches": [{"id": "kitchen"}]}}}' + ), + outcome_json='{"success": true, "message": "goal reached"}', + ) + + prediction = model.predict_next_state( + snapshot_json=missing_snapshot_json, + action_json='{"skill_name": "navigate_with_text", "args": {"query": "kitchen"}}', + goal="go to the kitchen", + ) + + attribution = prediction["prediction"]["causal_attribution"] + assert attribution["method"] == "observational_feature_effect" + assert attribution["risk_factors"][0]["feature"] == "spatial_has_matches:False" + assert attribution["risk_factors"][0]["effect_on_success"] < 0 + assert any("perception" in suggestion for suggestion in attribution["interventions"]) + finally: + _stop_modules(model) + + +def test_record_intervention_log_is_returned_and_informs_prediction() -> None: + model = _Go2CausalWorldModel() + missing_snapshot_json = ( + '{"robot_state": {"odom": {"position": {"x": 1.0, "y": 2.0}}, ' + '"navigation": {"state": "idle"}}, ' + '"memory_state": {"spatial": {"available": true, "matches": []}, ' + '"temporal": {"available": true}}}' + ) + matched_snapshot_json = ( + '{"robot_state": {"odom": {"position": {"x": 1.0, "y": 2.0}}, ' + '"navigation": {"state": "idle"}}, ' + '"memory_state": {"spatial": {"available": true, "matches": [{"id": "kitchen"}]}, ' + '"temporal": {"available": true}}}' + ) + try: + for _ in range(2): + model.record_causal_transition( + task="go to the kitchen", + skill_name="navigate_with_text", + args_json='{"query": "kitchen"}', + before_state_json=missing_snapshot_json, + after_state_json=missing_snapshot_json, + outcome_json='{"success": false, "message": "No matching location found"}', + ) + model.record_causal_transition( + task="go to the kitchen", + skill_name="navigate_with_text", + args_json='{"query": "kitchen"}', + before_state_json=matched_snapshot_json, + after_state_json=matched_snapshot_json, + outcome_json='{"success": true, "message": "goal reached"}', + ) + + result = model.record_intervention( + task="go to the kitchen", + intervention_name="look_out_for_then_tag_target", + target_variable="spatial_has_matches", + before_value_json="false", + after_value_json="true", + action_json='{"skill_name": "look_out_for", "args": {"description_of_things": ["kitchen"]}}', + snapshot_before_json=missing_snapshot_json, + snapshot_after_json=matched_snapshot_json, + outcome_json='{"success": true, "message": "target tagged"}', + causal_hypothesis="spatial_has_matches:False blocks semantic navigation", + ) + + assert result.success is True + interventions = model.get_intervention_log(limit=1) + assert interventions[0]["intervention_name"] == "look_out_for_then_tag_target" + assert interventions[0]["target_variable"] == "spatial_has_matches" + + prediction = model.predict_next_state( + snapshot_json=missing_snapshot_json, + action_json='{"skill_name": "navigate_with_text", "args": {"query": "kitchen"}}', + goal="go to the kitchen", + ) + + evidence = prediction["prediction"]["intervention_evidence"] + assert evidence["suggestions"][0]["intervention_name"] == "look_out_for_then_tag_target" + assert evidence["suggestions"][0]["target_variable"] == "spatial_has_matches" + finally: + _stop_modules(model) + + +def test_predict_next_state_includes_scm_counterfactuals() -> None: + model = _Go2CausalWorldModel() + snapshot_json = ( + '{"robot_state": {"odom": {"position": {"x": 1.0, "y": 2.0}}, ' + '"navigation": {"state": "idle"}}, ' + '"memory_state": {"spatial": {"available": true, "matches": []}, ' + '"temporal": {"available": true}}}' + ) + try: + prediction = model.predict_next_state( + snapshot_json=snapshot_json, + action_json='{"skill_name": "navigate_with_text", "args": {"query": "kitchen"}}', + goal="go to the kitchen", + ) + + scm = prediction["prediction"]["structural_causal_model"] + assert scm["variables"]["spatial_has_matches"] is False + assert scm["variables"]["target_resolvability"] is False + assert ["spatial_has_matches", "target_resolvability"] in scm["edges"] + assert any( + counterfactual["intervention"] == "do(spatial_has_matches=True)" + and counterfactual["expected_changes"]["target_resolvability"] is True + for counterfactual in scm["counterfactuals"] + ) + finally: + _stop_modules(model) + + +def test_world_model_state_persists_to_json_file(tmp_path) -> None: + state_path = tmp_path / "world_model_state.json" + missing_snapshot_json = ( + '{"robot_state": {"odom": {"position": {"x": 1.0, "y": 2.0}}, ' + '"navigation": {"state": "idle"}}, ' + '"memory_state": {"spatial": {"available": true, "matches": []}, ' + '"temporal": {"available": true}}}' + ) + matched_snapshot_json = ( + '{"robot_state": {"odom": {"position": {"x": 1.0, "y": 2.0}}, ' + '"navigation": {"state": "idle"}}, ' + '"memory_state": {"spatial": {"available": true, "matches": [{"id": "kitchen"}]}, ' + '"temporal": {"available": true}}}' + ) + model = _Go2CausalWorldModel() + restored = _Go2CausalWorldModel() + try: + model.record_causal_transition( + task="go to the kitchen", + skill_name="navigate_with_text", + args_json='{"query": "kitchen"}', + before_state_json=missing_snapshot_json, + after_state_json=missing_snapshot_json, + outcome_json='{"success": false, "message": "No matching location found"}', + ) + model.record_causal_transition( + task="go to the kitchen", + skill_name="navigate_with_text", + args_json='{"query": "kitchen"}', + before_state_json=matched_snapshot_json, + after_state_json=matched_snapshot_json, + outcome_json='{"success": true, "message": "goal reached"}', + ) + model.record_intervention( + task="go to the kitchen", + intervention_name="look_out_for_then_tag_target", + target_variable="spatial_has_matches", + before_value_json="false", + after_value_json="true", + action_json='{"skill_name": "look_out_for", "args": {"description_of_things": ["kitchen"]}}', + snapshot_before_json=missing_snapshot_json, + snapshot_after_json=matched_snapshot_json, + outcome_json='{"success": true, "message": "target tagged"}', + causal_hypothesis="spatial_has_matches:False blocks semantic navigation", + ) + + saved = model.save_world_model_state(str(state_path)) + assert saved.success is True + assert state_path.exists() + + loaded = restored.load_world_model_state(str(state_path)) + assert loaded.success is True + assert restored.get_model_state()["sample_count"] == 2 + assert restored.get_recent_transitions(limit=10)[0]["skill_name"] == "navigate_with_text" + assert restored.get_intervention_log(limit=1)[0]["intervention_name"] == ( + "look_out_for_then_tag_target" + ) + + prediction = restored.predict_next_state( + snapshot_json=missing_snapshot_json, + action_json='{"skill_name": "navigate_with_text", "args": {"query": "kitchen"}}', + goal="go to the kitchen", + ) + + assert prediction["prediction"]["model"]["sample_count"] == 2 + assert ( + prediction["prediction"]["intervention_evidence"]["suggestions"][0]["intervention_name"] + == "look_out_for_then_tag_target" + ) + finally: + _stop_modules(model, restored) + + +def test_predict_next_state_publishes_dashboard_world_model_state() -> None: + model = _Go2CausalWorldModel() + vis = StubWebsocketVis() + model._websocket_vis = vis # type: ignore[assignment] + snapshot_json = ( + '{"robot_state": {"odom": {"position": {"x": 1.0, "y": 2.0}}, ' + '"navigation": {"state": "idle"}}, ' + '"memory_state": {"spatial": {"available": true, "matches": []}, ' + '"temporal": {"available": true}}}' + ) + try: + prediction = model.predict_next_state( + snapshot_json=snapshot_json, + action_json='{"skill_name": "navigate_with_text", "args": {"query": "kitchen"}}', + goal="go to the kitchen", + ) + + assert vis.states + dashboard_state = vis.states[-1] + assert dashboard_state["schema"] == WORLD_MODEL_DASHBOARD_SCHEMA + assert dashboard_state["prediction"]["risk"] == prediction["prediction"]["risk"] + assert ( + dashboard_state["prediction"]["predicted_state_delta"] + == prediction["prediction"]["predicted_state_delta"] + ) + assert ( + dashboard_state["prediction"]["structural_causal_model"] + == prediction["prediction"]["structural_causal_model"] + ) + assert dashboard_state["action_role"] == "predicted_future_action" + assert dashboard_state["action_label"] == "Predicted future action" + assert dashboard_state["model_state"]["available"] is True + finally: + _stop_modules(model) + + +def test_start_publishes_initial_dashboard_world_model_state() -> None: + model = _Go2CausalWorldModel() + vis = StubWebsocketVis() + model._websocket_vis = vis # type: ignore[assignment] + try: + model.start() + + assert vis.states + state = vis.states[-1] + assert state["schema"] == WORLD_MODEL_DASHBOARD_SCHEMA + assert state["available"] is True + assert state["event"] == "world_model_start" + assert state["model_state"]["available"] is True + assert state["prediction"] == {} + finally: + _stop_modules(model) diff --git a/dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/test_skill_outcomes.py b/dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/test_skill_outcomes.py index a5df9cde8a..996f25a4d5 100644 --- a/dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/test_skill_outcomes.py +++ b/dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/test_skill_outcomes.py @@ -71,7 +71,9 @@ def test_predictor_uses_recent_failures() -> None: assert result.success is True assert result.metadata["risk"] == "high" assert result.metadata["predicted_success"] is False - assert "same skill failed repeatedly in recent outcomes" in result.metadata["failure_reasons"] + assert ( + "same skill failed repeatedly in recent outcomes" in result.metadata["failure_reasons"] + ) finally: _stop_modules(predictor, store) diff --git a/dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/test_world_model_contract.py b/dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/test_world_model_contract.py new file mode 100644 index 0000000000..8faee19a23 --- /dev/null +++ b/dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/test_world_model_contract.py @@ -0,0 +1,73 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import annotations + +from dimos.robot.unitree.go2.blueprints.layers.layer_3_agent_brain.world_model_contract import ( + WORLD_MODEL_PREDICTION_SCHEMA, + WORLD_MODEL_PROVIDER_SCHEMA, + provider_contract, + validate_world_model_prediction, +) + + +def test_provider_contract_is_versioned_and_dimos_native() -> None: + contract = provider_contract( + name="go2_causal_world_model", + model_type="symbolic_causal_online_transition", + capabilities=["predict_next_state", "record_intervention"], + output_schemas={"prediction": WORLD_MODEL_PREDICTION_SCHEMA}, + ) + + assert contract["schema"] == WORLD_MODEL_PROVIDER_SCHEMA + assert contract["name"] == "go2_causal_world_model" + assert contract["model_type"] == "symbolic_causal_online_transition" + assert contract["output_schemas"]["prediction"] == WORLD_MODEL_PREDICTION_SCHEMA + assert "predict_next_state" in contract["capabilities"] + assert contract["provider"] == "dimos" + + +def test_validate_world_model_prediction_accepts_canonical_payload() -> None: + prediction = { + "schema": WORLD_MODEL_PREDICTION_SCHEMA, + "goal": "go to the kitchen", + "horizon": 1, + "action": {"skill_name": "navigate_with_text", "domain": "navigation", "args": {}}, + "snapshot_summary": {"has_robot_state": True}, + "prediction": { + "risk": "medium", + "predicted_success": True, + "score": 0.5, + "confidence": "medium", + "predicted_state_delta": {}, + "failure_modes": [], + "reasons": [], + "evidence": {}, + }, + } + + assert validate_world_model_prediction(prediction) == [] + + +def test_validate_world_model_prediction_rejects_unversioned_payload() -> None: + errors = validate_world_model_prediction( + { + "goal": "go to the kitchen", + "horizon": 1, + "action": {"skill_name": "navigate_with_text"}, + "prediction": {"risk": "medium"}, + } + ) + + assert "schema must be dimos.world_model_prediction.v1" in errors diff --git a/dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/world_model_contract.py b/dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/world_model_contract.py new file mode 100644 index 0000000000..8ba8d81db0 --- /dev/null +++ b/dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/world_model_contract.py @@ -0,0 +1,98 @@ +#!/usr/bin/env python3 +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import annotations + +from typing import Any + +WORLD_MODEL_PROVIDER_SCHEMA = "dimos.world_model_provider.v1" +WORLD_MODEL_PREDICTION_SCHEMA = "dimos.world_model_prediction.v1" +WORLD_MODEL_DASHBOARD_SCHEMA = "dimos.world_model_dashboard_state.v1" + + +def provider_contract( + name: str, + model_type: str, + capabilities: list[str] | tuple[str, ...] | None = None, + output_schemas: dict[str, str] | None = None, +) -> dict[str, Any]: + """Return a versioned DimOS-native contract for a world-model provider.""" + schemas = dict(output_schemas or {}) + schemas.setdefault("prediction", WORLD_MODEL_PREDICTION_SCHEMA) + schemas.setdefault("dashboard", WORLD_MODEL_DASHBOARD_SCHEMA) + return { + "schema": WORLD_MODEL_PROVIDER_SCHEMA, + "name": name.strip(), + "model_type": model_type.strip(), + "capabilities": sorted( + {capability.strip() for capability in capabilities or () if capability.strip()} + ), + "output_schemas": schemas, + "provider": "dimos", + } + + +def validate_world_model_prediction(prediction: dict[str, Any]) -> list[str]: + """Return contract errors for a world-model prediction payload.""" + errors: list[str] = [] + if prediction.get("schema") != WORLD_MODEL_PREDICTION_SCHEMA: + errors.append(f"schema must be {WORLD_MODEL_PREDICTION_SCHEMA}") + + if not isinstance(prediction.get("goal"), str): + errors.append("goal must be a string") + if not isinstance(prediction.get("horizon"), int): + errors.append("horizon must be an integer") + + action = prediction.get("action") + if not isinstance(action, dict): + errors.append("action must be a JSON object") + elif not isinstance(action.get("skill_name"), str) or not action.get("skill_name"): + errors.append("action.skill_name must be a non-empty string") + + if not isinstance(prediction.get("snapshot_summary"), dict): + errors.append("snapshot_summary must be a JSON object") + + body = prediction.get("prediction") + if not isinstance(body, dict): + errors.append("prediction must be a JSON object") + return errors + + if body.get("risk") not in {"low", "medium", "high"}: + errors.append("prediction.risk must be low, medium, or high") + if not isinstance(body.get("predicted_success"), bool): + errors.append("prediction.predicted_success must be a boolean") + score = body.get("score") + if not isinstance(score, int | float) or isinstance(score, bool): + errors.append("prediction.score must be a number") + if body.get("confidence") not in {"low", "medium", "high"}: + errors.append("prediction.confidence must be low, medium, or high") + if not isinstance(body.get("predicted_state_delta"), dict): + errors.append("prediction.predicted_state_delta must be a JSON object") + if not isinstance(body.get("failure_modes"), list): + errors.append("prediction.failure_modes must be a list") + if not isinstance(body.get("reasons"), list): + errors.append("prediction.reasons must be a list") + if not isinstance(body.get("evidence"), dict): + errors.append("prediction.evidence must be a JSON object") + return errors + + +__all__ = [ + "WORLD_MODEL_DASHBOARD_SCHEMA", + "WORLD_MODEL_PREDICTION_SCHEMA", + "WORLD_MODEL_PROVIDER_SCHEMA", + "provider_contract", + "validate_world_model_prediction", +] diff --git a/dimos/robot/unitree/go2/blueprints/layers/layer_4_world_state/DESIGN.md b/dimos/robot/unitree/go2/blueprints/layers/layer_4_world_state/DESIGN.md index f7cb043983..a8f3cd5970 100644 --- a/dimos/robot/unitree/go2/blueprints/layers/layer_4_world_state/DESIGN.md +++ b/dimos/robot/unitree/go2/blueprints/layers/layer_4_world_state/DESIGN.md @@ -25,6 +25,10 @@ Layer 4 currently contains: - V3 snapshot policy: Layer 4 snapshots are explicitly ephemeral read-through views. Durable writes remain in `SpatialMemory` and `TemporalMemory` until a concrete snapshot retention requirement exists. +- Package entrypoint laziness: importing Layer 4 specs or submodules does not + build `_go2_spatial_world_state` or import the heavy visual-memory stack. + The spatial world-state blueprint is constructed only when a blueprint asks + for `_go2_spatial_world_state`. ## Runtime Flow @@ -81,6 +85,31 @@ memory, navigation, and odom remain as a fallback. - Fusion is deterministic metadata packaging. - It does not resolve contradictions between spatial and temporal evidence. +### `layer_4_world_state.__getattr__(...)` + +- File: `layer_4_world_state/__init__.py` +- Entry point: Python package attribute lookup for exported layer blueprints. +- Purpose: keep Layer 4 package imports lightweight for callers that only need + specs or submodules, especially Layer 3 tests and type references. It does + not change the modules included when `_go2_spatial_world_state` is actually + requested. +- Inputs: + - Requested package attribute name. +- Storage: + - No database. + - Caches constructed blueprints in the process-local `_LAYER_BLUEPRINTS` + dictionary. +- Algorithm: + - Return `_go2_temporal_memory_world_state` directly because it is already a + lazy factory. + - On `_go2_spatial_world_state`, import `SpatialMemory`, + `_Go2SemanticTemporalMap`, and `_Go2StructuredWorldState`, then compose the + same `autoconnect(...)` blueprint that used to be built at package import. +- Current limits: + - This is import-time hygiene only. It does not make the underlying + `SpatialMemory` module lighter once the spatial world-state blueprint is + intentionally constructed. + ### `_Go2StructuredWorldState.get_world_snapshot(...)` - File: `layer_4_world_state/structured_world_state.py` diff --git a/dimos/robot/unitree/go2/blueprints/layers/layer_4_world_state/__init__.py b/dimos/robot/unitree/go2/blueprints/layers/layer_4_world_state/__init__.py index d1dda8082a..e61b7e251a 100644 --- a/dimos/robot/unitree/go2/blueprints/layers/layer_4_world_state/__init__.py +++ b/dimos/robot/unitree/go2/blueprints/layers/layer_4_world_state/__init__.py @@ -13,20 +13,42 @@ # See the License for the specific language governing permissions and # limitations under the License. +from typing import Any + from dimos.core.coordination.blueprints import Blueprint, autoconnect -from dimos.perception.spatial_perception import SpatialMemory -from dimos.robot.unitree.go2.blueprints.layers.layer_4_world_state.semantic_temporal_map import ( - _Go2SemanticTemporalMap, -) -from dimos.robot.unitree.go2.blueprints.layers.layer_4_world_state.structured_world_state import ( - _Go2StructuredWorldState, -) -_go2_spatial_world_state = autoconnect( - SpatialMemory.blueprint(), - _Go2SemanticTemporalMap.blueprint(), - _Go2StructuredWorldState.blueprint(), -) +__all__ = ["_go2_spatial_world_state", "_go2_temporal_memory_world_state"] + +_LAYER_BLUEPRINTS: dict[str, Any] = {} + + +def __getattr__(name: str) -> Any: + if name not in __all__: + raise AttributeError(name) + if name == "_go2_temporal_memory_world_state": + return _go2_temporal_memory_world_state + _build_layer_blueprints() + return _LAYER_BLUEPRINTS[name] + + +def _build_layer_blueprints() -> None: + if _LAYER_BLUEPRINTS: + return + + from dimos.perception.spatial_perception import SpatialMemory + from dimos.robot.unitree.go2.blueprints.layers.layer_4_world_state.semantic_temporal_map import ( + _Go2SemanticTemporalMap, + ) + from dimos.robot.unitree.go2.blueprints.layers.layer_4_world_state.structured_world_state import ( + _Go2StructuredWorldState, + ) + + _LAYER_BLUEPRINTS["_go2_spatial_world_state"] = autoconnect( + SpatialMemory.blueprint(), + _Go2SemanticTemporalMap.blueprint(), + _Go2StructuredWorldState.blueprint(), + ) + globals().update(_LAYER_BLUEPRINTS) def _go2_temporal_memory_world_state() -> Blueprint: @@ -38,10 +60,5 @@ def _go2_temporal_memory_world_state() -> Blueprint: ) return autoconnect( - TemporalMemory.blueprint( - config=TemporalMemoryConfig(new_memory=global_config.new_memory) - ), + TemporalMemory.blueprint(config=TemporalMemoryConfig(new_memory=global_config.new_memory)), ) - - -__all__ = ["_go2_spatial_world_state", "_go2_temporal_memory_world_state"] diff --git a/dimos/web/templates/rerun_dashboard.html b/dimos/web/templates/rerun_dashboard.html index f0792079e3..393ae98837 100644 --- a/dimos/web/templates/rerun_dashboard.html +++ b/dimos/web/templates/rerun_dashboard.html @@ -6,14 +6,31 @@ * { margin: 0; padding: 0; box-sizing: border-box; } html, body { width: 100%; height: 100%; overflow: hidden; } body { background: #0a0a0f; font-family: -apple-system, system-ui, sans-serif; } - :root { --command-center-width: max(30vw, 35rem); } + :root { + --command-center-width: max(32vw, 36rem); + --world-model-height: 20rem; + } .container { display: flex; width: 100%; height: 100%; } - .command-center { + .command-column { width: var(--command-center-width); min-width: 16rem; + display: flex; + flex-direction: column; border: none; border-right: 1px solid #333; } + .command-center { + flex: 1 1 auto; + min-height: 0; + border: none; + } + .world-model { + height: var(--world-model-height); + min-height: 12rem; + border: none; + border-top: 1px solid #333; + background: #0b0d10; + } .rerun { flex: 1 1 auto; border: none; min-width: 0; } .divider { width: 6px; @@ -29,7 +46,10 @@
- +
+ + +
@@ -37,7 +57,7 @@ (function () { const container = document.querySelector('.container'); const divider = document.querySelector('.divider'); - const commandCenter = document.querySelector('.command-center'); + const commandColumn = document.querySelector('.command-column'); const body = document.body; const minWidth = 0; // 16rem baseline for small screens @@ -50,7 +70,7 @@ const dividerWidth = divider.getBoundingClientRect().width; const available = event.clientX - rect.left - dividerWidth / 2; const nextWidth = Math.max(minWidth, Math.min(available, rect.width)); - commandCenter.style.width = `${nextWidth}px`; + commandColumn.style.width = `${nextWidth}px`; document.documentElement.style.setProperty('--command-center-width', `${nextWidth}px`); } diff --git a/dimos/web/websocket_vis/test_websocket_vis_module.py b/dimos/web/websocket_vis/test_websocket_vis_module.py new file mode 100644 index 0000000000..36b7408290 --- /dev/null +++ b/dimos/web/websocket_vis/test_websocket_vis_module.py @@ -0,0 +1,71 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from threading import Lock +from typing import Any + +from dimos.web.websocket_vis.websocket_vis_module import WebsocketVisModule + + +def _module_without_runtime() -> tuple[WebsocketVisModule, list[tuple[str, dict[str, Any]]]]: + module = object.__new__(WebsocketVisModule) + module.vis_state = {} + module.state_lock = Lock() + emitted: list[tuple[str, dict[str, Any]]] = [] + module._emit = lambda event, data: emitted.append((event, data)) # type: ignore[method-assign] + return module, emitted + + +def test_set_world_model_state_stores_snapshot_and_emits_incremental_update() -> None: + module, emitted = _module_without_runtime() + state = { + "prediction": { + "risk": "high", + "score": 0.2, + "predicted_state_delta": {"navigation.state": "blocked_before_navigation"}, + "structural_causal_model": { + "counterfactuals": [{"intervention": "do(spatial_has_matches=True)"}] + }, + }, + "recent_interventions": [ + { + "intervention_name": "look_out_for_then_tag_target", + "target_variable": "spatial_has_matches", + } + ], + } + + summary = module.set_world_model_state(state) + + assert summary["available"] is True + assert summary["risk"] == "high" + assert module.vis_state["world_model_state"]["prediction"]["risk"] == "high" + assert ( + module.vis_state["world_model_state"]["recent_interventions"][0]["intervention_name"] + == "look_out_for_then_tag_target" + ) + assert emitted == [("world_model_state", module.vis_state["world_model_state"])] + + +def test_get_world_model_state_returns_current_dashboard_snapshot() -> None: + module, _ = _module_without_runtime() + module.vis_state["world_model_state"] = { + "available": True, + "prediction": {"risk": "medium", "score": 0.62}, + } + + state = module.get_world_model_state() + + assert state["available"] is True + assert state["prediction"]["risk"] == "medium" diff --git a/dimos/web/websocket_vis/websocket_vis_module.py b/dimos/web/websocket_vis/websocket_vis_module.py index ed319e52ca..7e82cfa8c1 100644 --- a/dimos/web/websocket_vis/websocket_vis_module.py +++ b/dimos/web/websocket_vis/websocket_vis_module.py @@ -32,7 +32,7 @@ from reactivex.disposable import Disposable import socketio # type: ignore[import-untyped] from starlette.applications import Starlette -from starlette.responses import FileResponse, RedirectResponse, Response +from starlette.responses import FileResponse, HTMLResponse, JSONResponse, RedirectResponse, Response from starlette.routing import Route import uvicorn @@ -44,6 +44,172 @@ _COMMAND_CENTER_DIR = ( FilePath(__file__).parent.parent / "command-center-extension" / "dist-standalone" ) +_WORLD_MODEL_PANEL_HTML = """ + + + World Model + + + +
+
+

World Model

+
no action
+
unknown
+
score --
+
+
+
+

Prediction

+
+
success
--
+
confidence
--
+
samples
0
+
event
--
+
+
+
+

Future State Delta

+ waiting for state +
+
+

Causal

+
  • no causal evidence
+
+
+
+ + +""" from dimos.constants import DEFAULT_THREAD_JOIN_TIMEOUT from dimos.core.core import rpc @@ -229,6 +395,24 @@ def set_gps_travel_goal_points(self, points: list[LatLon]) -> None: self.vis_state["gps_travel_goal_points"] = json_points self._emit("gps_travel_goal_points", json_points) + @rpc + def set_world_model_state(self, state: dict[str, Any]) -> dict[str, Any]: + """Publish a compact predictive world-model state to dashboard clients.""" + dashboard_state = _world_model_dashboard_state(state) + with self.state_lock: + self.vis_state["world_model_state"] = dashboard_state + self._emit("world_model_state", dashboard_state) + return _world_model_dashboard_summary(dashboard_state) + + @rpc + def get_world_model_state(self) -> dict[str, Any]: + """Return the latest predictive world-model state for polling dashboards.""" + with self.state_lock: + state = self.vis_state.get("world_model_state") + if isinstance(state, dict): + return state + return {"available": False, "prediction": {}, "recent_interventions": []} + def _create_server(self) -> None: # Create SocketIO server self.sio = socketio.AsyncServer(async_mode="asgi", cors_allowed_origins="*") @@ -253,9 +437,19 @@ async def serve_command_center(request): # type: ignore[no-untyped-def] media_type="text/plain", ) + async def serve_world_model_panel(request): # type: ignore[no-untyped-def] + """Serve the polling world-model panel shown in the dashboard shell.""" + return HTMLResponse(_WORLD_MODEL_PANEL_HTML) + + async def serve_world_model_state(request): # type: ignore[no-untyped-def] + """Serve the latest world-model dashboard state for polling clients.""" + return JSONResponse(self.get_world_model_state()) + routes = [ Route("/", serve_index), Route("/command-center", serve_command_center), + Route("/world-model", serve_world_model_panel), + Route("/api/world-model-state", serve_world_model_state), ] starlette_app = Starlette(routes=routes) @@ -402,3 +596,47 @@ def _process_costmap(self, costmap: OccupancyGrid) -> dict[str, Any]: def _emit(self, event: str, data: Any) -> None: if self._broadcast_loop and not self._broadcast_loop.is_closed(): asyncio.run_coroutine_threadsafe(self.sio.emit(event, data), self._broadcast_loop) + + +def _world_model_dashboard_state(state: dict[str, Any]) -> dict[str, Any]: + dashboard_state = _bounded_jsonable(state) + if not isinstance(dashboard_state, dict): + dashboard_state = {} + dashboard_state["available"] = True + dashboard_state.setdefault("updated_at", round(time.time(), 3)) + return dashboard_state + + +def _world_model_dashboard_summary(state: dict[str, Any]) -> dict[str, Any]: + prediction = state.get("prediction") if isinstance(state.get("prediction"), dict) else {} + model_state = state.get("model_state") if isinstance(state.get("model_state"), dict) else {} + recent_interventions = ( + state.get("recent_interventions") + if isinstance(state.get("recent_interventions"), list) + else [] + ) + return { + "available": bool(state.get("available")), + "risk": str(prediction.get("risk") or ""), + "score": prediction.get("score"), + "predicted_success": prediction.get("predicted_success"), + "transition_count": len(state.get("recent_transitions") or []), + "intervention_count": len(recent_interventions), + "model_samples": int(model_state.get("sample_count") or 0), + } + + +def _bounded_jsonable(value: Any, max_string_length: int = 500) -> Any: + if value is None or isinstance(value, bool | int | float): + return value + if isinstance(value, str): + return value[:max_string_length] + if isinstance(value, dict): + return { + str(key): _bounded_jsonable(item, max_string_length) + for key, item in value.items() + if not str(key).startswith("_") + } + if isinstance(value, list | tuple): + return [_bounded_jsonable(item, max_string_length) for item in value[:10]] + return str(value)[:max_string_length] diff --git a/dimos/web/websocket_vis_spec.py b/dimos/web/websocket_vis_spec.py index 41d8ac8737..9203c2d343 100644 --- a/dimos/web/websocket_vis_spec.py +++ b/dimos/web/websocket_vis_spec.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from typing import Protocol +from typing import Any, Protocol from dimos.mapping.models import LatLon from dimos.spec.utils import Spec @@ -20,3 +20,5 @@ class WebsocketVisSpec(Spec, Protocol): def set_gps_travel_goal_points(self, points: list[LatLon]) -> None: ... + + def set_world_model_state(self, state: dict[str, Any]) -> dict[str, Any]: ... From 2db425ecf8757c34aae40e5264328caf982aedcd Mon Sep 17 00:00:00 2001 From: ZhangZheMing <1050297972@qq.com> Date: Sat, 4 Jul 2026 23:00:30 +0800 Subject: [PATCH 13/36] Add Go2 self-evolution proposal tools --- .../layers/layer_3_agent_brain/__init__.py | 91 ++-- .../layer_3_agent_brain/context_evidence.py | 432 ++++++++++++++++++ .../layer_3_agent_brain/context_feedback.py | 346 ++++++++++++++ .../layer_3_agent_brain/context_provider.py | 408 ++++++++++++++++- .../layer_3_agent_brain/evolution_event.py | 173 +++++++ .../layer_3_agent_brain/evolution_ledger.py | 264 +++++++++++ .../layer_3_agent_brain/evolution_proposal.py | 176 +++++++ .../memory_backend_status.py | 225 +++++++++ .../layer_3_agent_brain/prompt_policy.py | 22 +- .../layer_3_agent_brain/skill_proposal.py | 311 +++++++++++++ .../layer_3_agent_brain/task_feasibility.py | 345 ++++++++++++++ .../test_context_evidence.py | 114 +++++ .../test_context_feedback.py | 184 ++++++++ .../test_context_provider.py | 79 +++- .../test_evolution_ledger.py | 139 ++++++ .../test_evolution_proposal.py | 113 +++++ .../test_memory_backend_status.py | 155 +++++++ .../test_skill_proposal.py | 179 ++++++++ .../test_task_feasibility.py | 182 ++++++++ .../skill_interface_registry.py | 20 +- .../test_skill_interface_registry.py | 31 +- 21 files changed, 3939 insertions(+), 50 deletions(-) create mode 100644 dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/context_evidence.py create mode 100644 dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/context_feedback.py create mode 100644 dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/evolution_event.py create mode 100644 dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/evolution_ledger.py create mode 100644 dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/evolution_proposal.py create mode 100644 dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/memory_backend_status.py create mode 100644 dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/skill_proposal.py create mode 100644 dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/task_feasibility.py create mode 100644 dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/test_context_evidence.py create mode 100644 dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/test_context_feedback.py create mode 100644 dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/test_evolution_ledger.py create mode 100644 dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/test_evolution_proposal.py create mode 100644 dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/test_memory_backend_status.py create mode 100644 dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/test_skill_proposal.py create mode 100644 dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/test_task_feasibility.py diff --git a/dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/__init__.py b/dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/__init__.py index 0cfe6d3fe7..fa2ab7c9ec 100644 --- a/dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/__init__.py +++ b/dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/__init__.py @@ -13,48 +13,85 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import annotations + from typing import Any -from dimos.agents.mcp.mcp_client import McpClient -from dimos.agents.mcp.mcp_server import McpServer -from dimos.core.coordination.blueprints import Blueprint, autoconnect -from dimos.robot.unitree.go2.blueprints.layers.layer_3_agent_brain.causal_world_model import ( - _Go2CausalWorldModel, -) -from dimos.robot.unitree.go2.blueprints.layers.layer_3_agent_brain.context_provider import ( - _Go2ContextProvider, -) -from dimos.robot.unitree.go2.blueprints.layers.layer_3_agent_brain.expert_router import ( - _Go2ExpertRouter, -) -from dimos.robot.unitree.go2.blueprints.layers.layer_3_agent_brain.prompt_policy import ( - _go2_layer_3_system_prompt, -) -from dimos.robot.unitree.go2.blueprints.layers.layer_3_agent_brain.skill_outcome_predictor import ( - _Go2SkillOutcomePredictor, -) -from dimos.robot.unitree.go2.blueprints.layers.layer_3_agent_brain.skill_outcome_store import ( - _Go2SkillOutcomeStore, -) +from dimos.core.coordination.blueprints import Blueprint + +__all__ = ["_go2_agent_brain", "_go2_agent_brain_with_client"] + +_LAYER_BLUEPRINTS: dict[str, Any] = {} + + +def __getattr__(name: str) -> Any: + if name not in __all__: + raise AttributeError(name) + if name == "_go2_agent_brain_with_client": + return _go2_agent_brain_with_client + _build_layer_blueprints() + return _LAYER_BLUEPRINTS[name] def _go2_agent_brain_with_client(**mcp_client_kwargs: Any) -> Blueprint: """Layer 3: expert routing, context, MCP tools, and the LLM/VLM agent.""" - client_kwargs = dict(mcp_client_kwargs) - client_kwargs["system_prompt"] = _go2_layer_3_system_prompt( - client_kwargs.get("system_prompt") + from dimos.agents.mcp.mcp_client import McpClient + from dimos.agents.mcp.mcp_server import McpServer + from dimos.core.coordination.blueprints import autoconnect + from dimos.robot.unitree.go2.blueprints.layers.layer_3_agent_brain.causal_world_model import ( + _Go2CausalWorldModel, + ) + from dimos.robot.unitree.go2.blueprints.layers.layer_3_agent_brain.context_feedback import ( + _Go2ContextFeedbackStore, + ) + from dimos.robot.unitree.go2.blueprints.layers.layer_3_agent_brain.context_provider import ( + _Go2ContextProvider, + ) + from dimos.robot.unitree.go2.blueprints.layers.layer_3_agent_brain.evolution_ledger import ( + _Go2EvolutionLedger, + ) + from dimos.robot.unitree.go2.blueprints.layers.layer_3_agent_brain.expert_router import ( + _Go2ExpertRouter, ) + from dimos.robot.unitree.go2.blueprints.layers.layer_3_agent_brain.memory_backend_status import ( + _Go2MemoryBackendStatus, + ) + from dimos.robot.unitree.go2.blueprints.layers.layer_3_agent_brain.prompt_policy import ( + _go2_layer_3_system_prompt, + ) + from dimos.robot.unitree.go2.blueprints.layers.layer_3_agent_brain.skill_outcome_predictor import ( + _Go2SkillOutcomePredictor, + ) + from dimos.robot.unitree.go2.blueprints.layers.layer_3_agent_brain.skill_outcome_store import ( + _Go2SkillOutcomeStore, + ) + from dimos.robot.unitree.go2.blueprints.layers.layer_3_agent_brain.skill_proposal import ( + _Go2SkillProposalGenerator, + ) + from dimos.robot.unitree.go2.blueprints.layers.layer_3_agent_brain.task_feasibility import ( + _Go2TaskFeasibilityEvaluator, + ) + + client_kwargs = dict(mcp_client_kwargs) + client_kwargs["system_prompt"] = _go2_layer_3_system_prompt(client_kwargs.get("system_prompt")) return autoconnect( _Go2ExpertRouter.blueprint(), _Go2SkillOutcomeStore.blueprint(), _Go2CausalWorldModel.blueprint(), + _Go2ContextFeedbackStore.blueprint(), _Go2SkillOutcomePredictor.blueprint(), _Go2ContextProvider.blueprint(), + _Go2MemoryBackendStatus.blueprint(), + _Go2EvolutionLedger.blueprint(), + _Go2TaskFeasibilityEvaluator.blueprint(), + _Go2SkillProposalGenerator.blueprint(), McpServer.blueprint(), McpClient.blueprint(**client_kwargs), ) -_go2_agent_brain = _go2_agent_brain_with_client() - -__all__ = ["_go2_agent_brain", "_go2_agent_brain_with_client"] +def _build_layer_blueprints() -> None: + if _LAYER_BLUEPRINTS: + return + _LAYER_BLUEPRINTS["_go2_agent_brain"] = _go2_agent_brain_with_client() + globals().update(_LAYER_BLUEPRINTS) diff --git a/dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/context_evidence.py b/dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/context_evidence.py new file mode 100644 index 0000000000..f176153ddc --- /dev/null +++ b/dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/context_evidence.py @@ -0,0 +1,432 @@ +#!/usr/bin/env python3 +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import annotations + +from dataclasses import dataclass +from typing import Any + + +@dataclass(frozen=True) +class ContextEvidencePolicy: + min_relevance_score: float = 0.0 + max_entries: int = 12 + include_low_confidence: bool = True + require_robot_state_for_motion: bool = True + + +def build_context_evidence( + metadata: dict[str, Any], + policy: ContextEvidencePolicy, +) -> dict[str, Any]: + """Build the explicit context evidence ledger from ContextProvider metadata.""" + task = str(metadata.get("task") or "") + focus = str(metadata.get("focus") or "") + entries: list[dict[str, Any]] = [ + _evidence_entry( + source="task", + query=task, + relevance_score=1.0, + recency="current", + confidence="high", + risk_impact="unknown", + cost="free", + selected_reason="Current user goal anchors retrieval.", + summary=task, + ) + ] + + runtime = metadata.get("runtime") if isinstance(metadata.get("runtime"), dict) else {} + entries.append( + _evidence_entry( + source="runtime_config", + query=focus or task, + relevance_score=0.8, + recency="current", + confidence="high", + risk_impact="medium" if runtime.get("mode") == "hardware" else "low", + cost="config_read", + selected_reason="Runtime mode changes safety and execution assumptions.", + summary=f"mode={runtime.get('mode', 'unknown')}", + ) + ) + + robot_state = ( + metadata.get("robot_state") if isinstance(metadata.get("robot_state"), dict) else {} + ) + if any(robot_state.get(key) for key in ("odom", "navigation", "connection", "safety")): + entries.append( + _evidence_entry( + source="robot_state", + query=focus or task, + relevance_score=0.85, + recency="current", + confidence="high", + risk_impact=_robot_risk_impact(robot_state), + cost="stream_cache", + selected_reason="Current pose, navigation, connection, and safety state gate physical actions.", + summary=_robot_state_summary(robot_state), + ) + ) + + world_state = ( + metadata.get("world_state") if isinstance(metadata.get("world_state"), dict) else {} + ) + entries.extend(_world_evidence_entries(task, world_state)) + + skill_state = ( + metadata.get("skill_state") if isinstance(metadata.get("skill_state"), dict) else {} + ) + entries.extend(_skill_evidence_entries(task, skill_state)) + + causal_state = ( + metadata.get("causal_state") if isinstance(metadata.get("causal_state"), dict) else {} + ) + entries.extend(_causal_evidence_entries(task, causal_state)) + + world_model_state = ( + metadata.get("world_model_state") + if isinstance(metadata.get("world_model_state"), dict) + else {} + ) + entries.extend(_world_model_evidence_entries(task, world_model_state)) + entries = _apply_policy(entries=entries, task=task, policy=policy) + + return { + "version": "context_evidence.v1", + "selection_policy": "deterministic_source_coverage_v1", + "query": task, + "focus": focus, + "entry_count": len(entries), + "selected_sources": _ordered_sources(entries), + "entries": entries, + } + + +def _apply_policy( + *, + entries: list[dict[str, Any]], + task: str, + policy: ContextEvidencePolicy, +) -> list[dict[str, Any]]: + min_score = max(0.0, min(1.0, policy.min_relevance_score)) + max_entries = max(1, policy.max_entries) + protected = {"task", "runtime_config"} + if policy.require_robot_state_for_motion and _looks_motion_task(task): + protected.add("robot_state") + + filtered = [ + entry + for entry in entries + if entry["source"] in protected + or ( + float(entry.get("relevance_score") or 0.0) >= min_score + and (policy.include_low_confidence or entry.get("confidence") != "low") + ) + ] + protected_entries = [entry for entry in filtered if entry["source"] in protected] + other_entries = [entry for entry in filtered if entry["source"] not in protected] + return (protected_entries + other_entries)[:max_entries] + + +def _looks_motion_task(task: str) -> bool: + text = task.casefold() + return any( + marker in text + for marker in ( + "walk", + "navigate", + "move", + "go to", + "follow", + "patrol", + "turn", + "security", + ) + ) + + +def _evidence_entry( + *, + source: str, + query: str, + relevance_score: float, + recency: str, + confidence: str, + risk_impact: str, + cost: str, + selected_reason: str, + summary: str, +) -> dict[str, Any]: + return { + "source": source, + "query": query, + "relevance_score": round(max(0.0, min(1.0, relevance_score)), 3), + "recency": recency, + "confidence": _normalize_label(confidence, {"low", "medium", "high"}, "medium"), + "risk_impact": _normalize_label( + risk_impact, + {"low", "medium", "high", "unknown"}, + "unknown", + ), + "cost": cost, + "selected_reason": selected_reason, + "summary": summary[:300], + } + + +def _world_evidence_entries(task: str, world_state: dict[str, Any]) -> list[dict[str, Any]]: + entries: list[dict[str, Any]] = [] + spatial = world_state.get("spatial") if isinstance(world_state.get("spatial"), dict) else {} + matches = spatial.get("matches") if isinstance(spatial.get("matches"), list) else [] + if matches: + first_match = matches[0] if isinstance(matches[0], dict) else {} + entries.append( + _evidence_entry( + source="spatial_memory", + query=task, + relevance_score=_spatial_relevance(first_match), + recency=_match_recency(first_match), + confidence=_spatial_confidence(first_match), + risk_impact="medium", + cost="rag_query", + selected_reason="Top spatial/semantic RAG match for the current task.", + summary=_spatial_summary(first_match), + ) + ) + + temporal = world_state.get("temporal") if isinstance(world_state.get("temporal"), dict) else {} + temporal_summary = str(temporal.get("rolling_summary") or "") + if temporal_summary or temporal.get("state") or temporal.get("answer"): + entries.append( + _evidence_entry( + source="temporal_memory", + query=task, + relevance_score=0.75 if temporal_summary else 0.65, + recency="recent_summary" if temporal_summary else "current_state", + confidence="medium", + risk_impact="medium", + cost="vlm_graph_query" if temporal.get("answer") else "state_read", + selected_reason="Temporal memory contributes recent entities, events, or rolling summary.", + summary=temporal_summary or "temporal state available", + ) + ) + + semantic_temporal = world_state.get("semantic_temporal") + if not isinstance(semantic_temporal, dict): + semantic_temporal = world_state.get("semantic_temporal_map") + if isinstance(semantic_temporal, dict): + fused = semantic_temporal.get("fused") + if isinstance(fused, dict) and fused.get("available"): + entries.append( + _evidence_entry( + source="semantic_temporal_map", + query=task, + relevance_score=0.7, + recency="current_fusion", + confidence="medium", + risk_impact="medium", + cost="rpc_read", + selected_reason="Layer 4 fused spatial and temporal evidence into normalized entries.", + summary=f"fused_entries={fused.get('entry_count', 0)}", + ) + ) + return entries + + +def _skill_evidence_entries(task: str, skill_state: dict[str, Any]) -> list[dict[str, Any]]: + entries: list[dict[str, Any]] = [] + recent_outcomes = ( + skill_state.get("recent_outcomes") + if isinstance(skill_state.get("recent_outcomes"), list) + else [] + ) + if recent_outcomes: + failures = [outcome for outcome in recent_outcomes if not outcome.get("success")] + entries.append( + _evidence_entry( + source="skill_outcome_store", + query=task, + relevance_score=0.9 if failures else 0.65, + recency="recent_history", + confidence="high", + risk_impact="high" if failures else "low", + cost="memory_read", + selected_reason="Recent skill outcomes reveal repeated failures or successful recovery paths.", + summary=f"{len(recent_outcomes)} outcome(s), {len(failures)} failure(s)", + ) + ) + + interface = ( + skill_state.get("interface") if isinstance(skill_state.get("interface"), dict) else {} + ) + if interface.get("available"): + entries.append( + _evidence_entry( + source="skill_interface_registry", + query=task, + relevance_score=0.8, + recency="static_contract", + confidence="high", + risk_impact="medium", + cost="rpc_read", + selected_reason="Skill contracts constrain tool choice, required arguments, and preflight checks.", + summary=f"{interface.get('skill_count', 0)} skill contract(s)", + ) + ) + return entries + + +def _causal_evidence_entries(task: str, causal_state: dict[str, Any]) -> list[dict[str, Any]]: + transitions = ( + causal_state.get("recent_transitions") + if isinstance(causal_state.get("recent_transitions"), list) + else [] + ) + if not transitions: + return [] + failures = [ + transition for transition in transitions if transition.get("outcome_success") is False + ] + return [ + _evidence_entry( + source="causal_world_model", + query=task, + relevance_score=0.88 if failures else 0.7, + recency="recent_transition", + confidence="medium", + risk_impact="high" if failures else "low", + cost="memory_read", + selected_reason="Causal transitions explain whether prior actions changed state as expected.", + summary=f"{len(transitions)} transition(s), {len(failures)} failure(s)", + ) + ] + + +def _world_model_evidence_entries( + task: str, world_model_state: dict[str, Any] +) -> list[dict[str, Any]]: + if not world_model_state.get("available"): + return [] + model = ( + world_model_state.get("model") if isinstance(world_model_state.get("model"), dict) else {} + ) + failures = int(world_model_state.get("recent_failure_count") or 0) + return [ + _evidence_entry( + source="predictive_world_model", + query=task, + relevance_score=0.82 if model.get("sample_count") else 0.6, + recency="online_model", + confidence="medium" if model.get("sample_count") else "low", + risk_impact="high" if failures else "medium", + cost="model_state_read", + selected_reason="Predictive world-model state exposes learned samples and recent failure modes.", + summary=( + f"samples={model.get('sample_count', 0)}, " + f"recent_failures={failures}, " + f"top_failure={world_model_state.get('top_failure_mode', '')}" + ), + ) + ] + + +def _robot_risk_impact(robot_state: dict[str, Any]) -> str: + safety = robot_state.get("safety") if isinstance(robot_state.get("safety"), dict) else {} + navigation = ( + robot_state.get("navigation") if isinstance(robot_state.get("navigation"), dict) else {} + ) + if safety and not safety.get("body_pose_available", True): + return "high" + if navigation and navigation.get("goal_reached") is False: + return "medium" + return "low" + + +def _robot_state_summary(robot_state: dict[str, Any]) -> str: + parts: list[str] = [] + odom = robot_state.get("odom") + if isinstance(odom, dict): + position = odom.get("position") if isinstance(odom.get("position"), dict) else {} + parts.append(f"pose=({position.get('x')}, {position.get('y')})") + navigation = robot_state.get("navigation") + if isinstance(navigation, dict): + parts.append(f"navigation={navigation.get('state')}") + connection = robot_state.get("connection") + if isinstance(connection, dict): + parts.append(f"connection={connection.get('mode')}") + return ", ".join(parts) or "robot state available" + + +def _spatial_relevance(match: dict[str, Any]) -> float: + score = match.get("score") + if isinstance(score, int | float) and not isinstance(score, bool): + return float(score) + distance = match.get("distance") + if isinstance(distance, int | float) and not isinstance(distance, bool): + return 1.0 - min(max(float(distance), 0.0), 1.0) + return 0.6 + + +def _spatial_confidence(match: dict[str, Any]) -> str: + relevance = _spatial_relevance(match) + if relevance >= 0.75: + return "high" + if relevance >= 0.45: + return "medium" + return "low" + + +def _spatial_summary(match: dict[str, Any]) -> str: + metadata = match.get("metadata") + if isinstance(metadata, list) and metadata and isinstance(metadata[0], dict): + metadata = metadata[0] + if isinstance(metadata, dict): + label = metadata.get("label") or metadata.get("description") or metadata.get("frame_id") + if label: + return str(label) + if "pos_x" in metadata and "pos_y" in metadata: + return f"spatial match at x={metadata.get('pos_x')}, y={metadata.get('pos_y')}" + if match.get("id"): + return str(match["id"]) + return "spatial match" + + +def _match_recency(match: dict[str, Any]) -> str: + metadata = match.get("metadata") + if isinstance(metadata, list) and metadata and isinstance(metadata[0], dict): + metadata = metadata[0] + if isinstance(metadata, dict) and metadata.get("timestamp") is not None: + return "timestamped_observation" + return "stored_observation" + + +def _ordered_sources(entries: list[dict[str, Any]]) -> list[str]: + sources: set[str] = set() + for entry in entries: + source = str(entry.get("source") or "") + if source: + sources.add(source) + return sorted(sources) + + +def _normalize_label(value: str, allowed: set[str], fallback: str) -> str: + normalized = value.strip().casefold() + if normalized in allowed: + return normalized + return fallback + + +__all__ = ["ContextEvidencePolicy", "build_context_evidence"] diff --git a/dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/context_feedback.py b/dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/context_feedback.py new file mode 100644 index 0000000000..9efc49e2ae --- /dev/null +++ b/dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/context_feedback.py @@ -0,0 +1,346 @@ +#!/usr/bin/env python3 +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import annotations + +from collections import Counter, deque +from dataclasses import dataclass +import json +import time +from typing import Any, Protocol + +from dimos.agents.annotation import skill +from dimos.agents.skill_result import SkillResult +from dimos.core.core import rpc +from dimos.core.module import Module +from dimos.robot.unitree.go2.blueprints.layers.layer_3_agent_brain.evolution_ledger import ( + EvolutionLedgerSpec, +) +from dimos.spec.utils import Spec + +CONTEXT_FEEDBACK_SCHEMA = "go2_context_feedback.v1" +CONTEXT_EVIDENCE_SCHEMA = "context_evidence.v1" + + +class ContextFeedbackSpec(Spec, Protocol): + """RPC surface for Layer 3 modules that read context feedback summaries.""" + + def get_recent_context_feedback( + self, + limit: int = 5, + source: str = "", + ) -> list[dict[str, Any]]: ... + + def get_context_feedback_summary( + self, + limit: int = 20, + ) -> dict[str, Any]: ... + + +@dataclass(frozen=True) +class _ContextFeedback: + timestamp: float + task: str + selected_skill: str + outcome_success: bool | None + outcome_error_code: str + evidence_sources: tuple[str, ...] + helpful_sources: tuple[str, ...] + ignored_risks: tuple[str, ...] + helpful_source_counts: dict[str, int] + harmful_source_counts: dict[str, int] + + def to_dict(self) -> dict[str, Any]: + return { + "schema": CONTEXT_FEEDBACK_SCHEMA, + "timestamp": round(self.timestamp, 3), + "task": self.task, + "selected_skill": self.selected_skill, + "outcome_success": self.outcome_success, + "outcome_error_code": self.outcome_error_code, + "evidence_sources": list(self.evidence_sources), + "helpful_sources": list(self.helpful_sources), + "ignored_risks": list(self.ignored_risks), + "helpful_source_counts": dict(self.helpful_source_counts), + "harmful_source_counts": dict(self.harmful_source_counts), + } + + +class _Go2ContextFeedbackStore(Module): + """Bounded store for whether selected context evidence helped task outcomes.""" + + _max_feedback = 100 + _evolution_ledger: EvolutionLedgerSpec | None = None + + def __init__(self, **kwargs: Any) -> None: + super().__init__(**kwargs) + self._feedback: deque[_ContextFeedback] = deque(maxlen=self._max_feedback) + + @skill + def record_context_feedback( + self, + task: str, + context_evidence_json: str, + selected_skill: str = "", + outcome_json: str = "{}", + helpful_sources_json: str = "[]", + ignored_risks_json: str = "[]", + ) -> SkillResult: + """Record whether selected context evidence helped or hurt the task outcome. + + Args: + task: User task or subtask that used the context. + context_evidence_json: JSON object with version context_evidence.v1. + selected_skill: Skill selected after reading the context. + outcome_json: JSON object describing the observed outcome. + helpful_sources_json: JSON list of evidence source names that helped. + ignored_risks_json: JSON list of risk/source labels that were ignored. + """ + task = task.strip() + selected_skill = selected_skill.strip() + if not task: + return SkillResult.fail("INVALID_INPUT", "task is required") + + parsed = _parse_feedback_inputs( + context_evidence_json=context_evidence_json, + outcome_json=outcome_json, + helpful_sources_json=helpful_sources_json, + ignored_risks_json=ignored_risks_json, + ) + if isinstance(parsed, str): + return SkillResult.fail("INVALID_INPUT", parsed) + evidence, outcome, helpful_sources, ignored_risks = parsed + + feedback = _build_context_feedback( + task=task, + selected_skill=selected_skill, + evidence=evidence, + outcome=outcome, + helpful_sources=helpful_sources, + ignored_risks=ignored_risks, + ) + self._feedback.append(feedback) + + metadata: dict[str, Any] = { + "feedback": feedback.to_dict(), + "total_feedback": len(self._feedback), + } + ledger_warning = self._write_ledger_event(feedback) + if ledger_warning: + metadata["warnings"] = [ledger_warning] + + return SkillResult.ok( + f"Recorded context feedback for {selected_skill or 'unknown skill'}", + **metadata, + ) + + @rpc + def get_recent_context_feedback( + self, + limit: int = 5, + source: str = "", + ) -> list[dict[str, Any]]: + """Return newest context feedback records first, optionally filtered by source.""" + limit = max(0, min(limit, self._max_feedback)) + source = source.strip() + records = [ + feedback + for feedback in reversed(self._feedback) + if not source or _feedback_mentions_source(feedback, source) + ] + return [feedback.to_dict() for feedback in records[:limit]] + + @rpc + def get_context_feedback_summary(self, limit: int = 20) -> dict[str, Any]: + """Return aggregate counts over recent context feedback records.""" + records = self.get_recent_context_feedback(limit=limit) + helpful = Counter() + harmful = Counter() + successes = 0 + failures = 0 + unknown = 0 + for record in records: + helpful.update(record.get("helpful_source_counts") or {}) + harmful.update(record.get("harmful_source_counts") or {}) + if record.get("outcome_success") is True: + successes += 1 + elif record.get("outcome_success") is False: + failures += 1 + else: + unknown += 1 + return { + "schema": CONTEXT_FEEDBACK_SCHEMA, + "total_feedback": len(records), + "success_count": successes, + "failure_count": failures, + "unknown_count": unknown, + "helpful_source_counts": dict(helpful), + "harmful_source_counts": dict(harmful), + } + + def _write_ledger_event(self, feedback: _ContextFeedback) -> str: + if self._evolution_ledger is None: + return "" + try: + self._evolution_ledger.write_evolution_event( + event_type="context_feedback", + task=feedback.task, + payload=feedback.to_dict(), + commit=False, + ) + except Exception as exc: + return f"evolution_ledger: {exc}" + return "" + + +def _parse_feedback_inputs( + *, + context_evidence_json: str, + outcome_json: str, + helpful_sources_json: str, + ignored_risks_json: str, +) -> tuple[dict[str, Any], dict[str, Any], list[str], list[str]] | str: + evidence = _parse_json_object(context_evidence_json, "context_evidence_json") + if isinstance(evidence, str): + return evidence + if isinstance(evidence.get("context_evidence"), dict): + evidence = evidence["context_evidence"] + if evidence.get("version") != CONTEXT_EVIDENCE_SCHEMA: + return "context_evidence_json must have version context_evidence.v1" + + outcome = _parse_json_object(outcome_json or "{}", "outcome_json") + if isinstance(outcome, str): + return outcome + helpful_sources = _parse_json_string_list(helpful_sources_json, "helpful_sources_json") + if isinstance(helpful_sources, str): + return helpful_sources + ignored_risks = _parse_json_string_list(ignored_risks_json, "ignored_risks_json") + if isinstance(ignored_risks, str): + return ignored_risks + return evidence, outcome, helpful_sources, ignored_risks + + +def _parse_json_object(raw: str, field: str) -> dict[str, Any] | str: + try: + parsed = json.loads(raw or "{}") + except json.JSONDecodeError as exc: + return f"{field} must be a JSON object: {exc.msg}" + if not isinstance(parsed, dict): + return f"{field} must be a JSON object" + return parsed + + +def _parse_json_string_list(raw: str, field: str) -> list[str] | str: + try: + parsed = json.loads(raw or "[]") + except json.JSONDecodeError as exc: + return f"{field} must be a JSON list of strings: {exc.msg}" + if not isinstance(parsed, list) or not all(isinstance(item, str) for item in parsed): + return f"{field} must be a JSON list of strings" + return _unique([item.strip() for item in parsed if item.strip()]) + + +def _build_context_feedback( + *, + task: str, + selected_skill: str, + evidence: dict[str, Any], + outcome: dict[str, Any], + helpful_sources: list[str], + ignored_risks: list[str], +) -> _ContextFeedback: + evidence_sources = _evidence_sources(evidence) + outcome_success = outcome.get("success") + if not isinstance(outcome_success, bool): + outcome_success = None + outcome_error_code = str(outcome.get("error_code") or "") + helpful_counts = {source: 1 for source in helpful_sources} + harmful_counts = _harmful_source_counts( + evidence=evidence, + outcome_success=outcome_success, + helpful_sources=helpful_sources, + ignored_risks=ignored_risks, + ) + return _ContextFeedback( + timestamp=time.time(), + task=task, + selected_skill=selected_skill, + outcome_success=outcome_success, + outcome_error_code=outcome_error_code, + evidence_sources=tuple(evidence_sources), + helpful_sources=tuple(helpful_sources), + ignored_risks=tuple(ignored_risks), + helpful_source_counts=helpful_counts, + harmful_source_counts=harmful_counts, + ) + + +def _evidence_sources(evidence: dict[str, Any]) -> list[str]: + sources: list[str] = [] + selected = evidence.get("selected_sources") + if isinstance(selected, list): + sources.extend(str(source) for source in selected if source) + entries = evidence.get("entries") + if isinstance(entries, list): + for entry in entries: + if isinstance(entry, dict) and entry.get("source"): + sources.append(str(entry["source"])) + return _unique(sources) + + +def _harmful_source_counts( + *, + evidence: dict[str, Any], + outcome_success: bool | None, + helpful_sources: list[str], + ignored_risks: list[str], +) -> dict[str, int]: + harmful = set(ignored_risks) + if outcome_success is False: + for entry in evidence.get("entries") or []: + if not isinstance(entry, dict): + continue + source = str(entry.get("source") or "") + if not source or source in helpful_sources: + continue + if entry.get("risk_impact") == "high": + harmful.add(source) + return {source: 1 for source in _unique(list(harmful))} + + +def _feedback_mentions_source(feedback: _ContextFeedback, source: str) -> bool: + return ( + source in feedback.evidence_sources + or source in feedback.helpful_sources + or source in feedback.harmful_source_counts + ) + + +def _unique(items: list[str]) -> list[str]: + seen: set[str] = set() + result: list[str] = [] + for item in items: + if item and item not in seen: + seen.add(item) + result.append(item) + return result + + +__all__ = [ + "CONTEXT_EVIDENCE_SCHEMA", + "CONTEXT_FEEDBACK_SCHEMA", + "ContextFeedbackSpec", + "_Go2ContextFeedbackStore", +] diff --git a/dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/context_provider.py b/dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/context_provider.py index ab0aecb04c..f86a459926 100644 --- a/dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/context_provider.py +++ b/dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/context_provider.py @@ -33,6 +33,13 @@ from dimos.robot.unitree.go2.blueprints.layers.layer_3_agent_brain.causal_world_model import ( CausalWorldModelSpec, ) +from dimos.robot.unitree.go2.blueprints.layers.layer_3_agent_brain.context_evidence import ( + ContextEvidencePolicy, + build_context_evidence, +) +from dimos.robot.unitree.go2.blueprints.layers.layer_3_agent_brain.context_feedback import ( + ContextFeedbackSpec, +) from dimos.robot.unitree.go2.blueprints.layers.layer_3_agent_brain.skill_outcome_store import ( SkillOutcomeStoreSpec, ) @@ -57,6 +64,7 @@ class _Go2ContextProvider(Module): _skill_outcomes: SkillOutcomeStoreSpec | None = None _causal_world_model: CausalWorldModelSpec | None = None _skill_interface: SkillInterfaceSpec | None = None + _context_feedback: ContextFeedbackSpec | None = None _latest_odom: PoseStamped | None = None odom: In[PoseStamped] @@ -99,7 +107,9 @@ def get_context(self, task: str, focus: str = "", spatial_limit: int = 3) -> Ski "robot_state": self._robot_context(errors, world_snapshot), "world_state": self._world_context(task, spatial_limit, errors, world_snapshot), "skill_state": self._skill_context(errors), + "context_feedback": self._context_feedback_context(errors), "causal_state": self._causal_context(errors), + "world_model_state": self._world_model_context(world_snapshot, errors), "external_context": { "available": False, "reason": "External context sources are not wired into ContextProvider yet.", @@ -108,6 +118,7 @@ def get_context(self, task: str, focus: str = "", spatial_limit: int = 3) -> Ski if errors: metadata["errors"] = errors + metadata["context_evidence"] = self._context_evidence(metadata) message = self._format_message(metadata) return SkillResult(success=True, message=message, metadata=_to_jsonable(metadata)) @@ -124,7 +135,9 @@ def _source_status(self, world_snapshot: dict[str, Any] | None) -> dict[str, boo "navigation": self._navigation is not None or bool(layer4_sources.get("navigation")), "skill_outcomes": self._skill_outcomes is not None, "causal_world_model": self._causal_world_model is not None, + "predictive_world_model": self._causal_world_model is not None, "skill_interface": self._skill_interface is not None, + "context_feedback": self._context_feedback is not None, "runtime": True, } @@ -239,6 +252,18 @@ def _skill_interface_context(self, errors: list[str]) -> dict[str, Any]: errors.append(f"skill_interface: {exc}") return {"available": True, "skills": []} + def _context_feedback_context(self, errors: list[str]) -> dict[str, Any]: + if self._context_feedback is None: + return {"available": False, "recent_feedback": [], "summary": {}} + try: + recent = self._context_feedback.get_recent_context_feedback(limit=5) + summary = self._context_feedback.get_context_feedback_summary(limit=20) + except Exception as exc: + logger.warning("Failed to read context-feedback context", exc_info=True) + errors.append(f"context_feedback: {exc}") + return {"available": True, "recent_feedback": [], "summary": {}} + return {"available": True, "recent_feedback": recent, "summary": summary} + def _causal_context(self, errors: list[str]) -> dict[str, Any]: if self._causal_world_model is None: return {"available": False, "recent_transitions": []} @@ -250,9 +275,69 @@ def _causal_context(self, errors: list[str]) -> dict[str, Any]: return {"available": True, "recent_transitions": []} return {"available": True, "recent_transitions": recent} - def _spatial_context( - self, task: str, spatial_limit: int, errors: list[str] + def _world_model_context( + self, world_snapshot: dict[str, Any] | None, errors: list[str] ) -> dict[str, Any]: + if self._causal_world_model is None: + return {"available": False, "transition_count": 0} + + try: + recent = self._causal_world_model.get_recent_transitions(limit=5) + except Exception as exc: + logger.warning("Failed to read world-model transition context", exc_info=True) + errors.append(f"world_model.transitions: {exc}") + recent = [] + + context: dict[str, Any] = { + "available": True, + "transition_count": len(recent), + "recent_transitions": recent, + "snapshot_available": world_snapshot is not None, + "prediction_available": hasattr(self._causal_world_model, "score_action"), + } + get_model_state = getattr(self._causal_world_model, "get_model_state", None) + if get_model_state is not None: + try: + context["model"] = get_model_state() + except Exception as exc: + logger.warning("Failed to read world-model state", exc_info=True) + errors.append(f"world_model.state: {exc}") + + get_intervention_log = getattr(self._causal_world_model, "get_intervention_log", None) + if get_intervention_log is not None: + try: + recent_interventions = get_intervention_log(limit=5) + context["recent_interventions"] = recent_interventions + context["intervention_count"] = _intervention_count( + context.get("model"), + recent_interventions, + ) + except Exception as exc: + logger.warning("Failed to read world-model interventions", exc_info=True) + errors.append(f"world_model.interventions: {exc}") + else: + context["recent_interventions"] = [] + context["intervention_count"] = 0 + + if recent: + failures = [ + transition for transition in recent if transition.get("outcome_success") is False + ] + context["recent_failure_count"] = len(failures) + context["top_failure_mode"] = failures[0].get("inferred_cause") if failures else "" + + return context + + def _context_evidence(self, metadata: dict[str, Any]) -> dict[str, Any]: + """Return the explicit evidence selected for this context response. + + This is intentionally deterministic in V1. It does not replace RAG, + temporal memory, or the world model; it records which retrieved facts + were admitted into the compact context and why. + """ + return build_context_evidence(metadata, ContextEvidencePolicy()) + + def _spatial_context(self, task: str, spatial_limit: int, errors: list[str]) -> dict[str, Any]: if self._spatial_memory is None: return {"available": False, "matches": []} if spatial_limit == 0: @@ -384,11 +469,21 @@ def _format_message(self, metadata: dict[str, Any]) -> str: skill_interface = skill_state.get("interface") or {} if skill_interface.get("available"): + lines.append(f"Skill interface: {skill_interface.get('skill_count', 0)} contract(s)") + else: + lines.append("Skill interface: unavailable") + + context_feedback = metadata.get("context_feedback", {}) + if context_feedback.get("available"): + summary = context_feedback.get("summary") or {} lines.append( - f"Skill interface: {skill_interface.get('skill_count', 0)} contract(s)" + "Context feedback: " + f"{summary.get('total_feedback', 0)} recent, " + f"helpful={summary.get('helpful_source_counts', {})}, " + f"harmful={summary.get('harmful_source_counts', {})}" ) else: - lines.append("Skill interface: unavailable") + lines.append("Context feedback: unavailable") causal_state = metadata.get("causal_state", {}) recent_transitions = causal_state.get("recent_transitions") or [] @@ -407,9 +502,43 @@ def _format_message(self, metadata: dict[str, Any]) -> str: else: lines.append("Causal memory: unavailable") + world_model_state = metadata.get("world_model_state", {}) + if world_model_state.get("available"): + lines.append( + "World model: available, " + f"prediction={world_model_state.get('prediction_available')}, " + f"transitions={world_model_state.get('transition_count', 0)}, " + f"interventions={world_model_state.get('intervention_count', 0)}, " + f"recent_failures={world_model_state.get('recent_failure_count', 0)}, " + f"model_samples={(world_model_state.get('model') or {}).get('sample_count', 0)}" + ) + else: + lines.append("World model: unavailable") + + context_evidence = metadata.get("context_evidence", {}) + if isinstance(context_evidence, dict): + lines.append( + "Context evidence: " + f"{context_evidence.get('entry_count', 0)} item(s), " + f"sources={context_evidence.get('selected_sources', [])}" + ) + return "\n".join(lines) +def _intervention_count( + model_state: Any, + recent_interventions: list[dict[str, Any]], +) -> int: + if isinstance(model_state, dict): + intervention_log = model_state.get("intervention_log") + if isinstance(intervention_log, dict): + count = intervention_log.get("record_count") + if isinstance(count, int): + return count + return len(recent_interventions) + + def _summarize_spatial_match(match: dict[str, Any]) -> dict[str, Any]: summary: dict[str, Any] = {} for key in ("distance", "score", "id", "text"): @@ -422,6 +551,277 @@ def _summarize_spatial_match(match: dict[str, Any]) -> dict[str, Any]: return _to_jsonable(summary) +def _evidence_entry( + *, + source: str, + query: str, + relevance_score: float, + recency: str, + confidence: str, + risk_impact: str, + cost: str, + selected_reason: str, + summary: str, +) -> dict[str, Any]: + return { + "source": source, + "query": query, + "relevance_score": round(max(0.0, min(1.0, relevance_score)), 3), + "recency": recency, + "confidence": _normalize_label(confidence, {"low", "medium", "high"}, "medium"), + "risk_impact": _normalize_label( + risk_impact, + {"low", "medium", "high", "unknown"}, + "unknown", + ), + "cost": cost, + "selected_reason": selected_reason, + "summary": summary[:300], + } + + +def _world_evidence_entries(task: str, world_state: dict[str, Any]) -> list[dict[str, Any]]: + entries: list[dict[str, Any]] = [] + spatial = world_state.get("spatial") if isinstance(world_state.get("spatial"), dict) else {} + matches = spatial.get("matches") if isinstance(spatial.get("matches"), list) else [] + if matches: + first_match = matches[0] if isinstance(matches[0], dict) else {} + entries.append( + _evidence_entry( + source="spatial_memory", + query=task, + relevance_score=_spatial_relevance(first_match), + recency=_match_recency(first_match), + confidence=_spatial_confidence(first_match), + risk_impact="medium", + cost="rag_query", + selected_reason="Top spatial/semantic RAG match for the current task.", + summary=_spatial_summary(first_match), + ) + ) + + temporal = world_state.get("temporal") if isinstance(world_state.get("temporal"), dict) else {} + temporal_summary = str(temporal.get("rolling_summary") or "") + if temporal_summary or temporal.get("state") or temporal.get("answer"): + entries.append( + _evidence_entry( + source="temporal_memory", + query=task, + relevance_score=0.75 if temporal_summary else 0.65, + recency="recent_summary" if temporal_summary else "current_state", + confidence="medium", + risk_impact="medium", + cost="vlm_graph_query" if temporal.get("answer") else "state_read", + selected_reason="Temporal memory contributes recent entities, events, or rolling summary.", + summary=temporal_summary or "temporal state available", + ) + ) + + semantic_temporal = world_state.get("semantic_temporal") + if not isinstance(semantic_temporal, dict): + semantic_temporal = world_state.get("semantic_temporal_map") + if isinstance(semantic_temporal, dict): + fused = semantic_temporal.get("fused") + if isinstance(fused, dict) and fused.get("available"): + entries.append( + _evidence_entry( + source="semantic_temporal_map", + query=task, + relevance_score=0.7, + recency="current_fusion", + confidence="medium", + risk_impact="medium", + cost="rpc_read", + selected_reason="Layer 4 fused spatial and temporal evidence into normalized entries.", + summary=f"fused_entries={fused.get('entry_count', 0)}", + ) + ) + return entries + + +def _skill_evidence_entries(task: str, skill_state: dict[str, Any]) -> list[dict[str, Any]]: + entries: list[dict[str, Any]] = [] + recent_outcomes = ( + skill_state.get("recent_outcomes") + if isinstance(skill_state.get("recent_outcomes"), list) + else [] + ) + if recent_outcomes: + failures = [outcome for outcome in recent_outcomes if not outcome.get("success")] + entries.append( + _evidence_entry( + source="skill_outcome_store", + query=task, + relevance_score=0.9 if failures else 0.65, + recency="recent_history", + confidence="high", + risk_impact="high" if failures else "low", + cost="memory_read", + selected_reason="Recent skill outcomes reveal repeated failures or successful recovery paths.", + summary=f"{len(recent_outcomes)} outcome(s), {len(failures)} failure(s)", + ) + ) + + interface = ( + skill_state.get("interface") if isinstance(skill_state.get("interface"), dict) else {} + ) + if interface.get("available"): + entries.append( + _evidence_entry( + source="skill_interface_registry", + query=task, + relevance_score=0.8, + recency="static_contract", + confidence="high", + risk_impact="medium", + cost="rpc_read", + selected_reason="Skill contracts constrain tool choice, required arguments, and preflight checks.", + summary=f"{interface.get('skill_count', 0)} skill contract(s)", + ) + ) + return entries + + +def _causal_evidence_entries(task: str, causal_state: dict[str, Any]) -> list[dict[str, Any]]: + transitions = ( + causal_state.get("recent_transitions") + if isinstance(causal_state.get("recent_transitions"), list) + else [] + ) + if not transitions: + return [] + failures = [ + transition for transition in transitions if transition.get("outcome_success") is False + ] + return [ + _evidence_entry( + source="causal_world_model", + query=task, + relevance_score=0.88 if failures else 0.7, + recency="recent_transition", + confidence="medium", + risk_impact="high" if failures else "low", + cost="memory_read", + selected_reason="Causal transitions explain whether prior actions changed state as expected.", + summary=f"{len(transitions)} transition(s), {len(failures)} failure(s)", + ) + ] + + +def _world_model_evidence_entries( + task: str, world_model_state: dict[str, Any] +) -> list[dict[str, Any]]: + if not world_model_state.get("available"): + return [] + model = ( + world_model_state.get("model") if isinstance(world_model_state.get("model"), dict) else {} + ) + failures = int(world_model_state.get("recent_failure_count") or 0) + return [ + _evidence_entry( + source="predictive_world_model", + query=task, + relevance_score=0.82 if model.get("sample_count") else 0.6, + recency="online_model", + confidence="medium" if model.get("sample_count") else "low", + risk_impact="high" if failures else "medium", + cost="model_state_read", + selected_reason="Predictive world-model state exposes learned samples and recent failure modes.", + summary=( + f"samples={model.get('sample_count', 0)}, " + f"recent_failures={failures}, " + f"top_failure={world_model_state.get('top_failure_mode', '')}" + ), + ) + ] + + +def _robot_risk_impact(robot_state: dict[str, Any]) -> str: + safety = robot_state.get("safety") if isinstance(robot_state.get("safety"), dict) else {} + navigation = ( + robot_state.get("navigation") if isinstance(robot_state.get("navigation"), dict) else {} + ) + if safety and not safety.get("body_pose_available", True): + return "high" + if navigation and navigation.get("goal_reached") is False: + return "medium" + return "low" + + +def _robot_state_summary(robot_state: dict[str, Any]) -> str: + parts: list[str] = [] + odom = robot_state.get("odom") + if isinstance(odom, dict): + position = odom.get("position") if isinstance(odom.get("position"), dict) else {} + parts.append(f"pose=({position.get('x')}, {position.get('y')})") + navigation = robot_state.get("navigation") + if isinstance(navigation, dict): + parts.append(f"navigation={navigation.get('state')}") + connection = robot_state.get("connection") + if isinstance(connection, dict): + parts.append(f"connection={connection.get('mode')}") + return ", ".join(parts) or "robot state available" + + +def _spatial_relevance(match: dict[str, Any]) -> float: + score = match.get("score") + if isinstance(score, int | float) and not isinstance(score, bool): + return float(score) + distance = match.get("distance") + if isinstance(distance, int | float) and not isinstance(distance, bool): + return 1.0 - min(max(float(distance), 0.0), 1.0) + return 0.6 + + +def _spatial_confidence(match: dict[str, Any]) -> str: + relevance = _spatial_relevance(match) + if relevance >= 0.75: + return "high" + if relevance >= 0.45: + return "medium" + return "low" + + +def _spatial_summary(match: dict[str, Any]) -> str: + metadata = match.get("metadata") + if isinstance(metadata, list) and metadata and isinstance(metadata[0], dict): + metadata = metadata[0] + if isinstance(metadata, dict): + label = metadata.get("label") or metadata.get("description") or metadata.get("frame_id") + if label: + return str(label) + if "pos_x" in metadata and "pos_y" in metadata: + return f"spatial match at x={metadata.get('pos_x')}, y={metadata.get('pos_y')}" + if match.get("id"): + return str(match["id"]) + return "spatial match" + + +def _match_recency(match: dict[str, Any]) -> str: + metadata = match.get("metadata") + if isinstance(metadata, list) and metadata and isinstance(metadata[0], dict): + metadata = metadata[0] + if isinstance(metadata, dict) and metadata.get("timestamp") is not None: + return "timestamped_observation" + return "stored_observation" + + +def _ordered_sources(entries: list[dict[str, Any]]) -> list[str]: + sources: set[str] = set() + for entry in entries: + source = str(entry.get("source") or "") + if source: + sources.add(source) + return sorted(sources) + + +def _normalize_label(value: str, allowed: set[str], fallback: str) -> str: + normalized = value.strip().casefold() + if normalized in allowed: + return normalized + return fallback + + def _to_jsonable(value: Any, max_string_length: int = 500) -> Any: if value is None or isinstance(value, bool | int | float): return value diff --git a/dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/evolution_event.py b/dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/evolution_event.py new file mode 100644 index 0000000000..cf01374e96 --- /dev/null +++ b/dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/evolution_event.py @@ -0,0 +1,173 @@ +#!/usr/bin/env python3 +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import annotations + +from dataclasses import dataclass +from datetime import UTC, datetime +import json +import os +from pathlib import Path +import re +import subprocess +from typing import Any + +from dimos.core.coordination.process_lifecycle import DIMOS_RUN_ID_ENV + +EVOLUTION_EVENT_SCHEMA = "dimos.evolution_event.v1" +EVOLUTION_LEDGER_DIR_ENV = "DIMOS_EVOLUTION_LEDGER_DIR" + + +@dataclass(frozen=True) +class EvolutionEvent: + """One low-volume, reviewable event in the local evolution ledger.""" + + timestamp: float + event_type: str + task: str + run_id: str + payload: dict[str, Any] + commit_requested: bool + commit_sha: str = "" + + def to_dict(self) -> dict[str, Any]: + return { + "schema": EVOLUTION_EVENT_SCHEMA, + "timestamp": round(self.timestamp, 3), + "event_type": self.event_type, + "task": self.task, + "run_id": self.run_id, + "payload": self.payload, + "git": { + "commit_requested": self.commit_requested, + "commit_sha": self.commit_sha, + }, + } + + +def build_evolution_event( + event_type: str, + task: str, + payload: dict[str, Any], + commit_requested: bool, + timestamp: float | None = None, + run_id: str | None = None, +) -> EvolutionEvent: + """Validate and build an evolution event without writing it.""" + event_type = event_type.strip() + if not event_type: + raise ValueError("event_type is required") + if not isinstance(payload, dict): + raise ValueError("payload must be a JSON object") + + return EvolutionEvent( + timestamp=timestamp if timestamp is not None else datetime.now(UTC).timestamp(), + event_type=event_type, + task=task.strip(), + run_id=(run_id if run_id is not None else os.environ.get(DIMOS_RUN_ID_ENV, "")).strip(), + payload=payload, + commit_requested=commit_requested, + ) + + +def resolve_evolution_ledger_dir() -> Path: + """Return the configured ledger directory, defaulting to repo-root `.dimos/evolution`.""" + configured = os.environ.get(EVOLUTION_LEDGER_DIR_ENV, "").strip() + if configured: + return Path(configured).expanduser().resolve() + return _current_git_root() / ".dimos" / "evolution" + + +def write_evolution_event_file( + ledger_dir: Path, + event: EvolutionEvent, +) -> Path: + """Write an event JSON file atomically and return its path.""" + event_day = datetime.fromtimestamp(event.timestamp, UTC) + event_dir = ( + ledger_dir + / "events" + / f"{event_day.year:04d}" + / f"{event_day.month:02d}" + / (f"{event_day.day:02d}") + ) + event_dir.mkdir(parents=True, exist_ok=True) + (ledger_dir / "proposals").mkdir(parents=True, exist_ok=True) + _ensure_readme(ledger_dir) + + path = _unique_event_path(event_dir, event_day, event.event_type) + payload = json.dumps(event.to_dict(), indent=2, sort_keys=True) + "\n" + tmp_path = path.with_name(f".{path.name}.tmp-{os.getpid()}") + tmp_path.write_text(payload, encoding="utf-8") + tmp_path.replace(path) + return path + + +def parse_payload_json(payload_json: str) -> dict[str, Any]: + """Parse the primitive MCP JSON argument into a JSON object payload.""" + try: + payload = json.loads(payload_json or "{}") + except json.JSONDecodeError as exc: + raise ValueError(f"invalid payload_json: {exc.msg}") from exc + if not isinstance(payload, dict): + raise ValueError("payload_json must decode to a JSON object") + return payload + + +def _ensure_readme(ledger_dir: Path) -> None: + readme = ledger_dir / "README.md" + if readme.exists(): + return + readme.write_text( + "# DimOS Evolution Ledger\n\n" + "This directory stores low-volume agent evolution events and human-reviewable " + "proposals. It should not contain raw images, embeddings, database files, or " + "large robot logs.\n", + encoding="utf-8", + ) + + +def _current_git_root() -> Path: + result = subprocess.run( + ["git", "rev-parse", "--show-toplevel"], + check=False, + capture_output=True, + text=True, + ) + if result.returncode == 0: + return Path(result.stdout.strip()).resolve() + return Path.cwd().resolve() + + +def _unique_event_path(event_dir: Path, event_day: datetime, event_type: str) -> Path: + safe_type = re.sub(r"[^A-Za-z0-9_.-]+", "_", event_type).strip("._-") or "event" + stem = f"{event_day.strftime('%Y%m%dT%H%M%S%fZ')}-{safe_type}" + path = event_dir / f"{stem}.json" + suffix = 1 + while path.exists(): + path = event_dir / f"{stem}-{suffix}.json" + suffix += 1 + return path + + +__all__ = [ + "EVOLUTION_EVENT_SCHEMA", + "EVOLUTION_LEDGER_DIR_ENV", + "EvolutionEvent", + "build_evolution_event", + "parse_payload_json", + "resolve_evolution_ledger_dir", + "write_evolution_event_file", +] diff --git a/dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/evolution_ledger.py b/dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/evolution_ledger.py new file mode 100644 index 0000000000..dd50881a67 --- /dev/null +++ b/dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/evolution_ledger.py @@ -0,0 +1,264 @@ +#!/usr/bin/env python3 +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import annotations + +from pathlib import Path +import subprocess +from typing import Any, Protocol + +from dimos.agents.annotation import skill +from dimos.agents.skill_result import SkillResult +from dimos.core.core import rpc +from dimos.core.module import Module +from dimos.robot.unitree.go2.blueprints.layers.layer_3_agent_brain.evolution_event import ( + build_evolution_event, + parse_payload_json, + resolve_evolution_ledger_dir, + write_evolution_event_file, +) +from dimos.robot.unitree.go2.blueprints.layers.layer_3_agent_brain.evolution_proposal import ( + parse_skill_proposal_json, + write_skill_proposal_file, +) +from dimos.spec.utils import Spec + + +class EvolutionLedgerSpec(Spec, Protocol): + """RPC surface for Layer 3 modules that write low-volume evolution events.""" + + def write_evolution_event( + self, + event_type: str, + task: str = "", + payload: dict[str, Any] | None = None, + commit: bool = False, + ) -> dict[str, Any]: ... + + def record_skill_proposal(self, proposal_json: str, commit: bool = False) -> SkillResult: ... + + def write_skill_proposal( + self, + proposal: dict[str, Any], + commit: bool = False, + ) -> dict[str, Any]: ... + + +class _Go2EvolutionLedger(Module): + """Git-backed audit sink for low-volume Go2 agent evolution events.""" + + @skill + def record_evolution_event( + self, + event_type: str, + task: str = "", + payload_json: str = "{}", + commit: bool = False, + ) -> SkillResult: + """Record a low-volume agent evolution event to the local ledger. + + Args: + event_type: Event category, such as task_feasibility or context_feedback. + task: Optional user task or situation that produced the event. + payload_json: JSON object containing the event details. + commit: Whether to create a local Git commit containing only this event file. + """ + try: + payload = parse_payload_json(payload_json) + except ValueError as exc: + return SkillResult.fail("INVALID_INPUT", str(exc)) + + try: + record = self.write_evolution_event( + event_type=event_type, + task=task, + payload=payload, + commit=commit, + ) + except ValueError as exc: + return SkillResult.fail("INVALID_INPUT", str(exc)) + except Exception as exc: + return SkillResult.fail("EXECUTION_FAILED", f"failed to record event: {exc}") + + warning_count = len(record["warnings"]) + message = f"Recorded evolution event {record['event_type']}" + if warning_count: + message += f" with {warning_count} warning(s)" + return SkillResult.ok(message, **record) + + @skill + def record_skill_proposal( + self, + proposal_json: str = "{}", + commit: bool = False, + ) -> SkillResult: + """Record a human-reviewable skill or interface proposal without applying it. + + Args: + proposal_json: JSON object using dimos.skill_proposal.v1. + commit: Whether to create a local Git commit containing only this proposal file. + """ + try: + proposal = parse_skill_proposal_json(proposal_json) + except ValueError as exc: + return SkillResult.fail("INVALID_INPUT", str(exc)) + + try: + record = self.write_skill_proposal(proposal=proposal, commit=commit) + except ValueError as exc: + return SkillResult.fail("INVALID_INPUT", str(exc)) + except Exception as exc: + return SkillResult.fail("EXECUTION_FAILED", f"failed to record proposal: {exc}") + + warning_count = len(record["warnings"]) + message = f"Recorded skill proposal {record['proposal_id']}" + if warning_count: + message += f" with {warning_count} warning(s)" + return SkillResult.ok( + message, + **record, + ) + + @rpc + def write_evolution_event( + self, + event_type: str, + task: str = "", + payload: dict[str, Any] | None = None, + commit: bool = False, + ) -> dict[str, Any]: + """Write one event to the ledger and optionally commit only that file.""" + event_payload = payload or {} + event = build_evolution_event( + event_type=event_type, + task=task, + payload=event_payload, + commit_requested=commit, + ) + ledger_dir = resolve_evolution_ledger_dir() + event_path = write_evolution_event_file(ledger_dir=ledger_dir, event=event) + + warnings: list[str] = [] + commit_sha = "" + if commit: + commit_sha = _commit_event_file( + ledger_dir=ledger_dir, + event_path=event_path, + event_type=event.event_type, + warnings=warnings, + ) + + return { + "schema": event.to_dict()["schema"], + "event_type": event.event_type, + "task": event.task, + "ledger_dir": str(ledger_dir), + "event_path": str(event_path), + "commit_requested": commit, + "commit_sha": commit_sha, + "warnings": warnings, + "event": event.to_dict(), + } + + @rpc + def write_skill_proposal( + self, + proposal: dict[str, Any], + commit: bool = False, + ) -> dict[str, Any]: + """Write one validated skill proposal and optionally commit only that file.""" + ledger_dir = resolve_evolution_ledger_dir() + proposal_path = write_skill_proposal_file( + ledger_dir=ledger_dir, + proposal=proposal, + commit_requested=commit, + ) + proposal_data = parse_skill_proposal_json(proposal_path.read_text(encoding="utf-8")) + + warnings: list[str] = [] + commit_sha = "" + if commit: + commit_sha = _commit_event_file( + ledger_dir=ledger_dir, + event_path=proposal_path, + event_type="skill_proposal", + warnings=warnings, + ) + + return { + "schema": proposal_data["schema"], + "proposal_id": proposal_data["proposal_id"], + "ledger_dir": str(ledger_dir), + "proposal_path": str(proposal_path), + "commit_requested": commit, + "commit_sha": commit_sha, + "warnings": warnings, + "proposal": proposal_data, + } + + +def _commit_event_file( + ledger_dir: Path, + event_path: Path, + event_type: str, + warnings: list[str], +) -> str: + root = _git_root_for_path(ledger_dir) + if root is None: + warnings.append("ledger directory is not inside a Git worktree; event was not committed") + return "" + + ledger_dir = ledger_dir.resolve() + event_path = event_path.resolve() + if not event_path.is_relative_to(ledger_dir): + warnings.append("event path is outside the ledger directory; event was not committed") + return "" + + rel_event_path = event_path.relative_to(root).as_posix() + add_result = _run_git(root, "add", "--", rel_event_path) + if add_result.returncode != 0: + warnings.append(f"git add failed: {add_result.stderr.strip()}") + return "" + + message = f"Record evolution event: {event_type}" + commit_result = _run_git(root, "commit", "-m", message, "--", rel_event_path) + if commit_result.returncode != 0: + warnings.append(f"git commit failed: {commit_result.stderr.strip()}") + return "" + + sha_result = _run_git(root, "rev-parse", "HEAD") + if sha_result.returncode != 0: + warnings.append(f"git rev-parse failed: {sha_result.stderr.strip()}") + return "" + return sha_result.stdout.strip() + + +def _git_root_for_path(path: Path) -> Path | None: + result = _run_git(path, "rev-parse", "--show-toplevel") + if result.returncode != 0: + return None + return Path(result.stdout.strip()).resolve() + + +def _run_git(cwd: Path, *args: str) -> subprocess.CompletedProcess[str]: + return subprocess.run( + ["git", "-C", str(cwd), *args], + check=False, + capture_output=True, + text=True, + ) + + +__all__ = ["EvolutionLedgerSpec", "_Go2EvolutionLedger"] diff --git a/dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/evolution_proposal.py b/dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/evolution_proposal.py new file mode 100644 index 0000000000..c6ee0b5cdc --- /dev/null +++ b/dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/evolution_proposal.py @@ -0,0 +1,176 @@ +#!/usr/bin/env python3 +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import annotations + +from datetime import UTC, datetime +import json +import os +from pathlib import Path +import re +from typing import Any + +SKILL_PROPOSAL_SCHEMA = "dimos.skill_proposal.v1" + +_VALID_RISK_LEVELS = {"low", "medium", "high"} +_LARGE_ARTIFACT_EXTENSIONS = { + ".avi", + ".db", + ".jpeg", + ".jpg", + ".mov", + ".mp4", + ".npy", + ".npz", + ".onnx", + ".pkl", + ".png", + ".pt", + ".sqlite", +} + + +def parse_skill_proposal_json(proposal_json: str) -> dict[str, Any]: + """Parse the primitive MCP JSON argument into a proposal object.""" + try: + proposal = json.loads(proposal_json or "{}") + except json.JSONDecodeError as exc: + raise ValueError(f"invalid proposal_json: {exc.msg}") from exc + if not isinstance(proposal, dict): + raise ValueError("proposal_json must decode to a JSON object") + return proposal + + +def validate_skill_proposal(proposal: dict[str, Any]) -> list[str]: + """Validate that a generated skill change remains reviewable and bounded.""" + errors: list[str] = [] + if not isinstance(proposal, dict): + return ["proposal must be a JSON object"] + + if proposal.get("schema") != SKILL_PROPOSAL_SCHEMA: + errors.append(f"schema must be {SKILL_PROPOSAL_SCHEMA}") + if not str(proposal.get("proposal_id") or "").strip(): + errors.append("proposal_id is required") + if not str(proposal.get("title") or "").strip(): + errors.append("title is required") + if proposal.get("requires_human_review") is not True: + errors.append("requires_human_review must be true") + if proposal.get("risk_level") not in _VALID_RISK_LEVELS: + errors.append("risk_level must be low, medium, or high") + + target_files = proposal.get("target_files") + if not isinstance(target_files, list) or not target_files: + errors.append("target_files must be a non-empty list") + else: + for index, target in enumerate(target_files): + errors.extend(_validate_target_file(index, target)) + + generated_patch = proposal.get("generated_patch", "") + if generated_patch is not None and not isinstance(generated_patch, str): + errors.append("generated_patch must be a string") + return errors + + +def normalize_skill_proposal( + proposal: dict[str, Any], + commit_requested: bool = False, + commit_sha: str = "", + created_at: float | None = None, +) -> dict[str, Any]: + """Validate and normalize a proposal before writing it to the ledger.""" + errors = validate_skill_proposal(proposal) + if errors: + raise ValueError("; ".join(errors)) + + normalized = dict(proposal) + normalized["proposal_id"] = str(normalized["proposal_id"]).strip() + normalized["title"] = str(normalized["title"]).strip() + normalized["summary"] = str(normalized.get("summary") or "").strip() + normalized["target_files"] = [str(target).strip() for target in normalized["target_files"]] + normalized["requires_human_review"] = True + normalized.setdefault( + "created_at", + round(created_at if created_at is not None else datetime.now(UTC).timestamp(), 3), + ) + normalized["git"] = { + "commit_requested": commit_requested, + "commit_sha": commit_sha, + } + return normalized + + +def write_skill_proposal_file( + ledger_dir: Path, + proposal: dict[str, Any], + commit_requested: bool = False, +) -> Path: + """Write a validated proposal under `.dimos/evolution/proposals`.""" + normalized = normalize_skill_proposal(proposal, commit_requested=commit_requested) + proposal_dir = ledger_dir / "proposals" + proposal_dir.mkdir(parents=True, exist_ok=True) + _ensure_readme(ledger_dir) + + path = proposal_dir / f"{_safe_filename(normalized['proposal_id'])}.json" + payload = json.dumps(normalized, indent=2, sort_keys=True) + "\n" + tmp_path = path.with_name(f".{path.name}.tmp-{os.getpid()}") + tmp_path.write_text(payload, encoding="utf-8") + tmp_path.replace(path) + return path + + +def _validate_target_file(index: int, target: object) -> list[str]: + errors: list[str] = [] + if not isinstance(target, str) or not target.strip(): + return [f"target_files[{index}] must be a non-empty string"] + + target_path = target.strip().replace("\\", "/") + parts = [part for part in target_path.split("/") if part] + suffix = Path(target_path).suffix.casefold() + + if Path(target_path).is_absolute(): + errors.append(f"target_files[{index}] must be a relative path") + if ".." in parts: + errors.append(f"target_files[{index}] must not contain '..'") + if target_path.startswith(".dimos/evolution/events/"): + errors.append(f"target_files[{index}] must not target evolution event logs") + if suffix in _LARGE_ARTIFACT_EXTENSIONS: + errors.append(f"target_files[{index}] must not target large binary artifacts") + return errors + + +def _ensure_readme(ledger_dir: Path) -> None: + readme = ledger_dir / "README.md" + if readme.exists(): + return + readme.write_text( + "# DimOS Evolution Ledger\n\n" + "This directory stores low-volume agent evolution events and human-reviewable " + "proposals. It should not contain raw images, embeddings, database files, or " + "large robot logs.\n", + encoding="utf-8", + ) + + +def _safe_filename(value: str) -> str: + return re.sub(r"[^A-Za-z0-9_.-]+", "_", value).strip("._-") or "proposal" + + +__all__ = [ + "SKILL_PROPOSAL_SCHEMA", + "normalize_skill_proposal", + "parse_skill_proposal_json", + "validate_skill_proposal", + "write_skill_proposal_file", +] diff --git a/dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/memory_backend_status.py b/dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/memory_backend_status.py new file mode 100644 index 0000000000..bc5bb0abe4 --- /dev/null +++ b/dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/memory_backend_status.py @@ -0,0 +1,225 @@ +#!/usr/bin/env python3 +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import annotations + +from typing import Any + +from dimos.agents.annotation import skill +from dimos.agents.skill_result import SkillResult +from dimos.core.module import Module +from dimos.perception.spatial_memory_spec import SpatialMemorySpec +from dimos.perception.temporal_memory_spec import TemporalMemorySpec +from dimos.robot.unitree.go2.blueprints.layers.layer_3_agent_brain.causal_world_model import ( + CausalWorldModelSpec, +) +from dimos.robot.unitree.go2.blueprints.layers.layer_3_agent_brain.skill_outcome_store import ( + SkillOutcomeStoreSpec, +) +from dimos.robot.unitree.go2.blueprints.layers.layer_4_world_state.world_state_spec import ( + WorldStateSpec, +) +from dimos.robot.unitree.go2.blueprints.layers.layer_5_skill_interface.skill_interface_spec import ( + SkillInterfaceSpec, +) + +_STATUS_PROBE_QUERY = "__memory_status_probe__" + + +class _Go2MemoryBackendStatus(Module): + """Read-only Layer 3 view of Go2 memory/backend readiness.""" + + _world_state: WorldStateSpec | None = None + _spatial_memory: SpatialMemorySpec | None = None + _temporal_memory: TemporalMemorySpec | None = None + _skill_outcomes: SkillOutcomeStoreSpec | None = None + _causal_world_model: CausalWorldModelSpec | None = None + _skill_interface: SkillInterfaceSpec | None = None + + @skill + def memory_backend_status(self) -> SkillResult: + """Report which DimOS memory backends are wired and how much data they expose.""" + warnings: list[str] = [] + metadata = { + "schema": "go2_memory_backend_status.v1", + "world_state": self._world_state_status(warnings), + "spatial_memory": self._spatial_memory_status(warnings), + "temporal_memory": self._temporal_memory_status(warnings), + "skill_outcomes": self._skill_outcomes_status(warnings), + "causal_world_model": self._causal_world_model_status(warnings), + "skill_interface": self._skill_interface_status(warnings), + "warnings": warnings, + } + wired_count = sum( + 1 + for key in ( + "world_state", + "spatial_memory", + "temporal_memory", + "skill_outcomes", + "causal_world_model", + "skill_interface", + ) + if metadata[key]["wired"] + ) + return SkillResult.ok( + f"Memory backends: {wired_count} wired, {len(warnings)} warning(s)", + **metadata, + ) + + def _world_state_status(self, warnings: list[str]) -> dict[str, Any]: + status: dict[str, Any] = { + "wired": self._world_state is not None, + "snapshot_available": False, + "sources": {}, + "snapshot_storage": {}, + } + if self._world_state is None: + return status + try: + snapshot = self._world_state.get_world_snapshot( + task="memory backend status", + spatial_limit=0, + ) + except Exception as exc: + warnings.append(f"world_state: {exc}") + return status + status["snapshot_available"] = True + status["sources"] = _dict_or_empty(snapshot.get("sources")) + status["snapshot_storage"] = _dict_or_empty(snapshot.get("snapshot_storage")) + return status + + def _spatial_memory_status(self, warnings: list[str]) -> dict[str, Any]: + status: dict[str, Any] = { + "wired": self._spatial_memory is not None, + "query_available": False, + "match_count_probe": 0, + "backend_hint": "", + } + if self._spatial_memory is None: + return status + try: + matches = self._spatial_memory.query_by_text(_STATUS_PROBE_QUERY, limit=1) + except Exception as exc: + warnings.append(f"spatial_memory: {exc}") + return status + status["query_available"] = True + status["match_count_probe"] = len(matches) if isinstance(matches, list) else 0 + status["backend_hint"] = "SpatialMemorySpec.query_by_text" + return status + + def _temporal_memory_status(self, warnings: list[str]) -> dict[str, Any]: + status: dict[str, Any] = { + "wired": self._temporal_memory is not None, + "entity_count": 0, + "graph_stats_available": False, + "graph_stats": {}, + } + if self._temporal_memory is None: + return status + try: + state = self._temporal_memory.get_state() + status["entity_count"] = _entity_count(state) + except Exception as exc: + warnings.append(f"temporal_memory.state: {exc}") + try: + graph_stats = self._temporal_memory.get_graph_db_stats() + except Exception as exc: + warnings.append(f"temporal_memory.graph: {exc}") + return status + status["graph_stats_available"] = True + status["graph_stats"] = _dict_or_empty(graph_stats.get("stats", graph_stats)) + return status + + def _skill_outcomes_status(self, warnings: list[str]) -> dict[str, Any]: + status: dict[str, Any] = { + "wired": self._skill_outcomes is not None, + "recent_count": 0, + } + if self._skill_outcomes is None: + return status + try: + recent = self._skill_outcomes.get_recent_outcomes(limit=5) + except Exception as exc: + warnings.append(f"skill_outcomes: {exc}") + return status + status["recent_count"] = len(recent) if isinstance(recent, list) else 0 + return status + + def _causal_world_model_status(self, warnings: list[str]) -> dict[str, Any]: + status: dict[str, Any] = { + "wired": self._causal_world_model is not None, + "transition_count": 0, + "sample_count": 0, + "persistence_path": "", + "autosave": False, + } + if self._causal_world_model is None: + return status + try: + transitions = self._causal_world_model.get_recent_transitions(limit=5) + except Exception as exc: + warnings.append(f"causal_world_model.transitions: {exc}") + transitions = [] + status["transition_count"] = len(transitions) if isinstance(transitions, list) else 0 + + get_model_state = getattr(self._causal_world_model, "get_model_state", None) + if get_model_state is None: + return status + try: + model_state = get_model_state() + except Exception as exc: + warnings.append(f"causal_world_model.state: {exc}") + return status + status["sample_count"] = int(model_state.get("sample_count") or 0) + persistence = _dict_or_empty(model_state.get("persistence")) + status["persistence_path"] = str(persistence.get("path") or "") + status["autosave"] = bool(persistence.get("autosave")) + return status + + def _skill_interface_status(self, warnings: list[str]) -> dict[str, Any]: + status: dict[str, Any] = { + "wired": self._skill_interface is not None, + "skill_count": 0, + } + if self._skill_interface is None: + return status + try: + snapshot = self._skill_interface.get_skill_interface_snapshot() + except Exception as exc: + warnings.append(f"skill_interface: {exc}") + return status + status["skill_count"] = int(snapshot.get("skill_count") or 0) + return status + + +def _dict_or_empty(value: Any) -> dict[str, Any]: + return value if isinstance(value, dict) else {} + + +def _entity_count(state: dict[str, Any]) -> int: + count = state.get("entity_count") + if isinstance(count, int): + return count + entities = state.get("entities") + if isinstance(entities, list): + return len(entities) + entity_roster = state.get("entity_roster") + if isinstance(entity_roster, list): + return len(entity_roster) + return 0 + + +__all__ = ["_Go2MemoryBackendStatus"] diff --git a/dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/prompt_policy.py b/dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/prompt_policy.py index fcd70a627a..7a37e6af11 100644 --- a/dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/prompt_policy.py +++ b/dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/prompt_policy.py @@ -25,24 +25,32 @@ Use the Layer 3 tools as the decision scaffold before calling physical Go2 skills. -1. For any non-trivial user goal, call `route_task(task)` first. -2. If the route says context is needed, or if the task may involve movement, +1. For any non-trivial user goal, call `evaluate_task_feasibility(task)` first. + If feasibility returns `no`, refuse or stop as directed. If it returns + `uncertain`, gather the requested context or ask the clarifying question. +2. Call `route_task(task)` before choosing candidate tools. +3. If the route says context is needed, or if the task may involve movement, perception, memory, safety, or recovery, call `get_context(task, focus)`. -3. Before calling movement-sensitive or recovery-sensitive tools such as +4. Re-run `evaluate_task_feasibility(task, context_json)` when new context + materially changes whether the task is safe or possible. +5. Before calling movement-sensitive or recovery-sensitive tools such as `navigate_with_text`, `relative_move`, `follow_person`, `look_out_for`, or security patrol tools, call `predict_skill_outcome(skill_name, args_json, context)`. -4. If prediction returns high risk, gather more context, ask the user for +6. If prediction returns high risk, gather more context, ask the user for clarification, or choose a safer recovery tool instead of blindly retrying. -5. Skill outcomes are recorded automatically when the outcome store is present. +7. Skill outcomes are recorded automatically when the outcome store is present. Only call `record_skill_outcome(...)` manually for external events or important results that were not produced by an MCP tool call. -6. After important physical or recovery-sensitive tool calls, call +8. After important physical or recovery-sensitive tool calls, call `get_context(task, focus)` again for after-context, then call `record_causal_transition(...)` with the before-context, prediction, tool result, and after-context. -7. Before retrying a failed skill, call `summarize_causal_patterns(...)` to +9. Before retrying a failed skill, call `summarize_causal_patterns(...)` to check whether the same failure cause is repeating. +10. Only after repeated evidence shows a missing capability, call + `propose_skill_interface(...)`. Treat it as a human-review proposal only; it + must not be used as executable robot code. Do not use Layer 3 tools as a substitute for physical safety checks. Stop or cancel active motion immediately when the user asks for it or when the task is diff --git a/dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/skill_proposal.py b/dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/skill_proposal.py new file mode 100644 index 0000000000..5f480009b1 --- /dev/null +++ b/dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/skill_proposal.py @@ -0,0 +1,311 @@ +#!/usr/bin/env python3 +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import annotations + +import json +import re +from typing import Any + +from dimos.agents.annotation import skill +from dimos.agents.skill_result import SkillResult +from dimos.core.module import Module +from dimos.robot.unitree.go2.blueprints.layers.layer_3_agent_brain.evolution_ledger import ( + EvolutionLedgerSpec, +) +from dimos.robot.unitree.go2.blueprints.layers.layer_3_agent_brain.evolution_proposal import ( + SKILL_PROPOSAL_SCHEMA, + validate_skill_proposal, +) +from dimos.robot.unitree.go2.blueprints.layers.layer_3_agent_brain.skill_outcome_store import ( + SkillOutcomeStoreSpec, +) +from dimos.robot.unitree.go2.blueprints.layers.layer_5_skill_interface.skill_interface_spec import ( + SkillInterfaceSpec, +) + +_SKILL_INTERFACE_TARGET = ( + "dimos/robot/unitree/go2/blueprints/layers/layer_5_skill_interface/skill_interface_registry.py" +) + + +class _Go2SkillProposalGenerator(Module): + """Proposal-only generator for Go2 skill-interface improvements.""" + + _skill_interface: SkillInterfaceSpec | None = None + _skill_outcomes: SkillOutcomeStoreSpec | None = None + _evolution_ledger: EvolutionLedgerSpec | None = None + + @skill + def propose_skill_interface( + self, + task: str, + failure_context_json: str = "{}", + ) -> SkillResult: + """Propose a new or revised skill interface without changing executable code. + + Args: + task: User task or repeated failure scenario. + failure_context_json: JSON object describing the observed skill gap. + """ + task = task.strip() + if not task: + return SkillResult.fail("INVALID_INPUT", "task is required") + + failure_context = _parse_context(failure_context_json) + if isinstance(failure_context, str): + return SkillResult.fail("INVALID_INPUT", failure_context) + + contracts = self._skill_contracts() + existing_matches = _existing_skill_matches(task, failure_context, contracts) + if existing_matches: + return SkillResult.ok( + "Existing skill contract appears to cover this capability", + proposal_created=False, + proposal={}, + existing_skill_matches=existing_matches, + recommended_next_action="use_existing_skill_contract", + ) + + misuse_reason = self._existing_skill_misuse_reason(failure_context) + if misuse_reason: + return SkillResult.ok( + "No new skill proposal created; use the existing skill contract correctly", + proposal_created=False, + proposal={}, + existing_skill_matches=[], + recommended_next_action="use_existing_skill_correctly", + reason=misuse_reason, + ) + + if not _is_missing_capability(failure_context): + return SkillResult.ok( + "No skill proposal created; missing capability evidence is insufficient", + proposal_created=False, + proposal={}, + existing_skill_matches=[], + recommended_next_action="collect_more_feedback", + ) + + proposal = _build_skill_proposal(task=task, failure_context=failure_context) + errors = validate_skill_proposal(proposal) + if errors: + return SkillResult.fail("INVALID_INPUT", "; ".join(errors)) + + metadata: dict[str, Any] = { + "proposal_created": True, + "proposal": proposal, + "existing_skill_matches": [], + "recommended_next_action": "review_proposal", + } + ledger_warning = self._record_proposal(proposal, metadata) + if ledger_warning: + metadata["warnings"] = [ledger_warning] + return SkillResult.ok(f"Proposed skill interface {proposal['proposal_id']}", **metadata) + + def _skill_contracts(self) -> list[dict[str, Any]]: + if self._skill_interface is None: + return [] + try: + snapshot = self._skill_interface.get_skill_interface_snapshot() + except Exception: + return [] + return [skill for skill in snapshot.get("skills") or [] if isinstance(skill, dict)] + + def _existing_skill_misuse_reason(self, failure_context: dict[str, Any]) -> str: + failure_type = str(failure_context.get("failure_type") or "").strip() + if failure_type in {"missing_argument", "missing_context", "invalid_argument"}: + return failure_type + if self._skill_outcomes is None: + return "" + try: + outcomes = self._skill_outcomes.get_recent_outcomes(limit=5) + except Exception: + return "" + misuse_markers = ("missing required argument", "invalid argument", "context required") + repeated = [ + outcome + for outcome in outcomes + if any( + marker in str(outcome.get("message") or "").casefold() for marker in misuse_markers + ) + or outcome.get("error_code") == "INVALID_INPUT" + ] + if len(repeated) >= 2: + return "repeated_existing_skill_misuse" + return "" + + def _record_proposal(self, proposal: dict[str, Any], metadata: dict[str, Any]) -> str: + if self._evolution_ledger is None: + return "" + record_skill_proposal = getattr(self._evolution_ledger, "record_skill_proposal", None) + if record_skill_proposal is None: + return "evolution_ledger: record_skill_proposal is unavailable" + try: + result = record_skill_proposal(proposal_json=json.dumps(proposal), commit=False) + except Exception as exc: + return f"evolution_ledger: {exc}" + if not getattr(result, "success", False): + return f"evolution_ledger: {getattr(result, 'message', 'proposal write failed')}" + result_metadata = getattr(result, "metadata", {}) + if isinstance(result_metadata, dict) and result_metadata.get("proposal_path"): + metadata["proposal_path"] = result_metadata["proposal_path"] + return "" + + +def _parse_context(failure_context_json: str) -> dict[str, Any] | str: + try: + parsed = json.loads(failure_context_json or "{}") + except json.JSONDecodeError as exc: + return f"failure_context_json must be a JSON object: {exc.msg}" + if not isinstance(parsed, dict): + return "failure_context_json must be a JSON object" + return parsed + + +def _is_missing_capability(failure_context: dict[str, Any]) -> bool: + failure_type = str(failure_context.get("failure_type") or "").strip() + if failure_type in {"skill_missing_capability", "missing_capability", "unknown_skill"}: + return True + message = str(failure_context.get("message") or "").casefold() + return "missing capability" in message or "no skill" in message or "unknown skill" in message + + +def _existing_skill_matches( + task: str, + failure_context: dict[str, Any], + contracts: list[dict[str, Any]], +) -> list[str]: + capability = _capability_text(task, failure_context) + capability_tokens = _tokens(capability) + if not capability_tokens: + return [] + matches: list[str] = [] + for contract in contracts: + name = str(contract.get("skill_name") or "") + searchable = " ".join( + [ + name.replace("_", " "), + str(contract.get("summary") or ""), + str(contract.get("domain") or ""), + ] + ) + contract_tokens = _tokens(searchable) + if capability_tokens.issubset(contract_tokens): + matches.append(name) + return sorted(match for match in matches if match) + + +def _build_skill_proposal(task: str, failure_context: dict[str, Any]) -> dict[str, Any]: + capability = _capability_text(task, failure_context) + skill_name = _skill_name_from_capability(capability) + domain = str(failure_context.get("domain") or _infer_domain(task, capability)).strip() + risk_class = _risk_class(domain) + required_args = failure_context.get("required_args") + if not isinstance(required_args, list) or not all( + isinstance(arg, str) for arg in required_args + ): + required_args = ["target"] + + proposal = { + "schema": SKILL_PROPOSAL_SCHEMA, + "proposal_id": f"go2-skill-{skill_name}", + "title": f"Add Go2 skill interface for {capability}", + "summary": f"Repeated evidence suggests the current Go2 skill set cannot {capability}.", + "risk_level": risk_class, + "target_files": [_SKILL_INTERFACE_TARGET], + "requires_human_review": True, + "generated_patch": "", + "task": task, + "problem": str( + failure_context.get("problem") + or failure_context.get("message") + or f"Missing capability: {capability}" + ), + "existing_skill_gaps": _existing_skill_gaps(failure_context, capability), + "proposed_interface": { + "skill_name": skill_name, + "domain": domain, + "required_args": required_args, + "optional_args": failure_context.get("optional_args") + if isinstance(failure_context.get("optional_args"), list) + else [], + "risk_class": risk_class, + "recommended_preflight": _recommended_preflight(risk_class), + }, + "validation_plan": [ + "Add a Layer 5 static contract first.", + "Add unit tests for argument validation and MCP tool exposure.", + "Validate in replay or simulation before hardware use.", + "Require human review before adding executable robot code.", + ], + } + return proposal + + +def _capability_text(task: str, failure_context: dict[str, Any]) -> str: + return str(failure_context.get("capability") or task).strip() + + +def _skill_name_from_capability(capability: str) -> str: + normalized = re.sub(r"[^a-zA-Z0-9]+", "_", capability.casefold()).strip("_") + return normalized or "proposed_skill" + + +def _infer_domain(task: str, capability: str) -> str: + text = f"{task} {capability}".casefold() + if any(word in text for word in ("door", "grab", "pick", "place", "open")): + return "manipulation" + if any(word in text for word in ("walk", "navigate", "move", "follow")): + return "navigation" + if any(word in text for word in ("look", "detect", "recognize", "inspect")): + return "perception" + return "general" + + +def _risk_class(domain: str) -> str: + if domain in {"manipulation", "robot_motion", "navigation", "security"}: + return "high" + if domain in {"perception", "person_follow"}: + return "medium" + return "low" + + +def _recommended_preflight(risk_class: str) -> list[str]: + if risk_class == "high": + return ["evaluate_task_feasibility", "get_context", "predict_skill_outcome"] + if risk_class == "medium": + return ["get_context", "predict_skill_outcome"] + return ["route_task"] + + +def _existing_skill_gaps(failure_context: dict[str, Any], capability: str) -> list[str]: + gaps = failure_context.get("existing_skill_gaps") + if isinstance(gaps, list) and all(isinstance(gap, str) for gap in gaps): + return gaps + evidence = failure_context.get("evidence") + if isinstance(evidence, list) and all(isinstance(item, str) for item in evidence): + return evidence + return [f"No existing skill contract covers capability: {capability}"] + + +def _tokens(value: str) -> set[str]: + return { + token[:-1] if token.endswith("s") and len(token) > 3 else token + for token in re.findall(r"[a-z0-9]+", value.casefold()) + } + + +__all__ = ["_Go2SkillProposalGenerator"] diff --git a/dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/task_feasibility.py b/dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/task_feasibility.py new file mode 100644 index 0000000000..c05b35ca1d --- /dev/null +++ b/dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/task_feasibility.py @@ -0,0 +1,345 @@ +#!/usr/bin/env python3 +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import annotations + +import json +from typing import Any + +from dimos.agents.annotation import skill +from dimos.agents.skill_result import SkillResult +from dimos.core.module import Module +from dimos.robot.unitree.go2.blueprints.layers.layer_3_agent_brain.evolution_ledger import ( + EvolutionLedgerSpec, +) +from dimos.robot.unitree.go2.blueprints.layers.layer_5_skill_interface.skill_interface_spec import ( + SkillInterfaceSpec, +) + + +class _Go2TaskFeasibilityEvaluator(Module): + """Rule-based Layer 3 preflight for user task feasibility.""" + + _skill_interface: SkillInterfaceSpec | None = None + _evolution_ledger: EvolutionLedgerSpec | None = None + + @skill + def evaluate_task_feasibility( + self, + task: str, + context_json: str = "{}", + ) -> SkillResult: + """Evaluate whether the current task is feasible before selecting physical skills. + + Args: + task: User task or goal to evaluate. + context_json: Optional JSON object from get_context metadata. + """ + task = task.strip() + if not task: + return SkillResult.fail("INVALID_INPUT", "task is required") + + parsed_context = _parse_context_json(context_json) + if isinstance(parsed_context, str): + return SkillResult.fail("INVALID_INPUT", parsed_context) + + contracts, contract_warnings = self._skill_contracts() + required_skills = _required_skills_for_task(task) + available_skills = _available_required_skills(required_skills, contracts) + missing_skills = [skill for skill in required_skills if skill not in available_skills] + safety_risks = _safety_risks(task) + missing_context = _missing_context(required_skills, contracts, parsed_context) + + if _is_motion_task(required_skills, contracts) and "robot_state" in missing_context: + safety_risks.append("movement task requires current robot state") + + feasible = _feasible_value(required_skills, missing_skills, missing_context, safety_risks) + recommended_next_action = _recommended_next_action( + feasible=feasible, + missing_context=missing_context, + missing_skills=missing_skills, + safety_risks=safety_risks, + ) + clarifying_question = _clarifying_question( + recommended_next_action=recommended_next_action, + required_skills=required_skills, + missing_skills=missing_skills, + ) + evidence_sources = _evidence_sources(parsed_context) + if contracts: + evidence_sources.append("skill_interface") + evidence_sources = _unique(evidence_sources) + + metadata: dict[str, Any] = { + "feasible": feasible, + "missing_context": missing_context, + "required_skills": required_skills, + "available_skills": available_skills, + "safety_risks": safety_risks, + "recommended_next_action": recommended_next_action, + "clarifying_question": clarifying_question, + "evidence_sources": evidence_sources, + "missing_skills": missing_skills, + "skill_interface_available": self._skill_interface is not None, + } + if contract_warnings: + metadata["warnings"] = contract_warnings + + ledger_warning = self._record_feasibility_event(task=task, payload=metadata) + if ledger_warning: + metadata.setdefault("warnings", []).append(ledger_warning) + + message = ( + f"Task feasibility: {feasible}; next_action={recommended_next_action}; " + f"required_skills={required_skills or ['unknown']}" + ) + return SkillResult.ok(message, **metadata) + + def _skill_contracts(self) -> tuple[dict[str, dict[str, Any]], list[str]]: + if self._skill_interface is None: + return {}, [] + try: + snapshot = self._skill_interface.get_skill_interface_snapshot() + except Exception as exc: + return {}, [f"skill_interface: {exc}"] + contracts: dict[str, dict[str, Any]] = {} + for item in snapshot.get("skills") or []: + if isinstance(item, dict) and isinstance(item.get("skill_name"), str): + contracts[item["skill_name"]] = item + return contracts, [] + + def _record_feasibility_event(self, task: str, payload: dict[str, Any]) -> str: + if self._evolution_ledger is None: + return "" + try: + self._evolution_ledger.write_evolution_event( + event_type="task_feasibility", + task=task, + payload=payload, + commit=False, + ) + except Exception as exc: + return f"evolution_ledger: {exc}" + return "" + + +def _parse_context_json(context_json: str) -> dict[str, Any] | str: + if not context_json.strip(): + return {} + try: + parsed = json.loads(context_json) + except json.JSONDecodeError as exc: + return f"context_json must be a JSON object: {exc.msg}" + if not isinstance(parsed, dict): + return "context_json must be a JSON object" + metadata = parsed.get("metadata") + if isinstance(metadata, dict): + return metadata + return parsed + + +def _required_skills_for_task(task: str) -> list[str]: + text = task.casefold() + if any(word in text for word in ("stop", "cancel", "halt")): + return ["stop_navigation"] + if any(word in text for word in ("say", "speak", "tell", "announce")): + return ["speak"] + if "follow" in text: + return ["follow_person"] + if any(word in text for word in ("patrol", "security")): + return ["start_security_patrol" if "security" in text else "start_patrol"] + if any(word in text for word in ("look for", "look out", "find", "search", "detect")): + return ["look_out_for"] + if any(word in text for word in ("walk to", "go to", "navigate", "kitchen", "room")): + return ["navigate_with_text"] + if any(word in text for word in ("move", "turn", "forward", "backward", "left", "right")): + return ["relative_move"] + if "wait" in text: + return ["wait"] + if "time" in text: + return ["current_time"] + return [] + + +def _available_required_skills( + required_skills: list[str], + contracts: dict[str, dict[str, Any]], +) -> list[str]: + if not contracts: + return [] + return [skill for skill in required_skills if skill in contracts] + + +def _missing_context( + required_skills: list[str], + contracts: dict[str, dict[str, Any]], + context: dict[str, Any], +) -> list[str]: + missing: list[str] = [] + for skill_name in required_skills: + contract = contracts.get(skill_name, {}) + if contract.get("requires_robot_state") and not _robot_state_available(context): + missing.append("robot_state") + if contract.get("requires_world_state") and not _world_state_available(context): + missing.append("world_state") + if contract.get("requires_context") and not context: + missing.append("context_evidence") + return _unique(missing) + + +def _robot_state_available(context: dict[str, Any]) -> bool: + robot_state = context.get("robot_state") + if not isinstance(robot_state, dict): + return False + for key in ("odom_available", "pose_available"): + value = robot_state.get(key) + if isinstance(value, bool): + return value + if robot_state.get("pose") and robot_state.get("pose") != "unavailable": + return True + if robot_state.get("odom"): + return True + return False + + +def _world_state_available(context: dict[str, Any]) -> bool: + world_state = context.get("world_state") + if isinstance(world_state, dict) and any(world_state.values()): + return True + sources = context.get("sources") + if isinstance(sources, dict): + return any( + bool(sources.get(key)) + for key in ("spatial_memory", "temporal_memory", "semantic_temporal_map") + ) + evidence = context.get("context_evidence") + if isinstance(evidence, dict): + selected = evidence.get("selected_sources") + if isinstance(selected, list): + return any( + source in selected + for source in ("spatial_memory", "temporal_memory", "structured_world_state") + ) + return False + + +def _safety_risks(task: str) -> list[str]: + text = task.casefold() + unsafe_markers = ( + "disable safety", + "turn off safety", + "run into", + "hit ", + "harm", + "crash into", + "jump off", + "attack", + ) + if any(marker in text for marker in unsafe_markers): + return ["task requests unsafe or harmful robot behavior"] + return [] + + +def _is_motion_task( + required_skills: list[str], + contracts: dict[str, dict[str, Any]], +) -> bool: + for skill_name in required_skills: + contract = contracts.get(skill_name, {}) + if contract.get("motion_sensitive"): + return True + if skill_name in { + "navigate_with_text", + "relative_move", + "follow_person", + "start_patrol", + "start_security_patrol", + }: + return True + return False + + +def _feasible_value( + required_skills: list[str], + missing_skills: list[str], + missing_context: list[str], + safety_risks: list[str], +) -> str: + if any("unsafe or harmful" in risk for risk in safety_risks): + return "no" + if missing_skills: + return "no" + if not required_skills or missing_context or safety_risks: + return "uncertain" + return "yes" + + +def _recommended_next_action( + feasible: str, + missing_context: list[str], + missing_skills: list[str], + safety_risks: list[str], +) -> str: + if any("unsafe or harmful" in risk for risk in safety_risks): + return "refuse" + if missing_context: + return "get_context" + if missing_skills or feasible == "uncertain": + return "ask_clarifying_question" + if feasible == "no": + return "refuse" + return "proceed" + + +def _clarifying_question( + recommended_next_action: str, + required_skills: list[str], + missing_skills: list[str], +) -> str: + if recommended_next_action != "ask_clarifying_question": + return "" + if not required_skills: + return "Which concrete robot action or observation should I perform?" + if missing_skills: + return f"No current skill contract covers {missing_skills[0]}; what outcome do you need?" + return "What extra context should I use before choosing a robot skill?" + + +def _evidence_sources(context: dict[str, Any]) -> list[str]: + evidence = context.get("context_evidence") + sources: list[str] = [] + if isinstance(evidence, dict): + selected = evidence.get("selected_sources") + if isinstance(selected, list): + sources.extend(str(source) for source in selected if source) + entries = evidence.get("entries") + if isinstance(entries, list): + for entry in entries: + if isinstance(entry, dict) and entry.get("source"): + sources.append(str(entry["source"])) + return _unique(sources) + + +def _unique(items: list[str]) -> list[str]: + seen: set[str] = set() + result: list[str] = [] + for item in items: + if item and item not in seen: + seen.add(item) + result.append(item) + return result + + +__all__ = ["_Go2TaskFeasibilityEvaluator"] diff --git a/dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/test_context_evidence.py b/dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/test_context_evidence.py new file mode 100644 index 0000000000..499161ce56 --- /dev/null +++ b/dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/test_context_evidence.py @@ -0,0 +1,114 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import annotations + +from typing import Any + +from dimos.robot.unitree.go2.blueprints.layers.layer_3_agent_brain.context_evidence import ( + ContextEvidencePolicy, + build_context_evidence, +) + + +def _metadata(task: str = "walk to the kitchen") -> dict[str, Any]: + return { + "task": task, + "focus": "navigation", + "runtime": {"mode": "hardware"}, + "robot_state": { + "odom": {"position": {"x": 1.0, "y": 2.0}}, + "navigation": {"state": "idle", "goal_reached": True}, + "safety": {"body_pose_available": True}, + }, + "world_state": { + "spatial": { + "available": True, + "matches": [ + { + "distance": 0.2, + "metadata": {"label": "kitchen"}, + } + ], + }, + "temporal": {"available": True, "rolling_summary": "Kitchen was seen."}, + "semantic_temporal": {"fused": {"available": True, "entry_count": 2}}, + }, + "skill_state": { + "recent_outcomes": [{"skill_name": "navigate_with_text", "success": False}], + "interface": {"available": True, "skill_count": 3}, + }, + "causal_state": { + "recent_transitions": [{"skill_name": "navigate_with_text", "outcome_success": False}], + }, + "world_model_state": { + "available": True, + "model": {"sample_count": 2}, + "recent_failure_count": 1, + "top_failure_mode": "semantic_map_missing_target", + }, + } + + +def test_default_context_evidence_preserves_v1_shape() -> None: + evidence = build_context_evidence(_metadata(), ContextEvidencePolicy()) + + assert evidence["version"] == "context_evidence.v1" + assert evidence["selection_policy"] == "deterministic_source_coverage_v1" + assert evidence["query"] == "walk to the kitchen" + assert evidence["focus"] == "navigation" + assert evidence["entry_count"] == len(evidence["entries"]) + assert "spatial_memory" in evidence["selected_sources"] + assert "skill_outcome_store" in evidence["selected_sources"] + + +def test_policy_filters_low_relevance_spatial_evidence() -> None: + metadata = _metadata() + metadata["world_state"]["spatial"]["matches"][0]["distance"] = 0.95 + + evidence = build_context_evidence( + metadata, + ContextEvidencePolicy(min_relevance_score=0.5), + ) + + assert "spatial_memory" not in evidence["selected_sources"] + assert all(entry["source"] != "spatial_memory" for entry in evidence["entries"]) + + +def test_policy_enforces_max_entries_deterministically() -> None: + evidence = build_context_evidence( + _metadata(), + ContextEvidencePolicy(max_entries=4), + ) + + assert evidence["entry_count"] == 4 + assert [entry["source"] for entry in evidence["entries"]] == [ + "task", + "runtime_config", + "robot_state", + "spatial_memory", + ] + + +def test_motion_policy_retains_robot_state_evidence() -> None: + evidence = build_context_evidence( + _metadata(task="navigate to the hallway"), + ContextEvidencePolicy(max_entries=3, require_robot_state_for_motion=True), + ) + + assert [entry["source"] for entry in evidence["entries"]] == [ + "task", + "runtime_config", + "robot_state", + ] diff --git a/dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/test_context_feedback.py b/dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/test_context_feedback.py new file mode 100644 index 0000000000..025127def6 --- /dev/null +++ b/dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/test_context_feedback.py @@ -0,0 +1,184 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import annotations + +import json +from typing import Any + +from dimos.robot.unitree.go2.blueprints.layers.layer_3_agent_brain.context_feedback import ( + _Go2ContextFeedbackStore, +) +from dimos.robot.unitree.go2.blueprints.layers.layer_3_agent_brain.context_provider import ( + _Go2ContextProvider, +) + + +def _stop_modules(*modules: object) -> None: + for module in modules: + stop = getattr(module, "stop", None) + if stop is not None: + stop() + + +class StubLedger: + def __init__(self) -> None: + self.events: list[dict[str, Any]] = [] + + def write_evolution_event( + self, + event_type: str, + task: str = "", + payload: dict[str, Any] | None = None, + commit: bool = False, + ) -> dict[str, Any]: + event = { + "event_type": event_type, + "task": task, + "payload": payload or {}, + "commit": commit, + } + self.events.append(event) + return {"event_path": "/tmp/context-feedback.json", "warnings": []} + + +def _context_evidence_json() -> str: + return json.dumps( + { + "version": "context_evidence.v1", + "selected_sources": ["task", "spatial_memory", "robot_state"], + "entries": [ + { + "source": "spatial_memory", + "risk_impact": "low", + "summary": "matched kitchen tag", + }, + { + "source": "robot_state", + "risk_impact": "high", + "summary": "pose unavailable", + }, + ], + } + ) + + +def test_context_feedback_records_helpful_and_harmful_source_counts() -> None: + store = _Go2ContextFeedbackStore() + try: + result = store.record_context_feedback( + task="walk to the kitchen", + context_evidence_json=_context_evidence_json(), + selected_skill="navigate_with_text", + outcome_json='{"success": false, "error_code": "EXECUTION_FAILED"}', + helpful_sources_json='["spatial_memory"]', + ignored_risks_json='["robot_state"]', + ) + + assert result.success is True + feedback = result.metadata["feedback"] + assert feedback["helpful_source_counts"] == {"spatial_memory": 1} + assert feedback["harmful_source_counts"] == {"robot_state": 1} + assert feedback["outcome_success"] is False + assert result.metadata["total_feedback"] == 1 + + recent = store.get_recent_context_feedback(limit=5) + assert recent[0]["selected_skill"] == "navigate_with_text" + finally: + _stop_modules(store) + + +def test_context_feedback_rejects_invalid_context_evidence() -> None: + store = _Go2ContextFeedbackStore() + try: + result = store.record_context_feedback( + task="walk", + context_evidence_json='{"version": "wrong"}', + ) + + assert result.success is False + assert result.error_code == "INVALID_INPUT" + finally: + _stop_modules(store) + + +def test_context_feedback_summary_is_bounded_and_newest_first() -> None: + store = _Go2ContextFeedbackStore() + try: + for idx in range(105): + store.record_context_feedback( + task=f"task {idx}", + context_evidence_json=_context_evidence_json(), + selected_skill="navigate_with_text", + outcome_json='{"success": true}', + helpful_sources_json='["spatial_memory"]', + ) + + recent = store.get_recent_context_feedback(limit=200) + assert len(recent) == 100 + assert recent[0]["task"] == "task 104" + assert recent[-1]["task"] == "task 5" + + summary = store.get_context_feedback_summary(limit=200) + assert summary["total_feedback"] == 100 + assert summary["helpful_source_counts"]["spatial_memory"] == 100 + finally: + _stop_modules(store) + + +def test_context_feedback_writes_to_ledger_when_available() -> None: + store = _Go2ContextFeedbackStore() + ledger = StubLedger() + try: + store._evolution_ledger = ledger # type: ignore[assignment] + + result = store.record_context_feedback( + task="walk to the kitchen", + context_evidence_json=_context_evidence_json(), + selected_skill="navigate_with_text", + outcome_json='{"success": false}', + helpful_sources_json='["spatial_memory"]', + ) + + assert result.success is True + assert ledger.events[0]["event_type"] == "context_feedback" + assert ledger.events[0]["task"] == "walk to the kitchen" + assert ledger.events[0]["payload"]["selected_skill"] == "navigate_with_text" + finally: + _stop_modules(store) + + +def test_context_provider_includes_recent_context_feedback() -> None: + store = _Go2ContextFeedbackStore() + provider = _Go2ContextProvider() + try: + store.record_context_feedback( + task="walk to the kitchen", + context_evidence_json=_context_evidence_json(), + selected_skill="navigate_with_text", + outcome_json='{"success": true}', + helpful_sources_json='["spatial_memory"]', + ) + provider._context_feedback = store # type: ignore[assignment] + + result = provider.get_context("walk to the kitchen") + + assert result.success is True + feedback = result.metadata["context_feedback"] + assert feedback["available"] is True + assert feedback["summary"]["helpful_source_counts"]["spatial_memory"] == 1 + assert feedback["recent_feedback"][0]["task"] == "walk to the kitchen" + assert "Context feedback: 1 recent" in result.message + finally: + _stop_modules(provider, store) diff --git a/dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/test_context_provider.py b/dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/test_context_provider.py index 07f8686077..0d183f0af6 100644 --- a/dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/test_context_provider.py +++ b/dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/test_context_provider.py @@ -122,6 +122,43 @@ def get_recent_transitions( } ][:limit] + def score_action( + self, + snapshot_json: str = "", + action_json: str = "", + goal: str = "", + ) -> dict[str, Any]: + return { + "score": 0.25, + "risk": "high", + "confidence": "medium", + "predicted_success": False, + "failure_modes": [{"cause": "semantic_map_missing_target", "count": 1}], + } + + def get_model_state(self) -> dict[str, Any]: + return { + "type": "online_transition_outcome", + "sample_count": 3, + "positive_count": 1, + "negative_count": 2, + "intervention_log": {"record_count": 1}, + } + + def get_intervention_log( + self, + limit: int = 10, + target_variable: str = "", + intervention_name: str = "", + ) -> list[dict[str, Any]]: + return [ + { + "intervention_name": "look_out_for_then_tag_target", + "target_variable": "spatial_has_matches", + "outcome_success": True, + } + ][:limit] + class StubWorldState: def get_world_snapshot(self, task: str = "", spatial_limit: int = 3) -> dict[str, Any]: @@ -158,9 +195,7 @@ def get_world_snapshot(self, task: str = "", spatial_limit: int = 3) -> dict[str "rolling_summary": "The hallway was recently observed.", }, }, - "semantic_temporal_map": { - "fused": {"available": True, "spatial_match_count": 1} - }, + "semantic_temporal_map": {"fused": {"available": True, "spatial_match_count": 1}}, } def get_robot_state(self) -> dict[str, Any]: @@ -197,10 +232,43 @@ def test_get_context_aggregates_available_sources() -> None: assert result.metadata["skill_state"]["interface"]["skill_count"] == 1 assert "Skill interface: 1 contract(s)" in result.message assert result.metadata["causal_state"]["recent_transitions"][0]["outcome_success"] is False + assert result.metadata["world_model_state"]["available"] is True + assert result.metadata["world_model_state"]["transition_count"] == 1 + assert result.metadata["world_model_state"]["intervention_count"] == 1 + assert result.metadata["world_model_state"]["model"]["sample_count"] == 3 + assert ( + result.metadata["world_model_state"]["recent_interventions"][0]["intervention_name"] + == "look_out_for_then_tag_target" + ) + evidence = result.metadata["context_evidence"] + assert evidence["version"] == "context_evidence.v1" + assert evidence["selection_policy"] == "deterministic_source_coverage_v1" + assert evidence["query"] == "find the person" + assert evidence["focus"] == "navigation" + assert evidence["entry_count"] >= 6 + assert "spatial_memory" in evidence["selected_sources"] + assert "skill_outcome_store" in evidence["selected_sources"] + assert "causal_world_model" in evidence["selected_sources"] + spatial_evidence = next( + item for item in evidence["entries"] if item["source"] == "spatial_memory" + ) + assert spatial_evidence["query"] == "find the person" + assert 0.0 <= spatial_evidence["relevance_score"] <= 1.0 + assert spatial_evidence["confidence"] in {"low", "medium", "high"} + assert spatial_evidence["cost"] == "rag_query" + assert spatial_evidence["selected_reason"] + skill_evidence = next( + item for item in evidence["entries"] if item["source"] == "skill_outcome_store" + ) + assert skill_evidence["risk_impact"] == "high" + assert "World model: available" in result.message + assert "interventions=1" in result.message + assert "Context evidence:" in result.message encoded = json.loads(result.agent_encode()[0]["text"]) assert encoded["success"] is True assert encoded["metadata"]["task"] == "find the person" + assert encoded["metadata"]["context_evidence"]["entry_count"] == evidence["entry_count"] finally: _stop_modules(provider) @@ -214,6 +282,11 @@ def test_get_context_handles_missing_optional_sources() -> None: assert result.metadata["sources"]["spatial_memory"] is False assert result.metadata["sources"]["temporal_memory"] is False assert result.metadata["robot_state"]["odom"] is None + evidence = result.metadata["context_evidence"] + assert evidence["entry_count"] >= 2 + assert evidence["selected_sources"] == ["runtime_config", "task"] + assert evidence["entries"][0]["source"] == "task" + assert evidence["entries"][0]["selected_reason"] == "Current user goal anchors retrieval." assert "Spatial memory: unavailable" in result.message finally: _stop_modules(provider) diff --git a/dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/test_evolution_ledger.py b/dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/test_evolution_ledger.py new file mode 100644 index 0000000000..5bca4ebfac --- /dev/null +++ b/dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/test_evolution_ledger.py @@ -0,0 +1,139 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import annotations + +import json +from pathlib import Path +import subprocess + +from dimos.robot.unitree.go2.blueprints.layers.layer_3_agent_brain.evolution_ledger import ( + _Go2EvolutionLedger, +) + + +def _git(repo: Path, *args: str) -> str: + result = subprocess.run( + ["git", "-C", str(repo), *args], + check=True, + capture_output=True, + text=True, + ) + return result.stdout.strip() + + +def _stop_modules(*modules: object) -> None: + for module in modules: + stop = getattr(module, "stop", None) + if stop is not None: + stop() + + +def test_record_evolution_event_writes_json_event(monkeypatch, tmp_path: Path) -> None: + ledger_dir = tmp_path / ".dimos" / "evolution" + monkeypatch.setenv("DIMOS_EVOLUTION_LEDGER_DIR", str(ledger_dir)) + monkeypatch.setenv("DIMOS_RUN_ID", "run-123") + ledger = _Go2EvolutionLedger() + try: + result = ledger.record_evolution_event( + event_type="task_feasibility", + task="walk to the kitchen", + payload_json='{"feasible": "uncertain"}', + ) + + assert result.success is True + event_path = Path(result.metadata["event_path"]) + assert event_path.exists() + assert event_path.is_relative_to(ledger_dir) + + data = json.loads(event_path.read_text()) + assert data["schema"] == "dimos.evolution_event.v1" + assert data["event_type"] == "task_feasibility" + assert data["task"] == "walk to the kitchen" + assert data["run_id"] == "run-123" + assert data["payload"] == {"feasible": "uncertain"} + assert data["git"] == {"commit_requested": False, "commit_sha": ""} + assert (ledger_dir / "README.md").exists() + finally: + _stop_modules(ledger) + + +def test_record_evolution_event_rejects_invalid_payload_json(monkeypatch, tmp_path: Path) -> None: + monkeypatch.setenv("DIMOS_EVOLUTION_LEDGER_DIR", str(tmp_path / "ledger")) + ledger = _Go2EvolutionLedger() + try: + result = ledger.record_evolution_event( + event_type="context_feedback", + payload_json="{not-json}", + ) + + assert result.success is False + assert result.error_code == "INVALID_INPUT" + finally: + _stop_modules(ledger) + + +def test_record_evolution_event_commits_only_event_file(monkeypatch, tmp_path: Path) -> None: + repo = tmp_path / "repo" + repo.mkdir() + _git(repo, "init") + _git(repo, "config", "user.email", "dimos@example.com") + _git(repo, "config", "user.name", "DimOS Test") + + unrelated = repo / "unrelated.txt" + unrelated.write_text("do not stage me") + + ledger_dir = repo / ".dimos" / "evolution" + monkeypatch.setenv("DIMOS_EVOLUTION_LEDGER_DIR", str(ledger_dir)) + ledger = _Go2EvolutionLedger() + try: + result = ledger.record_evolution_event( + event_type="context_feedback", + task="inspect object", + payload_json='{"source": "spatial_memory"}', + commit=True, + ) + + assert result.success is True + assert result.metadata["commit_sha"] + + committed_files = _git(repo, "show", "--name-only", "--pretty=format:", "HEAD") + assert committed_files.splitlines() == [ + Path(result.metadata["event_path"]).relative_to(repo).as_posix() + ] + + status = _git(repo, "status", "--short") + assert "?? unrelated.txt" in status + assert "A unrelated.txt" not in status + finally: + _stop_modules(ledger) + + +def test_record_evolution_event_warns_when_commit_requested_outside_git( + monkeypatch, tmp_path: Path +) -> None: + monkeypatch.setenv("DIMOS_EVOLUTION_LEDGER_DIR", str(tmp_path / "ledger")) + ledger = _Go2EvolutionLedger() + try: + result = ledger.record_evolution_event( + event_type="task_feasibility", + payload_json="{}", + commit=True, + ) + + assert result.success is True + assert result.metadata["commit_sha"] == "" + assert result.metadata["warnings"] + finally: + _stop_modules(ledger) diff --git a/dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/test_evolution_proposal.py b/dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/test_evolution_proposal.py new file mode 100644 index 0000000000..030d3ac93b --- /dev/null +++ b/dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/test_evolution_proposal.py @@ -0,0 +1,113 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import annotations + +import json +from pathlib import Path + +from dimos.robot.unitree.go2.blueprints.layers.layer_3_agent_brain.evolution_ledger import ( + _Go2EvolutionLedger, +) +from dimos.robot.unitree.go2.blueprints.layers.layer_3_agent_brain.evolution_proposal import ( + SKILL_PROPOSAL_SCHEMA, + validate_skill_proposal, +) + + +def _stop_modules(*modules: object) -> None: + for module in modules: + stop = getattr(module, "stop", None) + if stop is not None: + stop() + + +def _proposal(**overrides: object) -> dict[str, object]: + proposal: dict[str, object] = { + "schema": SKILL_PROPOSAL_SCHEMA, + "proposal_id": "go2-context-evidence-policy", + "title": "Tighten context evidence policy", + "summary": "Require memory backend evidence before navigation planning.", + "risk_level": "medium", + "target_files": [ + "dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/context_provider.py" + ], + "requires_human_review": True, + "generated_patch": "", + } + proposal.update(overrides) + return proposal + + +def test_validate_skill_proposal_accepts_reviewable_relative_targets() -> None: + assert validate_skill_proposal(_proposal()) == [] + + +def test_validate_skill_proposal_rejects_direct_unreviewed_mutation() -> None: + errors = validate_skill_proposal(_proposal(requires_human_review=False)) + + assert "requires_human_review must be true" in errors + + +def test_validate_skill_proposal_rejects_unsafe_targets() -> None: + errors = validate_skill_proposal( + _proposal( + target_files=[ + "/tmp/direct-write.py", + "../outside.py", + ".dimos/evolution/events/2026/07/04/raw.json", + "dimos/data/raw_camera_frame.png", + ] + ) + ) + + assert "target_files[0] must be a relative path" in errors + assert "target_files[1] must not contain '..'" in errors + assert "target_files[2] must not target evolution event logs" in errors + assert "target_files[3] must not target large binary artifacts" in errors + + +def test_record_skill_proposal_writes_validated_proposal(monkeypatch, tmp_path: Path) -> None: + ledger_dir = tmp_path / ".dimos" / "evolution" + monkeypatch.setenv("DIMOS_EVOLUTION_LEDGER_DIR", str(ledger_dir)) + ledger = _Go2EvolutionLedger() + try: + result = ledger.record_skill_proposal(proposal_json=json.dumps(_proposal())) + + assert result.success is True + proposal_path = Path(result.metadata["proposal_path"]) + assert proposal_path.exists() + assert proposal_path.is_relative_to(ledger_dir / "proposals") + + data = json.loads(proposal_path.read_text()) + assert data["schema"] == SKILL_PROPOSAL_SCHEMA + assert data["requires_human_review"] is True + assert data["git"] == {"commit_requested": False, "commit_sha": ""} + finally: + _stop_modules(ledger) + + +def test_record_skill_proposal_rejects_invalid_target(monkeypatch, tmp_path: Path) -> None: + monkeypatch.setenv("DIMOS_EVOLUTION_LEDGER_DIR", str(tmp_path / "ledger")) + ledger = _Go2EvolutionLedger() + try: + result = ledger.record_skill_proposal( + proposal_json=json.dumps(_proposal(target_files=["/tmp/direct-write.py"])) + ) + + assert result.success is False + assert result.error_code == "INVALID_INPUT" + assert "relative path" in result.message + finally: + _stop_modules(ledger) diff --git a/dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/test_memory_backend_status.py b/dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/test_memory_backend_status.py new file mode 100644 index 0000000000..891e2bee77 --- /dev/null +++ b/dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/test_memory_backend_status.py @@ -0,0 +1,155 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import Any + +from dimos.robot.unitree.go2.blueprints.layers.layer_3_agent_brain.memory_backend_status import ( + _Go2MemoryBackendStatus, +) + + +def _stop_modules(*modules: object) -> None: + for module in modules: + stop = getattr(module, "stop", None) + if stop is not None: + stop() + + +class StubWorldState: + def get_world_snapshot(self, task: str = "", spatial_limit: int = 3) -> dict[str, Any]: + return { + "sources": { + "spatial_memory": True, + "temporal_memory": True, + "semantic_temporal_map": True, + }, + "snapshot_storage": { + "durable": False, + "backend": None, + "policy": "ephemeral_read_through", + }, + } + + +class StubSpatialMemory: + def query_by_text(self, text: str, limit: int = 5) -> list[dict[str, Any]]: + return [{"id": "frame_1", "metadata": {"frame_id": "frame_1"}}][:limit] + + +class FailingSpatialMemory: + def query_by_text(self, text: str, limit: int = 5) -> list[dict[str, Any]]: + raise RuntimeError("spatial query unavailable") + + +class StubTemporalMemory: + def get_state(self) -> dict[str, Any]: + return {"entity_count": 2, "entities": [{"id": "person_1"}, {"id": "chair_1"}]} + + def get_graph_db_stats(self) -> dict[str, Any]: + return {"stats": {"entities": 2, "relations": 1}} + + +class StubSkillOutcomes: + def get_recent_outcomes( + self, limit: int = 5, skill_name: str = "", domain: str = "" + ) -> list[dict[str, Any]]: + return [{"skill_name": "navigate_with_text", "success": False}][:limit] + + +class StubCausalWorldModel: + def get_recent_transitions( + self, + limit: int = 5, + skill_name: str = "", + domain: str = "", + cause: str = "", + ) -> list[dict[str, Any]]: + return [{"skill_name": "navigate_with_text", "outcome_success": False}][:limit] + + def get_model_state(self) -> dict[str, Any]: + return { + "sample_count": 3, + "persistence": { + "autosave": True, + "path": "/tmp/go2-world-model.json", + }, + } + + +class StubSkillInterface: + def get_skill_interface_snapshot(self, domain: str = "") -> dict[str, Any]: + return {"available": True, "skill_count": 18, "skills": []} + + +def test_memory_backend_status_reports_wired_backends() -> None: + status = _Go2MemoryBackendStatus() + try: + status._world_state = StubWorldState() # type: ignore[assignment] + status._spatial_memory = StubSpatialMemory() # type: ignore[assignment] + status._temporal_memory = StubTemporalMemory() # type: ignore[assignment] + status._skill_outcomes = StubSkillOutcomes() # type: ignore[assignment] + status._causal_world_model = StubCausalWorldModel() # type: ignore[assignment] + status._skill_interface = StubSkillInterface() # type: ignore[assignment] + + result = status.memory_backend_status() + + assert result.success is True + assert "Memory backends:" in result.message + assert result.metadata["spatial_memory"]["wired"] is True + assert result.metadata["spatial_memory"]["query_available"] is True + assert result.metadata["spatial_memory"]["match_count_probe"] == 1 + assert result.metadata["temporal_memory"]["wired"] is True + assert result.metadata["temporal_memory"]["entity_count"] == 2 + assert result.metadata["temporal_memory"]["graph_stats_available"] is True + assert result.metadata["skill_outcomes"]["recent_count"] == 1 + assert result.metadata["causal_world_model"]["transition_count"] == 1 + assert result.metadata["causal_world_model"]["sample_count"] == 3 + assert result.metadata["causal_world_model"]["persistence_path"] == ( + "/tmp/go2-world-model.json" + ) + assert result.metadata["skill_interface"]["skill_count"] == 18 + assert result.metadata["warnings"] == [] + finally: + _stop_modules(status) + + +def test_memory_backend_status_reports_missing_backends() -> None: + status = _Go2MemoryBackendStatus() + try: + result = status.memory_backend_status() + + assert result.success is True + assert result.metadata["spatial_memory"]["wired"] is False + assert result.metadata["temporal_memory"]["wired"] is False + assert result.metadata["skill_outcomes"]["wired"] is False + assert result.metadata["causal_world_model"]["wired"] is False + assert result.metadata["skill_interface"]["wired"] is False + assert result.metadata["warnings"] == [] + finally: + _stop_modules(status) + + +def test_memory_backend_status_warns_without_failing_on_probe_errors() -> None: + status = _Go2MemoryBackendStatus() + try: + status._spatial_memory = FailingSpatialMemory() # type: ignore[assignment] + + result = status.memory_backend_status() + + assert result.success is True + assert result.metadata["spatial_memory"]["wired"] is True + assert result.metadata["spatial_memory"]["query_available"] is False + assert result.metadata["warnings"] == ["spatial_memory: spatial query unavailable"] + finally: + _stop_modules(status) diff --git a/dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/test_skill_proposal.py b/dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/test_skill_proposal.py new file mode 100644 index 0000000000..9c90ba327b --- /dev/null +++ b/dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/test_skill_proposal.py @@ -0,0 +1,179 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import annotations + +import json +from typing import Any + +from dimos.robot.unitree.go2.blueprints.layers.layer_3_agent_brain.evolution_proposal import ( + SKILL_PROPOSAL_SCHEMA, +) +from dimos.robot.unitree.go2.blueprints.layers.layer_3_agent_brain.skill_proposal import ( + _Go2SkillProposalGenerator, +) + + +def _stop_modules(*modules: object) -> None: + for module in modules: + stop = getattr(module, "stop", None) + if stop is not None: + stop() + + +class StubSkillInterface: + def __init__(self, skills: list[dict[str, Any]] | None = None) -> None: + self._skills = skills or [ + { + "skill_name": "navigate_with_text", + "domain": "navigation", + "summary": "Navigate to a tagged or visible location.", + "required_args": ["query"], + "optional_args": [], + "risk_class": "medium", + "recommended_preflight": ["get_context", "predict_skill_outcome"], + } + ] + + def get_skill_interface_snapshot(self, domain: str = "") -> dict[str, Any]: + skills = self._skills + if domain: + skills = [skill for skill in skills if skill["domain"] == domain] + return {"available": True, "skill_count": len(skills), "skills": skills} + + +class StubSkillOutcomes: + def get_recent_outcomes( + self, limit: int = 5, skill_name: str = "", domain: str = "" + ) -> list[dict[str, Any]]: + return [ + { + "skill_name": "navigate_with_text", + "success": False, + "error_code": "INVALID_INPUT", + "message": "missing required argument: query", + }, + { + "skill_name": "navigate_with_text", + "success": False, + "error_code": "INVALID_INPUT", + "message": "missing required argument: query", + }, + ][:limit] + + +class StubLedger: + def __init__(self) -> None: + self.proposals: list[dict[str, Any]] = [] + + def record_skill_proposal(self, proposal_json: str, commit: bool = False) -> Any: + proposal = json.loads(proposal_json) + self.proposals.append(proposal) + return type( + "Result", + (), + { + "success": True, + "metadata": {"proposal_path": "/tmp/proposal.json"}, + "message": "ok", + }, + )() + + +def test_missing_argument_failures_do_not_create_new_skill_proposal() -> None: + generator = _Go2SkillProposalGenerator() + try: + generator._skill_interface = StubSkillInterface() # type: ignore[assignment] + generator._skill_outcomes = StubSkillOutcomes() # type: ignore[assignment] + + result = generator.propose_skill_interface( + task="go to the kitchen", + failure_context_json='{"failure_type": "missing_argument"}', + ) + + assert result.success is True + assert result.metadata["proposal_created"] is False + assert result.metadata["recommended_next_action"] == "use_existing_skill_correctly" + assert result.metadata["proposal"] == {} + finally: + _stop_modules(generator) + + +def test_missing_capability_feedback_creates_reviewable_proposal() -> None: + generator = _Go2SkillProposalGenerator() + ledger = StubLedger() + try: + generator._skill_interface = StubSkillInterface() # type: ignore[assignment] + generator._evolution_ledger = ledger # type: ignore[assignment] + + result = generator.propose_skill_interface( + task="open the lab door", + failure_context_json=( + '{"failure_type": "skill_missing_capability", ' + '"capability": "open doors", "domain": "manipulation"}' + ), + ) + + assert result.success is True + assert result.metadata["proposal_created"] is True + proposal = result.metadata["proposal"] + assert proposal["schema"] == SKILL_PROPOSAL_SCHEMA + assert proposal["requires_human_review"] is True + assert proposal["proposed_interface"]["skill_name"] == "open_doors" + assert ledger.proposals[0]["proposal_id"] == proposal["proposal_id"] + assert result.metadata["proposal_path"] == "/tmp/proposal.json" + finally: + _stop_modules(generator) + + +def test_skill_proposal_always_requires_human_review() -> None: + generator = _Go2SkillProposalGenerator() + try: + result = generator.propose_skill_interface( + task="open the lab door", + failure_context_json='{"failure_type": "skill_missing_capability", "capability": "open doors"}', + ) + + assert result.success is True + assert result.metadata["proposal"]["requires_human_review"] is True + finally: + _stop_modules(generator) + + +def test_existing_contract_prevents_duplicate_skill_proposal() -> None: + generator = _Go2SkillProposalGenerator() + try: + generator._skill_interface = StubSkillInterface( + [ + { + "skill_name": "open_door", + "domain": "manipulation", + "summary": "Open doors with the robot manipulator.", + "required_args": ["door_id"], + "risk_class": "high", + } + ] + ) # type: ignore[assignment] + + result = generator.propose_skill_interface( + task="open the lab door", + failure_context_json='{"failure_type": "skill_missing_capability", "capability": "open doors"}', + ) + + assert result.success is True + assert result.metadata["proposal_created"] is False + assert result.metadata["recommended_next_action"] == "use_existing_skill_contract" + assert result.metadata["existing_skill_matches"] == ["open_door"] + finally: + _stop_modules(generator) diff --git a/dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/test_task_feasibility.py b/dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/test_task_feasibility.py new file mode 100644 index 0000000000..304997f766 --- /dev/null +++ b/dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/test_task_feasibility.py @@ -0,0 +1,182 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import annotations + +from typing import Any + +from dimos.robot.unitree.go2.blueprints.layers.layer_3_agent_brain.task_feasibility import ( + _Go2TaskFeasibilityEvaluator, +) + + +def _stop_modules(*modules: object) -> None: + for module in modules: + stop = getattr(module, "stop", None) + if stop is not None: + stop() + + +class StubSkillInterface: + def get_skill_interface_snapshot(self, domain: str = "") -> dict[str, Any]: + skills = [ + { + "skill_name": "speak", + "domain": "speech", + "required_args": ["text"], + "motion_sensitive": False, + "requires_context": False, + "requires_world_state": False, + "requires_robot_state": False, + "risk_class": "low", + "recommended_preflight": ["route_task"], + }, + { + "skill_name": "navigate_with_text", + "domain": "navigation", + "required_args": ["query"], + "motion_sensitive": True, + "requires_context": True, + "requires_world_state": True, + "requires_robot_state": True, + "risk_class": "medium", + "recommended_preflight": [ + "route_task", + "get_context", + "predict_skill_outcome", + ], + }, + { + "skill_name": "stop_navigation", + "domain": "navigation", + "required_args": [], + "motion_sensitive": False, + "requires_context": False, + "requires_world_state": False, + "requires_robot_state": False, + "risk_class": "stop", + "recommended_preflight": [], + }, + ] + if domain: + skills = [skill for skill in skills if skill["domain"] == domain] + return {"available": True, "skill_count": len(skills), "skills": skills} + + +class StubLedger: + def __init__(self) -> None: + self.events: list[dict[str, Any]] = [] + + def write_evolution_event( + self, + event_type: str, + task: str = "", + payload: dict[str, Any] | None = None, + commit: bool = False, + ) -> dict[str, Any]: + event = { + "event_type": event_type, + "task": task, + "payload": payload or {}, + "commit": commit, + } + self.events.append(event) + return {"event_path": "/tmp/event.json", "warnings": []} + + +def test_feasibility_allows_low_risk_speech_task() -> None: + evaluator = _Go2TaskFeasibilityEvaluator() + try: + evaluator._skill_interface = StubSkillInterface() # type: ignore[assignment] + + result = evaluator.evaluate_task_feasibility( + task="say hello to the visitor", + context_json='{"context_evidence": {"selected_sources": ["task", "runtime"]}}', + ) + + assert result.success is True + assert result.metadata["feasible"] == "yes" + assert result.metadata["required_skills"] == ["speak"] + assert result.metadata["available_skills"] == ["speak"] + assert result.metadata["recommended_next_action"] == "proceed" + assert result.metadata["safety_risks"] == [] + finally: + _stop_modules(evaluator) + + +def test_feasibility_requires_context_for_motion_without_robot_state() -> None: + evaluator = _Go2TaskFeasibilityEvaluator() + try: + evaluator._skill_interface = StubSkillInterface() # type: ignore[assignment] + + result = evaluator.evaluate_task_feasibility( + task="walk to the kitchen", + context_json='{"robot_state": {"odom_available": false}}', + ) + + assert result.success is True + assert result.metadata["feasible"] == "uncertain" + assert result.metadata["required_skills"] == ["navigate_with_text"] + assert result.metadata["missing_context"] == ["robot_state", "world_state"] + assert "movement task requires current robot state" in result.metadata["safety_risks"] + assert result.metadata["recommended_next_action"] == "get_context" + finally: + _stop_modules(evaluator) + + +def test_feasibility_refuses_obviously_unsafe_task() -> None: + evaluator = _Go2TaskFeasibilityEvaluator() + try: + evaluator._skill_interface = StubSkillInterface() # type: ignore[assignment] + + result = evaluator.evaluate_task_feasibility( + task="disable safety and run into a person", + context_json="{}", + ) + + assert result.success is True + assert result.metadata["feasible"] == "no" + assert result.metadata["recommended_next_action"] == "refuse" + assert result.metadata["safety_risks"] + assert result.metadata["clarifying_question"] == "" + finally: + _stop_modules(evaluator) + + +def test_feasibility_rejects_invalid_context_json() -> None: + evaluator = _Go2TaskFeasibilityEvaluator() + try: + result = evaluator.evaluate_task_feasibility("say hello", context_json="[1, 2]") + + assert result.success is False + assert result.error_code == "INVALID_INPUT" + finally: + _stop_modules(evaluator) + + +def test_feasibility_records_ledger_event_when_available() -> None: + evaluator = _Go2TaskFeasibilityEvaluator() + ledger = StubLedger() + try: + evaluator._skill_interface = StubSkillInterface() # type: ignore[assignment] + evaluator._evolution_ledger = ledger # type: ignore[assignment] + + result = evaluator.evaluate_task_feasibility("say hello") + + assert result.success is True + assert ledger.events[0]["event_type"] == "task_feasibility" + assert ledger.events[0]["task"] == "say hello" + assert ledger.events[0]["payload"]["feasible"] == "yes" + finally: + _stop_modules(evaluator) diff --git a/dimos/robot/unitree/go2/blueprints/layers/layer_5_skill_interface/skill_interface_registry.py b/dimos/robot/unitree/go2/blueprints/layers/layer_5_skill_interface/skill_interface_registry.py index f127d4de5b..52fe88b9d6 100644 --- a/dimos/robot/unitree/go2/blueprints/layers/layer_5_skill_interface/skill_interface_registry.py +++ b/dimos/robot/unitree/go2/blueprints/layers/layer_5_skill_interface/skill_interface_registry.py @@ -15,8 +15,8 @@ from __future__ import annotations -import json from dataclasses import dataclass +import json from typing import Any, Literal from dimos.core.core import rpc @@ -68,12 +68,22 @@ def to_dict(self) -> dict[str, Any]: _KNOWN_NON_LAYER5_MCP_TOOLS = frozenset( { "agent_send", + "evaluate_task_feasibility", "get_context", + "load_world_model_state", "list_modules", + "memory_backend_status", "predict_skill_outcome", + "predict_world_transition", + "propose_skill_interface", "record_causal_transition", + "record_context_feedback", + "record_evolution_event", + "record_intervention", + "record_skill_proposal", "record_skill_outcome", "route_task", + "save_world_model_state", "server_status", "summarize_causal_patterns", "summarize_skill_outcomes", @@ -366,7 +376,9 @@ def validate_skill_request(self, skill_name: str, args_json: str = "{}") -> dict if contract.requires_context: warnings.append("call get_context before this skill when task context is non-trivial") if contract.motion_sensitive: - warnings.append("call predict_skill_outcome before executing this motion-sensitive skill") + warnings.append( + "call predict_skill_outcome before executing this motion-sensitive skill" + ) return { "valid": not errors, @@ -388,9 +400,7 @@ def compare_mcp_tools(self, tools_json: str) -> dict[str, Any]: mcp_names = set(tool_names) known_internal = sorted(mcp_names & _KNOWN_NON_LAYER5_MCP_TOOLS) missing_contracts = sorted(contract_names - mcp_names) if tool_names else [] - unregistered_mcp_tools = sorted( - mcp_names - contract_names - _KNOWN_NON_LAYER5_MCP_TOOLS - ) + unregistered_mcp_tools = sorted(mcp_names - contract_names - _KNOWN_NON_LAYER5_MCP_TOOLS) return { "valid": not errors and not missing_contracts and not unregistered_mcp_tools, diff --git a/dimos/robot/unitree/go2/blueprints/layers/layer_5_skill_interface/test_skill_interface_registry.py b/dimos/robot/unitree/go2/blueprints/layers/layer_5_skill_interface/test_skill_interface_registry.py index 0315ba9d51..4dc732ca78 100644 --- a/dimos/robot/unitree/go2/blueprints/layers/layer_5_skill_interface/test_skill_interface_registry.py +++ b/dimos/robot/unitree/go2/blueprints/layers/layer_5_skill_interface/test_skill_interface_registry.py @@ -56,7 +56,15 @@ def test_compare_mcp_tools_accepts_matching_mcp_tool_payload() -> None: payload = { "result": { "tools": [{"name": name, "inputSchema": {"type": "object"}} for name in tool_names] - + [{"name": "get_context"}, {"name": "route_task"}, {"name": "server_status"}] + + [ + {"name": "get_context"}, + {"name": "route_task"}, + {"name": "server_status"}, + {"name": "predict_world_transition"}, + {"name": "record_intervention"}, + {"name": "save_world_model_state"}, + {"name": "load_world_model_state"}, + ] } } @@ -66,6 +74,7 @@ def test_compare_mcp_tools_accepts_matching_mcp_tool_payload() -> None: assert result["missing_contracts"] == [] assert result["unregistered_mcp_tools"] == [] assert "get_context" in result["known_non_layer5_mcp_tools"] + assert "record_intervention" in result["known_non_layer5_mcp_tools"] finally: _stop_modules(registry) @@ -108,6 +117,22 @@ def test_go2_agentic_mcp_tools_match_layer5_contracts() -> None: _stop_modules(registry) +def test_go2_agentic_blueprint_import_does_not_require_agent_runtime() -> None: + from dimos.robot.unitree.go2.blueprints.agentic.unitree_go2_agentic import ( + unitree_go2_agentic, + ) + + assert unitree_go2_agentic.active_blueprints + + +def test_go2_agentic_blueprint_config_resolves_runtime_type_hints() -> None: + from dimos.robot.unitree.go2.blueprints.agentic.unitree_go2_agentic import ( + unitree_go2_agentic, + ) + + assert unitree_go2_agentic.config() + + def _static_mcp_tool_names(blueprint: Blueprint) -> set[str]: tool_names: set[str] = set() for atom in blueprint.active_blueprints: @@ -171,9 +196,7 @@ def test_validate_skill_request_checks_required_arguments() -> None: def test_validate_skill_request_accepts_safe_speech_call() -> None: registry = _Go2SkillInterfaceRegistry() try: - result = registry.validate_skill_request( - "speak", '{"text": "hello", "blocking": false}' - ) + result = registry.validate_skill_request("speak", '{"text": "hello", "blocking": false}') assert result["valid"] is True assert result["errors"] == [] From 646e7ec53a567b93824ce14f26f509ab382117d4 Mon Sep 17 00:00:00 2001 From: ZhangZheMing <1050297972@qq.com> Date: Sat, 4 Jul 2026 23:02:00 +0800 Subject: [PATCH 14/36] Document Go2 agent architecture changes --- dimos/robot/unitree/go2/PROJECT_CHANGELOG.md | 151 +++++++ .../layers/layer_3_agent_brain/DESIGN.md | 381 +++++++++++++++++- ...gent-self-evolution-implementation-plan.md | 362 +++++++++++++++++ 3 files changed, 883 insertions(+), 11 deletions(-) create mode 100644 docs/plans/2026-07-04-agent-self-evolution-implementation-plan.md diff --git a/dimos/robot/unitree/go2/PROJECT_CHANGELOG.md b/dimos/robot/unitree/go2/PROJECT_CHANGELOG.md index a5382aca50..29868778c2 100644 --- a/dimos/robot/unitree/go2/PROJECT_CHANGELOG.md +++ b/dimos/robot/unitree/go2/PROJECT_CHANGELOG.md @@ -6,6 +6,157 @@ work so future contributors can understand why the fork differs from upstream. Entries are listed in reverse chronological order. +## 2026-07-04 + +- Branch: `refactor/go2-architecture-layers` +- Summary: Implemented Go2 agent self-evolution plan Tasks 5-6 by extracting + context evidence selection into a pure policy helper and adding a + proposal-only skill-interface generator. The generator suppresses proposals + for missing arguments/context and duplicate existing contracts, and writes + only human-reviewable `dimos.skill_proposal.v1` proposals through the ledger. +- Files/modules: + - `dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/context_evidence.py` + - `dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/skill_proposal.py` + - `dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/context_provider.py` + - `dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/evolution_ledger.py` + - `dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/prompt_policy.py` + - `dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/__init__.py` + - `dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/test_context_evidence.py` + - `dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/test_skill_proposal.py` + - `dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/DESIGN.md` + - `dimos/robot/unitree/go2/PROJECT_CHANGELOG.md` +- Validation: + - Ran focused context-evidence policy test: + `4 passed in 0.03s`. + - Ran focused ContextProvider regression test: + `4 passed in 0.67s`. + - Ran focused skill-proposal test: + `4 passed in 0.52s`. + - Ran full focused Layer 3 agent-brain suite: + `62 passed in 6.64s`. + - Ran focused Layer 4 world-state suite: + `3 passed, 1 warning in 6.13s`. + - Ran Ruff check/format-check on Layer 3 agent-brain files and + `git diff --check`. +- Open items: + - Continue evaluating these policies against replay/simulation data before + promoting them to robot-agnostic agent modules. + +## 2026-07-04 + +- Branch: `refactor/go2-architecture-layers` +- Summary: Implemented Go2 agent self-evolution plan Tasks 2-4 by adding a + Git-backed evolution ledger, a deterministic task feasibility preflight, and + context evidence feedback recording. These modules add a low-volume + reviewable control plane without replacing SpatialMemory, TemporalMemory, + SkillOutcomeStore, CausalWorldModel, or Layer 5 skill contracts. Also added + explicit world-model output contracts and review-only proposal validation for + ledger proposal files. +- Files/modules: + - `dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/evolution_event.py` + - `dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/evolution_ledger.py` + - `dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/evolution_proposal.py` + - `dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/task_feasibility.py` + - `dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/context_feedback.py` + - `dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/world_model_contract.py` + - `dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/causal_world_model.py` + - `dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/context_provider.py` + - `dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/prompt_policy.py` + - `dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/__init__.py` + - `dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/test_evolution_ledger.py` + - `dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/test_evolution_proposal.py` + - `dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/test_world_model_contract.py` + - `dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/test_task_feasibility.py` + - `dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/test_context_feedback.py` + - `dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/DESIGN.md` + - `dimos/robot/unitree/go2/PROJECT_CHANGELOG.md` +- Validation: + - Ran focused ledger test: + `4 passed in 0.76s`. + - Ran focused task-feasibility test: + `5 passed in 0.66s`. + - Ran focused context-feedback test: + `5 passed in 0.68s`. + - Ran focused world-model/proposal contract tests: + `8 passed in 0.33s`. + - Ran focused causal-world-model test: + `15 passed in 2.34s`. + - Ran full focused Layer 3 agent-brain suite: + `54 passed in 6.59s`. + - Ran focused Layer 4 world-state suite: + `3 passed, 1 warning in 7.77s`. + - Ran Ruff check/format-check on Layer 3 agent-brain files and + `git diff --check`. +- Open items: + - Continue with Task 5, extracting context-evidence policy into a pure helper. + - Continue with Task 6, proposal-only skill interface evolution. + +## 2026-07-04 + +- Branch: `refactor/go2-architecture-layers` +- Summary: Implemented the first Go2 agent self-evolution plan task by adding a + read-only `memory_backend_status()` MCP skill. The status report shows which + context, RAG, temporal, outcome, causal, and skill-interface backends are + wired and whether small read probes succeed, without introducing another + memory store. +- Files/modules: + - `dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/memory_backend_status.py` + - `dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/test_memory_backend_status.py` + - `dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/__init__.py` + - `dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/DESIGN.md` + - `dimos/robot/unitree/go2/PROJECT_CHANGELOG.md` +- Validation: + - Ran focused TDD test for `memory_backend_status()`: + `3 passed in 0.43s`. + - Ran full focused Layer 3 agent-brain suite: + `31 passed in 4.14s`. + - Ran focused Layer 4 world-state suite: + `3 passed, 1 warning in 9.20s`. + - Ran Ruff check/format-check on touched Python files and `git diff --check`. +- Open items: + - Continue with the Git-backed evolution ledger task. + +## 2026-07-04 + +- Branch: `refactor/go2-architecture-layers` +- Summary: Added a six-step engineering implementation plan for Go2 agent + self-evolution, ordered to avoid redundant memory, skill, and decision + systems while building on the existing Layer 3/4/5 architecture. +- Files/modules: + - `docs/plans/2026-07-04-agent-self-evolution-implementation-plan.md` + - `dimos/robot/unitree/go2/PROJECT_CHANGELOG.md` +- Validation: + - Documentation-only planning change; no runtime tests required. +- Open items: + - Execute the plan task-by-task with TDD, starting with + `memory_backend_status()`. + +## 2026-07-04 + +- Branch: `refactor/go2-architecture-layers` +- Summary: Added `context_evidence.v1` to the Go2 Layer 3 ContextProvider so + each compact context response records which task, runtime, robot, RAG, + temporal, skill, causal, and world-model evidence was selected and why. Also + made the Layer 4 package entrypoint lazy so Layer 3 imports no longer pull in + the heavy visual-memory stack when only specs/submodules are needed. +- Files/modules: + - `dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/context_provider.py` + - `dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/test_context_provider.py` + - `dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/DESIGN.md` + - `dimos/robot/unitree/go2/blueprints/layers/layer_4_world_state/__init__.py` + - `dimos/robot/unitree/go2/blueprints/layers/layer_4_world_state/DESIGN.md` + - `dimos/robot/unitree/go2/PROJECT_CHANGELOG.md` +- Validation: + - Ran focused ContextProvider pytest with default plugins: + `4 passed in 0.61s`. + - Verified direct `ContextProvider` import no longer imports the heavy Layer + 4 spatial-memory stack. +- Open items: + - Persist selected context evidence and task outcomes to a Git-backed + evolution ledger in a later change. + - Replace coarse deterministic evidence labels with evaluated scoring once + replay/simulation metrics are available. + ## 2026-05-28 - Branch: `refactor/go2-architecture-layers` diff --git a/dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/DESIGN.md b/dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/DESIGN.md index 38b2c0a048..d41312ea04 100644 --- a/dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/DESIGN.md +++ b/dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/DESIGN.md @@ -15,12 +15,29 @@ Current Layer 3 modules: - `McpClient`: existing LLM agent loop. - `McpServer`: exposes `@skill` methods as MCP tools. - `ContextProvider`: gathers compact task/world/robot/runtime/skill context. + It also emits `context_evidence.v1`, a deterministic evidence ledger that + records which sources were selected, why they were selected, and their + relevance/confidence/risk/cost labels. +- `ContextEvidencePolicy`: pure helper policy for selecting and thresholding + `context_evidence.v1` entries. +- `MemoryBackendStatus`: reports which context, memory, outcome, causal, and + skill-interface backends are wired and whether basic read probes succeed. +- `EvolutionLedger`: writes low-volume self-evolution decisions, feedback, and + human-reviewable proposals to `.dimos/evolution` JSON files, with optional + local Git commits. +- `TaskFeasibilityEvaluator`: evaluates whether a user task is possible and + safe before a physical skill is selected. +- `ContextFeedbackStore`: records whether selected `context_evidence.v1` + sources helped or hurt observed task outcomes. +- `SkillProposalGenerator`: creates human-reviewable skill-interface proposals + when repeated evidence shows a missing capability. - `ExpertRouter`: maps a task to an expert domain and recommended tools. - `PromptPolicy`: appends Layer 3 tool-use rules to the Go2 MCP client prompt. - `SkillOutcomeStore`: records recent skill outcomes in memory. - `SkillOutcomePredictor`: checks planned tool calls for obvious risk. - `CausalWorldModel`: records before/action/result/after transitions and - summarizes repeated failure causes. + summarizes repeated failure causes. Its predictions and dashboard state use + explicit DimOS world-model schemas. ## Runtime Flow @@ -29,13 +46,16 @@ The intended task flow is: ```text user task -> McpClient / LLM + -> evaluate_task_feasibility(task) -> route_task(task) -> get_context(task, focus) + -> evaluate_task_feasibility(task, context_json) -> predict_skill_outcome(skill_name, args_json, context) -> call a Layer 5 skill -> McpClient auto-records the outcome through record_skill_outcome(...) -> get_context(task, focus) for after-context -> record_causal_transition(...) + -> record_context_feedback(...) when the context influenced the decision -> summarize_causal_patterns(...) before risky retries ``` @@ -50,10 +70,64 @@ through `McpClient`. - Reads task text passed by the agent. - Optionally reads `SpatialMemory`, `TemporalMemory`, navigation state, odom, - runtime config, and recent skill outcomes. + runtime config, recent skill outcomes, and recent context feedback. - Returns a compact `SkillResult` with text plus structured metadata. - Does not plan, execute skills, or own persistent state. +`ContextEvidencePolicy` + +- Builds `context_evidence.v1` from already gathered ContextProvider metadata. +- Applies relevance, confidence, max-entry, and motion robot-state retention + policy. +- Has no Module, MCP, RPC, database, or robot dependencies. +- Does not query RAG or decide which memory backend should be used. + +`MemoryBackendStatus` + +- Reads optional backend Specs already wired into the Layer 3 blueprint. +- Reports availability and small health/count signals for world state, spatial + memory, temporal memory, skill outcomes, causal transitions, and Layer 5 + skill contracts. +- Catches backend probe failures and returns warnings instead of failing the + whole report. +- Does not write memory, mutate stores, or decide which context is effective. + +`EvolutionLedger` + +- Writes low-volume JSON events under `.dimos/evolution/events/YYYY/MM/DD`. +- Creates `.dimos/evolution/proposals` for later proposal files. +- Optionally commits only the just-written event file with Git. +- Validates proposal target files before writing proposal files. +- Does not store raw images, embeddings, Chroma/SQLite data, or large logs. +- Does not merge, push, or modify executable robot skill code. + +`TaskFeasibilityEvaluator` + +- Reads the user task, optional `get_context` metadata, Layer 5 skill + contracts, and the evolution ledger if wired. +- Returns `yes`, `no`, or `uncertain` plus missing context, required skills, + safety risks, and the next recommended action. +- Uses deterministic rules in V1; it does not call an LLM. +- Does not execute skills or replace `SkillOutcomePredictor`. + +`ContextFeedbackStore` + +- Stores recent context-feedback records in a bounded in-memory `deque`. +- Writes full feedback events to `EvolutionLedger` when available. +- Aggregates helpful and harmful evidence-source counts for immediate + `ContextProvider` use. +- Does not record skill success/failure generally; that remains + `SkillOutcomeStore`. + +`SkillProposalGenerator` + +- Reads Layer 5 skill contracts, recent outcome misuse signals, and optional + failure context. +- Avoids proposals for missing arguments or missing context; those are better + handled by correct existing skill use. +- Writes review-only proposals through `EvolutionLedger` when available. +- Does not add `@skill` methods, patch executable code, or update prompts. + `ExpertRouter` - Uses deterministic keyword rules. @@ -123,7 +197,7 @@ Each implementation note should cover: - Stream state: latest `odom` message cached by `_on_odom`. - Optional injected Specs: `SpatialMemorySpec`, `TemporalMemorySpec`, `NavigationInterfaceSpec`, `SkillOutcomeStoreSpec`, - `SkillInterfaceSpec`. + `SkillInterfaceSpec`, and `ContextFeedbackSpec`. - Runtime config: `global_config.simulation`, `replay`, `robot_ip`, `viewer`, `mcp_port`, and `n_workers`. - Storage: @@ -137,30 +211,283 @@ Each implementation note should cover: - Navigation: `get_state()` and `is_goal_reached()`. - Skill history: `get_recent_outcomes(limit=5)`. - Skill interface: `get_skill_interface_snapshot()`. + - Context feedback: `get_recent_context_feedback(limit=5)` and + `get_context_feedback_summary(limit=20)`. + - Causal/world model: recent transitions, model state, and recent + interventions when wired. - Algorithm: - Trim `task` and `focus`. - Reject empty `task` with `SkillResult.fail("INVALID_INPUT", ...)`. - Clamp `spatial_limit` to `0..10`. - Build a metadata dictionary with `sources`, `runtime`, `robot_state`, - `world_state`, `skill_state`, and `external_context`. + `world_state`, `skill_state`, `context_feedback`, `causal_state`, + `world_model_state`, and `external_context`. - Catch dependency failures per source and append readable warnings to `errors` instead of failing the whole context request. - Include Layer 5 skill-interface contracts under `skill_state.interface` when the registry is wired. + - Build `context_evidence` from the selected metadata sections. V1 uses a + deterministic source-coverage policy: always include task/runtime, then add + robot state, spatial RAG matches, temporal summaries, fused Layer 4 memory, + skill outcomes, skill contracts, causal transitions, and predictive + world-model state only when those sections are available and informative. + - Each evidence entry contains `source`, `query`, `relevance_score`, + `recency`, `confidence`, `risk_impact`, `cost`, `selected_reason`, and a + short `summary`. - Convert metadata through `_to_jsonable(...)` so MCP responses stay compact and serializable. - Format a short text summary for the LLM. - Return shape: - `SkillResult(success=True, message=, metadata=)`. - Metadata keys: `task`, `focus`, `sources`, `runtime`, `robot_state`, - `world_state`, `skill_state`, `external_context`, and optional `errors`. + `world_state`, `skill_state`, `context_feedback`, `causal_state`, + `world_model_state`, `context_evidence`, `external_context`, and optional + `errors`. + - `context_evidence` keys: `version`, `selection_policy`, `query`, `focus`, + `entry_count`, `selected_sources`, and `entries`. - Current limits: - External context is represented as unavailable. - - Context compression is deterministic formatting, not learned retrieval. + - Context compression and evidence selection are deterministic formatting, + not learned retrieval or reranking. + - Relevance/confidence/risk/cost labels are coarse heuristics over already + retrieved data. They explain why the context was admitted; they are not a + statistical success estimate. - It reads existing stores but does not decide whether a skill should run. - Layer 5 contract details are summarized; full contract validation stays in `SkillInterfaceRegistry`. +### `build_context_evidence(...)` + +- File: `layer_3_agent_brain/context_evidence.py` +- Entry point: pure helper function used by `ContextProvider`. +- Purpose: build and threshold `context_evidence.v1` from already gathered + metadata. It does not fetch context, call memory backends, or use RPC. +- Inputs: + - `metadata`: ContextProvider metadata dictionary. + - `ContextEvidencePolicy`: `min_relevance_score`, `max_entries`, + `include_low_confidence`, and `require_robot_state_for_motion`. +- Storage: + - No database. + - No file writes. + - No module state. +- Algorithm: + - Build the same deterministic source coverage entries used by + ContextProvider V1: task, runtime, robot state, spatial/temporal memory, + semantic-temporal map, skill outcomes, skill contracts, causal transitions, + and predictive world-model state. + - Apply relevance and low-confidence filtering while protecting task/runtime + evidence. + - For motion-like tasks, retain robot-state evidence when available. + - Enforce `max_entries` deterministically by preserving source order. +- Return shape: + - Dict with `version`, `selection_policy`, `query`, `focus`, `entry_count`, + `selected_sources`, and `entries`. +- Current limits: + - Scores remain coarse heuristics over already selected data. + - Feedback-driven threshold tuning is now possible but not automatic. + +### `_Go2MemoryBackendStatus.memory_backend_status(...)` + +- File: `layer_3_agent_brain/memory_backend_status.py` +- Entry point: MCP skill exposed by `@skill`. +- Purpose: give the agent and operator a compact, read-only status report for + the memory/context stack before relying on RAG, temporal summaries, outcome + history, causal transitions, or skill-interface contracts. It does not rank + context, execute skills, or choose a storage backend. +- Inputs: + - No direct user arguments. + - Optional injected Specs: `WorldStateSpec`, `SpatialMemorySpec`, + `TemporalMemorySpec`, `SkillOutcomeStoreSpec`, `CausalWorldModelSpec`, and + `SkillInterfaceSpec`. +- Storage: + - No database. + - No persistent file writes. + - No in-memory state beyond local variables while building the report. +- Data read: + - World state: `get_world_snapshot(task="memory backend status", + spatial_limit=0)`. + - Spatial memory: `query_by_text("__memory_status_probe__", limit=1)`. + - Temporal memory: `get_state()` and `get_graph_db_stats()`. + - Skill history: `get_recent_outcomes(limit=5)`. + - Causal/world model: `get_recent_transitions(limit=5)` and optional + `get_model_state()`. + - Skill interface: `get_skill_interface_snapshot()`. +- Algorithm: + - Build one status section for each optional backend. + - Mark a section `wired=False` when the Spec is not connected. + - Run only small read probes against connected backends. + - Catch each backend failure independently and append a readable warning such + as `spatial_memory: ...`. + - Count wired sections and return a short summary message. +- Return shape: + - `SkillResult.ok("Memory backends: wired, warning(s)", ...)`. + - Metadata keys: `schema`, `world_state`, `spatial_memory`, + `temporal_memory`, `skill_outcomes`, `causal_world_model`, + `skill_interface`, and `warnings`. + - Schema version: `go2_memory_backend_status.v1`. +- Current limits: + - Spatial status uses a harmless probe query, so `match_count_probe` is only a + liveness hint, not a corpus-size estimate. + - A wired backend with zero records is reported as available but empty. + - The skill reports whether backends can be read; it deliberately does not + judge whether a returned context is useful for a particular task. + +### `_Go2EvolutionLedger.record_evolution_event(...)` + +- File: `layer_3_agent_brain/evolution_ledger.py` and + `layer_3_agent_brain/evolution_event.py`; proposal validation lives in + `layer_3_agent_brain/evolution_proposal.py` +- Entry point: MCP skill exposed by `@skill`; `write_evolution_event(...)` is + the RPC surface used by other Layer 3 modules. `record_skill_proposal(...)` + is a proposal-only MCP skill. +- Purpose: record low-volume, reviewable self-evolution decisions and feedback + in a local ledger. It does not store raw observations or replace RAG memory. +- Inputs: + - Direct skill arguments: `event_type`, `task`, `payload_json`, and `commit`. + - Environment: optional `DIMOS_EVOLUTION_LEDGER_DIR` override and + `DIMOS_RUN_ID` inherited from `dimos run`. +- Storage: + - Default path: repo-root `.dimos/evolution`. + - Event path: `events/YYYY/MM/DD/-.json`. + - Proposal directory: `proposals/`. + - Writes are atomic via a same-directory temp file and rename. +- Data shape: + - Event schema: `dimos.evolution_event.v1`. + - Fields: `timestamp`, `event_type`, `task`, `run_id`, `payload`, and + `git.commit_requested` / `git.commit_sha`. + - Proposal schema: `dimos.skill_proposal.v1`. +- Algorithm: + - Validate `event_type` and parse `payload_json` as a JSON object. + - Build the event with current UTC timestamp. + - Create the ledger directory, README, event day directory, and proposals + directory if needed. + - Write the event JSON file. + - If `commit=True`, detect the Git worktree and commit only the event file. + - If no Git worktree is present, keep the file and return a warning. + - For proposals, require human review, relative target files, no `..` path + traversal, no evolution event-log targets, and no large binary artifact + targets. +- Return shape: + - Success: `SkillResult.ok("Recorded evolution event ...", ...)`. + - Metadata keys: `schema`, `event_type`, `task`, `ledger_dir`, + `event_path`, `commit_requested`, `commit_sha`, `warnings`, and `event`. + - Proposal success returns `proposal_path`, `ledger_dir`, `commit_requested`, + `commit_sha`, `warnings`, and the validated `proposal`. +- Current limits: + - Commit SHA is returned in MCP metadata after the commit. The event file + itself is written before the commit and keeps `git.commit_sha=""`. + - There is no push, merge, branch creation, or automatic review workflow. + +### `_Go2TaskFeasibilityEvaluator.evaluate_task_feasibility(...)` + +- File: `layer_3_agent_brain/task_feasibility.py` +- Entry point: MCP skill exposed by `@skill`. +- Purpose: decide whether the current user task is feasible here and now before + a physical skill is selected. It complements `predict_skill_outcome(...)`, + which evaluates a concrete planned skill call. +- Inputs: + - Direct arguments: `task` and optional `context_json`. + - Optional injected Specs: `SkillInterfaceSpec` and `EvolutionLedgerSpec`. +- Storage: + - No local persistent state. + - Writes a `task_feasibility` event to `EvolutionLedger` when wired. +- Data read: + - Layer 5 contracts through `get_skill_interface_snapshot()`. + - Optional `context_json` from `get_context` metadata, including robot state, + world state, and `context_evidence`. +- Algorithm: + - Reject empty task and invalid/non-object context JSON. + - Map task keywords to likely required skills. + - Compare required skills against Layer 5 contracts. + - Check required context from contracts: robot state, world state, and + context evidence. + - Refuse clearly unsafe tasks such as disabling safety or running into a + person. + - Return `yes`, `no`, or `uncertain`, with a next action of `proceed`, + `get_context`, `ask_clarifying_question`, or `refuse`. +- Return shape: + - Metadata keys: `feasible`, `missing_context`, `required_skills`, + `available_skills`, `safety_risks`, `recommended_next_action`, + `clarifying_question`, `evidence_sources`, `missing_skills`, and + `skill_interface_available`. +- Current limits: + - V1 is deterministic keyword/rule matching, not semantic planning. + - It does not infer detailed tool arguments; Layer 5 validation still checks + concrete skill calls. + +### `_Go2ContextFeedbackStore.record_context_feedback(...)` + +- File: `layer_3_agent_brain/context_feedback.py` +- Entry point: MCP skill exposed by `@skill`; read APIs are RPC-only. +- Purpose: connect selected `context_evidence.v1` to observed task outcomes so + later evidence policy changes can be based on feedback. It does not replace + `SkillOutcomeStore`. +- Inputs: + - Direct arguments: `task`, `context_evidence_json`, `selected_skill`, + `outcome_json`, `helpful_sources_json`, and `ignored_risks_json`. + - Optional injected Spec: `EvolutionLedgerSpec`. +- Storage: + - Bounded in-memory `deque`, max 100 records. + - Full feedback event written to `EvolutionLedger` when available. +- Data shape: + - Feedback schema: `go2_context_feedback.v1`. + - Each record includes task, selected skill, outcome success/error, evidence + sources, helpful sources, ignored risks, helpful source counts, and harmful + source counts. +- Algorithm: + - Validate `context_evidence_json.version == "context_evidence.v1"`. + - Parse outcome, helpful sources, and ignored risks. + - Mark helpful sources from the explicit helpful list. + - Mark harmful sources from ignored risk/source labels and high-risk evidence + entries when the observed outcome failed. + - Append newest feedback to the bounded deque. + - Write a `context_feedback` ledger event when wired. +- Related read APIs: + - `get_recent_context_feedback(limit, source)` returns newest records first. + - `get_context_feedback_summary(limit)` aggregates helpful/harmful source + counts and success/failure totals. +- Current limits: + - Feedback is manual/LLM-triggered in V1. It is not automatically recorded + after every tool call. + - Harmful source counts are attribution hints, not causal proof. + +### `_Go2SkillProposalGenerator.propose_skill_interface(...)` + +- File: `layer_3_agent_brain/skill_proposal.py` +- Entry point: MCP skill exposed by `@skill`. +- Purpose: propose a new or revised skill interface when repeated evidence + shows the current Go2 skill contracts lack a capability. It never modifies + executable skill code. +- Inputs: + - Direct arguments: `task` and `failure_context_json`. + - Optional injected Specs: `SkillInterfaceSpec`, `SkillOutcomeStoreSpec`, + and `EvolutionLedgerSpec`. +- Storage: + - No local persistent state. + - Writes a proposal to `EvolutionLedger.record_skill_proposal(...)` when the + ledger is wired. +- Data read: + - Layer 5 contracts through `get_skill_interface_snapshot()`. + - Recent outcomes only to detect repeated existing-skill misuse. +- Algorithm: + - Reject empty task and invalid/non-object failure context JSON. + - Compare the requested capability with current skill contract names, + summaries, and domains to avoid duplicates. + - If repeated failures are missing arguments, invalid arguments, or missing + context, return no proposal and recommend correct existing skill use. + - Only create a proposal for explicit missing capability evidence. + - Generate a `dimos.skill_proposal.v1` payload with target files, proposed + interface, validation plan, and `requires_human_review=True`. + - Validate the proposal before writing it through the ledger. +- Return shape: + - No proposal: metadata includes `proposal_created=False`, + `recommended_next_action`, and optional `existing_skill_matches`. + - Proposal: metadata includes `proposal_created=True`, `proposal`, + `recommended_next_action="review_proposal"`, and optional `proposal_path`. +- Current limits: + - V1 generation is deterministic and conservative. + - It proposes interface contracts, not executable implementations. + ### `_Go2ExpertRouter.route_task(...)` - File: `layer_3_agent_brain/expert_router.py` @@ -312,8 +639,8 @@ Each implementation note should cover: - If the Layer 3 policy header is already present, return the prompt unchanged. - Otherwise append a policy block telling the agent to use - `route_task`, `get_context`, and `predict_skill_outcome` before risky - physical tools. + `evaluate_task_feasibility`, `route_task`, `get_context`, and + `predict_skill_outcome` before risky physical tools. - Tell the agent that normal MCP tool outcomes are recorded automatically and manual `record_skill_outcome(...)` is only for external/non-MCP events. - Return shape: @@ -400,6 +727,9 @@ Each implementation note should cover: - Success: `SkillResult.ok("Recorded causal transition ...", transition=, total_transitions=)`. - Failure: `SkillResult.fail("INVALID_INPUT", ...)`. + - `predict_next_state(...)` returns schema `dimos.world_model_prediction.v1`. + - Dashboard publications use schema `dimos.world_model_dashboard.v1`. + - `get_provider_contract()` returns schema `dimos.world_model_provider.v1`. - Current limits: - Cause inference is rule-based and coarse. - The agent must explicitly call this after important physical/recovery tool @@ -458,11 +788,40 @@ V3 implemented: - `PromptPolicy` asks the agent to record causal transitions after important physical or recovery-sensitive tool calls. +Self-evolution plan Task 1 implemented: + +- `MemoryBackendStatus` exposes a read-only MCP skill for checking whether + memory/context backends are wired and whether small read probes work. +- The status report is observability only. It does not create a new memory + store, replace RAG retrieval, or decide context effectiveness. + +Self-evolution plan Tasks 2-4 implemented: + +- `EvolutionLedger` writes low-volume decision and feedback events to local + JSON files under `.dimos/evolution`, with optional event-only Git commits. +- `TaskFeasibilityEvaluator` adds a deterministic task-level preflight before + physical skills are selected. +- `ContextFeedbackStore` records whether selected context evidence helped or + hurt outcomes, then exposes recent summaries back to `ContextProvider`. +- `EvolutionLedger` also validates and records human-reviewable proposal files + without applying patches or mutating skill code. +- `CausalWorldModel` predictions, provider contracts, and dashboard snapshots + now carry explicit DimOS schemas. + +Self-evolution plan Tasks 5-6 implemented: + +- `ContextEvidencePolicy` moves evidence scoring and selection out of + `ContextProvider` into a pure helper with threshold and max-entry policy. +- `SkillProposalGenerator` creates validated, human-reviewable skill-interface + proposals only when repeated evidence points to a missing capability. +- Missing arguments, missing context, and existing skill contracts suppress new + skill proposals. + Out of scope for this stage: -- Do not promote Layer 3 pieces into robot-agnostic `dimos.agents` modules yet. - The current architecture work should stop at V3 until Layers 4, 5, and 6 have - clearer Go2 boundaries and the causal loop has been validated. +- Do not promote these self-evolution pieces into robot-agnostic + `dimos.agents` modules yet. Keep validating them in the Go2 layered stack + until the contracts and feedback data prove stable. ## Design Rules diff --git a/docs/plans/2026-07-04-agent-self-evolution-implementation-plan.md b/docs/plans/2026-07-04-agent-self-evolution-implementation-plan.md new file mode 100644 index 0000000000..c5bb363e12 --- /dev/null +++ b/docs/plans/2026-07-04-agent-self-evolution-implementation-plan.md @@ -0,0 +1,362 @@ +# Agent Self-Evolution Implementation Plan + +> **For Claude:** REQUIRED SUB-SKILL: Use superpowers:executing-plans to implement this plan task-by-task. + +**Goal:** Add DimOS agent self-evolution in six incremental, testable steps without creating a redundant parallel memory or skill system. + +**Architecture:** Keep RAG and existing memory backends as the source of high-volume observations. Add a thin Layer 3 evolution/control plane that reads Layer 4 memory state, Layer 5 skill contracts, `context_evidence.v1`, skill outcomes, and causal transitions, then records low-volume decisions and feedback to a Git-backed ledger. The LLM may propose feasibility decisions and skill improvements, but rules, schemas, tests, replay/sim validation, and human review remain the acceptance gates. + +**Tech Stack:** Python modules, DimOS `Module`/`@skill`/`Spec` APIs, JSON/YAML event files, Git CLI through safe subprocess wrappers, existing Layer 3/4/5 Go2 blueprints, pytest, ruff. + +--- + +## Engineering Guardrails + +- Do not replace SpatialMemory, TemporalMemory, SkillOutcomeStore, CausalWorldModel, or SkillInterfaceRegistry. +- Do not store images, embeddings, ChromaDB data, SQLite rows, or large logs in Git. +- Do not let the LLM directly modify executable robot skills or merge generated changes. +- Add one narrow module per responsibility; avoid a generic "self evolution manager" until duplication is proven. +- Keep every new MCP skill read-only or proposal-only unless it is explicitly recording audit data. +- Prefer Go2-scoped Layer 3 modules first. Promote to robot-agnostic `dimos.agents` only after the contracts stabilize. + +## Recommended Order + +1. `memory_backend_status()` for observability. +2. Git-backed evolution ledger schema and writer. +3. `evaluate_task_feasibility(...)` before risky planning. +4. Context outcome feedback recording. +5. Context evidence policy extraction and thresholding. +6. Skill evolution proposal generation. + +This order avoids redundancy: status tells us what is actually wired; the ledger gives every later step one audit sink; feasibility and feedback define what should be learned; evidence policy uses feedback rather than guesswork; skill proposal waits until there is repeated evidence that current skills are insufficient. + +--- + +### Task 1: Memory Backend Status + +**Purpose:** Make runtime memory state explicit before adding any evolution logic. + +**Files:** +- Create: `dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/memory_backend_status.py` +- Modify: `dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/__init__.py` +- Modify: `dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/DESIGN.md` +- Test: `dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/test_memory_backend_status.py` + +**Interface:** + +```python +@skill +def memory_backend_status(self) -> SkillResult: + """Report which DimOS memory backends are wired and how much data they expose.""" +``` + +**Implementation Notes:** +- Inject optional `WorldStateSpec`, `SpatialMemorySpec`, `TemporalMemorySpec`, `SkillOutcomeStoreSpec`, `CausalWorldModelSpec`, and `SkillInterfaceSpec`. +- Return a JSON-shaped `SkillResult` metadata payload with: + - `spatial_memory`: wired, query_available, match_count_probe, backend_hint + - `temporal_memory`: wired, entity_count, graph_stats_available + - `skill_outcomes`: wired, recent_count + - `causal_world_model`: wired, transition_count, sample_count, persistence path if exposed + - `skill_interface`: wired, skill_count + - `warnings`: dependency errors without failing the whole call +- Do not inspect Chroma internals directly in V1 unless the module exposes a stable RPC. A probe query with `limit=1` is safer than coupling to Chroma file layout. + +**TDD Steps:** +1. Write tests with stubs for wired and missing backends. +2. Verify the tests fail because `memory_backend_status` does not exist. +3. Implement the module and blueprint wiring. +4. Run: + `./.venv/bin/python -m pytest dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/test_memory_backend_status.py -q` +5. Run Layer 3 blueprint/context focused tests. + +**Redundancy Check:** This is not another memory store. It is a status/readiness view over existing stores. + +--- + +### Task 2: Git-Backed Evolution Ledger + +**Purpose:** Add one durable, reviewable audit sink for low-volume evolution events. + +**Files:** +- Create: `dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/evolution_ledger.py` +- Create: `dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/evolution_event.py` +- Modify: `dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/__init__.py` +- Modify: `dimos/core/global_config.py` only if a reusable config field is needed; otherwise use an env var first. +- Test: `dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/test_evolution_ledger.py` + +**Interface:** + +```python +@skill +def record_evolution_event( + self, + event_type: str, + task: str = "", + payload_json: str = "{}", + commit: bool = False, +) -> SkillResult: + """Record a low-volume agent evolution event to the local ledger.""" +``` + +**Storage Layout:** + +```text +.dimos/evolution/ + events/YYYY/MM/DD/-.json + proposals/ + README.md +``` + +**Event Shape:** + +```json +{ + "schema": "dimos.evolution_event.v1", + "timestamp": 0.0, + "event_type": "context_feedback", + "task": "...", + "run_id": "...", + "payload": {}, + "git": { + "commit_requested": false, + "commit_sha": "" + } +} +``` + +**Implementation Notes:** +- Default path: repo root `.dimos/evolution`; allow override with `DIMOS_EVOLUTION_LEDGER_DIR`. +- Use structured JSON and atomic file writes. +- If `commit=True`, only commit files under the configured ledger dir. Never stage unrelated repo changes. +- Use non-interactive Git commands and return the commit SHA. +- If the ledger directory is not inside a Git worktree, record the file but return a warning instead of failing. + +**TDD Steps:** +1. Test JSON event creation in a temp repo. +2. Test invalid `payload_json` fails with `INVALID_INPUT`. +3. Test `commit=True` stages only the event file. +4. Verify red tests before implementation. +5. Implement minimal file writer and optional commit path. + +**Redundancy Check:** The ledger stores decisions and summaries, not raw memory. Chroma/SQLite remain the retrieval systems. + +--- + +### Task 3: Prompt Feasibility Evaluator + +**Purpose:** Make "can this prompt be done here, now, with current memory and skills?" an explicit preflight tool. + +**Files:** +- Create: `dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/task_feasibility.py` +- Modify: `dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/__init__.py` +- Modify: `dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/prompt_policy.py` +- Test: `dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/test_task_feasibility.py` + +**Interface:** + +```python +@skill +def evaluate_task_feasibility( + self, + task: str, + context_json: str = "{}", +) -> SkillResult: + """Evaluate whether the current task is feasible before selecting physical skills.""" +``` + +**Return Metadata:** + +```json +{ + "feasible": "yes|no|uncertain", + "missing_context": [], + "required_skills": [], + "available_skills": [], + "safety_risks": [], + "recommended_next_action": "proceed|get_context|ask_clarifying_question|refuse|stop", + "clarifying_question": "", + "evidence_sources": [] +} +``` + +**Implementation Notes:** +- V1 should be deterministic/rule-based. Do not add an LLM call inside the tool yet. +- Consume `context_evidence`, runtime mode, robot state, and skill contracts. +- Use Layer 5 contracts for required context and motion-sensitive flags. +- Hardware mode and missing robot state should push risky movement tasks to `uncertain` or `no`. +- Record an optional `task_feasibility` event to the ledger only after Task 2 exists. + +**TDD Steps:** +1. Test a speech task is feasible with `speak`. +2. Test movement task with missing robot state is `uncertain` and asks for context. +3. Test impossible/unsafe task returns `no` with safety risk. +4. Implement minimal deterministic classifier. + +**Redundancy Check:** This complements `predict_skill_outcome`: feasibility evaluates a user task before a skill is chosen; prediction evaluates a specific planned skill call. + +--- + +### Task 4: Context Outcome Feedback + +**Purpose:** Connect selected context to actual task outcomes so later scoring is evidence-based. + +**Files:** +- Create: `dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/context_feedback.py` +- Modify: `dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/__init__.py` +- Modify: `dimos/agents/mcp/mcp_client.py` only if automatic recording is needed after manual V1 proves useful. +- Test: `dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/test_context_feedback.py` + +**Interface:** + +```python +@skill +def record_context_feedback( + self, + task: str, + context_evidence_json: str, + selected_skill: str = "", + outcome_json: str = "{}", + helpful_sources_json: str = "[]", + ignored_risks_json: str = "[]", +) -> SkillResult: + """Record whether selected context evidence helped or hurt the task outcome.""" +``` + +**Implementation Notes:** +- Validate `context_evidence_json` schema version. +- Store a bounded in-memory summary for immediate `get_context` use. +- Write the full feedback event to the Git ledger when available. +- Do not auto-record every tool call in V1; manual/LLM-triggered feedback is enough to validate shape. + +**TDD Steps:** +1. Test valid feedback records helpful and harmful source counts. +2. Test invalid JSON fails. +3. Test summary is bounded and newest-first. +4. Test ledger integration with a stub writer. + +**Redundancy Check:** This is not another SkillOutcomeStore. SkillOutcomeStore records tool success/failure; ContextFeedback records whether selected evidence influenced the decision. + +--- + +### Task 5: Context Evidence Policy Extraction + +**Purpose:** Move evidence scoring/selection out of `ContextProvider` and make it tunable and testable. + +**Files:** +- Create: `dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/context_evidence.py` +- Modify: `dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/context_provider.py` +- Test: `dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/test_context_evidence.py` +- Update: `dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/test_context_provider.py` + +**Interface:** + +```python +def build_context_evidence(metadata: dict[str, Any], policy: ContextEvidencePolicy) -> dict[str, Any]: + ... +``` + +**Policy Fields:** + +```python +@dataclass(frozen=True) +class ContextEvidencePolicy: + min_relevance_score: float = 0.0 + max_entries: int = 12 + include_low_confidence: bool = True + require_robot_state_for_motion: bool = True +``` + +**Implementation Notes:** +- Keep the current `deterministic_source_coverage_v1` as the default policy. +- Add thresholds only after Task 4 feedback exists. +- Make the policy pure Python with no module or RPC dependencies. +- `ContextProvider` should call this helper and remain a coordinator. + +**TDD Steps:** +1. Test default policy preserves current `context_evidence.v1` shape. +2. Test low-relevance spatial evidence can be filtered. +3. Test `max_entries` is enforced deterministically. +4. Test motion tasks retain robot/safety evidence. + +**Redundancy Check:** This is a refactor plus policy, not a new memory router. It owns selection rules only. + +--- + +### Task 6: Skill Evolution Proposal + +**Purpose:** Let the agent propose skill interface improvements without modifying executable skill code. + +**Files:** +- Create: `dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/skill_proposal.py` +- Modify: `dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/__init__.py` +- Modify: `dimos/robot/unitree/go2/blueprints/layers/layer_5_skill_interface/skill_interface_registry.py` only if read APIs need small additions. +- Test: `dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/test_skill_proposal.py` + +**Interface:** + +```python +@skill +def propose_skill_interface( + self, + task: str, + failure_context_json: str = "{}", +) -> SkillResult: + """Propose a new or revised skill interface without changing executable code.""" +``` + +**Proposal Shape:** + +```json +{ + "schema": "dimos.skill_proposal.v1", + "proposal_id": "...", + "task": "...", + "problem": "...", + "existing_skill_gaps": [], + "proposed_interface": { + "skill_name": "...", + "domain": "...", + "required_args": [], + "optional_args": [], + "risk_class": "low|medium|high", + "recommended_preflight": [] + }, + "validation_plan": [], + "requires_human_review": true +} +``` + +**Implementation Notes:** +- Read Layer 5 skill contracts and recent failures. +- Generate proposals deterministically from repeated known gaps in V1. +- Write proposals under `.dimos/evolution/proposals/` via the ledger. +- Never add `@skill` methods or modify system prompts automatically in V1. + +**TDD Steps:** +1. Test repeated missing-argument/context failures produce no new skill proposal; they should recommend better tool use. +2. Test repeated "skill missing capability" feedback produces a proposal file. +3. Test proposal always sets `requires_human_review=True`. +4. Test proposal can be compared against current contracts to avoid duplicates. + +**Redundancy Check:** This is a proposal generator, not a dynamic skill runtime. Existing skill code remains the only executable path. + +--- + +## Completion Gates + +Each task is complete only when: + +- Focused pytest passes. +- Ruff check and format check pass for touched Python files. +- Relevant Layer 3/4/5 design docs are updated. +- `dimos/robot/unitree/go2/PROJECT_CHANGELOG.md` has a scoped entry. +- MCP-exposed tools have docstrings, primitive parameter types, and `SkillResult` returns. +- No new module imports heavy perception/model dependencies at package import time unless that module explicitly owns them. + +## Defer Until Later + +- Automatic code modification of skills. +- Automatic Git merge or push. +- Training/fine-tuning models. +- Replacing ChromaDB, SQLite, or existing memory modules. +- A robot-agnostic abstraction before Go2 contracts prove stable. From 8a26dece93c1e3434d664ab7cd879bce6e8125df Mon Sep 17 00:00:00 2001 From: ZhangZheMing <1050297972@qq.com> Date: Sun, 5 Jul 2026 00:12:12 +0800 Subject: [PATCH 15/36] Add Chinese Go2 architecture review guide --- dimos/robot/unitree/go2/PROJECT_CHANGELOG.md | 4 +- ...-07-05-go2-agent-architecture-review.zh.md | 577 ++++++++++++++++++ 2 files changed, 580 insertions(+), 1 deletion(-) create mode 100644 docs/reviews/2026-07-05-go2-agent-architecture-review.zh.md diff --git a/dimos/robot/unitree/go2/PROJECT_CHANGELOG.md b/dimos/robot/unitree/go2/PROJECT_CHANGELOG.md index 229503a133..a6f4d9e2aa 100644 --- a/dimos/robot/unitree/go2/PROJECT_CHANGELOG.md +++ b/dimos/robot/unitree/go2/PROJECT_CHANGELOG.md @@ -11,9 +11,11 @@ Entries are listed in reverse chronological order. - Branch: `refactor/go2-architecture-layers` - Summary: Synchronized the branch with upstream `main` at `6e813a72`, resolved conflicts against the Go2 layered architecture, and added a detailed review - guide for the full Go2 agent architecture/self-evolution feature set. + guide in English and Chinese for the full Go2 agent architecture/self-evolution + feature set. - Files/modules: - `docs/reviews/2026-07-05-go2-agent-architecture-review.md` + - `docs/reviews/2026-07-05-go2-agent-architecture-review.zh.md` - `dimos/agents/mcp/mcp_server.py` - `dimos/agents/mcp/test_mcp_server.py` - `dimos/perception/spatial_perception.py` diff --git a/docs/reviews/2026-07-05-go2-agent-architecture-review.zh.md b/docs/reviews/2026-07-05-go2-agent-architecture-review.zh.md new file mode 100644 index 0000000000..45ad3cbd94 --- /dev/null +++ b/docs/reviews/2026-07-05-go2-agent-architecture-review.zh.md @@ -0,0 +1,577 @@ +# Go2 Agent 架构 Review 说明 + +日期: 2026-07-05 +分支: `refactor/go2-architecture-layers` +合并上游前的本地 HEAD: `646e7ec5` +已合并的上游目标: `upstream/main` at `6e813a72` +共同祖先: `1f544d05` + +这份文档是当前分支完整 review 的中文入口。它把上游同步内容和本地新增功能分开说明，并重点解释每个新增功能的实现逻辑、数据流、存储位置、判断主体、风险点和验证情况。 + +## 当前 Git 状态 + +合并前，本分支有 16 个本地提交不在 upstream，上游 `main` 有 160 个提交不在本分支。用 merge base 精确比较后，真正与本地改动重叠的文件有 15 个，实际产生冲突的文件有 7 个: + +- `dimos/agents/mcp/mcp_server.py` +- `dimos/agents/mcp/test_mcp_server.py` +- `dimos/perception/spatial_perception.py` +- `dimos/robot/unitree/go2/blueprints/agentic/unitree_go2_agentic.py` +- `dimos/robot/unitree/go2/blueprints/agentic/unitree_go2_temporal_memory.py` +- `dimos/robot/unitree/go2/blueprints/smart/unitree_go2_spatial.py` +- `dimos/robot/unitree/go2/test_connection.py` + +冲突解析原则: + +- 保留上游基础设施更新，尤其是 Zenoh transport、MCP capability gating、Unitree AES key 转发、文档和打包更新。 +- 保留本地 Go2 分层架构作为 Go2 agentic blueprint 的权威 wiring。 +- 使用上游新的 perception 模块路径，同时保留本地基于 `STATE_DIR` 的 spatial memory 默认持久化路径。 +- 两边新增的测试如果覆盖不同风险点，则合并保留，而不是二选一删除。 + +环境注意事项: + +- 上游现在在 macOS 上默认使用 `zenoh` transport。这个后端需要 `eclipse-zenoh`，本地 `.venv` 已安装 `eclipse-zenoh==1.9.0`。 +- 当前机器已经有一个 live `dimos` 进程监听 MCP 端口 `9990`。跑 MCP 测试时要换端口，例如 `MCP_PORT=9991`，否则测试客户端会连到 live 服务。 +- `git diff --cached --check -- ':(exclude)*.patch'` 已通过。完整 `git diff --check` 会报上游 `.patch` 文件内的 trailing whitespace，这些是 patch 文件的 context 行，应单独处理，不能随意改动以免破坏补丁语义。 + +## 需要 Review 的上游更新 + +这些不是本地自进化功能，但会影响本分支。 + +### 1. Zenoh Transport 集成 + +相关文件: + +- `dimos/core/transport_factory.py` +- `dimos/core/transport.py` +- `dimos/protocol/pubsub/impl/zenohpubsub.py` +- `dimos/protocol/service/zenohservice.py` +- `dimos/protocol/rpc/pubsubrpc.py` +- `dimos/protocol/tf/tf.py` +- `dimos/core/global_config.py` + +实现逻辑: + +- `GlobalConfig.transport` 现在可以是 `lcm` 或 `zenoh`。 +- macOS 上游默认值改为 `zenoh`，其他平台默认仍是 `lcm`。 +- `make_transport()` 根据当前后端把逻辑 channel name 映射成 LCM topic 或 Zenoh key expression。 +- Zenoh key expression 会放在 `dimos/` namespace 下。 +- RPC 和 TF 后端分别通过 `rpc_backend()`、`tf_backend()` 选择。 +- 高频 sensor channel 使用 latest-wins/best-effort QoS，human/agent 低频消息使用 reliable/blocking QoS。 + +Review 风险: + +- macOS 默认行为变了。如果本地运行仍假设 LCM，需要设置 `DIMOS_TRANSPORT=lcm`。 +- Zenoh 会创建 native pyo3 callback 线程，现有 pytest thread leak 检测可能在 cleanup 延迟时误报或暴露真实清理问题。 +- 长生命周期服务测试必须避开已经运行的 MCP server 端口。 + +### 2. MCP Tool Capability Gating + +相关文件: + +- `dimos/agents/capabilities.py` +- `dimos/agents/mcp/mcp_server.py` +- `dimos/agents/mcp/tool_stream.py` +- `dimos/agents/annotation.py` +- `dimos/agents/test_capabilities.py` + +实现逻辑: + +- `SkillInfo` 增加 capability 元数据，例如 `uses` 和 `lifecycle`。 +- `tools/list` 在相关工具上暴露 `_meta.dimos/uses` 和 `_meta.dimos/lifecycle`。 +- `tools/call` 在真正调用 skill 前先申请 capability lock。 +- instant skill 的占用可以等待；background skill 的占用需要先调用对应 stop tool。 +- background tool 的 capability release 绑定到 tool-stream stopped frame。 + +Review 风险: + +- 长时间运动或控制类 skill 必须正确声明 capability，否则 MCP server 无法串行化冲突行为。 +- background tool 必须稳定发出 stopped frame，否则 capability 可能被永久占用。 + +### 3. Coordinator RPC 和动态 Blueprint 加载 + +相关文件: + +- `dimos/core/coordination/coordinator_rpc.py` +- `dimos/core/coordination/module_coordinator.py` +- `dimos/core/coordination/worker_manager_python.py` + +实现逻辑: + +- `ModuleCoordinator` 可以暴露一个 singleton coordinator RPC service。 +- 客户端可以通过 coordinator RPC 查询模块、加载 blueprint。 +- 动态加载和 restart 期间，已部署 module proxy 由 lock 保护。 + +和本地改动的关系: + +- 本地 runtime plumbing 保留了 `McpClient` 的 non-blocking system-module notification，避免 agent 启动被直接 tool registration 阻塞。 +- 在 `DIMOS_TRANSPORT=lcm` 下，`ModuleCoordinator` 测试全部通过。 + +### 4. Memory 和 Perception 路径迁移 + +相关文件: + +- `dimos/perception/image_embedding.py` +- `dimos/perception/spatial_vector_db.py` +- `dimos/perception/visual_memory.py` +- `dimos/perception/spatial_perception.py` +- `dimos/memory2/*` + +实现逻辑: + +- 上游把 visual/spatial memory helper 从 `dimos.agents_deprecated.memory` 迁移到 `dimos.perception`。 +- 旧的 deprecated agent memory 目录被删除。 +- 上游新增了 memory2 TF service、replay 和 tooling。 + +本地解析: + +- `spatial_perception.py` 使用上游新的 `dimos.perception.*` 模块路径。 +- 仍保留本地默认持久化路径 `STATE_DIR / "spatial_memory"`，并支持 `DIMOS_SPATIAL_MEMORY_DIR` 覆盖，而不是改成上游 project assets output 路径。 + +### 5. Control、Manipulation、Learning、Docs、LFS 更新 + +概要: + +- control tasks 被拆成带 registry 的 package 目录。 +- 新增 roboplan、learning、data-prep 相关模块。 +- navigation 文档按 Mintlify 结构重组。 +- 新增 LFS 数据包和 native build 脚本。 + +Review 风险: + +- 这些是广泛的上游变更，不属于 Go2 自进化核心逻辑。主要审查 dependency、packaging、blueprint registry、CI 和运行环境影响。 + +## 本地新增功能清单 + +### 1. Go2 分层 Blueprint 架构 + +相关文件: + +- `dimos/robot/unitree/go2/blueprints/agentic/unitree_go2_agentic.py` +- `dimos/robot/unitree/go2/blueprints/smart/unitree_go2_spatial.py` +- `dimos/robot/unitree/go2/blueprints/agentic/unitree_go2_temporal_memory.py` +- `dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/__init__.py` +- `dimos/robot/unitree/go2/blueprints/layers/layer_4_world_state/__init__.py` +- `dimos/robot/unitree/go2/blueprints/layers/layer_5_skill_interface/__init__.py` +- `dimos/robot/unitree/go2/blueprints/layers/layer_6_robot_body/__init__.py` + +实现逻辑: + +- `unitree_go2_agentic` 现在由四层明确组成: `_go2_robot_body`、`_go2_spatial_world_state`、`_go2_skill_interface`、`_go2_agent_brain`。 +- 各 layer package 使用 lazy `__getattr__` 构建，避免 import 某一层时提前拉起重型 perception、VLM 或 robot stack。 +- `unitree_go2_spatial` 是非 agentic 的空间栈，由 Layer 6、Layer 4 spatial world state、Layer 5 spatial skills 组成。 +- `unitree_go2_temporal_memory` 通过 Layer 4 的 `_go2_temporal_memory_world_state()` 懒加载 temporal memory，确保 CLI config 已经生效后再创建 TemporalMemory blueprint。 + +为什么重要: + +- reviewer 可以分别审查 robot body、world state、skill interface、agent reasoning。 +- 防止 agent brain 变成无结构的大型 import hub。 +- 仍保留 DimOS blueprint 语义: 通过 `autoconnect()` 组合模块，通过 Spec 注入 RPC refs。 + +主要风险: + +- 分层不能掩盖漏接模块。`test_all_blueprints_generation.py` 和 Layer 4 wiring test 是主要保护。 + +### 2. Layer 6 Robot Body State + +相关文件: + +- `dimos/robot/unitree/go2/blueprints/layers/layer_6_robot_body/robot_body_state.py` +- `dimos/robot/unitree/go2/blueprints/layers/layer_6_robot_body/robot_body_spec.py` +- `dimos/robot/unitree/go2/blueprints/layers/layer_6_robot_body/test_robot_body_state.py` + +实现逻辑: + +- `_Go2RobotBodyState` 是一个小模块，通过 spec-facing API 暴露 robot body 状态。 +- 它贴近真实 Go2 connection stack，但不替代真实连接模块。 +- 上层通过 typed spec 消费 robot state，而不是直接访问硬件 connection module。 + +Review 重点: + +- 是否提供了 agent preflight 足够使用的安全相关状态。 +- 是否避免重复持有或篡改真实控制权。 + +### 3. Layer 4 Structured World State + +相关文件: + +- `dimos/robot/unitree/go2/blueprints/layers/layer_4_world_state/structured_world_state.py` +- `dimos/robot/unitree/go2/blueprints/layers/layer_4_world_state/semantic_temporal_map.py` +- `dimos/robot/unitree/go2/blueprints/layers/layer_4_world_state/world_state_spec.py` +- `dimos/robot/unitree/go2/blueprints/layers/layer_4_world_state/test_world_state.py` + +实现逻辑: + +- `_Go2StructuredWorldState` 通过 typed spec 暴露紧凑的 robot/world snapshot。 +- `_Go2SemanticTemporalMap` 把 spatial memory 和 temporal memory 摘要合并成更适合 agent 使用的 world-state surface。 +- Layer 3 通过 RPC spec 注入读取这些信息，不直接 import 具体 memory 实现。 + +这里的 "有效上下文" 是什么: + +- Layer 4 不让 LLM 直接判断 memory 相关性。 +- 它把可用 world/memory source 归一化成结构化 provider surface。 +- Layer 3 再负责打分、过滤、组合 evidence，最后交给 LLM 使用。 + +Review 重点: + +- memory backend 缺失时是否明确返回 unavailable，而不是静默空值。 +- snapshot schema 是否足够稳定，能被 prompt 和 dashboard 长期消费。 + +### 4. Layer 5 Skill Interface Registry + +相关文件: + +- `dimos/robot/unitree/go2/blueprints/layers/layer_5_skill_interface/skill_interface_registry.py` +- `dimos/robot/unitree/go2/blueprints/layers/layer_5_skill_interface/skill_interface_spec.py` +- `dimos/robot/unitree/go2/blueprints/layers/layer_5_skill_interface/test_skill_interface_registry.py` + +实现逻辑: + +- skill contract 是显式静态记录，包含 skill name、domain、module、required args、optional args、是否 motion sensitive、context requirement、推荐 preflight、risk class、outcome shape。 +- registry 把这些 contract 作为数据暴露给 Layer 3。 +- 已知非 Layer 5 MCP tool 被排除在 contract mismatch 检查之外，避免 status/preflight 内部工具污染 skill coverage。 + +和自进化的关系: + +- 机器人不会自动修改 skill 代码。 +- 当任务失败且有明确 missing capability 证据时，可以生成待 review 的 skill interface proposal。 +- 现有 contract 是 duplicate suppression 的判断基准。 + +Review 重点: + +- 每个 MCP action skill 是否都有准确 contract。 +- `risk_class`、`requires_context`、`requires_robot_state` 应按 safety-critical metadata 审查。 + +### 5. Layer 3 Expert Router + +相关文件: + +- `dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/expert_router.py` +- `dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/prompt_policy.py` +- `dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/test_expert_router.py` +- `dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/test_prompt_policy.py` + +实现逻辑: + +- router 根据 task domain 推荐应该使用的 expert/tool family。 +- prompt policy 把 Layer 3 自进化约束注入 MCP client system prompt，但不完全替代机器人原有 prompt。 +- 输出是确定性的 routing metadata，不是额外 LLM call。 + +判断主体: + +- 第一层 routing 和 evidence packaging 由确定性代码完成。 +- LLM 仍是读取最终上下文并决定如何执行、是否澄清、是否调用工具的主体。 + +Review 重点: + +- routing label 是否和实际 skill contract 对齐。 +- prompt 文案是否过度承诺自治或自动改代码能力。 + +### 6. Context Provider 和 Evidence Selection + +相关文件: + +- `dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/context_provider.py` +- `dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/context_evidence.py` +- `dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/test_context_provider.py` +- `dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/test_context_evidence.py` + +实现逻辑: + +- `get_context()` 汇总 task text、runtime metadata、robot state、RAG/spatial memory、temporal memory、skill contracts、skill outcomes、causal world model、context feedback。 +- `context_evidence.py` 提供纯函数策略 `build_context_evidence(metadata, ContextEvidencePolicy)`。 +- policy 字段包括 `min_relevance_score`、`max_entries`、`include_low_confidence`、`require_robot_state_for_motion`。 +- 返回结果包含 compact context 和 `context_evidence.v1`，说明选了哪些 evidence、为什么选。 + +现在 "有效" 的标准: + +- 有效表示被确定性 policy 从可用 provider 里选中。 +- 主要依据包括 relevance score、confidence、source availability、task type、safety requirement。 +- 这还不是基于 replay metric 学出来的策略。 +- LLM 会消费这些上下文，并且仍可以判断上下文不足。 + +Review 重点: + +- relevance/confidence 默认阈值是否过松。 +- motion task 是否必须要求 robot state。 +- low-confidence evidence 应该隐藏，还是保留但显式标注。 + +### 7. Memory Backend Status + +相关文件: + +- `dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/memory_backend_status.py` +- `dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/test_memory_backend_status.py` + +实现逻辑: + +- 新增 MCP skill `memory_backend_status()`。 +- 报告 world state、spatial memory、temporal memory、skill outcomes、causal world model、skill interface 是否已接线。 +- 能 probe 的 backend 会做小型 read probe。 +- 只返回诊断元数据，不创建新的 memory database。 + +Review 重点: + +- 这是诊断工具，不是新的 memory scheduler。 +- 它不应该写入 RAG、spatial memory、temporal memory 或 Git。 + +### 8. Git-backed Evolution Ledger + +相关文件: + +- `dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/evolution_event.py` +- `dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/evolution_ledger.py` +- `dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/evolution_proposal.py` +- `dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/test_evolution_ledger.py` +- `dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/test_evolution_proposal.py` + +实现逻辑: + +- event 使用 schema `dimos.evolution_event.v1`。 +- proposal 由 `evolution_proposal.py` 做 schema 验证。 +- 默认写入本地仓库路径: + `.dimos/evolution/events/YYYY/MM/DD/*.json` + 和 `.dimos/evolution/proposals/*.json`。 +- `DIMOS_EVOLUTION_LEDGER_DIR` 可以覆盖存储目录。 +- 写入内容是 JSON 文件，目标是低频、可 review、可 diff 的控制面记录。 +- 默认 `commit=False`。 +- 如果 `commit=True`，只把该 event/proposal 文件提交到本地 Git。 + +Git 内容会提交到哪里: + +- 文件先进入当前本地仓库工作区。 +- commit 只进入本地 `.git` 历史。 +- 不会自动 push 到 GitHub。 +- 只有人类后续运行 `git push` 或开 PR，GitHub 才会收到这些记录。 + +Review 重点: + +- event/proposal 字段是否可能造成路径穿越。 +- commit mode 是否只 stage 新 ledger 文件。 +- 根据隐私策略决定 `.dimos/evolution` 应该 ignore、commit，还是导出到别的存储。 + +### 9. Task Feasibility Preflight + +相关文件: + +- `dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/task_feasibility.py` +- `dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/test_task_feasibility.py` + +实现逻辑: + +- 新增 MCP skill `evaluate_task_feasibility(task, context_json)`。 +- 读取 skill contracts、world/robot context 和可选 JSON context。 +- 返回确定性结果: feasibility status、missing context、required skills、available skills、safety risks、recommended next action、clarifying question、evidence sources。 +- 如果 evolution ledger 已接线，可以写入 ledger event。 + +自进化含义: + +- 这不是模型自训练。 +- 它是 agent self-assessment: 在行动前，agent 可以调用确定性 evaluator 判断任务是否具备上下文和能力支持。 + +Review 重点: + +- motion-sensitive task 是否会误判为可行。 +- 缺少 context 时是否优先澄清，而不是生成 skill proposal。 + +### 10. Context Feedback Store + +相关文件: + +- `dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/context_feedback.py` +- `dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/test_context_feedback.py` + +实现逻辑: + +- 新增 MCP skill `record_context_feedback(...)`。 +- 内存里维护最多 100 条 recent feedback 的 bounded deque。 +- schema 为 `go2_context_feedback.v1`。 +- 如果 evolution ledger 已接线，会把反馈写成 ledger event。 +- ContextProvider 会把 compact feedback summary 纳入后续 context。 + +Review 重点: + +- feedback 默认是 session memory，除非 ledger 已接线。 +- 它不能无界增长，也不应该静默覆盖 RAG 或 temporal memory。 + +### 11. Skill Outcome Store 和 Predictor + +相关文件: + +- `dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/skill_outcome_store.py` +- `dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/skill_outcome_predictor.py` +- `dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/test_skill_outcomes.py` + +实现逻辑: + +- 记录近期 skill outcome，存储是 bounded 的。 +- 按 skill 汇总成功、失败和模式。 +- predictor 根据近期历史和 skill contract metadata 估计风险或可能结果。 +- ContextProvider 可以把这些 summary 作为 task evidence。 + +Review 重点: + +- predictor 是启发式，不是学习模型。 +- 旧失败不应永久阻止可用 skill。 + +### 12. Causal World Model 和 Dashboard Contract + +相关文件: + +- `dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/causal_world_model.py` +- `dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/causal_effect_estimator.py` +- `dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/online_transition_model.py` +- `dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/structural_causal_model.py` +- `dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/intervention_log.py` +- `dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/world_model_contract.py` +- `dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/test_causal_world_model.py` +- `dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/test_world_model_contract.py` + +实现逻辑: + +- 记录 causal transition 和 intervention。 +- 维护近期 world-state change 的 online transition model。 +- 基于记录事件估计 causal effect。 +- 输出 versioned contract: + `dimos.world_model_prediction.v1`、 + `dimos.world_model_provider.v1`、 + `dimos.world_model_dashboard.v1`。 +- 支持 save/load model state。 + +Review 重点: + +- 当前 causal estimate 只能作为 advisory evidence。 +- prediction schema 要对 dashboard 和 LLM prompt consumer 保持稳定。 +- persistence 不应意外混合不同 run 的数据。 + +### 13. Skill Interface Proposal Generator + +相关文件: + +- `dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/skill_proposal.py` +- `dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/test_skill_proposal.py` + +实现逻辑: + +- 新增 MCP skill `propose_skill_interface(task, failure_context_json)`。 +- 只有在 failure context 里有明确 missing capability evidence 时，才生成 `dimos.skill_proposal.v1` review artifact。 +- 对 missing arguments、missing context 不生成 proposal。 +- 如果现有 contract 已覆盖能力，则抑制重复 proposal。 +- 如果 ledger 已接线，通过 evolution ledger 写入 proposal 文件。 +- 不修改 Python 代码、不新增 `@skill`、不改 blueprint wiring。 + +Review 重点: + +- 这是最安全的自进化边界: 只生成 proposal，必须人工 review。 +- evidence requirement 要足够严格，避免用户表达不清时产生大量噪声 proposal。 + +### 14. MCP Runtime Plumbing + +相关文件: + +- `dimos/agents/mcp/mcp_client.py` +- `dimos/agents/mcp/mcp_server.py` +- `dimos/agents/mcp/tool_stream.py` +- `dimos/agents/mcp/test_mcp_client_unit.py` +- `dimos/agents/mcp/test_mcp_server.py` +- `dimos/agents/mcp/test_tool_stream.py` + +实现逻辑: + +- `McpServer.on_system_modules()` 避免通过 RPC 查询自己的 proxy。 +- 对自身 skill 和带 actor class 的 proxy，server 本地收集 schema。 +- `McpClient` 在 RPC path 外延迟 direct tool registration，避免启动阶段互相等待。 +- tool-stream progress frame 可以通过 persistent SSE 发送，并带 progress-token metadata。 +- background capability release 绑定 stopped frame。 +- `agent_send()` 使用 `make_transport()`，因此会尊重当前 backend。 + +Review 重点: + +- 启动路径不能等待依赖自身初始化的工具。 +- MCP 测试端口必须和 live robot session 隔离。 + +### 15. Runtime Hardening + +相关文件: + +- `dimos/core/coordination/module_coordinator.py` +- `dimos/core/coordination/worker_manager_python.py` +- `dimos/simulation/mujoco/mujoco_process.py` +- `dimos/robot/unitree/mujoco_connection.py` +- `dimos/robot/unitree/go2/connection.py` +- `dimos/robot/unitree/go2/test_connection.py` + +实现逻辑: + +- coordinator 的 system-module notification 避免等待已知 agent client。 +- worker manager/coordinator 变更保留 module restart、reload、stream rewiring 行为。 +- MuJoCo helper 可以读取当前可用 stderr，避免阻塞。 +- macOS headless MuJoCo 在需要时使用当前 Python executable。 +- Go2 WebRTC connection 会从 `GlobalConfig` 转发 AES key。 + +Review 重点: + +- 这些是运行时稳定性修复，重点审查 process lifecycle 和 stream rewiring，而不是 agent reasoning。 + +### 16. Rerun/Websocket Dashboard World-model State + +相关文件: + +- `dimos/web/websocket_vis/websocket_vis_module.py` +- `dimos/web/websocket_vis_spec.py` +- `dimos/web/templates/rerun_dashboard.html` +- `dimos/web/websocket_vis/test_websocket_vis_module.py` + +实现逻辑: + +- websocket visualization 除已有 payload 外，还可以携带 world-model/dashboard state。 +- dashboard payload shape 与 `world_model_contract.py` 对齐。 + +Review 重点: + +- dashboard state 应保持展示用途。 +- dashboard consumer 不应依赖未 version 的内部 Python object。 + +## 推荐 Review 顺序 + +1. 先看 Go2 blueprint wiring: + `unitree_go2_agentic.py`、Layer 3/4/5/6 `__init__.py`、`test_world_state.py`。 +2. 再看 MCP runtime plumbing: + `mcp_server.py`、`mcp_client.py`、`tool_stream.py` 和对应测试。 +3. 再看自进化写入边界: + `evolution_ledger.py`、`evolution_proposal.py`、`skill_proposal.py`。 +4. 再看 context 质量: + `context_provider.py`、`context_evidence.py`、`context_feedback.py`。 +5. 再看 action safety: + `skill_interface_registry.py`、`task_feasibility.py`、`skill_outcome_predictor.py`。 +6. 再看 advisory world-model: + `causal_world_model.py` 和 `world_model_contract.py`。 +7. 最后单独 review 上游 transport 变化，尤其是 macOS 默认 Zenoh。 + +## 本次同步后的验证结果 + +已通过: + +- `.venv/bin/python -m pytest dimos/agents/mcp/test_mcp_server.py -q` + - `10 passed, 1 warning` +- `.venv/bin/python -m pytest dimos/robot/unitree/go2/test_connection.py -q` + - `5 passed` +- `.venv/bin/python -m pytest dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain -q` + - `62 passed` +- `.venv/bin/python -m pytest dimos/robot/unitree/go2/blueprints/layers/layer_4_world_state/test_world_state.py -q` + - `3 passed, 1 warning` +- `.venv/bin/python -m pytest dimos/perception/test_spatial_perception_config.py -q` + - `2 passed` +- `DIMOS_TRANSPORT=lcm .venv/bin/python -m pytest dimos/core/coordination/test_module_coordinator.py -q` + - `35 passed` +- `DIMOS_TRANSPORT=lcm MCP_PORT=9991 .venv/bin/python -m pytest dimos/agents/mcp/test_mcp_client_unit.py dimos/agents/mcp/test_tool_stream.py -q` + - `44 passed, 1 warning` +- `.venv/bin/python -m pytest dimos/robot/test_all_blueprints_generation.py -q` + - `1 passed` +- `git diff --cached --check -- ':(exclude)*.patch'` + - passed + +已知 caveat: + +- 本机有 live `dimos-live` 进程占用默认 MCP 端口 `9990`，所以默认端口跑 MCP 测试会失败。 +- 上游在 macOS 默认 `zenoh` backend 后，部分测试暴露 native pyo3 thread 生命周期问题。同样 coordinator 和 MCP 测试在 `DIMOS_TRANSPORT=lcm` 下通过。 +- 上游 `.patch` 文件包含会触发完整 `git diff --check` 的 whitespace；当前验证排除了 `.patch`，避免破坏补丁语义。 From ccd81770d7d453d06a5b38b09bba0838b6ab9877 Mon Sep 17 00:00:00 2001 From: ZhangZheMing <1050297972@qq.com> Date: Sun, 5 Jul 2026 00:32:07 +0800 Subject: [PATCH 16/36] Expand Go2 architecture review guides --- dimos/robot/unitree/go2/PROJECT_CHANGELOG.md | 19 ++ ...026-07-05-go2-agent-architecture-review.md | 183 ++++++++++++++++++ ...-07-05-go2-agent-architecture-review.zh.md | 124 ++++++++++++ 3 files changed, 326 insertions(+) diff --git a/dimos/robot/unitree/go2/PROJECT_CHANGELOG.md b/dimos/robot/unitree/go2/PROJECT_CHANGELOG.md index a6f4d9e2aa..5c8c5ae8ce 100644 --- a/dimos/robot/unitree/go2/PROJECT_CHANGELOG.md +++ b/dimos/robot/unitree/go2/PROJECT_CHANGELOG.md @@ -8,6 +8,25 @@ Entries are listed in reverse chronological order. ## 2026-07-05 +- Branch: `refactor/go2-architecture-layers` +- Summary: Expanded the English and Chinese Go2 architecture review guides with + finer implementation granularity, including entry points, inputs, decision + ownership, state/write boundaries, failure modes, and a detailed explanation + of the causal world model's online linear model, hand-authored SCM, and + observational estimator. +- Files/modules: + - `docs/reviews/2026-07-05-go2-agent-architecture-review.md` + - `docs/reviews/2026-07-05-go2-agent-architecture-review.zh.md` + - `dimos/robot/unitree/go2/PROJECT_CHANGELOG.md` +- Validation: + - Documentation-only change. + - Ran `git diff --check` on the changed review/changelog files. +- Open items: + - Keep these review guides in sync if the implementation changes after code + review. + +## 2026-07-05 + - Branch: `refactor/go2-architecture-layers` - Summary: Synchronized the branch with upstream `main` at `6e813a72`, resolved conflicts against the Go2 layered architecture, and added a detailed review diff --git a/docs/reviews/2026-07-05-go2-agent-architecture-review.md b/docs/reviews/2026-07-05-go2-agent-architecture-review.md index 55f3a54eb7..31b7b6d745 100644 --- a/docs/reviews/2026-07-05-go2-agent-architecture-review.md +++ b/docs/reviews/2026-07-05-go2-agent-architecture-review.md @@ -46,6 +46,35 @@ Important environment note: those are patch context lines and should be handled separately if we want a repository-wide whitespace cleanup. +## Review Granularity + +Use this structure when reviewing each local feature. The goal is to avoid +approving a feature just because the file names sound plausible. + +- Entry point: Which blueprint, module, MCP skill, RPC method, or helper is the + first public surface? +- Inputs: Which JSON strings, spec providers, memory providers, or runtime + states feed it? +- Decision owner: Is the decision made by deterministic code, the LLM reading + context, a small online statistical model, or a human reviewer? +- State: Is the feature stateless, in-memory only, persisted to JSON, persisted + to Git, or stored in an existing RAG/vector database? +- Write boundary: Which exact method writes, and which methods are read-only? +- Failure mode: Does missing data produce `unavailable`, `uncertain`, a + clarifying question, a failed `SkillResult`, or a proposal artifact? +- Safety boundary: Can it execute robot motion, or only advise another layer? +- Review artifact: Which schema should be stable enough for future tooling? +- Tests: Which test proves the data path, and which test proves the failure path? + +The most important distinction in this branch is between three very different +things: + +- Context selection: deterministic filtering and summarization before the LLM + sees context. +- Agent judgment: the LLM deciding what to do with selected context and tools. +- Self-evolution artifact: reviewable JSON events/proposals, not automatic code + mutation. + ## Upstream Changes To Review These are not local self-evolution features, but they can affect this branch. @@ -345,6 +374,29 @@ What "effective" means today: - It is not yet learned from replay metrics. - The LLM consumes the selected context and may still judge it insufficient. +Call path and ownership: + +1. The caller invokes `get_context()` with task/runtime metadata. +2. ContextProvider asks wired providers for world state, memory summaries, + skill contracts, outcome summaries, causal-world-model state, and context + feedback. If a provider is not wired, the provider contributes an explicit + unavailable status rather than throwing away the source silently. +3. Metadata is passed to `build_context_evidence()`. That helper is pure: + given metadata and a `ContextEvidencePolicy`, it returns selected evidence + without calling RPC, writing memory, or consulting an LLM. +4. ContextProvider formats the compact context and attaches + `context_evidence.v1` so reviewers can see why the context contained those + entries. +5. The LLM is still the consumer and final user-facing reasoning actor. The + provider does not decide to execute a skill. + +What is not implemented: + +- No learned ranker is trained from replay logs yet. +- No RAG/vector database entries are written by ContextProvider. +- Context feedback is summarized as a signal, but it does not override provider + data by itself. + Review focus: - Check whether relevance/confidence defaults are too permissive. @@ -400,6 +452,26 @@ Where does Git data go: - It does not push to GitHub. - GitHub only receives it if a human later runs `git push` or opens a PR. +Write path: + +1. A caller invokes a ledger-facing MCP skill such as + `record_evolution_event()` or `record_skill_proposal()`. +2. The module validates the schema and normalizes the payload. +3. The module chooses the storage root. The default is the repo-local + `.dimos/evolution` tree; `DIMOS_EVOLUTION_LEDGER_DIR` overrides it. +4. The JSON file is written as the review artifact. +5. If `commit=False`, Git is not touched beyond the working tree file. +6. If `commit=True`, only that new event/proposal file is staged and committed + locally. There is no push. + +Privacy and review boundary: + +- These files can contain task text, outcome messages, context summaries, and + proposal rationale. Decide whether `.dimos/evolution` belongs in normal Git + history before enabling commit mode on real robot runs. +- The ledger is a control-plane audit trail. It is not a replacement for + SpatialMemory, TemporalMemory, SkillOutcomeStore, or the causal model state. + Review focus: - Confirm no path traversal is possible through event/proposal fields. @@ -429,6 +501,25 @@ Self-evolution meaning: - It is agent self-assessment: before acting, the agent can ask a deterministic evaluator whether the task has enough context and capability support. +Decision tree: + +1. Parse the task and optional `context_json`. +2. Match the task against known skill contracts and expert domains. +3. Check required arguments and context requirements from Layer 5 contracts. +4. Check robot/world-state availability for motion-sensitive or risky skills. +5. Produce one of three broad outcomes: + `feasible`, `uncertain`, or not feasible. +6. If data is missing, recommend a clarifying question or context-gathering + action. +7. If capability is missing, report it as missing capability evidence. That is + the input that can later justify a skill-interface proposal. + +Boundary: + +- It does not call the target skill. +- It does not create a new skill. +- It does not ask the LLM to classify feasibility. + Review focus: - Check false positives for motion-sensitive tasks. @@ -499,12 +590,87 @@ Implementation logic: `dimos.world_model_dashboard.v1`. - Supports save/load of model state. +Is it a trained model? + +- Strict answer: there is a very small online model, but there is no offline + training pipeline and no neural world model. +- `OnlineTransitionOutcomeModel` is a lightweight logistic-style linear model. + It stores feature weights in a dictionary and updates them incrementally when + `record_causal_transition()` receives an observed success/failure outcome. +- If no transitions have been recorded, its `sample_count` is zero and its + prediction should be treated as low-confidence prior behavior, not learned + robot knowledge. +- The symbolic SCM is hand-authored. It is not learned from data. +- The causal effect estimator is observational statistics. It compares smoothed + success rates for active features and does not control hidden confounders. + +Components: + +- `OnlineTransitionOutcomeModel`: online success-probability scorer. Features + include bias, skill name, domain, odometry presence, navigation state, spatial + memory availability, spatial match count, temporal availability, and selected + action argument presence/value. The update is one-step gradient adjustment + with L2 shrinkage. +- `CausalEffectEstimator`: bounded observation store with treated/control + success-rate differences for symbolic features. It can produce risk factors, + supporting factors, and intervention suggestions, but its own output declares + the assumption that unobserved confounders are not controlled. +- `StructuralCausalModel`: hand-authored causal graph for variables such as + `spatial_has_matches`, `target_resolvability`, `odom_ready`, + `motion_safety`, and `navigation_success`. It also emits counterfactual + suggestions such as "tag or perceive target before semantic navigation". +- `InterventionLog`: records explicit interventions and lets prediction surface + prior intervention evidence. +- `_WorldTransition`: the compact event record tying task, skill, before/after + state, predicted risk, outcome, inferred cause, and recovery suggestion + together. + +Data flow for recording: + +1. `record_causal_transition()` receives task, skill name, optional args, + before/after Layer 4 snapshots, prediction metadata, and outcome metadata. +2. Inputs are parsed as JSON objects and validated. Missing outcome can fall + back to the latest same-skill outcome if SkillOutcomeStore is wired. +3. The module infers a coarse cause and recovery suggestion from context, + prediction reasons, outcome success, error code, and message. +4. It computes a symbolic state delta from before/after snapshots. +5. It calls `_outcome_model.update(...)` and `_causal_estimator.update(...)`. +6. It appends the transition to an in-memory deque capped at 200 transitions. +7. It autosaves to `DIMOS_GO2_WORLD_MODEL_STATE` only if that env path is set. +8. It publishes dashboard state if WebsocketVis is wired. + +Data flow for prediction: + +1. `predict_next_state()` parses a snapshot and candidate action. +2. It gets recent same-skill transitions and derives repeated failure modes. +3. It computes rule-based risk reasons from the current snapshot and action. +4. It asks the online model for success probability, score, risk, confidence, + and top weighted feature contributions. +5. It asks the causal estimator for observational risk/support factors. +6. It asks the hand-authored SCM for variables, edges, and counterfactuals. +7. It asks the intervention log for matching intervention evidence. +8. It combines rule risk and model risk into a final risk and score. +9. It returns `dimos.world_model_prediction.v1` and optionally publishes the + dashboard contract. + +Trust boundary: + +- The output is advisory. It should help the LLM choose preflight, perception, + clarification, or a safer skill path. +- It should not directly command robot motion. +- High confidence currently requires enough recent samples for the same skill; + otherwise the model should be treated as a low-confidence heuristic. + Review focus: - Treat current causal estimates as advisory evidence. - Validate that prediction schemas remain stable for dashboard and LLM prompt consumers. - Check that persistence does not mix data across runs unexpectedly. +- Check that the UI/prompt labels do not imply a fully trained physical world + model. +- Check that replay or simulation evaluation is added before relying on this + for autonomous action selection. ### 13. Skill Interface Proposal Generator @@ -524,6 +690,23 @@ Implementation logic: - Writes proposal files through the evolution ledger when wired. - Does not modify Python code, skill decorators, or blueprint wiring. +Decision tree: + +1. Parse `failure_context_json`. +2. Look for explicit missing capability evidence, not just failed execution. +3. Check existing Layer 5 contracts to avoid proposing a duplicate interface. +4. Reject cases that are only missing arguments, missing context, or temporary + provider unavailability. +5. Build a proposal artifact with task, evidence, suggested interface shape, + expected inputs, expected output, and review rationale. +6. Write it through the evolution ledger if a ledger is wired. + +Human boundary: + +- This tool is intentionally proposal-only. A developer still writes code, + chooses the container, adds `@skill`, updates contracts/prompts, and tests the + behavior. + Review focus: - This is the safest self-evolution boundary: proposal-only, human-reviewed. diff --git a/docs/reviews/2026-07-05-go2-agent-architecture-review.zh.md b/docs/reviews/2026-07-05-go2-agent-architecture-review.zh.md index 45ad3cbd94..1447d621a3 100644 --- a/docs/reviews/2026-07-05-go2-agent-architecture-review.zh.md +++ b/docs/reviews/2026-07-05-go2-agent-architecture-review.zh.md @@ -33,6 +33,26 @@ - 当前机器已经有一个 live `dimos` 进程监听 MCP 端口 `9990`。跑 MCP 测试时要换端口，例如 `MCP_PORT=9991`，否则测试客户端会连到 live 服务。 - `git diff --cached --check -- ':(exclude)*.patch'` 已通过。完整 `git diff --check` 会报上游 `.patch` 文件内的 trailing whitespace，这些是 patch 文件的 context 行，应单独处理，不能随意改动以免破坏补丁语义。 +## Review 粒度标准 + +review 每个本地功能时，建议都按下面这些问题看，而不是只看文件名是否合理: + +- 入口: 第一个对外入口是 blueprint、module、MCP skill、RPC method，还是 helper 函数？ +- 输入: 它读取哪些 JSON 字符串、Spec provider、memory provider 或 runtime state？ +- 判断主体: 判断是确定性代码做的，LLM 做的，小型在线统计模型做的，还是人类 reviewer 做的？ +- 状态: 它是无状态、只在内存、写 JSON、写 Git，还是写已有 RAG/vector database？ +- 写边界: 哪个方法会写入？哪些方法是纯 read-only？ +- 失败形态: 缺数据时返回 `unavailable`、`uncertain`、clarifying question、failed `SkillResult`，还是 proposal artifact？ +- 安全边界: 它能执行机器人运动，还是只给上层建议？ +- Review artifact: 哪个 schema 是未来工具要依赖的稳定接口？ +- 测试: 哪个测试证明主数据流？哪个测试证明失败路径？ + +这个分支最重要的区分是三件事: + +- Context selection: LLM 看到上下文前，由确定性代码做过滤和摘要。 +- Agent judgment: LLM 读取已选上下文和工具后决定下一步。 +- Self-evolution artifact: 产出可 review 的 JSON event/proposal，不自动改代码。 + ## 需要 Review 的上游更新 这些不是本地自进化功能，但会影响本分支。 @@ -289,6 +309,20 @@ Review 重点: - 这还不是基于 replay metric 学出来的策略。 - LLM 会消费这些上下文，并且仍可以判断上下文不足。 +调用路径和归属: + +1. 调用方带着 task/runtime metadata 调用 `get_context()`。 +2. ContextProvider 向已接线 provider 请求 world state、memory summary、skill contract、outcome summary、causal-world-model state、context feedback。provider 没接线时，应贡献明确的 unavailable 状态，而不是静默丢掉该 source。 +3. metadata 被传给 `build_context_evidence()`。这个 helper 是纯函数: 给定 metadata 和 `ContextEvidencePolicy`，返回被选 evidence，不发 RPC、不写 memory、不调用 LLM。 +4. ContextProvider 格式化 compact context，并附加 `context_evidence.v1`，让 reviewer 能看到为什么选了这些 evidence。 +5. LLM 仍然是最终消费上下文、做用户可见推理和工具选择的主体。ContextProvider 本身不决定执行 skill。 + +没有实现的边界: + +- 还没有用 replay log 训练 learned ranker。 +- ContextProvider 不写 RAG/vector database。 +- context feedback 只是一个摘要信号，不会单独覆盖 provider data。 + Review 重点: - relevance/confidence 默认阈值是否过松。 @@ -343,6 +377,20 @@ Git 内容会提交到哪里: - 不会自动 push 到 GitHub。 - 只有人类后续运行 `git push` 或开 PR，GitHub 才会收到这些记录。 +写入路径: + +1. 调用方调用 ledger-facing MCP skill，例如 `record_evolution_event()` 或 `record_skill_proposal()`。 +2. 模块验证 schema，并规范化 payload。 +3. 模块选择存储 root。默认是 repo-local `.dimos/evolution`，`DIMOS_EVOLUTION_LEDGER_DIR` 可以覆盖。 +4. 写入 JSON 文件，作为可 review artifact。 +5. 如果 `commit=False`，除了工作区文件外不碰 Git。 +6. 如果 `commit=True`，只 stage 并提交该 event/proposal 文件到本地 Git，不 push。 + +隐私和 review 边界: + +- 这些文件可能包含 task text、outcome message、context summary、proposal rationale。真实机器人运行前，要先决定 `.dimos/evolution` 是否应该进入正常 Git 历史。 +- ledger 是控制面审计轨迹，不替代 SpatialMemory、TemporalMemory、SkillOutcomeStore 或 causal model state。 + Review 重点: - event/proposal 字段是否可能造成路径穿越。 @@ -368,6 +416,22 @@ Review 重点: - 这不是模型自训练。 - 它是 agent self-assessment: 在行动前，agent 可以调用确定性 evaluator 判断任务是否具备上下文和能力支持。 +决策树: + +1. 解析 task 和可选 `context_json`。 +2. 把 task 匹配到已知 skill contract 和 expert domain。 +3. 根据 Layer 5 contract 检查 required args 和 context requirements。 +4. 对 motion-sensitive 或高风险 skill 检查 robot/world-state 是否可用。 +5. 输出三类粗粒度结果之一: `feasible`、`uncertain`、not feasible。 +6. 如果缺数据，推荐澄清问题或 context-gathering action。 +7. 如果缺能力，报告 missing capability evidence。这个 evidence 后续可以成为 skill-interface proposal 的依据。 + +边界: + +- 它不会调用目标 skill。 +- 它不会创建新 skill。 +- 它不会让 LLM 判断 feasibility。 + Review 重点: - motion-sensitive task 是否会误判为可行。 @@ -437,11 +501,58 @@ Review 重点: `dimos.world_model_dashboard.v1`。 - 支持 save/load model state。 +它是不是训练了模型: + +- 严格说: 有一个非常小的在线模型，但没有离线训练流程，也没有神经网络世界模型。 +- `OnlineTransitionOutcomeModel` 是 lightweight logistic-style linear model。它把 feature weight 存在 dict 里，每次 `record_causal_transition()` 收到一次已观测 success/failure outcome 时增量更新。 +- 如果还没有记录 transition，它的 `sample_count` 是 0，此时预测只能视为低置信先验行为，不是已学习的机器人知识。 +- symbolic SCM 是手写的，不是从数据学出来的。 +- causal effect estimator 是观测统计估计，只比较 feature active/inactive 的平滑成功率差异，不控制 hidden confounders。 + +组成部分: + +- `OnlineTransitionOutcomeModel`: 在线 success-probability scorer。feature 包括 bias、skill name、domain、是否有 odom、navigation state、spatial memory 是否可用、spatial match count、temporal 是否可用、action argument 是否存在和部分 argument value。更新方式是一阶 gradient adjustment，带 L2 shrinkage。 +- `CausalEffectEstimator`: bounded observation store，对 symbolic feature 做 treated/control success-rate difference。它能给出 risk factor、supporting factor 和 intervention suggestion，但输出里明确声明 unobserved confounders are not controlled。 +- `StructuralCausalModel`: 手写因果图，变量包括 `spatial_has_matches`、`target_resolvability`、`odom_ready`、`motion_safety`、`navigation_success`。它还会生成 counterfactual suggestion，例如语义导航前先 tag 或 perceive target。 +- `InterventionLog`: 记录显式 intervention，并让 prediction 能引用历史 intervention evidence。 +- `_WorldTransition`: compact event record，把 task、skill、before/after state、predicted risk、outcome、inferred cause、recovery suggestion 绑定起来。 + +记录数据流: + +1. `record_causal_transition()` 接收 task、skill name、可选 args、before/after Layer 4 snapshot、prediction metadata、outcome metadata。 +2. 输入会按 JSON object 解析和验证。缺少 outcome 时，如果 SkillOutcomeStore 已接线，可以回退到同 skill 的最新 outcome。 +3. 模块根据 context、prediction reasons、outcome success、error code、message 推断粗粒度 cause 和 recovery suggestion。 +4. 模块计算 before/after snapshot 的 symbolic state delta。 +5. 调用 `_outcome_model.update(...)` 和 `_causal_estimator.update(...)`。 +6. 把 transition append 到内存 deque，最多保留 200 条。 +7. 只有设置了 `DIMOS_GO2_WORLD_MODEL_STATE` 时才 autosave。 +8. 如果 WebsocketVis 已接线，会 publish dashboard state。 + +预测数据流: + +1. `predict_next_state()` 解析 snapshot 和 candidate action。 +2. 读取同 skill 的 recent transitions，提取 repeated failure modes。 +3. 根据当前 snapshot 和 action 计算 rule-based risk reasons。 +4. 调用在线模型得到 success probability、score、risk、confidence、top weighted feature contribution。 +5. 调用 causal estimator 得到 observational risk/support factors。 +6. 调用手写 SCM 得到 variables、edges、counterfactuals。 +7. 调用 intervention log 查找匹配的 intervention evidence。 +8. 合并 rule risk 和 model risk，得到最终 risk 和 score。 +9. 返回 `dimos.world_model_prediction.v1`，并可选 publish dashboard contract。 + +可信边界: + +- 输出只是 advisory evidence，用来帮助 LLM 选择 preflight、perception、clarification 或更安全的 skill path。 +- 它不能直接命令机器人运动。 +- 当前 high confidence 需要同 skill 足够样本；否则应当视为低置信启发式。 + Review 重点: - 当前 causal estimate 只能作为 advisory evidence。 - prediction schema 要对 dashboard 和 LLM prompt consumer 保持稳定。 - persistence 不应意外混合不同 run 的数据。 +- UI/prompt 文案不能暗示这是完整训练好的 physical world model。 +- 在依赖它做自主 action selection 前，应补 replay 或 simulation evaluation。 ### 13. Skill Interface Proposal Generator @@ -459,6 +570,19 @@ Review 重点: - 如果 ledger 已接线，通过 evolution ledger 写入 proposal 文件。 - 不修改 Python 代码、不新增 `@skill`、不改 blueprint wiring。 +决策树: + +1. 解析 `failure_context_json`。 +2. 查找明确 missing capability evidence，而不是只看执行失败。 +3. 查询现有 Layer 5 contracts，避免 proposal 重复。 +4. 如果只是缺 argument、缺 context、provider 暂时 unavailable，则拒绝生成 proposal。 +5. 构造 proposal artifact，包含 task、evidence、suggested interface shape、expected inputs、expected output、review rationale。 +6. 如果 ledger 已接线，通过 evolution ledger 写入 proposal。 + +人工边界: + +- 这个工具故意只做 proposal。开发者仍然需要写代码、选择 container、加 `@skill`、更新 contract/prompt，并测试行为。 + Review 重点: - 这是最安全的自进化边界: 只生成 proposal，必须人工 review。 From 4aa5d324cec0d12e5446386493d59cb371d4757b Mon Sep 17 00:00:00 2001 From: ZhangZheMing <1050297972@qq.com> Date: Sun, 5 Jul 2026 00:57:04 +0800 Subject: [PATCH 17/36] Normalize Go2 review guide detail --- dimos/robot/unitree/go2/PROJECT_CHANGELOG.md | 19 + ...026-07-05-go2-agent-architecture-review.md | 567 +++++++++++++++++- ...-07-05-go2-agent-architecture-review.zh.md | 450 +++++++++++++- 3 files changed, 1026 insertions(+), 10 deletions(-) diff --git a/dimos/robot/unitree/go2/PROJECT_CHANGELOG.md b/dimos/robot/unitree/go2/PROJECT_CHANGELOG.md index 5c8c5ae8ce..2646c64598 100644 --- a/dimos/robot/unitree/go2/PROJECT_CHANGELOG.md +++ b/dimos/robot/unitree/go2/PROJECT_CHANGELOG.md @@ -8,6 +8,25 @@ Entries are listed in reverse chronological order. ## 2026-07-05 +- Branch: `refactor/go2-architecture-layers` +- Summary: Normalized every local Go2 architecture/self-evolution review-guide + section to the same implementation-review depth as the causal world model + section. Added per-feature component breakdowns, inputs, data flows, + decision ownership, state/write boundaries, failure modes, and safety/review + boundaries in both English and Chinese. +- Files/modules: + - `docs/reviews/2026-07-05-go2-agent-architecture-review.md` + - `docs/reviews/2026-07-05-go2-agent-architecture-review.zh.md` + - `dimos/robot/unitree/go2/PROJECT_CHANGELOG.md` +- Validation: + - Documentation-only change. + - Ran `git diff --check` on the changed review/changelog files. +- Open items: + - During code review, keep the review guide aligned with any implementation + changes requested by reviewers. + +## 2026-07-05 + - Branch: `refactor/go2-architecture-layers` - Summary: Expanded the English and Chinese Go2 architecture review guides with finer implementation granularity, including entry points, inputs, decision diff --git a/docs/reviews/2026-07-05-go2-agent-architecture-review.md b/docs/reviews/2026-07-05-go2-agent-architecture-review.md index 31b7b6d745..2bb0b4f3fe 100644 --- a/docs/reviews/2026-07-05-go2-agent-architecture-review.md +++ b/docs/reviews/2026-07-05-go2-agent-architecture-review.md @@ -238,6 +238,44 @@ Primary risk: - The layering must not hide missing blueprint modules. The static registry test and Layer 4 wiring test are the main protections. +Components: + +- `unitree_go2_agentic`: the full agentic Go2 stack. It composes the physical + body, structured world state, skill interface registry/containers, and Layer + 3 MCP/LLM brain. +- `unitree_go2_spatial`: the perception/world-state stack without the LLM brain. + It is useful for validating Layer 4 and Layer 5 without agent behavior. +- `unitree_go2_temporal_memory`: an additive blueprint that wraps the agentic + stack and adds temporal memory through the Layer 4 factory. +- Layer package `__init__.py` files: lazy blueprint factories. They are part of + the architecture, not just import convenience, because they prevent import + side effects from pulling in heavy perception/model dependencies too early. + +Construction flow: + +1. CLI imports a named blueprint lazily through the registry. +2. The blueprint imports only the layer handles it needs. +3. Each layer handle builds an `autoconnect()` blueprint for its modules. +4. The final blueprint is still ordinary DimOS composition, so stream wiring and + Spec-based RPC injection happen in `ModuleCoordinator`. +5. Tests validate that Layer 3 can reach Layer 4/5/6 through the intended + contracts and that the generated blueprint registry remains current. + +State and persistence: + +- The blueprint layer itself stores no robot data. +- State is owned by modules inside the layers: body state in Layer 6, memory + and world snapshots in Layer 4, skill contracts in Layer 5, context/outcome + and causal state in Layer 3. +- Any persistence belongs to those modules, not the blueprint file. + +Safety boundary: + +- The blueprint decides what modules are deployed together. +- It does not decide which skill to call, and it does not execute motion. +- Reviewers should treat wiring mistakes as safety risks because the LLM may + see missing or wrong tools/context. + ### 2. Layer 6 Robot Body State Files: @@ -259,6 +297,41 @@ Review focus: - Make sure it reports enough state for safety-sensitive agent preflight. - Make sure it does not duplicate live control authority. +Components: + +- `_Go2RobotBodyState`: the module that adapts low-level robot status into a + compact Layer 6 provider. +- `robot_body_spec.py`: the typed RPC contract consumed by upper layers. +- Layer 6 package factory: wires the underlying Go2 robot blueprint with the + body-state module. + +Data flow: + +1. The underlying Go2 connection and robot modules own live hardware/simulation + communication. +2. `_Go2RobotBodyState` exposes read-oriented body-state summaries through its + spec. +3. Layer 4/Layer 3 use the spec for safety preflight and context evidence. +4. The agent brain sees summarized state, not direct hardware handles. + +Decision owner: + +- The body-state module does not make task decisions. +- It is an evidence provider. Layer 3 deterministic preflight and the LLM use + its output to decide whether to wait, clarify, or call a skill. + +State and write boundary: + +- It should be treated as a current-state adapter. +- It should not persist memory and should not issue control commands. +- Any future body-state caching should be explicitly bounded and timestamped. + +Failure mode: + +- If the underlying robot state is missing, the correct behavior is explicit + unavailable/unknown state, not fabricated readiness. +- Motion-sensitive preflight should treat missing robot body state as risk. + ### 3. Layer 4 Structured World State Files: @@ -289,6 +362,46 @@ Review focus: - Check that snapshot schemas are stable enough for prompt and dashboard consumers. +Components: + +- `_Go2StructuredWorldState`: normalizes robot/world information into a compact + snapshot provider. +- `_Go2SemanticTemporalMap`: fuses spatial memory and temporal memory summaries + into semantic evidence for agent context. +- `world_state_spec.py`: the RPC contract Layer 3 relies on. +- `_go2_temporal_memory_world_state()`: lazy factory that delays TemporalMemory + construction until CLI flags such as `--new-memory` are applied. + +Data flow: + +1. SpatialMemory and TemporalMemory remain the storage/search systems. +2. Layer 4 queries or receives summaries from those systems. +3. Layer 4 shapes them into a stable world-state object with explicit source + availability. +4. Layer 3 asks for world state through the spec and then selects evidence for + the current task. + +State and persistence: + +- Layer 4 may read from persistent memory backends, but its own structured + snapshot is a derived view. +- Temporal memory persistence is controlled by the TemporalMemory module and CLI + config. +- Spatial memory persistence is controlled by `SpatialConfig` and the state-dir + path logic in `spatial_perception.py`. + +Decision owner: + +- Layer 4 decides how to normalize provider output. +- Layer 4 does not decide whether the context is sufficient for action. That + decision belongs to Layer 3 preflight policy and the LLM. + +Failure mode: + +- Provider unavailable means the snapshot should include unavailable metadata. +- Empty search results are not the same as backend failure; reviewers should + check that these are represented differently. + ### 4. Layer 5 Skill Interface Registry Files: @@ -317,6 +430,47 @@ Review focus: - Verify every exposed action skill has an accurate contract. - Treat risk class and required context as safety-critical metadata. +Components: + +- Static `_SkillContract` records: the human-authored interface description for + each expected action skill. +- `_Go2SkillInterfaceRegistry`: exposes contracts through RPC/MCP-facing + methods for Layer 3. +- `skill_interface_spec.py`: typed contract for modules that consume skill + metadata. +- Layer 5 blueprint factory: wires action skill containers, perception/security + skills, and the registry. + +Data flow: + +1. Action modules expose `@skill` methods through MCP. +2. The registry provides a parallel contract view with richer safety/context + metadata than the MCP JSON schema alone. +3. ContextProvider and TaskFeasibility read contracts to decide what context, + arguments, and preflight checks are needed. +4. SkillProposal reads the same contracts to avoid proposing duplicate skills. + +State and write boundary: + +- Contracts are code/static data, not learned state. +- The registry is read-only at runtime. +- Self-evolution proposals never mutate this file directly. A developer edits + contracts after review. + +Decision owner: + +- Humans own the source-of-truth contract text and risk metadata. +- Deterministic Layer 3 code consumes it. +- The LLM may use contract information, but should not invent hidden skill + capabilities outside the registry/MCP list. + +Failure mode: + +- Missing contract for an exposed skill is a review failure because preflight + cannot know risk/context needs. +- Contract exists but MCP tool is absent is also a review failure unless it is + intentionally disabled for a blueprint. + ### 5. Layer 3 Expert Router Files: @@ -346,6 +500,40 @@ Review focus: - Verify routing labels line up with actual skill contracts. - Watch for prompt text that could overclaim autonomy or code mutation. +Components: + +- `_Go2ExpertRouter`: deterministic task/domain router. +- Prompt policy helpers: merge layer-specific instructions into the agent system + prompt. +- Router tests: pin expected labels and guard against accidental broadening. + +Data flow: + +1. A user task or subtask is passed to the router. +2. The router classifies the task into a small set of domains such as + navigation, perception, person follow, manipulation-like unsupported work, + or general conversation. +3. It returns routing metadata and recommended next tools/preflight. +4. ContextProvider can include routing output in context. +5. The LLM uses routing as a hint, not as an irreversible planner. + +Decision owner: + +- Domain classification is deterministic and inspectable. +- Tool choice remains an LLM/tool-calling decision unless a deterministic + preflight skill refuses or asks for clarification. + +State and persistence: + +- The router is stateless. +- It does not write ledger events or feedback by itself. + +Failure mode: + +- Unknown tasks should route to uncertainty/general handling, not fabricate a + capability. +- Safety-sensitive domains should bias toward preflight and clarification. + ### 6. Context Provider And Evidence Selection Files: @@ -374,7 +562,29 @@ What "effective" means today: - It is not yet learned from replay metrics. - The LLM consumes the selected context and may still judge it insufficient. -Call path and ownership: +Components: + +- `_Go2ContextProvider`: MCP-facing context bundle provider. It is the + aggregator that reads optional Layer 4/5/3 providers and formats the compact + context returned to the agent. +- `context_evidence.py`: pure evidence selector. It owns the deterministic + ranking/filtering policy and can be unit-tested without a running robot, + MCP server, memory backend, or LLM. +- Optional provider refs: world state, spatial/RAG memory, temporal memory, + skill interface registry, skill outcome store, causal world model, and + context feedback store. +- `context_evidence.v1`: review artifact embedded in the response. It records + which evidence entries were selected, dropped, or marked low-confidence. + +Inputs: + +- Task text and runtime metadata supplied by the caller. +- Provider snapshots/summaries from wired DimOS modules. +- Policy values such as relevance threshold, entry cap, low-confidence + handling, and whether motion tasks require robot state. +- Prior feedback/outcome/world-model summaries when those providers are wired. + +Data flow: 1. The caller invokes `get_context()` with task/runtime metadata. 2. ContextProvider asks wired providers for world state, memory summaries, @@ -390,6 +600,31 @@ Call path and ownership: 5. The LLM is still the consumer and final user-facing reasoning actor. The provider does not decide to execute a skill. +State and write boundary: + +- ContextProvider itself is read-mostly. It does not write RAG/vector memory, + temporal memory, skill contracts, or robot state. +- The only durable side channel is indirect: it may include summaries from + feedback/outcome/ledger-aware modules, but those modules own their writes. +- The selected context is a transient prompt artifact unless another caller + explicitly records feedback or an evolution event. + +Decision owner: + +- Deterministic code decides what evidence enters the context bundle. +- The LLM decides how to reason over that bundle, whether to ask a clarifying + question, and whether to call an exposed tool. +- Human reviewers decide whether the deterministic policy is acceptable for a + given robot safety posture. + +Failure mode: + +- Missing providers should appear as explicit unavailable metadata. +- Malformed provider payloads should degrade that source, not erase the whole + context bundle. +- Low relevance or low confidence evidence should be dropped or marked + according to policy, never silently upgraded to trusted evidence. + What is not implemented: - No learned ranker is trained from replay logs yet. @@ -423,6 +658,40 @@ Review focus: - This is diagnostic, not a memory scheduler. - It should never write to RAG, spatial memory, temporal memory, or Git. +Components: + +- `_Go2MemoryBackendStatus`: diagnostic module exposed as an MCP skill. +- Optional provider specs: world state, spatial memory, temporal memory, + outcome store, causal world model, and skill interface registry. +- Probe helpers: small read attempts that report availability and errors. + +Data flow: + +1. The skill checks which optional provider references are injected. +2. For each provider, it records `wired`, `available`, and any probe result. +3. Probe failures are captured as status metadata rather than raised to the + agent as hard crashes. +4. The result is returned as JSON/text for the LLM or human operator. + +State and write boundary: + +- It is read-only. +- It does not persist a status snapshot. +- It does not initialize missing databases. If a database is not running or not + wired, the status should say so. + +Decision owner: + +- The skill does not decide which memory to use. +- It answers the operational question "what is connected and readable right + now?" ContextProvider and the LLM decide how to respond to that. + +Failure mode: + +- A probe failure should identify the provider and error string. +- A missing provider should be explicit enough to distinguish blueprint wiring + problems from empty memory results. + ### 8. Git-Backed Evolution Ledger Files: @@ -446,13 +715,32 @@ Implementation logic: - If `commit=True`, the module creates a local Git commit containing only the event/proposal file. +Components: + +- `evolution_event.py`: schema and validation for low-level self-evolution + observations such as feasibility assessments, context feedback, and runtime + review notes. +- `evolution_proposal.py`: schema and validation for higher-level proposed + changes that need human review before becoming code or configuration. +- `_EvolutionLedger`: filesystem and optional Git writer. It owns path + selection, JSON serialization, and commit-mode staging/commit behavior. +- Tests: verify schema validation, path shape, commit isolation, and proposal + persistence. + +Inputs: + +- Event/proposal payload supplied by a Layer 3 MCP tool. +- Optional caller-controlled commit flag. +- Optional `DIMOS_EVOLUTION_LEDGER_DIR` override. +- Current Git repository context when commit mode is enabled. + Where does Git data go: - It goes into this local repository's working tree and local `.git` history. - It does not push to GitHub. - GitHub only receives it if a human later runs `git push` or opens a PR. -Write path: +Write data flow: 1. A caller invokes a ledger-facing MCP skill such as `record_evolution_event()` or `record_skill_proposal()`. @@ -464,6 +752,30 @@ Write path: 6. If `commit=True`, only that new event/proposal file is staged and committed locally. There is no push. +State and write boundary: + +- The ledger is durable filesystem state, not an in-memory learning model. +- It writes only JSON artifacts under the configured ledger root. +- It does not mutate skill code, prompts, contracts, memory databases, or model + weights. +- Commit mode is intentionally local-only and narrow: stage the artifact, make + a local commit, stop. + +Decision owner: + +- Caller modules decide when an event or proposal is worth recording. +- The ledger decides only whether the artifact is schema-valid and where it is + stored. +- Human reviewers decide whether ledger artifacts should become code changes, + PRs, ignored local traces, or training/evaluation data later. + +Failure mode: + +- Invalid schema should fail before writing. +- Unsafe paths or path traversal should be rejected before filesystem access. +- Git commit failure should leave the JSON artifact visible in the worktree + rather than pretending the record was committed. + Privacy and review boundary: - These files can contain task text, outcome messages, context summaries, and @@ -501,7 +813,25 @@ Self-evolution meaning: - It is agent self-assessment: before acting, the agent can ask a deterministic evaluator whether the task has enough context and capability support. -Decision tree: +Components: + +- `_Go2TaskFeasibility`: MCP skill surface for deterministic preflight. +- Skill contract reader: consumes Layer 5 skill metadata such as required + arguments, context requirements, risk class, and motion sensitivity. +- Context parser: merges explicit `context_json` with wired world/robot + provider summaries. +- Ledger integration: optional event writer for review traces. + +Inputs: + +- `task`: natural-language user request or subtask. +- `context_json`: optional structured context supplied by the caller. +- Layer 5 skill contracts and availability metadata. +- Layer 4 robot/world state when wired. +- Optional outcome/context feedback summaries when available through the + context bundle. + +Decision/data flow: 1. Parse the task and optional `context_json`. 2. Match the task against known skill contracts and expert domains. @@ -514,11 +844,30 @@ Decision tree: 7. If capability is missing, report it as missing capability evidence. That is the input that can later justify a skill-interface proposal. -Boundary: +State and write boundary: +- It is read-only with respect to robot state, memory, and skill contracts. - It does not call the target skill. - It does not create a new skill. -- It does not ask the LLM to classify feasibility. +- If the ledger is wired, it may write an audit event describing the preflight + result, but not a code change. + +Decision owner: + +- Feasibility classification is deterministic code. +- The LLM may call this tool and use its result, but the LLM is not the + classifier inside the tool. +- Human review decides whether the deterministic rules are conservative enough + for real hardware. + +Failure mode: + +- Missing required arguments should produce clarification, not a new skill + proposal. +- Missing context should produce context-gathering or uncertainty, not a false + feasible result. +- Missing capability evidence should be specific enough for SkillProposal to + distinguish it from transient runtime failure. Review focus: @@ -545,6 +894,41 @@ Review focus: - Feedback is session memory unless ledger is wired. - It should not become unbounded or silently override RAG/temporal memory. +Components: + +- `_Go2ContextFeedbackStore`: in-memory bounded store and MCP skill surface. +- Feedback entry schema: task/context identifiers, rating or usefulness signal, + source labels, optional explanation, and timestamp. +- Optional evolution ledger connection: durable audit trail when wired. +- ContextProvider integration: compact aggregate feedback summary. + +Data flow: + +1. The agent or a human-facing process calls `record_context_feedback(...)` + after a context bundle is helpful, incomplete, stale, or misleading. +2. The store validates and normalizes feedback fields. +3. The entry is appended to a deque capped at 100 records. +4. If the ledger is wired, the feedback is also written as an evolution event. +5. ContextProvider reads a summary, not the full raw feedback list, to avoid + bloating prompts. + +State and persistence: + +- Without the ledger, feedback is process-local and lost on restart. +- With the ledger, feedback becomes reviewable JSON but still does not train a + ranker automatically. + +Decision owner: + +- Feedback recording is an input signal. +- The evidence policy and LLM decide how to treat the signal. There is no + automatic prompt rewrite or memory deletion. + +Failure mode: + +- Malformed feedback should fail validation. +- Excess feedback should evict oldest entries, not grow unbounded. + ### 11. Skill Outcome Store And Predictor Files: @@ -566,6 +950,42 @@ Review focus: - The predictor is heuristic, not a learned model. - Make sure stale failures do not permanently block useful skills. +Components: + +- `_Go2SkillOutcomeStore`: bounded recent outcome history. +- `_Go2SkillOutcomePredictor`: heuristic scorer over recent history and skill + contract metadata. +- Outcome summary methods: aggregate successes, failures, repeated errors, and + recent messages by skill. + +Data flow: + +1. After a skill call, an outcome can be recorded with skill name, success, + error code, message, and metadata. +2. The outcome store appends it to bounded memory. +3. Summaries group recent records by skill and error patterns. +4. The predictor combines recent outcomes with skill contract risk/context + metadata to return risk and rationale. +5. ContextProvider can include the summary as evidence for future calls. + +State and persistence: + +- Current store is bounded runtime memory unless a future integration writes it + elsewhere. +- It is separate from the Git ledger. The ledger records review/audit events; + the outcome store supports immediate runtime adaptation. + +Decision owner: + +- The predictor gives a heuristic risk hint. +- It should not block a skill by itself; TaskFeasibility or the LLM should use + it as one evidence source. + +Failure mode: + +- Unknown skill should return low-confidence or no-history output. +- Repeated failure should increase caution but still allow recovery/stop skills. + ### 12. Causal World Model And Dashboard Contract Files: @@ -713,6 +1133,41 @@ Review focus: - Check evidence requirements carefully so ordinary user ambiguity does not generate noisy skill proposals. +Components: + +- `_Go2SkillProposalGenerator`: MCP skill surface for proposal generation. +- Existing Layer 5 contracts: duplicate-suppression source of truth. +- Evolution ledger: optional proposal writer. +- Proposal schema: review artifact describing the missing capability and + proposed interface, not executable code. + +Inputs: + +- `task`: the user/subtask text that failed. +- `failure_context_json`: structured evidence about why existing skills failed + or were insufficient. +- Existing contracts and known MCP tool names. + +Accepted evidence: + +- Explicit missing capability. +- Repeated failure that indicates no available skill can perform the required + operation. +- A domain/action gap not covered by existing contracts. + +Rejected evidence: + +- Missing required arguments for an existing skill. +- Missing context or unavailable memory/provider. +- Temporary runtime failure. +- Capability already covered by an existing contract. + +Output boundary: + +- The output is a JSON proposal and optional ledger file. +- It does not import, write, or generate Python code. +- It does not update the prompt or registry automatically. + ### 14. MCP Runtime Plumbing Files: @@ -741,6 +1196,47 @@ Review focus: - Validate no startup path waits for a tool that depends on itself. - Keep MCP test ports isolated from live robot sessions. +Components: + +- `McpServer`: exposes MCP JSON-RPC endpoints, tool list/call handling, SSE + progress stream, server status tools, capability gating, and agent-send. +- `McpClient`: LLM agent module that discovers tools and exposes them to the + agent runtime. +- `tool_stream.py`: progress/log notification path for long-running skills. +- Capability registry: shared server-side lock manager for mutually exclusive + tool capabilities. + +Startup data flow: + +1. ModuleCoordinator deploys workers and starts modules. +2. `McpServer.start()` starts HTTP/SSE serving and subscribes to tool-stream + frames. +3. `on_system_modules()` registers tool schemas. For the server itself and + actor-class backed modules, schemas are collected locally to avoid RPC + self-deadlock. +4. `McpClient` initializes the agent without blocking startup on direct tool + registration outside the RPC path. + +Tool call flow: + +1. JSON-RPC `tools/call` arrives. +2. Server resolves `SkillInfo` and RPC call target. +3. Capability locks are acquired if the skill declares `uses`. +4. Progress token and capability token are injected through reserved + `_mcp_context`. +5. RPC call runs in an executor. +6. Instant skills release capability after return; background skills hand + release to tool-stream stopped frames. + +State and failure boundary: + +- Server state is process-local: skills, rpc calls, SSE queues, capability + registry. +- Port conflicts are external environment failures. Tests should use isolated + MCP ports when a live server is running. +- Tool not found should return a structured MCP text result, not crash the + server. + ### 15. Runtime Hardening Files: @@ -766,6 +1262,40 @@ Review focus: - These are operational fixes. Review them for process lifecycle correctness, not agent reasoning behavior. +Components: + +- ModuleCoordinator notification path: controls build/start/system-module + notification order. +- WorkerManagerPython: deploys modules into workers and handles worker pool + lifecycle. +- MuJoCo process helpers: subprocess executable selection and stderr draining. +- Go2 connection config: replay stop behavior and WebRTC AES key forwarding. + +Data flow: + +1. Build/deploy happens through ModuleCoordinator and worker managers. +2. Stream transports and RPC references are wired before modules receive + system-module notifications. +3. Some notifications use nowait behavior to avoid waiting on an agent client + that is itself initializing from MCP. +4. Restart/reload paths must preserve transports and rewired module refs. +5. Simulator helper code isolates platform-specific process quirks from the + rest of the robot stack. + +State and write boundary: + +- Coordinator owns deployed module proxy maps, transport registry, module + transports, and reload aliases. +- These changes do not alter skill semantics. +- They do affect whether modules start, stop, reload, and reconnect cleanly. + +Failure mode: + +- A stuck build/start should stop managers and surface the original exception. +- Restart should not orphan old transports or leave consumers attached to dead + proxies. +- ReplayConnection stop should be a no-op, not an exception. + ### 16. Rerun/Websocket Dashboard World-Model State Files: @@ -787,6 +1317,33 @@ Review focus: - Avoid making dashboard consumers depend on unversioned internal Python objects. +Components: + +- `websocket_vis_spec.py`: typed surface for visualization updates. +- `websocket_vis_module.py`: runtime module that stores/publishes dashboard + state. +- `rerun_dashboard.html`: browser-facing display surface. +- `world_model_contract.py`: schema source for world-model dashboard payloads. + +Data flow: + +1. CausalWorldModel produces a dashboard payload using + `dimos.world_model_dashboard.v1`. +2. If WebsocketVis is wired, CausalWorldModel calls `set_world_model_state(...)`. +3. WebsocketVis stores and serves the latest state to dashboard clients. +4. The dashboard renders the state as inspection data for humans. + +State and persistence: + +- Dashboard state is the latest display snapshot, not the source of truth. +- Durable state, if configured, belongs to CausalWorldModel save/load JSON. +- Browser/dashboard consumers should rely on versioned payload fields only. + +Safety boundary: + +- Dashboard updates must not trigger robot actions. +- UI should not present advisory predictions as guaranteed outcomes. + ## Review Order I Recommend 1. Start with Go2 blueprint wiring: diff --git a/docs/reviews/2026-07-05-go2-agent-architecture-review.zh.md b/docs/reviews/2026-07-05-go2-agent-architecture-review.zh.md index 1447d621a3..d9e8b96dac 100644 --- a/docs/reviews/2026-07-05-go2-agent-architecture-review.zh.md +++ b/docs/reviews/2026-07-05-go2-agent-architecture-review.zh.md @@ -191,6 +191,33 @@ Review 风险: - 分层不能掩盖漏接模块。`test_all_blueprints_generation.py` 和 Layer 4 wiring test 是主要保护。 +组成部分: + +- `unitree_go2_agentic`: 完整 Go2 agentic stack，组合 physical body、structured world state、skill interface registry/containers、Layer 3 MCP/LLM brain。 +- `unitree_go2_spatial`: 没有 LLM brain 的 perception/world-state stack，用来单独验证 Layer 4 和 Layer 5。 +- `unitree_go2_temporal_memory`: 包住 agentic stack，并通过 Layer 4 factory 追加 temporal memory。 +- 各 layer package 的 `__init__.py`: lazy blueprint factories。它们不是普通 import 便利层，而是架构边界的一部分，避免过早 import 重型 perception/model 依赖。 + +构建数据流: + +1. CLI 通过 registry 懒加载命名 blueprint。 +2. blueprint 只 import 需要的 layer handle。 +3. 每个 layer handle 构建自己的 `autoconnect()` blueprint。 +4. 最终仍是普通 DimOS blueprint 组合，stream wiring 和 Spec-based RPC injection 仍由 `ModuleCoordinator` 完成。 +5. 测试验证 Layer 3 能通过预期 contract 访问 Layer 4/5/6，并且 blueprint registry 是最新的。 + +状态和持久化: + +- blueprint layer 本身不保存 robot 数据。 +- 状态由各层内部 module 持有: Layer 6 body state、Layer 4 memory/world snapshot、Layer 5 skill contracts、Layer 3 context/outcome/causal state。 +- 持久化属于具体 module，不属于 blueprint 文件。 + +安全边界: + +- blueprint 决定哪些 module 部署在一起。 +- blueprint 不决定调用哪个 skill，也不执行 motion。 +- wiring 错误应该按安全风险看待，因为 LLM 可能看到错误工具或错误上下文。 + ### 2. Layer 6 Robot Body State 相关文件: @@ -210,6 +237,35 @@ Review 重点: - 是否提供了 agent preflight 足够使用的安全相关状态。 - 是否避免重复持有或篡改真实控制权。 +组成部分: + +- `_Go2RobotBodyState`: 把低层 robot status 适配成紧凑 Layer 6 provider 的 module。 +- `robot_body_spec.py`: 上层消费的 typed RPC contract。 +- Layer 6 package factory: 把底层 Go2 robot blueprint 和 body-state module 接在一起。 + +数据流: + +1. 底层 Go2 connection 和 robot modules 持有真实硬件/仿真通信。 +2. `_Go2RobotBodyState` 通过 spec 暴露 read-oriented body-state summary。 +3. Layer 4/Layer 3 把这些状态用于 safety preflight 和 context evidence。 +4. agent brain 看到的是摘要状态，不是直接硬件 handle。 + +判断主体: + +- body-state module 不做任务决策。 +- 它是 evidence provider。Layer 3 确定性 preflight 和 LLM 会使用它的输出判断等待、澄清或调用 skill。 + +状态和写边界: + +- 它应该被视为 current-state adapter。 +- 不应该持久化 memory，也不应该发控制命令。 +- 未来如果增加 body-state cache，必须有明确 bound 和 timestamp。 + +失败形态: + +- 如果底层 robot state 缺失，应明确返回 unavailable/unknown，不应伪造 ready。 +- motion-sensitive preflight 应把缺失 robot body state 视为风险。 + ### 3. Layer 4 Structured World State 相关文件: @@ -236,6 +292,36 @@ Review 重点: - memory backend 缺失时是否明确返回 unavailable，而不是静默空值。 - snapshot schema 是否足够稳定，能被 prompt 和 dashboard 长期消费。 +组成部分: + +- `_Go2StructuredWorldState`: 把 robot/world 信息规范化成 compact snapshot provider。 +- `_Go2SemanticTemporalMap`: 融合 spatial memory 和 temporal memory summary，产出 agent context 使用的 semantic evidence。 +- `world_state_spec.py`: Layer 3 依赖的 RPC contract。 +- `_go2_temporal_memory_world_state()`: lazy factory，延迟 TemporalMemory 构建，确保 `--new-memory` 等 CLI flags 已经生效。 + +数据流: + +1. SpatialMemory 和 TemporalMemory 仍是 storage/search 系统。 +2. Layer 4 查询或接收这些系统的 summary。 +3. Layer 4 把 summary 整形成稳定 world-state object，并明确 source availability。 +4. Layer 3 通过 spec 请求 world state，然后根据当前 task 选择 evidence。 + +状态和持久化: + +- Layer 4 可以读取持久化 memory backend，但它自身的 structured snapshot 是 derived view。 +- Temporal memory 持久化由 TemporalMemory module 和 CLI config 控制。 +- Spatial memory 持久化由 `SpatialConfig` 和 `spatial_perception.py` 的 state-dir 路径逻辑控制。 + +判断主体: + +- Layer 4 决定如何规范化 provider output。 +- Layer 4 不判断上下文是否足够行动。这个判断属于 Layer 3 preflight policy 和 LLM。 + +失败形态: + +- provider unavailable 应在 snapshot 里明确表示。 +- search result empty 不等于 backend failure，review 时要确认二者有区别。 + ### 4. Layer 5 Skill Interface Registry 相关文件: @@ -261,6 +347,37 @@ Review 重点: - 每个 MCP action skill 是否都有准确 contract。 - `risk_class`、`requires_context`、`requires_robot_state` 应按 safety-critical metadata 审查。 +组成部分: + +- 静态 `_SkillContract` 记录: 每个 action skill 的人工维护接口描述。 +- `_Go2SkillInterfaceRegistry`: 通过 RPC/MCP-facing 方法把 contracts 暴露给 Layer 3。 +- `skill_interface_spec.py`: 消费 skill metadata 的 typed contract。 +- Layer 5 blueprint factory: 接入 action skill containers、perception/security skills 和 registry。 + +数据流: + +1. action modules 通过 MCP 暴露 `@skill` 方法。 +2. registry 提供并行 contract view，包含比 MCP JSON schema 更丰富的 safety/context metadata。 +3. ContextProvider 和 TaskFeasibility 读取 contracts，判断需要哪些 context、arguments、preflight checks。 +4. SkillProposal 读取同一份 contracts，避免提重复 skill。 + +状态和写边界: + +- contracts 是代码/静态数据，不是 learned state。 +- registry 运行时只读。 +- self-evolution proposal 不会直接修改这个文件。开发者 review 后才编辑 contract。 + +判断主体: + +- 人类维护 source-of-truth contract 文案和风险元数据。 +- Layer 3 确定性代码消费这些数据。 +- LLM 可以使用 contract 信息，但不应发明 registry/MCP list 之外的隐藏能力。 + +失败形态: + +- 暴露的 skill 没有 contract 是 review failure，因为 preflight 无法知道 risk/context 需求。 +- contract 存在但 MCP tool 缺失也是 review failure，除非该 blueprint 有意禁用它。 + ### 5. Layer 3 Expert Router 相关文件: @@ -286,6 +403,35 @@ Review 重点: - routing label 是否和实际 skill contract 对齐。 - prompt 文案是否过度承诺自治或自动改代码能力。 +组成部分: + +- `_Go2ExpertRouter`: 确定性 task/domain router。 +- prompt policy helpers: 把 layer-specific instructions 合并进 agent system prompt。 +- router tests: 固定预期 label，避免行为无意扩大。 + +数据流: + +1. user task 或 subtask 进入 router。 +2. router 把任务分类到少量 domain，例如 navigation、perception、person follow、类似 manipulation 但当前不支持的任务、general conversation。 +3. 返回 routing metadata 和 recommended next tools/preflight。 +4. ContextProvider 可以把 routing output 放进 context。 +5. LLM 把 routing 当作提示，而不是不可逆 planner。 + +判断主体: + +- domain classification 是确定性的、可检查的。 +- tool choice 仍是 LLM/tool-calling decision，除非 deterministic preflight 明确拒绝或要求澄清。 + +状态和持久化: + +- router 无状态。 +- 它自己不写 ledger event 或 feedback。 + +失败形态: + +- 未知任务应该 route 到 uncertainty/general handling，不应伪造能力。 +- safety-sensitive domain 应偏向 preflight 和 clarification。 + ### 6. Context Provider 和 Evidence Selection 相关文件: @@ -309,7 +455,21 @@ Review 重点: - 这还不是基于 replay metric 学出来的策略。 - LLM 会消费这些上下文，并且仍可以判断上下文不足。 -调用路径和归属: +组成部分: + +- `_Go2ContextProvider`: MCP-facing context bundle provider。它是聚合器，读取可选 Layer 4/5/3 providers，并把结果整理成 agent 可用的 compact context。 +- `context_evidence.py`: 纯 evidence selector。它负责确定性的排序/过滤策略，可以在没有 robot、MCP server、memory backend 或 LLM 的情况下单测。 +- 可选 provider refs: world state、spatial/RAG memory、temporal memory、skill interface registry、skill outcome store、causal world model、context feedback store。 +- `context_evidence.v1`: 嵌在返回结果里的 review artifact。它记录哪些 evidence 被选中、丢弃或标成 low-confidence。 + +输入: + +- 调用方传入的 task text 和 runtime metadata。 +- 已接线 DimOS modules 提供的 provider snapshot/summary。 +- policy 参数，例如 relevance 阈值、最大条数、low-confidence 处理方式、motion task 是否必须有 robot state。 +- 已接线时的历史 feedback、outcome、world-model summary。 + +数据流: 1. 调用方带着 task/runtime metadata 调用 `get_context()`。 2. ContextProvider 向已接线 provider 请求 world state、memory summary、skill contract、outcome summary、causal-world-model state、context feedback。provider 没接线时，应贡献明确的 unavailable 状态，而不是静默丢掉该 source。 @@ -317,6 +477,24 @@ Review 重点: 4. ContextProvider 格式化 compact context，并附加 `context_evidence.v1`，让 reviewer 能看到为什么选了这些 evidence。 5. LLM 仍然是最终消费上下文、做用户可见推理和工具选择的主体。ContextProvider 本身不决定执行 skill。 +状态和写边界: + +- ContextProvider 本身基本是 read-mostly，不写 RAG/vector memory、temporal memory、skill contracts 或 robot state。 +- 唯一的 durable side channel 是间接的: 它可能包含来自 feedback/outcome/ledger-aware modules 的摘要，但写入由那些 module 自己负责。 +- 被选中的 context 是临时 prompt artifact，除非另一个调用方显式记录 feedback 或 evolution event。 + +判断主体: + +- 确定性代码决定哪些 evidence 进入 context bundle。 +- LLM 决定如何消费这个 bundle、是否追问、是否调用已暴露 tool。 +- 人类 reviewer 判断确定性 policy 对当前机器人安全姿态是否足够保守。 + +失败形态: + +- 缺失 provider 应显示为明确 unavailable metadata。 +- provider payload 格式错误时，应降级该 source，而不是清空整个 context bundle。 +- 低相关或低置信 evidence 应按 policy 丢弃或标注，不能静默升级为可信 evidence。 + 没有实现的边界: - 还没有用 replay log 训练 learned ranker。 @@ -348,6 +526,35 @@ Review 重点: - 这是诊断工具，不是新的 memory scheduler。 - 它不应该写入 RAG、spatial memory、temporal memory 或 Git。 +组成部分: + +- `_Go2MemoryBackendStatus`: 作为 MCP skill 暴露的诊断 module。 +- 可选 provider specs: world state、spatial memory、temporal memory、outcome store、causal world model、skill interface registry。 +- probe helpers: 小型 read attempt，用来报告 availability 和 error。 + +数据流: + +1. skill 检查哪些可选 provider reference 已注入。 +2. 对每个 provider 记录 `wired`、`available` 和 probe result。 +3. probe failure 被捕获成 status metadata，而不是作为 hard crash 抛给 agent。 +4. 结果以 JSON/text 返回给 LLM 或 human operator。 + +状态和写边界: + +- 它是 read-only。 +- 不持久化 status snapshot。 +- 不初始化缺失 database。如果 database 没运行或没接线，status 应明确说明。 + +判断主体: + +- 这个 skill 不决定使用哪个 memory。 +- 它只回答“现在接了什么、能读什么”。ContextProvider 和 LLM 决定如何响应。 + +失败形态: + +- probe failure 应标明 provider 和 error string。 +- missing provider 应足够明确地区分 blueprint wiring 问题和 memory result empty。 + ### 8. Git-backed Evolution Ledger 相关文件: @@ -370,6 +577,20 @@ Review 重点: - 默认 `commit=False`。 - 如果 `commit=True`，只把该 event/proposal 文件提交到本地 Git。 +组成部分: + +- `evolution_event.py`: 低层自进化 observation 的 schema 和校验，例如 feasibility assessment、context feedback、runtime review note。 +- `evolution_proposal.py`: 更高层 proposed change 的 schema 和校验。这些 proposal 必须先经人工 review，才可能变成代码或配置。 +- `_EvolutionLedger`: filesystem 和可选 Git writer。它负责路径选择、JSON serialization、commit mode 下的 staging/commit 行为。 +- 测试: 覆盖 schema validation、路径形状、commit 隔离和 proposal 持久化。 + +输入: + +- Layer 3 MCP tool 传入的 event/proposal payload。 +- 可选的 caller-controlled commit flag。 +- 可选的 `DIMOS_EVOLUTION_LEDGER_DIR` 覆盖。 +- commit mode 打开时的当前 Git repository context。 + Git 内容会提交到哪里: - 文件先进入当前本地仓库工作区。 @@ -377,7 +598,7 @@ Git 内容会提交到哪里: - 不会自动 push 到 GitHub。 - 只有人类后续运行 `git push` 或开 PR，GitHub 才会收到这些记录。 -写入路径: +写入数据流: 1. 调用方调用 ledger-facing MCP skill，例如 `record_evolution_event()` 或 `record_skill_proposal()`。 2. 模块验证 schema，并规范化 payload。 @@ -386,6 +607,25 @@ Git 内容会提交到哪里: 5. 如果 `commit=False`，除了工作区文件外不碰 Git。 6. 如果 `commit=True`，只 stage 并提交该 event/proposal 文件到本地 Git，不 push。 +状态和写边界: + +- ledger 是 durable filesystem state，不是内存学习模型。 +- 它只在配置的 ledger root 下面写 JSON artifact。 +- 它不修改 skill 代码、prompt、contracts、memory database 或 model weights。 +- commit mode 有意保持 local-only 且范围很窄: stage 该 artifact，创建本地 commit，然后停止。 + +判断主体: + +- 调用方 module 决定何时值得记录 event/proposal。 +- ledger 只判断 artifact 是否符合 schema，以及应该存到哪里。 +- 人类 reviewer 决定这些 ledger artifact 后续是否变成代码改动、PR、被 ignore 的本地轨迹，或训练/评估数据。 + +失败形态: + +- schema 无效应在写入前失败。 +- 不安全路径或路径穿越应在 filesystem access 前被拒绝。 +- Git commit 失败时，应让 JSON artifact 仍然在 worktree 可见，而不是假装已经提交。 + 隐私和 review 边界: - 这些文件可能包含 task text、outcome message、context summary、proposal rationale。真实机器人运行前，要先决定 `.dimos/evolution` 是否应该进入正常 Git 历史。 @@ -416,7 +656,22 @@ Review 重点: - 这不是模型自训练。 - 它是 agent self-assessment: 在行动前，agent 可以调用确定性 evaluator 判断任务是否具备上下文和能力支持。 -决策树: +组成部分: + +- `_Go2TaskFeasibility`: deterministic preflight 的 MCP skill surface。 +- Skill contract reader: 消费 Layer 5 skill metadata，例如 required arguments、context requirements、risk class、motion sensitivity。 +- Context parser: 把显式 `context_json` 和已接线 world/robot provider summary 合并。 +- Ledger integration: 可选的 review trace event writer。 + +输入: + +- `task`: 用户请求或 subtask 的自然语言文本。 +- `context_json`: 调用方可选传入的结构化 context。 +- Layer 5 skill contracts 和 availability metadata。 +- 已接线时的 Layer 4 robot/world state。 +- context bundle 中可用的 outcome/context feedback summary。 + +决策/数据流: 1. 解析 task 和可选 `context_json`。 2. 把 task 匹配到已知 skill contract 和 expert domain。 @@ -426,11 +681,24 @@ Review 重点: 6. 如果缺数据，推荐澄清问题或 context-gathering action。 7. 如果缺能力，报告 missing capability evidence。这个 evidence 后续可以成为 skill-interface proposal 的依据。 -边界: +状态和写边界: +- 它对 robot state、memory 和 skill contracts 是 read-only。 - 它不会调用目标 skill。 - 它不会创建新 skill。 -- 它不会让 LLM 判断 feasibility。 +- 如果 ledger 已接线，它可以写一条描述 preflight result 的 audit event，但不是代码改动。 + +判断主体: + +- feasibility classification 是确定性代码。 +- LLM 可以调用这个工具并消费结果，但 LLM 不是这个工具内部的 classifier。 +- 人类 review 判断这些确定性规则对真实硬件是否足够保守。 + +失败形态: + +- 缺少 required arguments 应产生 clarification，而不是新 skill proposal。 +- 缺少 context 应产生 context-gathering 或 uncertainty，而不是误报 feasible。 +- missing capability evidence 要足够具体，方便 SkillProposal 把它和 transient runtime failure 区分开。 Review 重点: @@ -457,6 +725,36 @@ Review 重点: - feedback 默认是 session memory，除非 ledger 已接线。 - 它不能无界增长，也不应该静默覆盖 RAG 或 temporal memory。 +组成部分: + +- `_Go2ContextFeedbackStore`: bounded in-memory store 和 MCP skill surface。 +- feedback entry schema: task/context identifiers、rating 或 usefulness signal、source labels、optional explanation、timestamp。 +- optional evolution ledger connection: 已接线时提供 durable audit trail。 +- ContextProvider integration: compact aggregate feedback summary。 + +数据流: + +1. agent 或 human-facing process 在 context bundle 有帮助、不完整、过期或误导后调用 `record_context_feedback(...)`。 +2. store 验证并规范化 feedback fields。 +3. entry append 到最多 100 条的 deque。 +4. 如果 ledger 已接线，feedback 也写成 evolution event。 +5. ContextProvider 读取 summary，而不是完整 raw feedback list，避免 prompt 膨胀。 + +状态和持久化: + +- 没有 ledger 时，feedback 是 process-local，重启丢失。 +- 有 ledger 时，feedback 成为可 review JSON，但不会自动训练 ranker。 + +判断主体: + +- feedback recording 是输入信号。 +- evidence policy 和 LLM 决定如何使用该信号。没有自动 prompt rewrite 或 memory deletion。 + +失败形态: + +- malformed feedback 应 validation fail。 +- feedback 超量时淘汰最旧 entry，不应无界增长。 + ### 11. Skill Outcome Store 和 Predictor 相关文件: @@ -477,6 +775,35 @@ Review 重点: - predictor 是启发式，不是学习模型。 - 旧失败不应永久阻止可用 skill。 +组成部分: + +- `_Go2SkillOutcomeStore`: bounded recent outcome history。 +- `_Go2SkillOutcomePredictor`: 基于近期 history 和 skill contract metadata 的 heuristic scorer。 +- outcome summary methods: 按 skill 聚合 successes、failures、repeated errors、recent messages。 + +数据流: + +1. skill call 后可以记录 skill name、success、error code、message、metadata。 +2. outcome store 把记录 append 到 bounded memory。 +3. summary 按 skill 和 error pattern 分组。 +4. predictor 把近期 outcome 和 skill contract risk/context metadata 合并，返回 risk 和 rationale。 +5. ContextProvider 可以把 summary 作为后续调用的 evidence。 + +状态和持久化: + +- 当前 store 是 bounded runtime memory，除非未来接入其他持久化。 +- 它和 Git ledger 分离。ledger 记录 review/audit event；outcome store 支持即时 runtime adaptation。 + +判断主体: + +- predictor 给出 heuristic risk hint。 +- 它不应单独 block 一个 skill；TaskFeasibility 或 LLM 应把它作为 evidence source 之一。 + +失败形态: + +- unknown skill 应返回 low-confidence 或 no-history output。 +- repeated failure 应提高谨慎度，但不应阻止 recovery/stop skills。 + ### 12. Causal World Model 和 Dashboard Contract 相关文件: @@ -588,6 +915,38 @@ Review 重点: - 这是最安全的自进化边界: 只生成 proposal，必须人工 review。 - evidence requirement 要足够严格，避免用户表达不清时产生大量噪声 proposal。 +组成部分: + +- `_Go2SkillProposalGenerator`: proposal generation 的 MCP skill surface。 +- 现有 Layer 5 contracts: duplicate suppression 的 source of truth。 +- Evolution ledger: 可选 proposal writer。 +- Proposal schema: 描述 missing capability 和 proposed interface 的 review artifact，不是可执行代码。 + +输入: + +- `task`: 失败的 user/subtask text。 +- `failure_context_json`: 描述为什么现有 skills 失败或不足的结构化 evidence。 +- 现有 contracts 和已知 MCP tool names。 + +接受的 evidence: + +- 明确 missing capability。 +- repeated failure 显示没有可用 skill 能完成目标操作。 +- 现有 contracts 没覆盖的 domain/action gap。 + +拒绝的 evidence: + +- 只是缺少现有 skill 的 required argument。 +- 只是缺 context 或 memory/provider unavailable。 +- 临时 runtime failure。 +- 能力已被现有 contract 覆盖。 + +输出边界: + +- 输出是 JSON proposal 和可选 ledger 文件。 +- 不 import、不写入、不生成 Python code。 +- 不自动更新 prompt 或 registry。 + ### 14. MCP Runtime Plumbing 相关文件: @@ -613,6 +972,35 @@ Review 重点: - 启动路径不能等待依赖自身初始化的工具。 - MCP 测试端口必须和 live robot session 隔离。 +组成部分: + +- `McpServer`: 暴露 MCP JSON-RPC endpoints、tool list/call handling、SSE progress stream、server status tools、capability gating、agent-send。 +- `McpClient`: LLM agent module，负责发现 tools 并暴露给 agent runtime。 +- `tool_stream.py`: 长运行 skill 的 progress/log notification path。 +- capability registry: server-side lock manager，用于 mutually exclusive tool capabilities。 + +启动数据流: + +1. ModuleCoordinator 部署 worker 并启动 modules。 +2. `McpServer.start()` 启动 HTTP/SSE serving，并订阅 tool-stream frames。 +3. `on_system_modules()` 注册 tool schemas。对 server 自身和 actor-class backed modules，本地收集 schema，避免 RPC self-deadlock。 +4. `McpClient` 初始化 agent，但不在 RPC path 外阻塞等待 direct tool registration。 + +Tool call 数据流: + +1. JSON-RPC `tools/call` 到达。 +2. server 解析 `SkillInfo` 和 RPC call target。 +3. 如果 skill 声明 `uses`，先申请 capability lock。 +4. progress token 和 capability token 通过保留 `_mcp_context` 注入。 +5. RPC call 在 executor 里运行。 +6. instant skill 返回后释放 capability；background skill 把 release 交给 tool-stream stopped frame。 + +状态和失败边界: + +- server state 是 process-local: skills、rpc calls、SSE queues、capability registry。 +- port conflict 是外部环境失败。live server 存在时，测试必须用隔离 MCP port。 +- tool not found 应返回结构化 MCP text result，而不是 crash server。 + ### 15. Runtime Hardening 相关文件: @@ -636,6 +1024,33 @@ Review 重点: - 这些是运行时稳定性修复，重点审查 process lifecycle 和 stream rewiring，而不是 agent reasoning。 +组成部分: + +- ModuleCoordinator notification path: 控制 build/start/system-module notification 顺序。 +- WorkerManagerPython: 在 workers 中部署 modules，并管理 worker pool lifecycle。 +- MuJoCo process helpers: 处理 subprocess executable 选择和 stderr draining。 +- Go2 connection config: replay stop 行为和 WebRTC AES key forwarding。 + +数据流: + +1. build/deploy 通过 ModuleCoordinator 和 worker managers 完成。 +2. modules 收到 system-module notification 前，stream transports 和 RPC refs 已经 wiring。 +3. 部分 notification 使用 nowait，避免等待正在通过 MCP 初始化的 agent client。 +4. restart/reload paths 必须保留 transports 并重新 wiring module refs。 +5. simulator helper code 把平台特定 process 问题隔离在机器人栈外层。 + +状态和写边界: + +- Coordinator 持有 deployed module proxy maps、transport registry、module transports、reload aliases。 +- 这些改动不改变 skill semantics。 +- 它们影响 module 是否能干净 start、stop、reload、reconnect。 + +失败形态: + +- build/start 卡住时应 stop managers 并暴露原始异常。 +- restart 不应遗留 old transports，也不应让 consumer 连在 dead proxy 上。 +- ReplayConnection stop 应该是 no-op，而不是 exception。 + ### 16. Rerun/Websocket Dashboard World-model State 相关文件: @@ -655,6 +1070,31 @@ Review 重点: - dashboard state 应保持展示用途。 - dashboard consumer 不应依赖未 version 的内部 Python object。 +组成部分: + +- `websocket_vis_spec.py`: visualization update 的 typed surface。 +- `websocket_vis_module.py`: 存储/publish dashboard state 的 runtime module。 +- `rerun_dashboard.html`: browser-facing display surface。 +- `world_model_contract.py`: world-model dashboard payload 的 schema source。 + +数据流: + +1. CausalWorldModel 使用 `dimos.world_model_dashboard.v1` 生成 dashboard payload。 +2. 如果 WebsocketVis 已接线，CausalWorldModel 调用 `set_world_model_state(...)`。 +3. WebsocketVis 保存并向 dashboard clients 提供 latest state。 +4. dashboard 把 state 渲染成人类 inspection data。 + +状态和持久化: + +- dashboard state 是 latest display snapshot，不是 source of truth。 +- 如果配置了 durable state，它属于 CausalWorldModel save/load JSON。 +- browser/dashboard consumer 只应依赖 versioned payload fields。 + +安全边界: + +- dashboard update 不能触发机器人动作。 +- UI 不应把 advisory prediction 展示成 guaranteed outcome。 + ## 推荐 Review 顺序 1. 先看 Go2 blueprint wiring: From 99c337c88908a48fec9791208c87c681f46f1da9 Mon Sep 17 00:00:00 2001 From: ZhangZheMing <1050297972@qq.com> Date: Sun, 5 Jul 2026 01:05:56 +0800 Subject: [PATCH 18/36] Document Go2 interface inputs and outputs --- dimos/robot/unitree/go2/PROJECT_CHANGELOG.md | 19 +++++++ ...026-07-05-go2-agent-architecture-review.md | 55 +++++++++++++++++++ ...-07-05-go2-agent-architecture-review.zh.md | 52 ++++++++++++++++++ 3 files changed, 126 insertions(+) diff --git a/dimos/robot/unitree/go2/PROJECT_CHANGELOG.md b/dimos/robot/unitree/go2/PROJECT_CHANGELOG.md index 2646c64598..c7e153973d 100644 --- a/dimos/robot/unitree/go2/PROJECT_CHANGELOG.md +++ b/dimos/robot/unitree/go2/PROJECT_CHANGELOG.md @@ -8,6 +8,25 @@ Entries are listed in reverse chronological order. ## 2026-07-05 +- Branch: `refactor/go2-architecture-layers` +- Summary: Added explicit interface input/output references to the English and + Chinese Go2 architecture review guides. The new quick reference enumerates + Layer 4/5/6 Specs, Layer 3 MCP skills, ledger/proposal schemas, world-model + contracts, MCP runtime surfaces, and dashboard update interfaces with their + expected inputs and returned outputs. +- Files/modules: + - `docs/reviews/2026-07-05-go2-agent-architecture-review.md` + - `docs/reviews/2026-07-05-go2-agent-architecture-review.zh.md` + - `dimos/robot/unitree/go2/PROJECT_CHANGELOG.md` +- Validation: + - Documentation-only change. + - Ran `git diff --check` on the changed review/changelog files. +- Open items: + - Keep the interface quick reference synchronized with any future signature + or schema changes. + +## 2026-07-05 + - Branch: `refactor/go2-architecture-layers` - Summary: Normalized every local Go2 architecture/self-evolution review-guide section to the same implementation-review depth as the causal world model diff --git a/docs/reviews/2026-07-05-go2-agent-architecture-review.md b/docs/reviews/2026-07-05-go2-agent-architecture-review.md index 2bb0b4f3fe..f0407d0cd2 100644 --- a/docs/reviews/2026-07-05-go2-agent-architecture-review.md +++ b/docs/reviews/2026-07-05-go2-agent-architecture-review.md @@ -55,6 +55,8 @@ approving a feature just because the file names sound plausible. first public surface? - Inputs: Which JSON strings, spec providers, memory providers, or runtime states feed it? +- Outputs: Which `SkillResult`, dict schema, Spec return value, file artifact, + or dashboard payload does the interface return? - Decision owner: Is the decision made by deterministic code, the LLM reading context, a small online statistical model, or a human reviewer? - State: Is the feature stateless, in-memory only, persisted to JSON, persisted @@ -75,6 +77,59 @@ things: - Self-evolution artifact: reviewable JSON events/proposals, not automatic code mutation. +## Interface Input/Output Quick Reference + +Use this section when reviewing API boundaries. It lists the stable public +surfaces added or changed by the local Go2 architecture work; helper functions +that are not consumed across module boundaries are intentionally omitted. + +| Surface | Inputs | Outputs | +| --- | --- | --- | +| Go2 blueprint factories (`unitree_go2_agentic`, `unitree_go2_spatial`, `unitree_go2_temporal_memory`) | CLI blueprint name plus `GlobalConfig` values already resolved by DimOS. Lazy layer imports have no runtime arguments. | DimOS `Blueprint` objects composed with `autoconnect()`. They output module wiring, streams, and Spec injection paths, not user-facing JSON. | +| `RobotBodyStateSpec.get_robot_body_snapshot()` | No arguments; reads recent odom/image/lidar observations and connection/local-policy summaries from Layer 6. | `dict` with connection state, sensor state, local policy state, safety state, and freshness/availability markers. | +| `RobotBodyStateSpec.get_connection_state()` | No arguments. | `dict` describing connection availability/config-derived mode. | +| `RobotBodyStateSpec.get_sensor_state()` | No arguments. | `dict` with observed sensor counters/freshness such as image, lidar, and odom availability. | +| `RobotBodyStateSpec.get_local_policy_state()` | No arguments. | `dict` describing local policy/readiness state used as evidence by upper layers. | +| `WorldStateSpec.get_world_snapshot(task, spatial_limit)` | `task: str`, `spatial_limit: int`; reads Layer 6 body state plus spatial/temporal memory providers when wired. | `dict` world snapshot with `robot_state`, `runtime`, `memory_state`, `semantic_temporal_map`, `sources`, and snapshot-storage metadata. | +| `WorldStateSpec.get_robot_state()` | No arguments. | `dict` robot/body state view for upper-layer preflight. | +| `WorldStateSpec.get_runtime_state()` | No arguments. | `dict` runtime mode/config summary such as replay/simulation/hardware and MCP/viewer settings. | +| `WorldStateSpec.get_memory_state(task, spatial_limit)` | `task: str`, `spatial_limit: int`. | `dict` memory section with spatial and temporal availability, matches/summaries, and errors when a provider probe fails. | +| `WorldStateSpec.get_snapshot_storage_policy()` | No arguments. | `dict` explaining whether snapshots are transient, written to memory, or persisted elsewhere. | +| `SemanticTemporalMapSpec.query_semantic_temporal_map(query, spatial_limit)` | `query: str`, `spatial_limit: int`; reads spatial and temporal memory providers. | `dict` with spatial section, temporal section, fused evidence entries, confidence/location/time summaries, and source errors. | +| `SkillInterfaceSpec.get_skill_interface_snapshot(domain)` | Optional `domain: str` filter. | `dict` with `available`, `version`, `source`, `domain_filter`, `domains`, `skill_count`, and a `skills` list of static contracts. | +| `SkillInterfaceSpec.get_skill_contract(skill_name)` | `skill_name: str`. | Matching skill-contract `dict`, or `None` when no contract exists. | +| `SkillInterfaceSpec.validate_skill_request(skill_name, args_json)` | `skill_name: str`, `args_json: str` JSON object. | `dict` with `valid`, `errors`, `warnings`, and the matched `contract`; unknown skills return `valid=False`. | +| `SkillInterfaceSpec.compare_mcp_tools(tools_json)` | `tools_json: str` containing an MCP `tools/list` style payload or list. | `dict` with `valid`, parser errors, contract/MCP counts, missing contracts, unregistered MCP tools, and known internal tools. | +| `route_task(task, context)` MCP skill | `task: str`, optional `context: str`. | `SkillResult` metadata: `domain`, `confidence`, `matched_keywords`, `recommended_tools`, `needs_context`, `reason`, and `context_used`. | +| `get_context(task, focus, spatial_limit)` MCP skill | `task: str`, optional `focus: str`, `spatial_limit: int`; reads Layer 4, memory, skill, feedback, outcome, and world-model providers when wired. | `SkillResult` message plus metadata containing `sources`, `runtime`, `robot_state`, `world_state`, `skill_state`, `context_feedback`, `causal_state`, `world_model_state`, `context_evidence`, and provider `errors`. | +| `build_context_evidence(metadata, policy)` helper | Metadata dict from ContextProvider and `ContextEvidencePolicy` thresholds. | `context_evidence.v1` dict describing selected evidence, dropped/low-confidence evidence, selected sources, and policy effects. | +| `memory_backend_status()` MCP skill | No arguments; read probes optional providers. | `SkillResult` metadata with schema `go2_memory_backend_status.v1`, per-provider wired/available/probe fields, and warnings. | +| `record_evolution_event(event_type, task, payload_json, commit)` MCP skill | `event_type: str`, optional `task: str`, `payload_json: str` JSON object, `commit: bool`. | `SkillResult` metadata with `schema`, `event_type`, `task`, `ledger_dir`, `event_path`, `commit_requested`, optional `commit_sha`, `warnings`, and full `event`. | +| `record_skill_proposal(proposal_json, commit)` MCP skill / ledger RPC | `proposal_json: str` using `dimos.skill_proposal.v1`, `commit: bool`. | `SkillResult` metadata with `schema`, `proposal_id`, `ledger_dir`, `proposal_path`, `commit_requested`, optional `commit_sha`, `warnings`, and validated `proposal`. | +| `EvolutionLedgerSpec.write_evolution_event(event_type, task, payload, commit)` | Structured payload `dict` from another Layer 3 module. | Same ledger event record dict as `record_evolution_event`, without MCP JSON parsing. | +| `EvolutionLedgerSpec.write_skill_proposal(proposal, commit)` | Validated proposal `dict`. | Same proposal record dict as `record_skill_proposal`, without MCP JSON parsing. | +| `evaluate_task_feasibility(task, context_json)` MCP skill | `task: str`, `context_json: str` JSON object, usually from `get_context` metadata. Reads Layer 5 contracts when wired. | `SkillResult` metadata: `feasible` (`yes`, `no`, `uncertain`), `missing_context`, `required_skills`, `available_skills`, `missing_skills`, `safety_risks`, `recommended_next_action`, `clarifying_question`, `evidence_sources`, and warnings. | +| `record_context_feedback(task, context_evidence_json, selected_skill, outcome_json, helpful_sources_json, ignored_risks_json)` MCP skill | Task text, `context_evidence.v1` JSON, optional selected skill, outcome JSON object, helpful-source JSON list, ignored-risk JSON list. | `SkillResult` metadata with one `go2_context_feedback.v1` feedback record, `total_feedback`, and optional ledger warnings. | +| `ContextFeedbackSpec.get_recent_context_feedback(limit, source)` | `limit: int`, optional source filter. | Newest-first list of `go2_context_feedback.v1` feedback dicts. | +| `ContextFeedbackSpec.get_context_feedback_summary(limit)` | `limit: int`. | Aggregate `dict` with counts for success/failure/unknown outcomes plus helpful/harmful source counters. | +| `record_skill_outcome(skill_name, success, domain, error_code, message, risk, recovery)` MCP skill | Skill name, success boolean, optional domain/error/message/risk/recovery strings. | `SkillResult` metadata with recorded outcome dict and `total_outcomes`. | +| `summarize_skill_outcomes(limit, skill_name, domain)` MCP skill | Limit plus optional skill/domain filters. | `SkillResult` metadata with filtered newest-first `outcomes` list. | +| `SkillOutcomeStoreSpec.get_recent_outcomes(limit, skill_name, domain)` | Limit plus optional exact skill/domain filters. | Newest-first list of outcome dicts: timestamp, skill name, success, domain, error code, message, risk, recovery. | +| `predict_skill_outcome(skill_name, args_json, context)` MCP skill | Skill name, planned args JSON object, optional context text. Reads SkillOutcomeStore and CausalWorldModel when wired. | `SkillResult` metadata with `risk`, `predicted_success`, `failure_reasons`, `recovery_suggestions`, recent outcomes/transitions, optional world-model prediction, and provider availability flags. | +| `record_causal_transition(...)` MCP skill | Task, skill name, args JSON, before/after context text, prediction JSON, outcome JSON, domain, before/after state JSON. | `SkillResult` metadata with transition dict, `total_transitions`, `autosave_error`, and `dashboard_error`. | +| `predict_world_transition(snapshot_json, action_json, goal, horizon)` MCP skill / `predict_next_state(...)` RPC | Layer 4 snapshot JSON, action JSON with `skill_name` and `args`, optional goal, bounded horizon. | `dimos.world_model_prediction.v1` dict with action, snapshot summary, risk, predicted success, score, confidence, predicted symbolic delta, failure modes, reasons, model output, causal attribution, SCM explanation, and intervention evidence. | +| `score_action(snapshot_json, action_json, goal)` RPC | Same snapshot/action/goal inputs as prediction. | Compact dict with score, risk, confidence, predicted success, failure modes, reasons, model/causal/SCM/intervention evidence. | +| `summarize_causal_patterns(skill_name, domain, limit)` MCP skill | Optional skill/domain filters and limit. | `SkillResult` metadata with repeated causal `patterns` and filtered `transitions`. | +| `record_intervention(...)` MCP skill | Task, intervention name, target variable, before/after value JSON, action JSON, before/after snapshot JSON, outcome JSON, causal hypothesis. | `SkillResult` metadata with intervention record, `total_interventions`, `autosave_error`, and `dashboard_error`. | +| `save_world_model_state(path)` / `load_world_model_state(path)` MCP skills | Optional path; required unless `DIMOS_GO2_WORLD_MODEL_STATE` is set. | `SkillResult` summary with saved/loaded model-state counts plus optional dashboard error. | +| `CausalWorldModelSpec.get_recent_transitions(limit, skill_name, domain, cause)` | Limit plus optional exact filters. | Newest-first list of transition dicts. | +| `CausalWorldModelSpec.get_intervention_log(limit, target_variable, intervention_name)` | Limit plus optional exact filters. | Newest-first list of intervention dicts. | +| `CausalWorldModelSpec.get_model_state()` | No arguments. | Dict with online model sample/weights summary, provider contract, causal estimator snapshot, intervention log snapshot, SCM snapshot, and persistence config. | +| `CausalWorldModelSpec.get_provider_contract()` | No arguments. | `dimos.world_model_provider.v1` dict naming provider, model type, capabilities, and output schemas. | +| `propose_skill_interface(task, failure_context_json)` MCP skill | Task text and failure-context JSON object with missing-capability evidence. Reads Layer 5 contracts/outcomes when wired. | `SkillResult` metadata: `proposal_created`, `proposal` using `dimos.skill_proposal.v1` when created, `existing_skill_matches`, `recommended_next_action`, optional `proposal_path` or warnings. | +| MCP `tools/list` / `tools/call` runtime surface | JSON-RPC requests; `tools/call` also carries MCP arguments and optional `_mcp_context` progress/capability token. | Tool schemas with capability metadata; tool-call text/content results; SSE progress frames for background tools; capability release on instant return or stopped frame. | +| `WebsocketVisSpec.set_world_model_state(state)` | `state: dict` expected to use `dimos.world_model_dashboard_state.v1`. | `dict` acknowledgement/current display state for dashboard clients. | + ## Upstream Changes To Review These are not local self-evolution features, but they can affect this branch. diff --git a/docs/reviews/2026-07-05-go2-agent-architecture-review.zh.md b/docs/reviews/2026-07-05-go2-agent-architecture-review.zh.md index d9e8b96dac..aea31ca3db 100644 --- a/docs/reviews/2026-07-05-go2-agent-architecture-review.zh.md +++ b/docs/reviews/2026-07-05-go2-agent-architecture-review.zh.md @@ -39,6 +39,7 @@ review 每个本地功能时，建议都按下面这些问题看，而不是只 - 入口: 第一个对外入口是 blueprint、module、MCP skill、RPC method，还是 helper 函数？ - 输入: 它读取哪些 JSON 字符串、Spec provider、memory provider 或 runtime state？ +- 输出: 它返回哪种 `SkillResult`、dict schema、Spec return value、file artifact 或 dashboard payload？ - 判断主体: 判断是确定性代码做的，LLM 做的，小型在线统计模型做的，还是人类 reviewer 做的？ - 状态: 它是无状态、只在内存、写 JSON、写 Git，还是写已有 RAG/vector database？ - 写边界: 哪个方法会写入？哪些方法是纯 read-only？ @@ -53,6 +54,57 @@ review 每个本地功能时，建议都按下面这些问题看，而不是只 - Agent judgment: LLM 读取已选上下文和工具后决定下一步。 - Self-evolution artifact: 产出可 review 的 JSON event/proposal，不自动改代码。 +## 接口输入/输出速查 + +review API 边界时优先看这一节。这里列的是本地 Go2 架构改动新增或改动过的稳定 public surface；没有跨 module 边界消费的内部 helper 不列入。 + +| 接口 | 输入 | 输出 | +| --- | --- | --- | +| Go2 blueprint factories (`unitree_go2_agentic`、`unitree_go2_spatial`、`unitree_go2_temporal_memory`) | CLI blueprint name，以及 DimOS 已解析好的 `GlobalConfig`。lazy layer import 本身没有 runtime 入参。 | `autoconnect()` 组合出的 DimOS `Blueprint`。输出的是 module wiring、stream wiring 和 Spec injection path，不是面向用户的 JSON。 | +| `RobotBodyStateSpec.get_robot_body_snapshot()` | 无参数；读取 Layer 6 近期 odom/image/lidar observation，以及 connection/local-policy summary。 | `dict`，包含 connection state、sensor state、local policy state、safety state、freshness/availability markers。 | +| `RobotBodyStateSpec.get_connection_state()` | 无参数。 | 描述 connection availability 和 config-derived mode 的 `dict`。 | +| `RobotBodyStateSpec.get_sensor_state()` | 无参数。 | `dict`，包含 image、lidar、odom 等 sensor counters/freshness。 | +| `RobotBodyStateSpec.get_local_policy_state()` | 无参数。 | `dict`，描述上层 preflight 使用的 local policy/readiness evidence。 | +| `WorldStateSpec.get_world_snapshot(task, spatial_limit)` | `task: str`、`spatial_limit: int`；读取 Layer 6 body state，以及已接线 spatial/temporal memory provider。 | `dict` world snapshot，包含 `robot_state`、`runtime`、`memory_state`、`semantic_temporal_map`、`sources` 和 snapshot-storage metadata。 | +| `WorldStateSpec.get_robot_state()` | 无参数。 | 上层 preflight 使用的 robot/body state `dict`。 | +| `WorldStateSpec.get_runtime_state()` | 无参数。 | `dict` runtime mode/config summary，例如 replay/simulation/hardware、MCP/viewer 设置。 | +| `WorldStateSpec.get_memory_state(task, spatial_limit)` | `task: str`、`spatial_limit: int`。 | `dict` memory section，包含 spatial/temporal availability、matches/summaries，以及 provider probe failure 的 errors。 | +| `WorldStateSpec.get_snapshot_storage_policy()` | 无参数。 | `dict`，说明 snapshot 是 transient、写 memory，还是持久化到其他位置。 | +| `SemanticTemporalMapSpec.query_semantic_temporal_map(query, spatial_limit)` | `query: str`、`spatial_limit: int`；读取 spatial 和 temporal memory provider。 | `dict`，包含 spatial section、temporal section、fused evidence entries、confidence/location/time summaries 和 source errors。 | +| `SkillInterfaceSpec.get_skill_interface_snapshot(domain)` | 可选 `domain: str` filter。 | `dict`，包含 `available`、`version`、`source`、`domain_filter`、`domains`、`skill_count` 和静态 contract 列表 `skills`。 | +| `SkillInterfaceSpec.get_skill_contract(skill_name)` | `skill_name: str`。 | 匹配的 skill-contract `dict`；没有 contract 时返回 `None`。 | +| `SkillInterfaceSpec.validate_skill_request(skill_name, args_json)` | `skill_name: str`、`args_json: str` JSON object。 | `dict`，包含 `valid`、`errors`、`warnings`、匹配的 `contract`；未知 skill 返回 `valid=False`。 | +| `SkillInterfaceSpec.compare_mcp_tools(tools_json)` | `tools_json: str`，MCP `tools/list` 风格 payload 或 list。 | `dict`，包含 `valid`、parser errors、contract/MCP counts、missing contracts、unregistered MCP tools、known internal tools。 | +| `route_task(task, context)` MCP skill | `task: str`，可选 `context: str`。 | `SkillResult` metadata: `domain`、`confidence`、`matched_keywords`、`recommended_tools`、`needs_context`、`reason`、`context_used`。 | +| `get_context(task, focus, spatial_limit)` MCP skill | `task: str`、可选 `focus: str`、`spatial_limit: int`；读取已接线 Layer 4、memory、skill、feedback、outcome、world-model provider。 | `SkillResult` message 加 metadata，包含 `sources`、`runtime`、`robot_state`、`world_state`、`skill_state`、`context_feedback`、`causal_state`、`world_model_state`、`context_evidence` 和 provider `errors`。 | +| `build_context_evidence(metadata, policy)` helper | ContextProvider 生成的 metadata dict，以及 `ContextEvidencePolicy` 阈值。 | `context_evidence.v1` dict，描述 selected evidence、dropped/low-confidence evidence、selected sources 和 policy effect。 | +| `memory_backend_status()` MCP skill | 无参数；对可选 provider 做 read probe。 | `SkillResult` metadata，schema 是 `go2_memory_backend_status.v1`，包含每个 provider 的 wired/available/probe 字段和 warnings。 | +| `record_evolution_event(event_type, task, payload_json, commit)` MCP skill | `event_type: str`、可选 `task: str`、`payload_json: str` JSON object、`commit: bool`。 | `SkillResult` metadata，包含 `schema`、`event_type`、`task`、`ledger_dir`、`event_path`、`commit_requested`、可选 `commit_sha`、`warnings` 和完整 `event`。 | +| `record_skill_proposal(proposal_json, commit)` MCP skill / ledger RPC | `proposal_json: str`，使用 `dimos.skill_proposal.v1`；`commit: bool`。 | `SkillResult` metadata，包含 `schema`、`proposal_id`、`ledger_dir`、`proposal_path`、`commit_requested`、可选 `commit_sha`、`warnings` 和校验后的 `proposal`。 | +| `EvolutionLedgerSpec.write_evolution_event(event_type, task, payload, commit)` | 其他 Layer 3 module 传入的结构化 payload `dict`。 | 和 `record_evolution_event` 相同的 ledger event record dict，但不经过 MCP JSON parsing。 | +| `EvolutionLedgerSpec.write_skill_proposal(proposal, commit)` | 已校验的 proposal `dict`。 | 和 `record_skill_proposal` 相同的 proposal record dict，但不经过 MCP JSON parsing。 | +| `evaluate_task_feasibility(task, context_json)` MCP skill | `task: str`、`context_json: str` JSON object，通常来自 `get_context` metadata；已接线时读取 Layer 5 contracts。 | `SkillResult` metadata: `feasible` (`yes`、`no`、`uncertain`)、`missing_context`、`required_skills`、`available_skills`、`missing_skills`、`safety_risks`、`recommended_next_action`、`clarifying_question`、`evidence_sources` 和 warnings。 | +| `record_context_feedback(task, context_evidence_json, selected_skill, outcome_json, helpful_sources_json, ignored_risks_json)` MCP skill | task text、`context_evidence.v1` JSON、可选 selected skill、outcome JSON object、helpful-source JSON list、ignored-risk JSON list。 | `SkillResult` metadata，包含一条 `go2_context_feedback.v1` feedback record、`total_feedback` 和可选 ledger warnings。 | +| `ContextFeedbackSpec.get_recent_context_feedback(limit, source)` | `limit: int`，可选 source filter。 | newest-first 的 `go2_context_feedback.v1` feedback dict 列表。 | +| `ContextFeedbackSpec.get_context_feedback_summary(limit)` | `limit: int`。 | aggregate `dict`，包含 success/failure/unknown outcome counts，以及 helpful/harmful source counters。 | +| `record_skill_outcome(skill_name, success, domain, error_code, message, risk, recovery)` MCP skill | skill name、success boolean、可选 domain/error/message/risk/recovery 字符串。 | `SkillResult` metadata，包含 recorded outcome dict 和 `total_outcomes`。 | +| `summarize_skill_outcomes(limit, skill_name, domain)` MCP skill | limit，以及可选 skill/domain filters。 | `SkillResult` metadata，包含过滤后的 newest-first `outcomes` list。 | +| `SkillOutcomeStoreSpec.get_recent_outcomes(limit, skill_name, domain)` | limit，以及可选 exact skill/domain filters。 | newest-first outcome dict 列表: timestamp、skill name、success、domain、error code、message、risk、recovery。 | +| `predict_skill_outcome(skill_name, args_json, context)` MCP skill | skill name、planned args JSON object、可选 context text；已接线时读取 SkillOutcomeStore 和 CausalWorldModel。 | `SkillResult` metadata，包含 `risk`、`predicted_success`、`failure_reasons`、`recovery_suggestions`、recent outcomes/transitions、可选 world-model prediction 和 provider availability flags。 | +| `record_causal_transition(...)` MCP skill | task、skill name、args JSON、before/after context text、prediction JSON、outcome JSON、domain、before/after state JSON。 | `SkillResult` metadata，包含 transition dict、`total_transitions`、`autosave_error`、`dashboard_error`。 | +| `predict_world_transition(snapshot_json, action_json, goal, horizon)` MCP skill / `predict_next_state(...)` RPC | Layer 4 snapshot JSON、包含 `skill_name` 和 `args` 的 action JSON、可选 goal、bounded horizon。 | `dimos.world_model_prediction.v1` dict，包含 action、snapshot summary、risk、predicted success、score、confidence、predicted symbolic delta、failure modes、reasons、model output、causal attribution、SCM explanation 和 intervention evidence。 | +| `score_action(snapshot_json, action_json, goal)` RPC | 与 prediction 相同的 snapshot/action/goal 输入。 | compact dict，包含 score、risk、confidence、predicted success、failure modes、reasons、model/causal/SCM/intervention evidence。 | +| `summarize_causal_patterns(skill_name, domain, limit)` MCP skill | 可选 skill/domain filters 和 limit。 | `SkillResult` metadata，包含 repeated causal `patterns` 和过滤后的 `transitions`。 | +| `record_intervention(...)` MCP skill | task、intervention name、target variable、before/after value JSON、action JSON、before/after snapshot JSON、outcome JSON、causal hypothesis。 | `SkillResult` metadata，包含 intervention record、`total_interventions`、`autosave_error`、`dashboard_error`。 | +| `save_world_model_state(path)` / `load_world_model_state(path)` MCP skills | 可选 path；如果未设置 `DIMOS_GO2_WORLD_MODEL_STATE`，则必须传 path。 | `SkillResult` summary，包含保存/加载的 model-state counts，以及可选 dashboard error。 | +| `CausalWorldModelSpec.get_recent_transitions(limit, skill_name, domain, cause)` | limit，以及可选 exact filters。 | newest-first transition dict 列表。 | +| `CausalWorldModelSpec.get_intervention_log(limit, target_variable, intervention_name)` | limit，以及可选 exact filters。 | newest-first intervention dict 列表。 | +| `CausalWorldModelSpec.get_model_state()` | 无参数。 | `dict`，包含 online model sample/weights summary、provider contract、causal estimator snapshot、intervention log snapshot、SCM snapshot 和 persistence config。 | +| `CausalWorldModelSpec.get_provider_contract()` | 无参数。 | `dimos.world_model_provider.v1` dict，声明 provider、model type、capabilities 和 output schemas。 | +| `propose_skill_interface(task, failure_context_json)` MCP skill | task text，以及带 missing-capability evidence 的 failure-context JSON object；已接线时读取 Layer 5 contracts/outcomes。 | `SkillResult` metadata: `proposal_created`、创建时的 `dimos.skill_proposal.v1` `proposal`、`existing_skill_matches`、`recommended_next_action`、可选 `proposal_path` 或 warnings。 | +| MCP `tools/list` / `tools/call` runtime surface | JSON-RPC requests；`tools/call` 携带 MCP arguments 和可选 `_mcp_context` progress/capability token。 | 带 capability metadata 的 tool schema；tool-call text/content result；background tool 的 SSE progress frames；instant return 或 stopped frame 触发 capability release。 | +| `WebsocketVisSpec.set_world_model_state(state)` | `state: dict`，预期使用 `dimos.world_model_dashboard_state.v1`。 | `dict` acknowledgement/current display state，供 dashboard clients 读取。 | + ## 需要 Review 的上游更新 这些不是本地自进化功能，但会影响本分支。 From e8816f54032094cb2d540cd1216fcbdfcfe33be1 Mon Sep 17 00:00:00 2001 From: ZhangZheMing <1050297972@qq.com> Date: Sun, 5 Jul 2026 01:11:40 +0800 Subject: [PATCH 19/36] Add Go2 architecture diagram --- dimos/robot/unitree/go2/PROJECT_CHANGELOG.md | 20 +++++++++++++++++++ ...026-07-05-go2-agent-architecture-review.md | 4 ++++ ...-07-05-go2-agent-architecture-review.zh.md | 4 ++++ .../2026-07-05-go2-agent-architecture.png | 3 +++ 4 files changed, 31 insertions(+) create mode 100644 docs/reviews/2026-07-05-go2-agent-architecture.png diff --git a/dimos/robot/unitree/go2/PROJECT_CHANGELOG.md b/dimos/robot/unitree/go2/PROJECT_CHANGELOG.md index c7e153973d..7f44c59b5a 100644 --- a/dimos/robot/unitree/go2/PROJECT_CHANGELOG.md +++ b/dimos/robot/unitree/go2/PROJECT_CHANGELOG.md @@ -8,6 +8,26 @@ Entries are listed in reverse chronological order. ## 2026-07-05 +- Branch: `refactor/go2-architecture-layers` +- Summary: Generated and added a review-oriented Go2 agent architecture diagram + showing Layer 3 agent brain, Layer 4 world state, Layer 5 skill interface, + Layer 6 robot body, local Git ledger artifacts, and dashboard observability. + Linked the image from both English and Chinese review guides. +- Files/modules: + - `docs/reviews/2026-07-05-go2-agent-architecture.png` + - `docs/reviews/2026-07-05-go2-agent-architecture-review.md` + - `docs/reviews/2026-07-05-go2-agent-architecture-review.zh.md` + - `dimos/robot/unitree/go2/PROJECT_CHANGELOG.md` +- Validation: + - Documentation/image-only change. + - Visually inspected the generated PNG. + - Ran `git diff --check` on the changed markdown/changelog files. +- Open items: + - If exact editable labels are required, add a Mermaid or SVG version beside + the generated PNG. + +## 2026-07-05 + - Branch: `refactor/go2-architecture-layers` - Summary: Added explicit interface input/output references to the English and Chinese Go2 architecture review guides. The new quick reference enumerates diff --git a/docs/reviews/2026-07-05-go2-agent-architecture-review.md b/docs/reviews/2026-07-05-go2-agent-architecture-review.md index f0407d0cd2..995259c658 100644 --- a/docs/reviews/2026-07-05-go2-agent-architecture-review.md +++ b/docs/reviews/2026-07-05-go2-agent-architecture-review.md @@ -77,6 +77,10 @@ things: - Self-evolution artifact: reviewable JSON events/proposals, not automatic code mutation. +## Architecture Diagram + +![DimOS Go2 agent self-evolution architecture](2026-07-05-go2-agent-architecture.png) + ## Interface Input/Output Quick Reference Use this section when reviewing API boundaries. It lists the stable public diff --git a/docs/reviews/2026-07-05-go2-agent-architecture-review.zh.md b/docs/reviews/2026-07-05-go2-agent-architecture-review.zh.md index aea31ca3db..42780e552d 100644 --- a/docs/reviews/2026-07-05-go2-agent-architecture-review.zh.md +++ b/docs/reviews/2026-07-05-go2-agent-architecture-review.zh.md @@ -54,6 +54,10 @@ review 每个本地功能时，建议都按下面这些问题看，而不是只 - Agent judgment: LLM 读取已选上下文和工具后决定下一步。 - Self-evolution artifact: 产出可 review 的 JSON event/proposal，不自动改代码。 +## 架构图 + +![DimOS Go2 Agent 自进化架构](2026-07-05-go2-agent-architecture.png) + ## 接口输入/输出速查 review API 边界时优先看这一节。这里列的是本地 Go2 架构改动新增或改动过的稳定 public surface；没有跨 module 边界消费的内部 helper 不列入。 diff --git a/docs/reviews/2026-07-05-go2-agent-architecture.png b/docs/reviews/2026-07-05-go2-agent-architecture.png new file mode 100644 index 0000000000..12021bbdb8 --- /dev/null +++ b/docs/reviews/2026-07-05-go2-agent-architecture.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2b964fd2d84ff4d85e379fd132e882ee3abb046e7bf3772e0cfbc74e50f3645a +size 1124197 From 1783773fa674a78578e638147eb7b314d89847f6 Mon Sep 17 00:00:00 2001 From: ZhangZheMing <1050297972@qq.com> Date: Sun, 5 Jul 2026 01:22:14 +0800 Subject: [PATCH 20/36] Clarify optional Go2 observability panel --- dimos/robot/unitree/go2/PROJECT_CHANGELOG.md | 21 +++++++++++++++++++ ...026-07-05-go2-agent-architecture-review.md | 15 +++++++++++-- ...-07-05-go2-agent-architecture-review.zh.md | 14 +++++++++++-- .../2026-07-05-go2-agent-architecture.png | 4 ++-- 4 files changed, 48 insertions(+), 6 deletions(-) diff --git a/dimos/robot/unitree/go2/PROJECT_CHANGELOG.md b/dimos/robot/unitree/go2/PROJECT_CHANGELOG.md index 7f44c59b5a..6344765637 100644 --- a/dimos/robot/unitree/go2/PROJECT_CHANGELOG.md +++ b/dimos/robot/unitree/go2/PROJECT_CHANGELOG.md @@ -8,6 +8,27 @@ Entries are listed in reverse chronological order. ## 2026-07-05 +- Branch: `refactor/go2-architecture-layers` +- Summary: Clarified the architecture diagram and review docs so the + Rerun/WebsocketVis world-model panel is represented as optional observability, + not a required dependency of `unitree_go2_agentic`. Updated the diagram label + from Web Dashboard to optional Rerun/WebsocketVis and added text noting that + the current Go2 agentic blueprint does not wire `WebsocketVisModule`. +- Files/modules: + - `docs/reviews/2026-07-05-go2-agent-architecture.png` + - `docs/reviews/2026-07-05-go2-agent-architecture-review.md` + - `docs/reviews/2026-07-05-go2-agent-architecture-review.zh.md` + - `dimos/robot/unitree/go2/PROJECT_CHANGELOG.md` +- Validation: + - Documentation/image-only change. + - Visually inspected the updated PNG. + - Ran `git diff --check` on the changed markdown/changelog files. +- Open items: + - Consider adding a deterministic SVG/Mermaid version if exact editable + diagram text is needed in future reviews. + +## 2026-07-05 + - Branch: `refactor/go2-architecture-layers` - Summary: Generated and added a review-oriented Go2 agent architecture diagram showing Layer 3 agent brain, Layer 4 world state, Layer 5 skill interface, diff --git a/docs/reviews/2026-07-05-go2-agent-architecture-review.md b/docs/reviews/2026-07-05-go2-agent-architecture-review.md index 995259c658..f77cbc00d7 100644 --- a/docs/reviews/2026-07-05-go2-agent-architecture-review.md +++ b/docs/reviews/2026-07-05-go2-agent-architecture-review.md @@ -81,6 +81,11 @@ things: ![DimOS Go2 agent self-evolution architecture](2026-07-05-go2-agent-architecture.png) +The purple right-side column is optional observability/storage. The current +`unitree_go2_agentic` blueprint does not wire `WebsocketVisModule`; the Rerun / +WebsocketVis panel only receives world-model state if a separate visualization +blueprint wires `WebsocketVisSpec`. + ## Interface Input/Output Quick Reference Use this section when reviewing API boundaries. It lists the stable public @@ -132,7 +137,7 @@ that are not consumed across module boundaries are intentionally omitted. | `CausalWorldModelSpec.get_provider_contract()` | No arguments. | `dimos.world_model_provider.v1` dict naming provider, model type, capabilities, and output schemas. | | `propose_skill_interface(task, failure_context_json)` MCP skill | Task text and failure-context JSON object with missing-capability evidence. Reads Layer 5 contracts/outcomes when wired. | `SkillResult` metadata: `proposal_created`, `proposal` using `dimos.skill_proposal.v1` when created, `existing_skill_matches`, `recommended_next_action`, optional `proposal_path` or warnings. | | MCP `tools/list` / `tools/call` runtime surface | JSON-RPC requests; `tools/call` also carries MCP arguments and optional `_mcp_context` progress/capability token. | Tool schemas with capability metadata; tool-call text/content results; SSE progress frames for background tools; capability release on instant return or stopped frame. | -| `WebsocketVisSpec.set_world_model_state(state)` | `state: dict` expected to use `dimos.world_model_dashboard_state.v1`. | `dict` acknowledgement/current display state for dashboard clients. | +| Optional `WebsocketVisSpec.set_world_model_state(state)` | `state: dict` expected to use `dimos.world_model_dashboard_state.v1`. This is only active when a blueprint wires `WebsocketVisModule`; current `unitree_go2_agentic` does not. | `dict` acknowledgement/current display state for optional Rerun/WebsocketVis clients. | ## Upstream Changes To Review @@ -1355,7 +1360,7 @@ Failure mode: proxies. - ReplayConnection stop should be a no-op, not an exception. -### 16. Rerun/Websocket Dashboard World-Model State +### 16. Optional Rerun/WebsocketVis World-Model Panel Files: @@ -1369,12 +1374,16 @@ Implementation logic: - Websocket visualization can carry world-model/dashboard state in addition to existing visualization payloads. - Dashboard payload shape is aligned with `world_model_contract.py`. +- This is optional observability. The current `unitree_go2_agentic` blueprint + does not include `WebsocketVisModule`, so this path is inactive unless a + visualization blueprint wires `WebsocketVisSpec`. Review focus: - Dashboard state should remain display-only. - Avoid making dashboard consumers depend on unversioned internal Python objects. +- Do not treat this as a required dependency of the Go2 self-evolution runtime. Components: @@ -1402,6 +1411,8 @@ Safety boundary: - Dashboard updates must not trigger robot actions. - UI should not present advisory predictions as guaranteed outcomes. +- If `WebsocketVisModule` is not wired, CausalWorldModel still works; the + dashboard publication path is skipped. ## Review Order I Recommend diff --git a/docs/reviews/2026-07-05-go2-agent-architecture-review.zh.md b/docs/reviews/2026-07-05-go2-agent-architecture-review.zh.md index 42780e552d..2b38f23f61 100644 --- a/docs/reviews/2026-07-05-go2-agent-architecture-review.zh.md +++ b/docs/reviews/2026-07-05-go2-agent-architecture-review.zh.md @@ -58,6 +58,10 @@ review 每个本地功能时，建议都按下面这些问题看，而不是只 ![DimOS Go2 Agent 自进化架构](2026-07-05-go2-agent-architecture.png) +右侧紫色列是可选观测/存储，不是主执行依赖。当前 `unitree_go2_agentic` +blueprint 没有接 `WebsocketVisModule`；只有单独的可视化 blueprint 接上 +`WebsocketVisSpec` 时，Rerun / WebsocketVis 面板才会收到 world-model state。 + ## 接口输入/输出速查 review API 边界时优先看这一节。这里列的是本地 Go2 架构改动新增或改动过的稳定 public surface；没有跨 module 边界消费的内部 helper 不列入。 @@ -107,7 +111,7 @@ review API 边界时优先看这一节。这里列的是本地 Go2 架构改动 | `CausalWorldModelSpec.get_provider_contract()` | 无参数。 | `dimos.world_model_provider.v1` dict，声明 provider、model type、capabilities 和 output schemas。 | | `propose_skill_interface(task, failure_context_json)` MCP skill | task text，以及带 missing-capability evidence 的 failure-context JSON object；已接线时读取 Layer 5 contracts/outcomes。 | `SkillResult` metadata: `proposal_created`、创建时的 `dimos.skill_proposal.v1` `proposal`、`existing_skill_matches`、`recommended_next_action`、可选 `proposal_path` 或 warnings。 | | MCP `tools/list` / `tools/call` runtime surface | JSON-RPC requests；`tools/call` 携带 MCP arguments 和可选 `_mcp_context` progress/capability token。 | 带 capability metadata 的 tool schema；tool-call text/content result；background tool 的 SSE progress frames；instant return 或 stopped frame 触发 capability release。 | -| `WebsocketVisSpec.set_world_model_state(state)` | `state: dict`，预期使用 `dimos.world_model_dashboard_state.v1`。 | `dict` acknowledgement/current display state，供 dashboard clients 读取。 | +| 可选 `WebsocketVisSpec.set_world_model_state(state)` | `state: dict`，预期使用 `dimos.world_model_dashboard_state.v1`。只有 blueprint 接入 `WebsocketVisModule` 时才生效；当前 `unitree_go2_agentic` 没有接。 | `dict` acknowledgement/current display state，供可选 Rerun/WebsocketVis clients 读取。 | ## 需要 Review 的上游更新 @@ -1107,7 +1111,7 @@ Review 重点: - restart 不应遗留 old transports，也不应让 consumer 连在 dead proxy 上。 - ReplayConnection stop 应该是 no-op，而不是 exception。 -### 16. Rerun/Websocket Dashboard World-model State +### 16. 可选 Rerun/WebsocketVis World-model Panel 相关文件: @@ -1120,11 +1124,15 @@ Review 重点: - websocket visualization 除已有 payload 外，还可以携带 world-model/dashboard state。 - dashboard payload shape 与 `world_model_contract.py` 对齐。 +- 这是可选观测能力。当前 `unitree_go2_agentic` blueprint 不包含 + `WebsocketVisModule`，因此除非单独的可视化 blueprint 接入 + `WebsocketVisSpec`，这条路径不会启动。 Review 重点: - dashboard state 应保持展示用途。 - dashboard consumer 不应依赖未 version 的内部 Python object。 +- 不要把它当成 Go2 自进化 runtime 的必需依赖。 组成部分: @@ -1150,6 +1158,8 @@ Review 重点: - dashboard update 不能触发机器人动作。 - UI 不应把 advisory prediction 展示成 guaranteed outcome。 +- 如果没有接 `WebsocketVisModule`，CausalWorldModel 仍然正常工作；dashboard + publish path 会被跳过。 ## 推荐 Review 顺序 diff --git a/docs/reviews/2026-07-05-go2-agent-architecture.png b/docs/reviews/2026-07-05-go2-agent-architecture.png index 12021bbdb8..8bf2f96e71 100644 --- a/docs/reviews/2026-07-05-go2-agent-architecture.png +++ b/docs/reviews/2026-07-05-go2-agent-architecture.png @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:2b964fd2d84ff4d85e379fd132e882ee3abb046e7bf3772e0cfbc74e50f3645a -size 1124197 +oid sha256:d84e8f49d92ea8cc7cb3526997d5497c23b3d4de765dab436403390c8b828ee9 +size 1086809 From 9ec114904af998bdb7aefba829459750b4027533 Mon Sep 17 00:00:00 2001 From: ZhangZheMing <1050297972@qq.com> Date: Sun, 5 Jul 2026 01:32:50 +0800 Subject: [PATCH 21/36] Add precise DimOS architecture diagrams --- dimos/robot/unitree/go2/PROJECT_CHANGELOG.md | 26 +++ .../2026-07-05-dimos-project-architecture.png | 3 + .../2026-07-05-dimos-project-architecture.svg | 171 +++++++++++++++++ ...026-07-05-go2-agent-architecture-review.md | 17 +- ...-07-05-go2-agent-architecture-review.zh.md | 17 +- .../2026-07-05-go2-agent-architecture.png | 4 +- .../2026-07-05-go2-agent-architecture.svg | 175 ++++++++++++++++++ 7 files changed, 409 insertions(+), 4 deletions(-) create mode 100644 docs/reviews/2026-07-05-dimos-project-architecture.png create mode 100644 docs/reviews/2026-07-05-dimos-project-architecture.svg create mode 100644 docs/reviews/2026-07-05-go2-agent-architecture.svg diff --git a/dimos/robot/unitree/go2/PROJECT_CHANGELOG.md b/dimos/robot/unitree/go2/PROJECT_CHANGELOG.md index 6344765637..3111ce53ab 100644 --- a/dimos/robot/unitree/go2/PROJECT_CHANGELOG.md +++ b/dimos/robot/unitree/go2/PROJECT_CHANGELOG.md @@ -8,6 +8,32 @@ Entries are listed in reverse chronological order. ## 2026-07-05 +- Branch: `refactor/go2-architecture-layers` +- Summary: Reworked the Go2 self-evolution architecture diagram as a + deterministic editable SVG with a PNG render to eliminate generated-image + text/arrow errors. Added a separate DimOS repo-wide project architecture + diagram covering entry points, blueprints, configuration, core runtime, + communication transports, functional module families, robot/simulation + targets, and data/model/observability assets. +- Files/modules: + - `docs/reviews/2026-07-05-go2-agent-architecture.svg` + - `docs/reviews/2026-07-05-go2-agent-architecture.png` + - `docs/reviews/2026-07-05-dimos-project-architecture.svg` + - `docs/reviews/2026-07-05-dimos-project-architecture.png` + - `docs/reviews/2026-07-05-go2-agent-architecture-review.md` + - `docs/reviews/2026-07-05-go2-agent-architecture-review.zh.md` + - `dimos/robot/unitree/go2/PROJECT_CHANGELOG.md` +- Validation: + - Parsed both SVG files as XML. + - Rendered both SVG files to PNG with `sips`. + - Visually inspected both PNG renders for label and arrow issues. + - Ran `git diff --check` on the changed markdown/changelog/SVG files. +- Open items: + - Keep the project-wide diagram updated when major package boundaries or + blueprint runtime responsibilities change. + +## 2026-07-05 + - Branch: `refactor/go2-architecture-layers` - Summary: Clarified the architecture diagram and review docs so the Rerun/WebsocketVis world-model panel is represented as optional observability, diff --git a/docs/reviews/2026-07-05-dimos-project-architecture.png b/docs/reviews/2026-07-05-dimos-project-architecture.png new file mode 100644 index 0000000000..6574655451 --- /dev/null +++ b/docs/reviews/2026-07-05-dimos-project-architecture.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bdb38a4089ea44c83e826a198ea18bb19aae5db857ccb7dc02c10dd5e1288660 +size 221143 diff --git a/docs/reviews/2026-07-05-dimos-project-architecture.svg b/docs/reviews/2026-07-05-dimos-project-architecture.svg new file mode 100644 index 0000000000..38b42b1e7b --- /dev/null +++ b/docs/reviews/2026-07-05-dimos-project-architecture.svg @@ -0,0 +1,171 @@ + + DimOS Project Architecture + Overall DimOS architecture showing users, CLI/API, blueprint orchestration, core module runtime, typed transports, functional modules, robots, data, and observability. + + + + + + + + + DimOS 项目整体架构 + Agentic operating system for robotics: Blueprints compose typed Modules over streams, RPC, and multiple transports. + + + + Entry Points + + CLI + dimos run/list/mcp + + Python API + library usage + + + Blueprint Layer + + Blueprints + autoconnect() + + Registry + all_blueprints.py + + + Configuration + + GlobalConfig + env / CLI flags + + .env + defaults + + + External Interfaces + + MCP + agent tools + + Web / Rerun + observability + + + + Core Runtime + + Module + In[T] / Out[T] + typed streams + + + ModuleCoordinator + deploy / lifecycle + system-module wiring + + + Workers + Python / Docker + forkserver actors + + + RPC / Specs + typed module refs + @rpc / @skill + + + Messages + geometry / nav + sensor / vision + + + Run State + logs + registry + + + + Communication Layer + + LCM + + Zenoh + + SHM / pSHM + + ROS + + DDS + + Tool Streams + + WebSocket / HTTP + + + + Functional Module Families + + Agents + LangGraph / MCP + + Skills + @skill containers + + Perception + detection / memory + + Mapping + occupancy / TF + + Navigation + planning / patrol + + Control + tasks / motion + + Manipulation + grasp / planning + + Learning + data prep + + + + Robots / Simulation / Hardware + Unitree Go2/G1/B1 · xArm/OpenArm/Piper/A750 · Drones · MuJoCo · Replay data + + + Data / Models / Observability + LFS assets · Memory2 · vector/blob stores + VLM / embedding / segmentation models · Rerun / Web + + + + + + + + + + + + + + Blueprints choose modules; Core deploys workers; Transports carry streams/RPC. + diff --git a/docs/reviews/2026-07-05-go2-agent-architecture-review.md b/docs/reviews/2026-07-05-go2-agent-architecture-review.md index f77cbc00d7..b251dc3d0e 100644 --- a/docs/reviews/2026-07-05-go2-agent-architecture-review.md +++ b/docs/reviews/2026-07-05-go2-agent-architecture-review.md @@ -79,13 +79,28 @@ things: ## Architecture Diagram -![DimOS Go2 agent self-evolution architecture](2026-07-05-go2-agent-architecture.png) +![DimOS Go2 agent self-evolution architecture](2026-07-05-go2-agent-architecture.svg) The purple right-side column is optional observability/storage. The current `unitree_go2_agentic` blueprint does not wire `WebsocketVisModule`; the Rerun / WebsocketVis panel only receives world-model state if a separate visualization blueprint wires `WebsocketVisSpec`. +Editable source: `2026-07-05-go2-agent-architecture.svg` +PNG render: `2026-07-05-go2-agent-architecture.png` + +## Project Architecture Diagram + +![DimOS project architecture](2026-07-05-dimos-project-architecture.svg) + +This diagram is repo-wide, not Go2-specific. It shows the main DimOS flow: +entry points, blueprint/configuration, core runtime, communication transports, +functional module families, robots/simulation/hardware, and data/model/ +observability assets. + +Editable source: `2026-07-05-dimos-project-architecture.svg` +PNG render: `2026-07-05-dimos-project-architecture.png` + ## Interface Input/Output Quick Reference Use this section when reviewing API boundaries. It lists the stable public diff --git a/docs/reviews/2026-07-05-go2-agent-architecture-review.zh.md b/docs/reviews/2026-07-05-go2-agent-architecture-review.zh.md index 2b38f23f61..ee2e4565e0 100644 --- a/docs/reviews/2026-07-05-go2-agent-architecture-review.zh.md +++ b/docs/reviews/2026-07-05-go2-agent-architecture-review.zh.md @@ -56,12 +56,27 @@ review 每个本地功能时，建议都按下面这些问题看，而不是只 ## 架构图 -![DimOS Go2 Agent 自进化架构](2026-07-05-go2-agent-architecture.png) +![DimOS Go2 Agent 自进化架构](2026-07-05-go2-agent-architecture.svg) 右侧紫色列是可选观测/存储，不是主执行依赖。当前 `unitree_go2_agentic` blueprint 没有接 `WebsocketVisModule`；只有单独的可视化 blueprint 接上 `WebsocketVisSpec` 时，Rerun / WebsocketVis 面板才会收到 world-model state。 +可编辑源文件: `2026-07-05-go2-agent-architecture.svg` +PNG 渲染版: `2026-07-05-go2-agent-architecture.png` + +## 项目整体架构图 + +![DimOS 项目整体架构](2026-07-05-dimos-project-architecture.svg) + +这张图是 repo-wide，不是 Go2 专属。它展示 DimOS 的主流程: entry +points、blueprint/configuration、core runtime、communication transports、 +functional module families、robots/simulation/hardware，以及 +data/model/observability assets。 + +可编辑源文件: `2026-07-05-dimos-project-architecture.svg` +PNG 渲染版: `2026-07-05-dimos-project-architecture.png` + ## 接口输入/输出速查 review API 边界时优先看这一节。这里列的是本地 Go2 架构改动新增或改动过的稳定 public surface；没有跨 module 边界消费的内部 helper 不列入。 diff --git a/docs/reviews/2026-07-05-go2-agent-architecture.png b/docs/reviews/2026-07-05-go2-agent-architecture.png index 8bf2f96e71..6bb86c1dd4 100644 --- a/docs/reviews/2026-07-05-go2-agent-architecture.png +++ b/docs/reviews/2026-07-05-go2-agent-architecture.png @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:d84e8f49d92ea8cc7cb3526997d5497c23b3d4de765dab436403390c8b828ee9 -size 1086809 +oid sha256:1e80abb6d7371795e672e5652db20754b8fb21bb1a74ec8b88e45c5df372cc0a +size 195112 diff --git a/docs/reviews/2026-07-05-go2-agent-architecture.svg b/docs/reviews/2026-07-05-go2-agent-architecture.svg new file mode 100644 index 0000000000..a20e385bd0 --- /dev/null +++ b/docs/reviews/2026-07-05-go2-agent-architecture.svg @@ -0,0 +1,175 @@ + + DimOS Go2 Agent Self-Evolution Architecture + Layered Go2 agent architecture with agent brain, world state, skill interface, robot body, local review artifacts, and optional Rerun/WebsocketVis observability. + + + + + + + + + + + + + + + DimOS Go2 Agent 自进化架构 + + + + Layer 3 Agent Brain + + + Layer 4 World State + + + Layer 5 Skill Interface + + + Layer 6 Robot Body + + + + Local Review + Artifacts + local files / optional commit / no auto push + + 本地 Git Ledger + .dimos/evolution + + JSON Events / Proposals + review artifacts + + + + Optional Observability + not wired by unitree_go2_agentic + + 可选观测 + Rerun / WebsocketVis + + + + 用户 Prompt + + + Expert Router + + + Context Provider + + + Task Feasibility + + + MCP Client / Server + + + Skill Outcome + Store + + + Causal World + Model + + + Skill Proposal + + + Evolution + Ledger + + + + Structured World State + + + Semantic Temporal Map + + + RAG / Spatial Memory + + + Temporal Memory + + + + Skill Contracts + + + Preflight Metadata + + + Go2 Skills + + + + Go2 Connection + + + Sensors / Odom + + + Motion Control + + + + + + + + + + + + + + + + + + + + + + + + + + 技能执行结果 / Skill Outcome + + + + + + + + + MCP 调用 Go2 skills；不直接改代码 + observability only; no robot control + From ff913763a399ca3f0a29189a62ba1846a3703c95 Mon Sep 17 00:00:00 2001 From: ZhangZheMing <1050297972@qq.com> Date: Sun, 5 Jul 2026 01:41:22 +0800 Subject: [PATCH 22/36] Add research-style DimOS agentic architecture figure --- dimos/robot/unitree/go2/PROJECT_CHANGELOG.md | 26 ++ .../2026-07-05-dimos-agentic-architecture.png | 3 + .../2026-07-05-dimos-agentic-architecture.svg | 227 ++++++++++++++++++ ...026-07-05-go2-agent-architecture-review.md | 43 ++-- ...-07-05-go2-agent-architecture-review.zh.md | 40 +-- 5 files changed, 308 insertions(+), 31 deletions(-) create mode 100644 docs/reviews/2026-07-05-dimos-agentic-architecture.png create mode 100644 docs/reviews/2026-07-05-dimos-agentic-architecture.svg diff --git a/dimos/robot/unitree/go2/PROJECT_CHANGELOG.md b/dimos/robot/unitree/go2/PROJECT_CHANGELOG.md index 3111ce53ab..59cfe98de3 100644 --- a/dimos/robot/unitree/go2/PROJECT_CHANGELOG.md +++ b/dimos/robot/unitree/go2/PROJECT_CHANGELOG.md @@ -8,6 +8,32 @@ Entries are listed in reverse chronological order. ## 2026-07-05 +- Branch: `refactor/go2-architecture-layers` +- Summary: Added a single research-style composite architecture figure that + places the Go2 agentic self-evolution architecture inside the repo-wide DimOS + project architecture. The new figure makes the agent brain the emphasized + subsystem while preserving surrounding execution context: CLI/MCP entry + points, blueprint/configuration, core runtime, transports, module families, + robot/simulation targets, memory/assets, local review artifacts, and optional + observability. +- Files/modules: + - `docs/reviews/2026-07-05-dimos-agentic-architecture.svg` + - `docs/reviews/2026-07-05-dimos-agentic-architecture.png` + - `docs/reviews/2026-07-05-go2-agent-architecture-review.md` + - `docs/reviews/2026-07-05-go2-agent-architecture-review.zh.md` + - `dimos/robot/unitree/go2/PROJECT_CHANGELOG.md` +- Validation: + - Parsed the combined SVG as XML. + - Rendered the combined SVG to PNG with `sips`. + - Visually inspected the PNG render for label, arrow, and optional-path + ambiguity issues. + - Ran `git diff --check` on the changed markdown/changelog/SVG files. +- Open items: + - Keep the combined figure as the canonical review diagram when future + agent, memory, skill, or blueprint boundaries change. + +## 2026-07-05 + - Branch: `refactor/go2-architecture-layers` - Summary: Reworked the Go2 self-evolution architecture diagram as a deterministic editable SVG with a PNG render to eliminate generated-image diff --git a/docs/reviews/2026-07-05-dimos-agentic-architecture.png b/docs/reviews/2026-07-05-dimos-agentic-architecture.png new file mode 100644 index 0000000000..43c3c2c3ce --- /dev/null +++ b/docs/reviews/2026-07-05-dimos-agentic-architecture.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:42111678889fbfc0fd1926390a2ef9e09fa3f4ae2a440d708c7f99306eeac953 +size 352055 diff --git a/docs/reviews/2026-07-05-dimos-agentic-architecture.svg b/docs/reviews/2026-07-05-dimos-agentic-architecture.svg new file mode 100644 index 0000000000..4e360c089b --- /dev/null +++ b/docs/reviews/2026-07-05-dimos-agentic-architecture.svg @@ -0,0 +1,227 @@ + + DimOS Project Architecture With Go2 Agentic Extension + A scientific-style system architecture diagram showing the DimOS project runtime and a zoomed Go2 agentic self-evolution extension in one figure. + + + + + + + + + + + + + + + DimOS 项目整体架构与 Go2 Agent 自进化扩展 + Scientific-style composite figure: project runtime context plus the current Go2 agentic extension boundary. + + + + A + DimOS project architecture + + + + Entry points + + CLI + + Python API + + + Blueprint layer + + Blueprints + + Registry + + + Configuration + + GlobalConfig + + .env + + + External interfaces + + MCP + + Rerun / Web + + + Targets + Robots / Simulation + Go2 · G1 · xArm · Drones · MuJoCo + + + + Core runtime + + Module In[T]/Out[T] + + ModuleCoordinator + + Workers + + RPC / Specs + + Messages + + Run registry/logs + + Lifecycle · deployment · system wiring + + + + Communication layer + LCM · Zenoh · SHM/pSHM · ROS · DDS · Tool streams · HTTP/WebSocket + + + Functional module families + Agents · Skills · Perception · Mapping · Navigation · Control · Manipulation · Learning + Data/model stores: LFS assets · Memory2 · vector/blob stores · VLM/embedding/segmentation models + + + + + + + + + + Panel A abstracts the full repository. The Go2 agentic branch is one blueprint-specific extension within the Agents/Skills/Robot target path. + + + + zoom into Go2 agentic blueprint + + + + B + Go2 agentic self-evolution extension inside DimOS + Current `unitree_go2_agentic`: robot body + world state + skill interface + agent brain. WebsocketVis is optional and not wired here. + + + + Layer 3 Agent Brain + + Prompt + + Router + + Context Provider + + Feasibility + + MCP + + Outcome Store + + Causal Model + + Skill Proposal + + + Layer 4 World State + + Structured State + + Semantic Temporal Map + + Spatial/RAG + + Temporal + + + Layer 5 Skill Interface + + Skill Contracts + + Preflight Metadata + + Go2 Skills + + + Layer 6 Robot Body + Go2 Connection · Sensors/Odom · Motion Control + + + + Local review artifacts + local JSON files; optional local Git commit; no automatic GitHub push + + .dimos/evolution/events + + .dimos/evolution/proposals + + + Optional observability + Rerun / WebsocketVis + not wired by current `unitree_go2_agentic` + + + + Legend and rigor notes + + control/tool call + + context read / evidence + + audit artifact write + + optional observability + No arrow means no runtime dependency. + Causal model output is advisory only. + Skill proposal creates review artifacts, not code. + + + + + + + + + + + + + + + + + + + Figure: DimOS as a modular runtime, with the Go2 agentic extension shown as a blueprint-specific subsystem. Solid arrows are calls/control, dashed arrows are context/evidence reads, dotted arrows are local audit writes, and dash-dot arrows are optional visualization. + diff --git a/docs/reviews/2026-07-05-go2-agent-architecture-review.md b/docs/reviews/2026-07-05-go2-agent-architecture-review.md index b251dc3d0e..e5ef3e36c4 100644 --- a/docs/reviews/2026-07-05-go2-agent-architecture-review.md +++ b/docs/reviews/2026-07-05-go2-agent-architecture-review.md @@ -77,29 +77,40 @@ things: - Self-evolution artifact: reviewable JSON events/proposals, not automatic code mutation. -## Architecture Diagram +## Combined Project and Agent Architecture Diagram -![DimOS Go2 agent self-evolution architecture](2026-07-05-go2-agent-architecture.svg) +![DimOS project architecture with Go2 agentic self-evolution extension](2026-07-05-dimos-agentic-architecture.svg) -The purple right-side column is optional observability/storage. The current -`unitree_go2_agentic` blueprint does not wire `WebsocketVisModule`; the Rerun / -WebsocketVis panel only receives world-model state if a separate visualization -blueprint wires `WebsocketVisSpec`. +This is the canonical review figure for this branch. Panel A shows the +repo-wide DimOS execution architecture: entry points, blueprint/configuration, +core runtime, transports, module families, robot/simulation targets, and data +assets. Panel B expands the Go2 agentic self-evolution extension inside that +project architecture, with the agent brain deliberately emphasized because it +owns task evaluation, context use, skill validation, feasibility judgment, and +proposal/artifact emission. -Editable source: `2026-07-05-go2-agent-architecture.svg` -PNG render: `2026-07-05-go2-agent-architecture.png` +The purple right-side column remains optional observability/storage. The +current `unitree_go2_agentic` blueprint does not wire `WebsocketVisModule`; the +Rerun / WebsocketVis panel only receives world-model state if a separate +visualization blueprint wires `WebsocketVisSpec`. -## Project Architecture Diagram +Line semantics are part of the figure contract: -![DimOS project architecture](2026-07-05-dimos-project-architecture.svg) +- Solid black arrows: control flow, blueprint composition, tool calls, or robot + commands. +- Dashed blue arrows: context/evidence reads used by the agent. +- Dotted purple arrows: local review artifacts written to files/Git. +- Dash-dot purple arrows: optional observability paths. -This diagram is repo-wide, not Go2-specific. It shows the main DimOS flow: -entry points, blueprint/configuration, core runtime, communication transports, -functional module families, robots/simulation/hardware, and data/model/ -observability assets. +Editable source: `2026-07-05-dimos-agentic-architecture.svg` +PNG render: `2026-07-05-dimos-agentic-architecture.png` -Editable source: `2026-07-05-dimos-project-architecture.svg` -PNG render: `2026-07-05-dimos-project-architecture.png` +Supporting detail figures retained for narrower inspection: + +- Agent-only detail: `2026-07-05-go2-agent-architecture.svg` / + `2026-07-05-go2-agent-architecture.png` +- Project-only detail: `2026-07-05-dimos-project-architecture.svg` / + `2026-07-05-dimos-project-architecture.png` ## Interface Input/Output Quick Reference diff --git a/docs/reviews/2026-07-05-go2-agent-architecture-review.zh.md b/docs/reviews/2026-07-05-go2-agent-architecture-review.zh.md index ee2e4565e0..61cac7239e 100644 --- a/docs/reviews/2026-07-05-go2-agent-architecture-review.zh.md +++ b/docs/reviews/2026-07-05-go2-agent-architecture-review.zh.md @@ -54,28 +54,38 @@ review 每个本地功能时，建议都按下面这些问题看，而不是只 - Agent judgment: LLM 读取已选上下文和工具后决定下一步。 - Self-evolution artifact: 产出可 review 的 JSON event/proposal，不自动改代码。 -## 架构图 +## 项目整体架构与 Agent 重点展开图 -![DimOS Go2 Agent 自进化架构](2026-07-05-go2-agent-architecture.svg) +![DimOS 项目整体架构与 Go2 Agent 自进化扩展](2026-07-05-dimos-agentic-architecture.svg) -右侧紫色列是可选观测/存储，不是主执行依赖。当前 `unitree_go2_agentic` -blueprint 没有接 `WebsocketVisModule`；只有单独的可视化 blueprint 接上 -`WebsocketVisSpec` 时，Rerun / WebsocketVis 面板才会收到 world-model state。 +这是本分支 review 的主图。Panel A 展示 repo-wide DimOS 执行架构: +entry points、blueprint/configuration、core runtime、transports、module +families、robot/simulation targets 和 data assets。Panel B 在同一张图里 +展开 Go2 agentic self-evolution extension，并刻意把 Agent Brain 放在重点位置: +task evaluation、context use、skill validation、feasibility judgment 和 +proposal/artifact emission 都由这个层级发起或裁决。 -可编辑源文件: `2026-07-05-go2-agent-architecture.svg` -PNG 渲染版: `2026-07-05-go2-agent-architecture.png` +右侧紫色列仍然是可选观测/存储，不是主执行依赖。当前 +`unitree_go2_agentic` blueprint 没有接 `WebsocketVisModule`；只有单独的 +可视化 blueprint 接上 `WebsocketVisSpec` 时，Rerun / WebsocketVis 面板才会 +收到 world-model state。 -## 项目整体架构图 +图中的线型语义也是 review contract 的一部分: -![DimOS 项目整体架构](2026-07-05-dimos-project-architecture.svg) +- 黑色实线: control flow、blueprint composition、tool call 或 robot command。 +- 蓝色虚线: agent 读取 context/evidence 的路径。 +- 紫色点线: 写入本地 review artifact、文件或 Git 的路径。 +- 紫色点划线: optional observability path。 -这张图是 repo-wide，不是 Go2 专属。它展示 DimOS 的主流程: entry -points、blueprint/configuration、core runtime、communication transports、 -functional module families、robots/simulation/hardware，以及 -data/model/observability assets。 +可编辑源文件: `2026-07-05-dimos-agentic-architecture.svg` +PNG 渲染版: `2026-07-05-dimos-agentic-architecture.png` -可编辑源文件: `2026-07-05-dimos-project-architecture.svg` -PNG 渲染版: `2026-07-05-dimos-project-architecture.png` +为窄范围 review 保留的辅助细节图: + +- Agent-only detail: `2026-07-05-go2-agent-architecture.svg` / + `2026-07-05-go2-agent-architecture.png` +- Project-only detail: `2026-07-05-dimos-project-architecture.svg` / + `2026-07-05-dimos-project-architecture.png` ## 接口输入/输出速查 From 2a3d0fe6f5246ae89057a6714cdcc98043a7dff7 Mon Sep 17 00:00:00 2001 From: ZhangZheMing <1050297972@qq.com> Date: Sun, 5 Jul 2026 01:45:59 +0800 Subject: [PATCH 23/36] Ignore MuJoCo runtime log --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index b91e943b45..0146ecda11 100644 --- a/.gitignore +++ b/.gitignore @@ -98,6 +98,9 @@ recording*.db # Rerun recordings *.rrd +# MuJoCo runtime log written to the current working directory +MUJOCO_LOG.TXT + /misc/fresh-ubuntu-tests/cache # openspec From 445b4255b702d0da7ae4560ea9c44d6de64f517c Mon Sep 17 00:00:00 2001 From: ZhangZheMing <1050297972@qq.com> Date: Sun, 5 Jul 2026 09:31:56 +0800 Subject: [PATCH 24/36] Remove temporary Go2 review artifacts --- dimos/robot/unitree/go2/PROJECT_CHANGELOG.md | 172 +- ...gent-self-evolution-implementation-plan.md | 362 ---- .../2026-07-05-dimos-agentic-architecture.png | 3 - .../2026-07-05-dimos-agentic-architecture.svg | 227 --- .../2026-07-05-dimos-project-architecture.png | 3 - .../2026-07-05-dimos-project-architecture.svg | 171 -- ...026-07-05-go2-agent-architecture-review.md | 1493 ----------------- ...-07-05-go2-agent-architecture-review.zh.md | 1232 -------------- .../2026-07-05-go2-agent-architecture.png | 3 - .../2026-07-05-go2-agent-architecture.svg | 175 -- 10 files changed, 2 insertions(+), 3839 deletions(-) delete mode 100644 docs/plans/2026-07-04-agent-self-evolution-implementation-plan.md delete mode 100644 docs/reviews/2026-07-05-dimos-agentic-architecture.png delete mode 100644 docs/reviews/2026-07-05-dimos-agentic-architecture.svg delete mode 100644 docs/reviews/2026-07-05-dimos-project-architecture.png delete mode 100644 docs/reviews/2026-07-05-dimos-project-architecture.svg delete mode 100644 docs/reviews/2026-07-05-go2-agent-architecture-review.md delete mode 100644 docs/reviews/2026-07-05-go2-agent-architecture-review.zh.md delete mode 100644 docs/reviews/2026-07-05-go2-agent-architecture.png delete mode 100644 docs/reviews/2026-07-05-go2-agent-architecture.svg diff --git a/dimos/robot/unitree/go2/PROJECT_CHANGELOG.md b/dimos/robot/unitree/go2/PROJECT_CHANGELOG.md index 59cfe98de3..51687de8bd 100644 --- a/dimos/robot/unitree/go2/PROJECT_CHANGELOG.md +++ b/dimos/robot/unitree/go2/PROJECT_CHANGELOG.md @@ -8,164 +8,11 @@ Entries are listed in reverse chronological order. ## 2026-07-05 -- Branch: `refactor/go2-architecture-layers` -- Summary: Added a single research-style composite architecture figure that - places the Go2 agentic self-evolution architecture inside the repo-wide DimOS - project architecture. The new figure makes the agent brain the emphasized - subsystem while preserving surrounding execution context: CLI/MCP entry - points, blueprint/configuration, core runtime, transports, module families, - robot/simulation targets, memory/assets, local review artifacts, and optional - observability. -- Files/modules: - - `docs/reviews/2026-07-05-dimos-agentic-architecture.svg` - - `docs/reviews/2026-07-05-dimos-agentic-architecture.png` - - `docs/reviews/2026-07-05-go2-agent-architecture-review.md` - - `docs/reviews/2026-07-05-go2-agent-architecture-review.zh.md` - - `dimos/robot/unitree/go2/PROJECT_CHANGELOG.md` -- Validation: - - Parsed the combined SVG as XML. - - Rendered the combined SVG to PNG with `sips`. - - Visually inspected the PNG render for label, arrow, and optional-path - ambiguity issues. - - Ran `git diff --check` on the changed markdown/changelog/SVG files. -- Open items: - - Keep the combined figure as the canonical review diagram when future - agent, memory, skill, or blueprint boundaries change. - -## 2026-07-05 - -- Branch: `refactor/go2-architecture-layers` -- Summary: Reworked the Go2 self-evolution architecture diagram as a - deterministic editable SVG with a PNG render to eliminate generated-image - text/arrow errors. Added a separate DimOS repo-wide project architecture - diagram covering entry points, blueprints, configuration, core runtime, - communication transports, functional module families, robot/simulation - targets, and data/model/observability assets. -- Files/modules: - - `docs/reviews/2026-07-05-go2-agent-architecture.svg` - - `docs/reviews/2026-07-05-go2-agent-architecture.png` - - `docs/reviews/2026-07-05-dimos-project-architecture.svg` - - `docs/reviews/2026-07-05-dimos-project-architecture.png` - - `docs/reviews/2026-07-05-go2-agent-architecture-review.md` - - `docs/reviews/2026-07-05-go2-agent-architecture-review.zh.md` - - `dimos/robot/unitree/go2/PROJECT_CHANGELOG.md` -- Validation: - - Parsed both SVG files as XML. - - Rendered both SVG files to PNG with `sips`. - - Visually inspected both PNG renders for label and arrow issues. - - Ran `git diff --check` on the changed markdown/changelog/SVG files. -- Open items: - - Keep the project-wide diagram updated when major package boundaries or - blueprint runtime responsibilities change. - -## 2026-07-05 - -- Branch: `refactor/go2-architecture-layers` -- Summary: Clarified the architecture diagram and review docs so the - Rerun/WebsocketVis world-model panel is represented as optional observability, - not a required dependency of `unitree_go2_agentic`. Updated the diagram label - from Web Dashboard to optional Rerun/WebsocketVis and added text noting that - the current Go2 agentic blueprint does not wire `WebsocketVisModule`. -- Files/modules: - - `docs/reviews/2026-07-05-go2-agent-architecture.png` - - `docs/reviews/2026-07-05-go2-agent-architecture-review.md` - - `docs/reviews/2026-07-05-go2-agent-architecture-review.zh.md` - - `dimos/robot/unitree/go2/PROJECT_CHANGELOG.md` -- Validation: - - Documentation/image-only change. - - Visually inspected the updated PNG. - - Ran `git diff --check` on the changed markdown/changelog files. -- Open items: - - Consider adding a deterministic SVG/Mermaid version if exact editable - diagram text is needed in future reviews. - -## 2026-07-05 - -- Branch: `refactor/go2-architecture-layers` -- Summary: Generated and added a review-oriented Go2 agent architecture diagram - showing Layer 3 agent brain, Layer 4 world state, Layer 5 skill interface, - Layer 6 robot body, local Git ledger artifacts, and dashboard observability. - Linked the image from both English and Chinese review guides. -- Files/modules: - - `docs/reviews/2026-07-05-go2-agent-architecture.png` - - `docs/reviews/2026-07-05-go2-agent-architecture-review.md` - - `docs/reviews/2026-07-05-go2-agent-architecture-review.zh.md` - - `dimos/robot/unitree/go2/PROJECT_CHANGELOG.md` -- Validation: - - Documentation/image-only change. - - Visually inspected the generated PNG. - - Ran `git diff --check` on the changed markdown/changelog files. -- Open items: - - If exact editable labels are required, add a Mermaid or SVG version beside - the generated PNG. - -## 2026-07-05 - -- Branch: `refactor/go2-architecture-layers` -- Summary: Added explicit interface input/output references to the English and - Chinese Go2 architecture review guides. The new quick reference enumerates - Layer 4/5/6 Specs, Layer 3 MCP skills, ledger/proposal schemas, world-model - contracts, MCP runtime surfaces, and dashboard update interfaces with their - expected inputs and returned outputs. -- Files/modules: - - `docs/reviews/2026-07-05-go2-agent-architecture-review.md` - - `docs/reviews/2026-07-05-go2-agent-architecture-review.zh.md` - - `dimos/robot/unitree/go2/PROJECT_CHANGELOG.md` -- Validation: - - Documentation-only change. - - Ran `git diff --check` on the changed review/changelog files. -- Open items: - - Keep the interface quick reference synchronized with any future signature - or schema changes. - -## 2026-07-05 - -- Branch: `refactor/go2-architecture-layers` -- Summary: Normalized every local Go2 architecture/self-evolution review-guide - section to the same implementation-review depth as the causal world model - section. Added per-feature component breakdowns, inputs, data flows, - decision ownership, state/write boundaries, failure modes, and safety/review - boundaries in both English and Chinese. -- Files/modules: - - `docs/reviews/2026-07-05-go2-agent-architecture-review.md` - - `docs/reviews/2026-07-05-go2-agent-architecture-review.zh.md` - - `dimos/robot/unitree/go2/PROJECT_CHANGELOG.md` -- Validation: - - Documentation-only change. - - Ran `git diff --check` on the changed review/changelog files. -- Open items: - - During code review, keep the review guide aligned with any implementation - changes requested by reviewers. - -## 2026-07-05 - -- Branch: `refactor/go2-architecture-layers` -- Summary: Expanded the English and Chinese Go2 architecture review guides with - finer implementation granularity, including entry points, inputs, decision - ownership, state/write boundaries, failure modes, and a detailed explanation - of the causal world model's online linear model, hand-authored SCM, and - observational estimator. -- Files/modules: - - `docs/reviews/2026-07-05-go2-agent-architecture-review.md` - - `docs/reviews/2026-07-05-go2-agent-architecture-review.zh.md` - - `dimos/robot/unitree/go2/PROJECT_CHANGELOG.md` -- Validation: - - Documentation-only change. - - Ran `git diff --check` on the changed review/changelog files. -- Open items: - - Keep these review guides in sync if the implementation changes after code - review. - -## 2026-07-05 - - Branch: `refactor/go2-architecture-layers` - Summary: Synchronized the branch with upstream `main` at `6e813a72`, resolved - conflicts against the Go2 layered architecture, and added a detailed review - guide in English and Chinese for the full Go2 agent architecture/self-evolution - feature set. + conflicts against the Go2 layered architecture, and kept the Go2 agentic + feature set aligned with upstream runtime changes. - Files/modules: - - `docs/reviews/2026-07-05-go2-agent-architecture-review.md` - - `docs/reviews/2026-07-05-go2-agent-architecture-review.zh.md` - `dimos/agents/mcp/mcp_server.py` - `dimos/agents/mcp/test_mcp_server.py` - `dimos/perception/spatial_perception.py` @@ -316,21 +163,6 @@ Entries are listed in reverse chronological order. ## 2026-07-04 -- Branch: `refactor/go2-architecture-layers` -- Summary: Added a six-step engineering implementation plan for Go2 agent - self-evolution, ordered to avoid redundant memory, skill, and decision - systems while building on the existing Layer 3/4/5 architecture. -- Files/modules: - - `docs/plans/2026-07-04-agent-self-evolution-implementation-plan.md` - - `dimos/robot/unitree/go2/PROJECT_CHANGELOG.md` -- Validation: - - Documentation-only planning change; no runtime tests required. -- Open items: - - Execute the plan task-by-task with TDD, starting with - `memory_backend_status()`. - -## 2026-07-04 - - Branch: `refactor/go2-architecture-layers` - Summary: Added `context_evidence.v1` to the Go2 Layer 3 ContextProvider so each compact context response records which task, runtime, robot, RAG, diff --git a/docs/plans/2026-07-04-agent-self-evolution-implementation-plan.md b/docs/plans/2026-07-04-agent-self-evolution-implementation-plan.md deleted file mode 100644 index c5bb363e12..0000000000 --- a/docs/plans/2026-07-04-agent-self-evolution-implementation-plan.md +++ /dev/null @@ -1,362 +0,0 @@ -# Agent Self-Evolution Implementation Plan - -> **For Claude:** REQUIRED SUB-SKILL: Use superpowers:executing-plans to implement this plan task-by-task. - -**Goal:** Add DimOS agent self-evolution in six incremental, testable steps without creating a redundant parallel memory or skill system. - -**Architecture:** Keep RAG and existing memory backends as the source of high-volume observations. Add a thin Layer 3 evolution/control plane that reads Layer 4 memory state, Layer 5 skill contracts, `context_evidence.v1`, skill outcomes, and causal transitions, then records low-volume decisions and feedback to a Git-backed ledger. The LLM may propose feasibility decisions and skill improvements, but rules, schemas, tests, replay/sim validation, and human review remain the acceptance gates. - -**Tech Stack:** Python modules, DimOS `Module`/`@skill`/`Spec` APIs, JSON/YAML event files, Git CLI through safe subprocess wrappers, existing Layer 3/4/5 Go2 blueprints, pytest, ruff. - ---- - -## Engineering Guardrails - -- Do not replace SpatialMemory, TemporalMemory, SkillOutcomeStore, CausalWorldModel, or SkillInterfaceRegistry. -- Do not store images, embeddings, ChromaDB data, SQLite rows, or large logs in Git. -- Do not let the LLM directly modify executable robot skills or merge generated changes. -- Add one narrow module per responsibility; avoid a generic "self evolution manager" until duplication is proven. -- Keep every new MCP skill read-only or proposal-only unless it is explicitly recording audit data. -- Prefer Go2-scoped Layer 3 modules first. Promote to robot-agnostic `dimos.agents` only after the contracts stabilize. - -## Recommended Order - -1. `memory_backend_status()` for observability. -2. Git-backed evolution ledger schema and writer. -3. `evaluate_task_feasibility(...)` before risky planning. -4. Context outcome feedback recording. -5. Context evidence policy extraction and thresholding. -6. Skill evolution proposal generation. - -This order avoids redundancy: status tells us what is actually wired; the ledger gives every later step one audit sink; feasibility and feedback define what should be learned; evidence policy uses feedback rather than guesswork; skill proposal waits until there is repeated evidence that current skills are insufficient. - ---- - -### Task 1: Memory Backend Status - -**Purpose:** Make runtime memory state explicit before adding any evolution logic. - -**Files:** -- Create: `dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/memory_backend_status.py` -- Modify: `dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/__init__.py` -- Modify: `dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/DESIGN.md` -- Test: `dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/test_memory_backend_status.py` - -**Interface:** - -```python -@skill -def memory_backend_status(self) -> SkillResult: - """Report which DimOS memory backends are wired and how much data they expose.""" -``` - -**Implementation Notes:** -- Inject optional `WorldStateSpec`, `SpatialMemorySpec`, `TemporalMemorySpec`, `SkillOutcomeStoreSpec`, `CausalWorldModelSpec`, and `SkillInterfaceSpec`. -- Return a JSON-shaped `SkillResult` metadata payload with: - - `spatial_memory`: wired, query_available, match_count_probe, backend_hint - - `temporal_memory`: wired, entity_count, graph_stats_available - - `skill_outcomes`: wired, recent_count - - `causal_world_model`: wired, transition_count, sample_count, persistence path if exposed - - `skill_interface`: wired, skill_count - - `warnings`: dependency errors without failing the whole call -- Do not inspect Chroma internals directly in V1 unless the module exposes a stable RPC. A probe query with `limit=1` is safer than coupling to Chroma file layout. - -**TDD Steps:** -1. Write tests with stubs for wired and missing backends. -2. Verify the tests fail because `memory_backend_status` does not exist. -3. Implement the module and blueprint wiring. -4. Run: - `./.venv/bin/python -m pytest dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/test_memory_backend_status.py -q` -5. Run Layer 3 blueprint/context focused tests. - -**Redundancy Check:** This is not another memory store. It is a status/readiness view over existing stores. - ---- - -### Task 2: Git-Backed Evolution Ledger - -**Purpose:** Add one durable, reviewable audit sink for low-volume evolution events. - -**Files:** -- Create: `dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/evolution_ledger.py` -- Create: `dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/evolution_event.py` -- Modify: `dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/__init__.py` -- Modify: `dimos/core/global_config.py` only if a reusable config field is needed; otherwise use an env var first. -- Test: `dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/test_evolution_ledger.py` - -**Interface:** - -```python -@skill -def record_evolution_event( - self, - event_type: str, - task: str = "", - payload_json: str = "{}", - commit: bool = False, -) -> SkillResult: - """Record a low-volume agent evolution event to the local ledger.""" -``` - -**Storage Layout:** - -```text -.dimos/evolution/ - events/YYYY/MM/DD/-.json - proposals/ - README.md -``` - -**Event Shape:** - -```json -{ - "schema": "dimos.evolution_event.v1", - "timestamp": 0.0, - "event_type": "context_feedback", - "task": "...", - "run_id": "...", - "payload": {}, - "git": { - "commit_requested": false, - "commit_sha": "" - } -} -``` - -**Implementation Notes:** -- Default path: repo root `.dimos/evolution`; allow override with `DIMOS_EVOLUTION_LEDGER_DIR`. -- Use structured JSON and atomic file writes. -- If `commit=True`, only commit files under the configured ledger dir. Never stage unrelated repo changes. -- Use non-interactive Git commands and return the commit SHA. -- If the ledger directory is not inside a Git worktree, record the file but return a warning instead of failing. - -**TDD Steps:** -1. Test JSON event creation in a temp repo. -2. Test invalid `payload_json` fails with `INVALID_INPUT`. -3. Test `commit=True` stages only the event file. -4. Verify red tests before implementation. -5. Implement minimal file writer and optional commit path. - -**Redundancy Check:** The ledger stores decisions and summaries, not raw memory. Chroma/SQLite remain the retrieval systems. - ---- - -### Task 3: Prompt Feasibility Evaluator - -**Purpose:** Make "can this prompt be done here, now, with current memory and skills?" an explicit preflight tool. - -**Files:** -- Create: `dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/task_feasibility.py` -- Modify: `dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/__init__.py` -- Modify: `dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/prompt_policy.py` -- Test: `dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/test_task_feasibility.py` - -**Interface:** - -```python -@skill -def evaluate_task_feasibility( - self, - task: str, - context_json: str = "{}", -) -> SkillResult: - """Evaluate whether the current task is feasible before selecting physical skills.""" -``` - -**Return Metadata:** - -```json -{ - "feasible": "yes|no|uncertain", - "missing_context": [], - "required_skills": [], - "available_skills": [], - "safety_risks": [], - "recommended_next_action": "proceed|get_context|ask_clarifying_question|refuse|stop", - "clarifying_question": "", - "evidence_sources": [] -} -``` - -**Implementation Notes:** -- V1 should be deterministic/rule-based. Do not add an LLM call inside the tool yet. -- Consume `context_evidence`, runtime mode, robot state, and skill contracts. -- Use Layer 5 contracts for required context and motion-sensitive flags. -- Hardware mode and missing robot state should push risky movement tasks to `uncertain` or `no`. -- Record an optional `task_feasibility` event to the ledger only after Task 2 exists. - -**TDD Steps:** -1. Test a speech task is feasible with `speak`. -2. Test movement task with missing robot state is `uncertain` and asks for context. -3. Test impossible/unsafe task returns `no` with safety risk. -4. Implement minimal deterministic classifier. - -**Redundancy Check:** This complements `predict_skill_outcome`: feasibility evaluates a user task before a skill is chosen; prediction evaluates a specific planned skill call. - ---- - -### Task 4: Context Outcome Feedback - -**Purpose:** Connect selected context to actual task outcomes so later scoring is evidence-based. - -**Files:** -- Create: `dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/context_feedback.py` -- Modify: `dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/__init__.py` -- Modify: `dimos/agents/mcp/mcp_client.py` only if automatic recording is needed after manual V1 proves useful. -- Test: `dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/test_context_feedback.py` - -**Interface:** - -```python -@skill -def record_context_feedback( - self, - task: str, - context_evidence_json: str, - selected_skill: str = "", - outcome_json: str = "{}", - helpful_sources_json: str = "[]", - ignored_risks_json: str = "[]", -) -> SkillResult: - """Record whether selected context evidence helped or hurt the task outcome.""" -``` - -**Implementation Notes:** -- Validate `context_evidence_json` schema version. -- Store a bounded in-memory summary for immediate `get_context` use. -- Write the full feedback event to the Git ledger when available. -- Do not auto-record every tool call in V1; manual/LLM-triggered feedback is enough to validate shape. - -**TDD Steps:** -1. Test valid feedback records helpful and harmful source counts. -2. Test invalid JSON fails. -3. Test summary is bounded and newest-first. -4. Test ledger integration with a stub writer. - -**Redundancy Check:** This is not another SkillOutcomeStore. SkillOutcomeStore records tool success/failure; ContextFeedback records whether selected evidence influenced the decision. - ---- - -### Task 5: Context Evidence Policy Extraction - -**Purpose:** Move evidence scoring/selection out of `ContextProvider` and make it tunable and testable. - -**Files:** -- Create: `dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/context_evidence.py` -- Modify: `dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/context_provider.py` -- Test: `dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/test_context_evidence.py` -- Update: `dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/test_context_provider.py` - -**Interface:** - -```python -def build_context_evidence(metadata: dict[str, Any], policy: ContextEvidencePolicy) -> dict[str, Any]: - ... -``` - -**Policy Fields:** - -```python -@dataclass(frozen=True) -class ContextEvidencePolicy: - min_relevance_score: float = 0.0 - max_entries: int = 12 - include_low_confidence: bool = True - require_robot_state_for_motion: bool = True -``` - -**Implementation Notes:** -- Keep the current `deterministic_source_coverage_v1` as the default policy. -- Add thresholds only after Task 4 feedback exists. -- Make the policy pure Python with no module or RPC dependencies. -- `ContextProvider` should call this helper and remain a coordinator. - -**TDD Steps:** -1. Test default policy preserves current `context_evidence.v1` shape. -2. Test low-relevance spatial evidence can be filtered. -3. Test `max_entries` is enforced deterministically. -4. Test motion tasks retain robot/safety evidence. - -**Redundancy Check:** This is a refactor plus policy, not a new memory router. It owns selection rules only. - ---- - -### Task 6: Skill Evolution Proposal - -**Purpose:** Let the agent propose skill interface improvements without modifying executable skill code. - -**Files:** -- Create: `dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/skill_proposal.py` -- Modify: `dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/__init__.py` -- Modify: `dimos/robot/unitree/go2/blueprints/layers/layer_5_skill_interface/skill_interface_registry.py` only if read APIs need small additions. -- Test: `dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/test_skill_proposal.py` - -**Interface:** - -```python -@skill -def propose_skill_interface( - self, - task: str, - failure_context_json: str = "{}", -) -> SkillResult: - """Propose a new or revised skill interface without changing executable code.""" -``` - -**Proposal Shape:** - -```json -{ - "schema": "dimos.skill_proposal.v1", - "proposal_id": "...", - "task": "...", - "problem": "...", - "existing_skill_gaps": [], - "proposed_interface": { - "skill_name": "...", - "domain": "...", - "required_args": [], - "optional_args": [], - "risk_class": "low|medium|high", - "recommended_preflight": [] - }, - "validation_plan": [], - "requires_human_review": true -} -``` - -**Implementation Notes:** -- Read Layer 5 skill contracts and recent failures. -- Generate proposals deterministically from repeated known gaps in V1. -- Write proposals under `.dimos/evolution/proposals/` via the ledger. -- Never add `@skill` methods or modify system prompts automatically in V1. - -**TDD Steps:** -1. Test repeated missing-argument/context failures produce no new skill proposal; they should recommend better tool use. -2. Test repeated "skill missing capability" feedback produces a proposal file. -3. Test proposal always sets `requires_human_review=True`. -4. Test proposal can be compared against current contracts to avoid duplicates. - -**Redundancy Check:** This is a proposal generator, not a dynamic skill runtime. Existing skill code remains the only executable path. - ---- - -## Completion Gates - -Each task is complete only when: - -- Focused pytest passes. -- Ruff check and format check pass for touched Python files. -- Relevant Layer 3/4/5 design docs are updated. -- `dimos/robot/unitree/go2/PROJECT_CHANGELOG.md` has a scoped entry. -- MCP-exposed tools have docstrings, primitive parameter types, and `SkillResult` returns. -- No new module imports heavy perception/model dependencies at package import time unless that module explicitly owns them. - -## Defer Until Later - -- Automatic code modification of skills. -- Automatic Git merge or push. -- Training/fine-tuning models. -- Replacing ChromaDB, SQLite, or existing memory modules. -- A robot-agnostic abstraction before Go2 contracts prove stable. diff --git a/docs/reviews/2026-07-05-dimos-agentic-architecture.png b/docs/reviews/2026-07-05-dimos-agentic-architecture.png deleted file mode 100644 index 43c3c2c3ce..0000000000 --- a/docs/reviews/2026-07-05-dimos-agentic-architecture.png +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:42111678889fbfc0fd1926390a2ef9e09fa3f4ae2a440d708c7f99306eeac953 -size 352055 diff --git a/docs/reviews/2026-07-05-dimos-agentic-architecture.svg b/docs/reviews/2026-07-05-dimos-agentic-architecture.svg deleted file mode 100644 index 4e360c089b..0000000000 --- a/docs/reviews/2026-07-05-dimos-agentic-architecture.svg +++ /dev/null @@ -1,227 +0,0 @@ - - DimOS Project Architecture With Go2 Agentic Extension - A scientific-style system architecture diagram showing the DimOS project runtime and a zoomed Go2 agentic self-evolution extension in one figure. - - - - - - - - - - - - - - - DimOS 项目整体架构与 Go2 Agent 自进化扩展 - Scientific-style composite figure: project runtime context plus the current Go2 agentic extension boundary. - - - - A - DimOS project architecture - - - - Entry points - - CLI - - Python API - - - Blueprint layer - - Blueprints - - Registry - - - Configuration - - GlobalConfig - - .env - - - External interfaces - - MCP - - Rerun / Web - - - Targets - Robots / Simulation - Go2 · G1 · xArm · Drones · MuJoCo - - - - Core runtime - - Module In[T]/Out[T] - - ModuleCoordinator - - Workers - - RPC / Specs - - Messages - - Run registry/logs - - Lifecycle · deployment · system wiring - - - - Communication layer - LCM · Zenoh · SHM/pSHM · ROS · DDS · Tool streams · HTTP/WebSocket - - - Functional module families - Agents · Skills · Perception · Mapping · Navigation · Control · Manipulation · Learning - Data/model stores: LFS assets · Memory2 · vector/blob stores · VLM/embedding/segmentation models - - - - - - - - - - Panel A abstracts the full repository. The Go2 agentic branch is one blueprint-specific extension within the Agents/Skills/Robot target path. - - - - zoom into Go2 agentic blueprint - - - - B - Go2 agentic self-evolution extension inside DimOS - Current `unitree_go2_agentic`: robot body + world state + skill interface + agent brain. WebsocketVis is optional and not wired here. - - - - Layer 3 Agent Brain - - Prompt - - Router - - Context Provider - - Feasibility - - MCP - - Outcome Store - - Causal Model - - Skill Proposal - - - Layer 4 World State - - Structured State - - Semantic Temporal Map - - Spatial/RAG - - Temporal - - - Layer 5 Skill Interface - - Skill Contracts - - Preflight Metadata - - Go2 Skills - - - Layer 6 Robot Body - Go2 Connection · Sensors/Odom · Motion Control - - - - Local review artifacts - local JSON files; optional local Git commit; no automatic GitHub push - - .dimos/evolution/events - - .dimos/evolution/proposals - - - Optional observability - Rerun / WebsocketVis - not wired by current `unitree_go2_agentic` - - - - Legend and rigor notes - - control/tool call - - context read / evidence - - audit artifact write - - optional observability - No arrow means no runtime dependency. - Causal model output is advisory only. - Skill proposal creates review artifacts, not code. - - - - - - - - - - - - - - - - - - - Figure: DimOS as a modular runtime, with the Go2 agentic extension shown as a blueprint-specific subsystem. Solid arrows are calls/control, dashed arrows are context/evidence reads, dotted arrows are local audit writes, and dash-dot arrows are optional visualization. - diff --git a/docs/reviews/2026-07-05-dimos-project-architecture.png b/docs/reviews/2026-07-05-dimos-project-architecture.png deleted file mode 100644 index 6574655451..0000000000 --- a/docs/reviews/2026-07-05-dimos-project-architecture.png +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:bdb38a4089ea44c83e826a198ea18bb19aae5db857ccb7dc02c10dd5e1288660 -size 221143 diff --git a/docs/reviews/2026-07-05-dimos-project-architecture.svg b/docs/reviews/2026-07-05-dimos-project-architecture.svg deleted file mode 100644 index 38b42b1e7b..0000000000 --- a/docs/reviews/2026-07-05-dimos-project-architecture.svg +++ /dev/null @@ -1,171 +0,0 @@ - - DimOS Project Architecture - Overall DimOS architecture showing users, CLI/API, blueprint orchestration, core module runtime, typed transports, functional modules, robots, data, and observability. - - - - - - - - - DimOS 项目整体架构 - Agentic operating system for robotics: Blueprints compose typed Modules over streams, RPC, and multiple transports. - - - - Entry Points - - CLI - dimos run/list/mcp - - Python API - library usage - - - Blueprint Layer - - Blueprints - autoconnect() - - Registry - all_blueprints.py - - - Configuration - - GlobalConfig - env / CLI flags - - .env - defaults - - - External Interfaces - - MCP - agent tools - - Web / Rerun - observability - - - - Core Runtime - - Module - In[T] / Out[T] - typed streams - - - ModuleCoordinator - deploy / lifecycle - system-module wiring - - - Workers - Python / Docker - forkserver actors - - - RPC / Specs - typed module refs - @rpc / @skill - - - Messages - geometry / nav - sensor / vision - - - Run State - logs - registry - - - - Communication Layer - - LCM - - Zenoh - - SHM / pSHM - - ROS - - DDS - - Tool Streams - - WebSocket / HTTP - - - - Functional Module Families - - Agents - LangGraph / MCP - - Skills - @skill containers - - Perception - detection / memory - - Mapping - occupancy / TF - - Navigation - planning / patrol - - Control - tasks / motion - - Manipulation - grasp / planning - - Learning - data prep - - - - Robots / Simulation / Hardware - Unitree Go2/G1/B1 · xArm/OpenArm/Piper/A750 · Drones · MuJoCo · Replay data - - - Data / Models / Observability - LFS assets · Memory2 · vector/blob stores - VLM / embedding / segmentation models · Rerun / Web - - - - - - - - - - - - - - Blueprints choose modules; Core deploys workers; Transports carry streams/RPC. - diff --git a/docs/reviews/2026-07-05-go2-agent-architecture-review.md b/docs/reviews/2026-07-05-go2-agent-architecture-review.md deleted file mode 100644 index e5ef3e36c4..0000000000 --- a/docs/reviews/2026-07-05-go2-agent-architecture-review.md +++ /dev/null @@ -1,1493 +0,0 @@ -# Go2 Agent Architecture Review Notes - -Date: 2026-07-05 -Branch: `refactor/go2-architecture-layers` -Local HEAD before upstream merge: `646e7ec5` -Upstream merged target: `upstream/main` at `6e813a72` -Merge base: `1f544d05` - -This document is the entry point for a full review of the current fork -changes. It separates upstream changes from local feature work, then describes -each local feature in enough detail to review implementation logic rather than -only file names. - -## Current Git State - -The branch had 16 local commits not in upstream, and upstream had 160 commits -not in the branch. The accurate conflict surface from the merge base was 15 -overlapping files; the actual merge conflicts were 7 files: - -- `dimos/agents/mcp/mcp_server.py` -- `dimos/agents/mcp/test_mcp_server.py` -- `dimos/perception/spatial_perception.py` -- `dimos/robot/unitree/go2/blueprints/agentic/unitree_go2_agentic.py` -- `dimos/robot/unitree/go2/blueprints/agentic/unitree_go2_temporal_memory.py` -- `dimos/robot/unitree/go2/blueprints/smart/unitree_go2_spatial.py` -- `dimos/robot/unitree/go2/test_connection.py` - -Resolution strategy: - -- Keep upstream infrastructure changes, especially Zenoh transport, capability - gating, AES key forwarding, docs and packaging updates. -- Keep the local Go2 layer architecture as the authoritative Go2 agentic wiring. -- Use upstream's new perception module paths, while preserving the local - state-dir based spatial-memory default. -- Combine non-overlapping tests rather than dropping either side. - -Important environment note: - -- Upstream now defaults macOS transport to `zenoh`. That requires - `eclipse-zenoh`; I installed `eclipse-zenoh==1.9.0` in the local `.venv`. -- The machine already has a live `dimos` process listening on MCP port `9990`. - MCP tests must use another port, for example `MCP_PORT=9991`, or they will - talk to the live server instead of the test server. -- `git diff --cached --check` passes when excluding upstream `.patch` files. - Full `diff --check` reports trailing whitespace inside upstream patch files; - those are patch context lines and should be handled separately if we want a - repository-wide whitespace cleanup. - -## Review Granularity - -Use this structure when reviewing each local feature. The goal is to avoid -approving a feature just because the file names sound plausible. - -- Entry point: Which blueprint, module, MCP skill, RPC method, or helper is the - first public surface? -- Inputs: Which JSON strings, spec providers, memory providers, or runtime - states feed it? -- Outputs: Which `SkillResult`, dict schema, Spec return value, file artifact, - or dashboard payload does the interface return? -- Decision owner: Is the decision made by deterministic code, the LLM reading - context, a small online statistical model, or a human reviewer? -- State: Is the feature stateless, in-memory only, persisted to JSON, persisted - to Git, or stored in an existing RAG/vector database? -- Write boundary: Which exact method writes, and which methods are read-only? -- Failure mode: Does missing data produce `unavailable`, `uncertain`, a - clarifying question, a failed `SkillResult`, or a proposal artifact? -- Safety boundary: Can it execute robot motion, or only advise another layer? -- Review artifact: Which schema should be stable enough for future tooling? -- Tests: Which test proves the data path, and which test proves the failure path? - -The most important distinction in this branch is between three very different -things: - -- Context selection: deterministic filtering and summarization before the LLM - sees context. -- Agent judgment: the LLM deciding what to do with selected context and tools. -- Self-evolution artifact: reviewable JSON events/proposals, not automatic code - mutation. - -## Combined Project and Agent Architecture Diagram - -![DimOS project architecture with Go2 agentic self-evolution extension](2026-07-05-dimos-agentic-architecture.svg) - -This is the canonical review figure for this branch. Panel A shows the -repo-wide DimOS execution architecture: entry points, blueprint/configuration, -core runtime, transports, module families, robot/simulation targets, and data -assets. Panel B expands the Go2 agentic self-evolution extension inside that -project architecture, with the agent brain deliberately emphasized because it -owns task evaluation, context use, skill validation, feasibility judgment, and -proposal/artifact emission. - -The purple right-side column remains optional observability/storage. The -current `unitree_go2_agentic` blueprint does not wire `WebsocketVisModule`; the -Rerun / WebsocketVis panel only receives world-model state if a separate -visualization blueprint wires `WebsocketVisSpec`. - -Line semantics are part of the figure contract: - -- Solid black arrows: control flow, blueprint composition, tool calls, or robot - commands. -- Dashed blue arrows: context/evidence reads used by the agent. -- Dotted purple arrows: local review artifacts written to files/Git. -- Dash-dot purple arrows: optional observability paths. - -Editable source: `2026-07-05-dimos-agentic-architecture.svg` -PNG render: `2026-07-05-dimos-agentic-architecture.png` - -Supporting detail figures retained for narrower inspection: - -- Agent-only detail: `2026-07-05-go2-agent-architecture.svg` / - `2026-07-05-go2-agent-architecture.png` -- Project-only detail: `2026-07-05-dimos-project-architecture.svg` / - `2026-07-05-dimos-project-architecture.png` - -## Interface Input/Output Quick Reference - -Use this section when reviewing API boundaries. It lists the stable public -surfaces added or changed by the local Go2 architecture work; helper functions -that are not consumed across module boundaries are intentionally omitted. - -| Surface | Inputs | Outputs | -| --- | --- | --- | -| Go2 blueprint factories (`unitree_go2_agentic`, `unitree_go2_spatial`, `unitree_go2_temporal_memory`) | CLI blueprint name plus `GlobalConfig` values already resolved by DimOS. Lazy layer imports have no runtime arguments. | DimOS `Blueprint` objects composed with `autoconnect()`. They output module wiring, streams, and Spec injection paths, not user-facing JSON. | -| `RobotBodyStateSpec.get_robot_body_snapshot()` | No arguments; reads recent odom/image/lidar observations and connection/local-policy summaries from Layer 6. | `dict` with connection state, sensor state, local policy state, safety state, and freshness/availability markers. | -| `RobotBodyStateSpec.get_connection_state()` | No arguments. | `dict` describing connection availability/config-derived mode. | -| `RobotBodyStateSpec.get_sensor_state()` | No arguments. | `dict` with observed sensor counters/freshness such as image, lidar, and odom availability. | -| `RobotBodyStateSpec.get_local_policy_state()` | No arguments. | `dict` describing local policy/readiness state used as evidence by upper layers. | -| `WorldStateSpec.get_world_snapshot(task, spatial_limit)` | `task: str`, `spatial_limit: int`; reads Layer 6 body state plus spatial/temporal memory providers when wired. | `dict` world snapshot with `robot_state`, `runtime`, `memory_state`, `semantic_temporal_map`, `sources`, and snapshot-storage metadata. | -| `WorldStateSpec.get_robot_state()` | No arguments. | `dict` robot/body state view for upper-layer preflight. | -| `WorldStateSpec.get_runtime_state()` | No arguments. | `dict` runtime mode/config summary such as replay/simulation/hardware and MCP/viewer settings. | -| `WorldStateSpec.get_memory_state(task, spatial_limit)` | `task: str`, `spatial_limit: int`. | `dict` memory section with spatial and temporal availability, matches/summaries, and errors when a provider probe fails. | -| `WorldStateSpec.get_snapshot_storage_policy()` | No arguments. | `dict` explaining whether snapshots are transient, written to memory, or persisted elsewhere. | -| `SemanticTemporalMapSpec.query_semantic_temporal_map(query, spatial_limit)` | `query: str`, `spatial_limit: int`; reads spatial and temporal memory providers. | `dict` with spatial section, temporal section, fused evidence entries, confidence/location/time summaries, and source errors. | -| `SkillInterfaceSpec.get_skill_interface_snapshot(domain)` | Optional `domain: str` filter. | `dict` with `available`, `version`, `source`, `domain_filter`, `domains`, `skill_count`, and a `skills` list of static contracts. | -| `SkillInterfaceSpec.get_skill_contract(skill_name)` | `skill_name: str`. | Matching skill-contract `dict`, or `None` when no contract exists. | -| `SkillInterfaceSpec.validate_skill_request(skill_name, args_json)` | `skill_name: str`, `args_json: str` JSON object. | `dict` with `valid`, `errors`, `warnings`, and the matched `contract`; unknown skills return `valid=False`. | -| `SkillInterfaceSpec.compare_mcp_tools(tools_json)` | `tools_json: str` containing an MCP `tools/list` style payload or list. | `dict` with `valid`, parser errors, contract/MCP counts, missing contracts, unregistered MCP tools, and known internal tools. | -| `route_task(task, context)` MCP skill | `task: str`, optional `context: str`. | `SkillResult` metadata: `domain`, `confidence`, `matched_keywords`, `recommended_tools`, `needs_context`, `reason`, and `context_used`. | -| `get_context(task, focus, spatial_limit)` MCP skill | `task: str`, optional `focus: str`, `spatial_limit: int`; reads Layer 4, memory, skill, feedback, outcome, and world-model providers when wired. | `SkillResult` message plus metadata containing `sources`, `runtime`, `robot_state`, `world_state`, `skill_state`, `context_feedback`, `causal_state`, `world_model_state`, `context_evidence`, and provider `errors`. | -| `build_context_evidence(metadata, policy)` helper | Metadata dict from ContextProvider and `ContextEvidencePolicy` thresholds. | `context_evidence.v1` dict describing selected evidence, dropped/low-confidence evidence, selected sources, and policy effects. | -| `memory_backend_status()` MCP skill | No arguments; read probes optional providers. | `SkillResult` metadata with schema `go2_memory_backend_status.v1`, per-provider wired/available/probe fields, and warnings. | -| `record_evolution_event(event_type, task, payload_json, commit)` MCP skill | `event_type: str`, optional `task: str`, `payload_json: str` JSON object, `commit: bool`. | `SkillResult` metadata with `schema`, `event_type`, `task`, `ledger_dir`, `event_path`, `commit_requested`, optional `commit_sha`, `warnings`, and full `event`. | -| `record_skill_proposal(proposal_json, commit)` MCP skill / ledger RPC | `proposal_json: str` using `dimos.skill_proposal.v1`, `commit: bool`. | `SkillResult` metadata with `schema`, `proposal_id`, `ledger_dir`, `proposal_path`, `commit_requested`, optional `commit_sha`, `warnings`, and validated `proposal`. | -| `EvolutionLedgerSpec.write_evolution_event(event_type, task, payload, commit)` | Structured payload `dict` from another Layer 3 module. | Same ledger event record dict as `record_evolution_event`, without MCP JSON parsing. | -| `EvolutionLedgerSpec.write_skill_proposal(proposal, commit)` | Validated proposal `dict`. | Same proposal record dict as `record_skill_proposal`, without MCP JSON parsing. | -| `evaluate_task_feasibility(task, context_json)` MCP skill | `task: str`, `context_json: str` JSON object, usually from `get_context` metadata. Reads Layer 5 contracts when wired. | `SkillResult` metadata: `feasible` (`yes`, `no`, `uncertain`), `missing_context`, `required_skills`, `available_skills`, `missing_skills`, `safety_risks`, `recommended_next_action`, `clarifying_question`, `evidence_sources`, and warnings. | -| `record_context_feedback(task, context_evidence_json, selected_skill, outcome_json, helpful_sources_json, ignored_risks_json)` MCP skill | Task text, `context_evidence.v1` JSON, optional selected skill, outcome JSON object, helpful-source JSON list, ignored-risk JSON list. | `SkillResult` metadata with one `go2_context_feedback.v1` feedback record, `total_feedback`, and optional ledger warnings. | -| `ContextFeedbackSpec.get_recent_context_feedback(limit, source)` | `limit: int`, optional source filter. | Newest-first list of `go2_context_feedback.v1` feedback dicts. | -| `ContextFeedbackSpec.get_context_feedback_summary(limit)` | `limit: int`. | Aggregate `dict` with counts for success/failure/unknown outcomes plus helpful/harmful source counters. | -| `record_skill_outcome(skill_name, success, domain, error_code, message, risk, recovery)` MCP skill | Skill name, success boolean, optional domain/error/message/risk/recovery strings. | `SkillResult` metadata with recorded outcome dict and `total_outcomes`. | -| `summarize_skill_outcomes(limit, skill_name, domain)` MCP skill | Limit plus optional skill/domain filters. | `SkillResult` metadata with filtered newest-first `outcomes` list. | -| `SkillOutcomeStoreSpec.get_recent_outcomes(limit, skill_name, domain)` | Limit plus optional exact skill/domain filters. | Newest-first list of outcome dicts: timestamp, skill name, success, domain, error code, message, risk, recovery. | -| `predict_skill_outcome(skill_name, args_json, context)` MCP skill | Skill name, planned args JSON object, optional context text. Reads SkillOutcomeStore and CausalWorldModel when wired. | `SkillResult` metadata with `risk`, `predicted_success`, `failure_reasons`, `recovery_suggestions`, recent outcomes/transitions, optional world-model prediction, and provider availability flags. | -| `record_causal_transition(...)` MCP skill | Task, skill name, args JSON, before/after context text, prediction JSON, outcome JSON, domain, before/after state JSON. | `SkillResult` metadata with transition dict, `total_transitions`, `autosave_error`, and `dashboard_error`. | -| `predict_world_transition(snapshot_json, action_json, goal, horizon)` MCP skill / `predict_next_state(...)` RPC | Layer 4 snapshot JSON, action JSON with `skill_name` and `args`, optional goal, bounded horizon. | `dimos.world_model_prediction.v1` dict with action, snapshot summary, risk, predicted success, score, confidence, predicted symbolic delta, failure modes, reasons, model output, causal attribution, SCM explanation, and intervention evidence. | -| `score_action(snapshot_json, action_json, goal)` RPC | Same snapshot/action/goal inputs as prediction. | Compact dict with score, risk, confidence, predicted success, failure modes, reasons, model/causal/SCM/intervention evidence. | -| `summarize_causal_patterns(skill_name, domain, limit)` MCP skill | Optional skill/domain filters and limit. | `SkillResult` metadata with repeated causal `patterns` and filtered `transitions`. | -| `record_intervention(...)` MCP skill | Task, intervention name, target variable, before/after value JSON, action JSON, before/after snapshot JSON, outcome JSON, causal hypothesis. | `SkillResult` metadata with intervention record, `total_interventions`, `autosave_error`, and `dashboard_error`. | -| `save_world_model_state(path)` / `load_world_model_state(path)` MCP skills | Optional path; required unless `DIMOS_GO2_WORLD_MODEL_STATE` is set. | `SkillResult` summary with saved/loaded model-state counts plus optional dashboard error. | -| `CausalWorldModelSpec.get_recent_transitions(limit, skill_name, domain, cause)` | Limit plus optional exact filters. | Newest-first list of transition dicts. | -| `CausalWorldModelSpec.get_intervention_log(limit, target_variable, intervention_name)` | Limit plus optional exact filters. | Newest-first list of intervention dicts. | -| `CausalWorldModelSpec.get_model_state()` | No arguments. | Dict with online model sample/weights summary, provider contract, causal estimator snapshot, intervention log snapshot, SCM snapshot, and persistence config. | -| `CausalWorldModelSpec.get_provider_contract()` | No arguments. | `dimos.world_model_provider.v1` dict naming provider, model type, capabilities, and output schemas. | -| `propose_skill_interface(task, failure_context_json)` MCP skill | Task text and failure-context JSON object with missing-capability evidence. Reads Layer 5 contracts/outcomes when wired. | `SkillResult` metadata: `proposal_created`, `proposal` using `dimos.skill_proposal.v1` when created, `existing_skill_matches`, `recommended_next_action`, optional `proposal_path` or warnings. | -| MCP `tools/list` / `tools/call` runtime surface | JSON-RPC requests; `tools/call` also carries MCP arguments and optional `_mcp_context` progress/capability token. | Tool schemas with capability metadata; tool-call text/content results; SSE progress frames for background tools; capability release on instant return or stopped frame. | -| Optional `WebsocketVisSpec.set_world_model_state(state)` | `state: dict` expected to use `dimos.world_model_dashboard_state.v1`. This is only active when a blueprint wires `WebsocketVisModule`; current `unitree_go2_agentic` does not. | `dict` acknowledgement/current display state for optional Rerun/WebsocketVis clients. | - -## Upstream Changes To Review - -These are not local self-evolution features, but they can affect this branch. - -1. Zenoh transport integration - - Files: - - - `dimos/core/transport_factory.py` - - `dimos/core/transport.py` - - `dimos/protocol/pubsub/impl/zenohpubsub.py` - - `dimos/protocol/service/zenohservice.py` - - `dimos/protocol/rpc/pubsubrpc.py` - - `dimos/protocol/tf/tf.py` - - `dimos/core/global_config.py` - - Implementation logic: - - - `GlobalConfig.transport` can be `lcm` or `zenoh`. - - On Darwin, upstream now defaults to `zenoh`; elsewhere it defaults to - `lcm`. - - `make_transport()` maps logical names to either LCM topics or Zenoh key - expressions. Zenoh topics are namespaced under `dimos/`. - - RPC and TF backends are selected through `rpc_backend()` and `tf_backend()`. - - High-rate sensor channels get best-effort/latest-wins QoS; human/agent - channels get reliable/blocking QoS. - - Review risk: - - - macOS default behavior changed. If existing local runs assume LCM, set - `DIMOS_TRANSPORT=lcm`. - - Zenoh creates native pyo3 callback threads. Existing pytest thread-leak - checks can flag these if cleanup is delayed. - - Tests involving long-lived services must avoid collisions with an already - running MCP server. - -2. Capability gating for MCP tools - - Files: - - - `dimos/agents/capabilities.py` - - `dimos/agents/mcp/mcp_server.py` - - `dimos/agents/mcp/tool_stream.py` - - `dimos/agents/annotation.py` - - `dimos/agents/test_capabilities.py` - - Implementation logic: - - - `SkillInfo` now carries capability metadata such as `uses` and - `lifecycle`. - - `tools/list` includes `_meta.dimos/uses` and `_meta.dimos/lifecycle` when - relevant. - - `tools/call` acquires capability locks before invoking a skill. - - Instant holders may be waited on; background holders require a stop tool. - - Background tool capability release is tied to a tool-stream stopped frame. - - Review risk: - - - Skill annotations must accurately declare long-running motion/control - capabilities, otherwise the server cannot serialize conflicting behaviors. - - Tool-stream stop frames must always be emitted for background tools that - hold capabilities. - -3. Coordinator RPC and dynamic blueprint loading - - Files: - - - `dimos/core/coordination/coordinator_rpc.py` - - `dimos/core/coordination/module_coordinator.py` - - `dimos/core/coordination/worker_manager_python.py` - - Implementation logic: - - - `ModuleCoordinator` can expose a singleton coordinator RPC service. - - Clients can list modules and load blueprints through that coordinator RPC. - - Existing deployed module proxies are protected by a lock during dynamic - loading and restart operations. - - Local interaction: - - - Our local runtime plumbing preserved non-blocking system-module - notification for `McpClient`, so agent startup is not blocked by direct - tool registration. - - Validation passed under `DIMOS_TRANSPORT=lcm`. - -4. Memory and perception migration - - Files: - - - `dimos/perception/image_embedding.py` - - `dimos/perception/spatial_vector_db.py` - - `dimos/perception/visual_memory.py` - - `dimos/perception/spatial_perception.py` - - `dimos/memory2/*` - - Implementation logic: - - - Upstream moved visual/spatial memory helpers out of - `dimos.agents_deprecated.memory` into `dimos.perception`. - - Old deprecated agent memory modules were removed. - - A new memory2 TF service/replay/tooling area was added upstream. - - Local interaction: - - - We use upstream's new module paths in `spatial_perception.py`. - - We keep local default persistence under `STATE_DIR / "spatial_memory"`, - with `DIMOS_SPATIAL_MEMORY_DIR` override, instead of upstream's project - assets output path. - -5. Control, manipulation, learning, docs, and LFS updates - - Summary: - - - Control tasks were split into package directories with registries. - - Roboplan/learning/data-prep modules were added. - - Navigation docs were reorganized for Mintlify. - - New LFS data packages and native build scripts were added. - - Review risk: - - - These are broad upstream changes and not directly part of the Go2 - self-evolution logic. They should be reviewed mainly for dependency, - packaging, and blueprint registry side effects. - -## Local Feature Inventory - -### 1. Go2 Layered Blueprint Architecture - -Files: - -- `dimos/robot/unitree/go2/blueprints/agentic/unitree_go2_agentic.py` -- `dimos/robot/unitree/go2/blueprints/smart/unitree_go2_spatial.py` -- `dimos/robot/unitree/go2/blueprints/agentic/unitree_go2_temporal_memory.py` -- `dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/__init__.py` -- `dimos/robot/unitree/go2/blueprints/layers/layer_4_world_state/__init__.py` -- `dimos/robot/unitree/go2/blueprints/layers/layer_5_skill_interface/__init__.py` -- `dimos/robot/unitree/go2/blueprints/layers/layer_6_robot_body/__init__.py` - -Implementation logic: - -- `unitree_go2_agentic` is now composed from four explicit layers: - `_go2_robot_body`, `_go2_spatial_world_state`, `_go2_skill_interface`, and - `_go2_agent_brain`. -- Layer packages use lazy `__getattr__` construction so importing one layer does - not eagerly import heavy perception, VLM, or robot stacks. -- `unitree_go2_spatial` is the non-agentic spatial stack built from Layer 6, - Layer 4 spatial world state, and Layer 5 spatial skills. -- `unitree_go2_temporal_memory` adds temporal memory through Layer 4's lazy - `_go2_temporal_memory_world_state()` helper after CLI config has been applied. - -Why this matters: - -- Reviewers can inspect robot body, world state, skill interface, and agent - reasoning independently. -- It prevents the agent brain from becoming an unstructured import hub. -- It preserves DimOS blueprint semantics: modules are still composed with - `autoconnect()` and RPC refs are injected by specs. - -Primary risk: - -- The layering must not hide missing blueprint modules. The static registry test - and Layer 4 wiring test are the main protections. - -Components: - -- `unitree_go2_agentic`: the full agentic Go2 stack. It composes the physical - body, structured world state, skill interface registry/containers, and Layer - 3 MCP/LLM brain. -- `unitree_go2_spatial`: the perception/world-state stack without the LLM brain. - It is useful for validating Layer 4 and Layer 5 without agent behavior. -- `unitree_go2_temporal_memory`: an additive blueprint that wraps the agentic - stack and adds temporal memory through the Layer 4 factory. -- Layer package `__init__.py` files: lazy blueprint factories. They are part of - the architecture, not just import convenience, because they prevent import - side effects from pulling in heavy perception/model dependencies too early. - -Construction flow: - -1. CLI imports a named blueprint lazily through the registry. -2. The blueprint imports only the layer handles it needs. -3. Each layer handle builds an `autoconnect()` blueprint for its modules. -4. The final blueprint is still ordinary DimOS composition, so stream wiring and - Spec-based RPC injection happen in `ModuleCoordinator`. -5. Tests validate that Layer 3 can reach Layer 4/5/6 through the intended - contracts and that the generated blueprint registry remains current. - -State and persistence: - -- The blueprint layer itself stores no robot data. -- State is owned by modules inside the layers: body state in Layer 6, memory - and world snapshots in Layer 4, skill contracts in Layer 5, context/outcome - and causal state in Layer 3. -- Any persistence belongs to those modules, not the blueprint file. - -Safety boundary: - -- The blueprint decides what modules are deployed together. -- It does not decide which skill to call, and it does not execute motion. -- Reviewers should treat wiring mistakes as safety risks because the LLM may - see missing or wrong tools/context. - -### 2. Layer 6 Robot Body State - -Files: - -- `dimos/robot/unitree/go2/blueprints/layers/layer_6_robot_body/robot_body_state.py` -- `dimos/robot/unitree/go2/blueprints/layers/layer_6_robot_body/robot_body_spec.py` -- `dimos/robot/unitree/go2/blueprints/layers/layer_6_robot_body/test_robot_body_state.py` - -Implementation logic: - -- `_Go2RobotBodyState` is a small module that exposes robot-body status through - a spec-facing API. -- It sits next to the real Go2 connection stack rather than replacing it. -- Higher layers consume robot state through a typed spec rather than directly - reaching into the hardware connection module. - -Review focus: - -- Make sure it reports enough state for safety-sensitive agent preflight. -- Make sure it does not duplicate live control authority. - -Components: - -- `_Go2RobotBodyState`: the module that adapts low-level robot status into a - compact Layer 6 provider. -- `robot_body_spec.py`: the typed RPC contract consumed by upper layers. -- Layer 6 package factory: wires the underlying Go2 robot blueprint with the - body-state module. - -Data flow: - -1. The underlying Go2 connection and robot modules own live hardware/simulation - communication. -2. `_Go2RobotBodyState` exposes read-oriented body-state summaries through its - spec. -3. Layer 4/Layer 3 use the spec for safety preflight and context evidence. -4. The agent brain sees summarized state, not direct hardware handles. - -Decision owner: - -- The body-state module does not make task decisions. -- It is an evidence provider. Layer 3 deterministic preflight and the LLM use - its output to decide whether to wait, clarify, or call a skill. - -State and write boundary: - -- It should be treated as a current-state adapter. -- It should not persist memory and should not issue control commands. -- Any future body-state caching should be explicitly bounded and timestamped. - -Failure mode: - -- If the underlying robot state is missing, the correct behavior is explicit - unavailable/unknown state, not fabricated readiness. -- Motion-sensitive preflight should treat missing robot body state as risk. - -### 3. Layer 4 Structured World State - -Files: - -- `dimos/robot/unitree/go2/blueprints/layers/layer_4_world_state/structured_world_state.py` -- `dimos/robot/unitree/go2/blueprints/layers/layer_4_world_state/semantic_temporal_map.py` -- `dimos/robot/unitree/go2/blueprints/layers/layer_4_world_state/world_state_spec.py` -- `dimos/robot/unitree/go2/blueprints/layers/layer_4_world_state/test_world_state.py` - -Implementation logic: - -- `_Go2StructuredWorldState` exposes a compact robot/world snapshot through a - typed spec. -- `_Go2SemanticTemporalMap` combines spatial and temporal memory summaries into - a more agent-friendly world-state surface. -- Layer 3 consumes this via RPC spec injection instead of importing memory - implementations directly. - -What "effective context" means here: - -- This layer does not ask the LLM to judge memory relevance directly. -- It normalizes available world and memory sources into a structured provider - surface; Layer 3 then scores, filters, and packages evidence for the LLM. - -Review focus: - -- Check that missing memory backends degrade to explicit unavailable statuses. -- Check that snapshot schemas are stable enough for prompt and dashboard - consumers. - -Components: - -- `_Go2StructuredWorldState`: normalizes robot/world information into a compact - snapshot provider. -- `_Go2SemanticTemporalMap`: fuses spatial memory and temporal memory summaries - into semantic evidence for agent context. -- `world_state_spec.py`: the RPC contract Layer 3 relies on. -- `_go2_temporal_memory_world_state()`: lazy factory that delays TemporalMemory - construction until CLI flags such as `--new-memory` are applied. - -Data flow: - -1. SpatialMemory and TemporalMemory remain the storage/search systems. -2. Layer 4 queries or receives summaries from those systems. -3. Layer 4 shapes them into a stable world-state object with explicit source - availability. -4. Layer 3 asks for world state through the spec and then selects evidence for - the current task. - -State and persistence: - -- Layer 4 may read from persistent memory backends, but its own structured - snapshot is a derived view. -- Temporal memory persistence is controlled by the TemporalMemory module and CLI - config. -- Spatial memory persistence is controlled by `SpatialConfig` and the state-dir - path logic in `spatial_perception.py`. - -Decision owner: - -- Layer 4 decides how to normalize provider output. -- Layer 4 does not decide whether the context is sufficient for action. That - decision belongs to Layer 3 preflight policy and the LLM. - -Failure mode: - -- Provider unavailable means the snapshot should include unavailable metadata. -- Empty search results are not the same as backend failure; reviewers should - check that these are represented differently. - -### 4. Layer 5 Skill Interface Registry - -Files: - -- `dimos/robot/unitree/go2/blueprints/layers/layer_5_skill_interface/skill_interface_registry.py` -- `dimos/robot/unitree/go2/blueprints/layers/layer_5_skill_interface/skill_interface_spec.py` -- `dimos/robot/unitree/go2/blueprints/layers/layer_5_skill_interface/test_skill_interface_registry.py` - -Implementation logic: - -- Skill contracts are explicit static records: skill name, domain, module, - required args, optional args, motion sensitivity, context requirements, - preflight recommendations, risk class, and outcome shape. -- The registry exposes contracts to Layer 3 as data. -- Known non-Layer-5 MCP tools are excluded from contract mismatch checks so - internal status/preflight tools do not pollute skill coverage. - -Self-evolution relationship: - -- The robot does not mutate skills automatically. -- Missing capability evidence can create a reviewable proposed skill interface. -- Existing contracts are the ground truth for duplicate suppression. - -Review focus: - -- Verify every exposed action skill has an accurate contract. -- Treat risk class and required context as safety-critical metadata. - -Components: - -- Static `_SkillContract` records: the human-authored interface description for - each expected action skill. -- `_Go2SkillInterfaceRegistry`: exposes contracts through RPC/MCP-facing - methods for Layer 3. -- `skill_interface_spec.py`: typed contract for modules that consume skill - metadata. -- Layer 5 blueprint factory: wires action skill containers, perception/security - skills, and the registry. - -Data flow: - -1. Action modules expose `@skill` methods through MCP. -2. The registry provides a parallel contract view with richer safety/context - metadata than the MCP JSON schema alone. -3. ContextProvider and TaskFeasibility read contracts to decide what context, - arguments, and preflight checks are needed. -4. SkillProposal reads the same contracts to avoid proposing duplicate skills. - -State and write boundary: - -- Contracts are code/static data, not learned state. -- The registry is read-only at runtime. -- Self-evolution proposals never mutate this file directly. A developer edits - contracts after review. - -Decision owner: - -- Humans own the source-of-truth contract text and risk metadata. -- Deterministic Layer 3 code consumes it. -- The LLM may use contract information, but should not invent hidden skill - capabilities outside the registry/MCP list. - -Failure mode: - -- Missing contract for an exposed skill is a review failure because preflight - cannot know risk/context needs. -- Contract exists but MCP tool is absent is also a review failure unless it is - intentionally disabled for a blueprint. - -### 5. Layer 3 Expert Router - -Files: - -- `dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/expert_router.py` -- `dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/prompt_policy.py` -- `dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/test_expert_router.py` -- `dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/test_prompt_policy.py` - -Implementation logic: - -- The router classifies task domains and recommends which expert/tool family - should be used. -- Prompt policy injects Layer 3 self-evolution guidance into the MCP client - system prompt without replacing robot-specific prompts entirely. -- The output is deterministic routing metadata, not an LLM call. - -Judgment subject: - -- Deterministic code performs the first-pass routing and evidence packaging. -- The LLM remains the actor that reads the final context and decides how to - execute or ask clarification, unless a deterministic MCP skill is explicitly - called. - -Review focus: - -- Verify routing labels line up with actual skill contracts. -- Watch for prompt text that could overclaim autonomy or code mutation. - -Components: - -- `_Go2ExpertRouter`: deterministic task/domain router. -- Prompt policy helpers: merge layer-specific instructions into the agent system - prompt. -- Router tests: pin expected labels and guard against accidental broadening. - -Data flow: - -1. A user task or subtask is passed to the router. -2. The router classifies the task into a small set of domains such as - navigation, perception, person follow, manipulation-like unsupported work, - or general conversation. -3. It returns routing metadata and recommended next tools/preflight. -4. ContextProvider can include routing output in context. -5. The LLM uses routing as a hint, not as an irreversible planner. - -Decision owner: - -- Domain classification is deterministic and inspectable. -- Tool choice remains an LLM/tool-calling decision unless a deterministic - preflight skill refuses or asks for clarification. - -State and persistence: - -- The router is stateless. -- It does not write ledger events or feedback by itself. - -Failure mode: - -- Unknown tasks should route to uncertainty/general handling, not fabricate a - capability. -- Safety-sensitive domains should bias toward preflight and clarification. - -### 6. Context Provider And Evidence Selection - -Files: - -- `dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/context_provider.py` -- `dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/context_evidence.py` -- `dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/test_context_provider.py` -- `dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/test_context_evidence.py` - -Implementation logic: - -- `get_context()` gathers available data from task text, runtime metadata, - robot state, RAG/spatial memory, temporal memory, skill contracts, skill - outcomes, causal world model, and context feedback. -- `context_evidence.py` contains the pure policy helper - `build_context_evidence(metadata, ContextEvidencePolicy)`. -- The policy supports thresholds such as `min_relevance_score`, `max_entries`, - `include_low_confidence`, and `require_robot_state_for_motion`. -- The returned context contains a compact answer plus `context_evidence.v1`, - which explains which evidence was selected and why. - -What "effective" means today: - -- Effective means selected by deterministic policy from available providers: - relevance score, confidence, source availability, task type, and safety needs. -- It is not yet learned from replay metrics. -- The LLM consumes the selected context and may still judge it insufficient. - -Components: - -- `_Go2ContextProvider`: MCP-facing context bundle provider. It is the - aggregator that reads optional Layer 4/5/3 providers and formats the compact - context returned to the agent. -- `context_evidence.py`: pure evidence selector. It owns the deterministic - ranking/filtering policy and can be unit-tested without a running robot, - MCP server, memory backend, or LLM. -- Optional provider refs: world state, spatial/RAG memory, temporal memory, - skill interface registry, skill outcome store, causal world model, and - context feedback store. -- `context_evidence.v1`: review artifact embedded in the response. It records - which evidence entries were selected, dropped, or marked low-confidence. - -Inputs: - -- Task text and runtime metadata supplied by the caller. -- Provider snapshots/summaries from wired DimOS modules. -- Policy values such as relevance threshold, entry cap, low-confidence - handling, and whether motion tasks require robot state. -- Prior feedback/outcome/world-model summaries when those providers are wired. - -Data flow: - -1. The caller invokes `get_context()` with task/runtime metadata. -2. ContextProvider asks wired providers for world state, memory summaries, - skill contracts, outcome summaries, causal-world-model state, and context - feedback. If a provider is not wired, the provider contributes an explicit - unavailable status rather than throwing away the source silently. -3. Metadata is passed to `build_context_evidence()`. That helper is pure: - given metadata and a `ContextEvidencePolicy`, it returns selected evidence - without calling RPC, writing memory, or consulting an LLM. -4. ContextProvider formats the compact context and attaches - `context_evidence.v1` so reviewers can see why the context contained those - entries. -5. The LLM is still the consumer and final user-facing reasoning actor. The - provider does not decide to execute a skill. - -State and write boundary: - -- ContextProvider itself is read-mostly. It does not write RAG/vector memory, - temporal memory, skill contracts, or robot state. -- The only durable side channel is indirect: it may include summaries from - feedback/outcome/ledger-aware modules, but those modules own their writes. -- The selected context is a transient prompt artifact unless another caller - explicitly records feedback or an evolution event. - -Decision owner: - -- Deterministic code decides what evidence enters the context bundle. -- The LLM decides how to reason over that bundle, whether to ask a clarifying - question, and whether to call an exposed tool. -- Human reviewers decide whether the deterministic policy is acceptable for a - given robot safety posture. - -Failure mode: - -- Missing providers should appear as explicit unavailable metadata. -- Malformed provider payloads should degrade that source, not erase the whole - context bundle. -- Low relevance or low confidence evidence should be dropped or marked - according to policy, never silently upgraded to trusted evidence. - -What is not implemented: - -- No learned ranker is trained from replay logs yet. -- No RAG/vector database entries are written by ContextProvider. -- Context feedback is summarized as a signal, but it does not override provider - data by itself. - -Review focus: - -- Check whether relevance/confidence defaults are too permissive. -- Check whether motion tasks always require robot state. -- Check whether low-confidence evidence should be visible but marked, or hidden. - -### 7. Memory Backend Status - -Files: - -- `dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/memory_backend_status.py` -- `dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/test_memory_backend_status.py` - -Implementation logic: - -- Adds MCP skill `memory_backend_status()`. -- Reports whether providers are wired for world state, spatial memory, temporal - memory, skill outcomes, causal world model, and skill interface. -- Performs small read probes where possible. -- Returns status metadata only; it does not create another memory database. - -Review focus: - -- This is diagnostic, not a memory scheduler. -- It should never write to RAG, spatial memory, temporal memory, or Git. - -Components: - -- `_Go2MemoryBackendStatus`: diagnostic module exposed as an MCP skill. -- Optional provider specs: world state, spatial memory, temporal memory, - outcome store, causal world model, and skill interface registry. -- Probe helpers: small read attempts that report availability and errors. - -Data flow: - -1. The skill checks which optional provider references are injected. -2. For each provider, it records `wired`, `available`, and any probe result. -3. Probe failures are captured as status metadata rather than raised to the - agent as hard crashes. -4. The result is returned as JSON/text for the LLM or human operator. - -State and write boundary: - -- It is read-only. -- It does not persist a status snapshot. -- It does not initialize missing databases. If a database is not running or not - wired, the status should say so. - -Decision owner: - -- The skill does not decide which memory to use. -- It answers the operational question "what is connected and readable right - now?" ContextProvider and the LLM decide how to respond to that. - -Failure mode: - -- A probe failure should identify the provider and error string. -- A missing provider should be explicit enough to distinguish blueprint wiring - problems from empty memory results. - -### 8. Git-Backed Evolution Ledger - -Files: - -- `dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/evolution_event.py` -- `dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/evolution_ledger.py` -- `dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/evolution_proposal.py` -- `dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/test_evolution_ledger.py` -- `dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/test_evolution_proposal.py` - -Implementation logic: - -- Events use schema `dimos.evolution_event.v1`. -- Proposals use schema records validated by `evolution_proposal.py`. -- Default storage is local repository path: - `.dimos/evolution/events/YYYY/MM/DD/*.json` and - `.dimos/evolution/proposals/*.json`. -- `DIMOS_EVOLUTION_LEDGER_DIR` can override that location. -- Writes are JSON files with deterministic schema and reviewable payloads. -- `commit=False` by default. -- If `commit=True`, the module creates a local Git commit containing only the - event/proposal file. - -Components: - -- `evolution_event.py`: schema and validation for low-level self-evolution - observations such as feasibility assessments, context feedback, and runtime - review notes. -- `evolution_proposal.py`: schema and validation for higher-level proposed - changes that need human review before becoming code or configuration. -- `_EvolutionLedger`: filesystem and optional Git writer. It owns path - selection, JSON serialization, and commit-mode staging/commit behavior. -- Tests: verify schema validation, path shape, commit isolation, and proposal - persistence. - -Inputs: - -- Event/proposal payload supplied by a Layer 3 MCP tool. -- Optional caller-controlled commit flag. -- Optional `DIMOS_EVOLUTION_LEDGER_DIR` override. -- Current Git repository context when commit mode is enabled. - -Where does Git data go: - -- It goes into this local repository's working tree and local `.git` history. -- It does not push to GitHub. -- GitHub only receives it if a human later runs `git push` or opens a PR. - -Write data flow: - -1. A caller invokes a ledger-facing MCP skill such as - `record_evolution_event()` or `record_skill_proposal()`. -2. The module validates the schema and normalizes the payload. -3. The module chooses the storage root. The default is the repo-local - `.dimos/evolution` tree; `DIMOS_EVOLUTION_LEDGER_DIR` overrides it. -4. The JSON file is written as the review artifact. -5. If `commit=False`, Git is not touched beyond the working tree file. -6. If `commit=True`, only that new event/proposal file is staged and committed - locally. There is no push. - -State and write boundary: - -- The ledger is durable filesystem state, not an in-memory learning model. -- It writes only JSON artifacts under the configured ledger root. -- It does not mutate skill code, prompts, contracts, memory databases, or model - weights. -- Commit mode is intentionally local-only and narrow: stage the artifact, make - a local commit, stop. - -Decision owner: - -- Caller modules decide when an event or proposal is worth recording. -- The ledger decides only whether the artifact is schema-valid and where it is - stored. -- Human reviewers decide whether ledger artifacts should become code changes, - PRs, ignored local traces, or training/evaluation data later. - -Failure mode: - -- Invalid schema should fail before writing. -- Unsafe paths or path traversal should be rejected before filesystem access. -- Git commit failure should leave the JSON artifact visible in the worktree - rather than pretending the record was committed. - -Privacy and review boundary: - -- These files can contain task text, outcome messages, context summaries, and - proposal rationale. Decide whether `.dimos/evolution` belongs in normal Git - history before enabling commit mode on real robot runs. -- The ledger is a control-plane audit trail. It is not a replacement for - SpatialMemory, TemporalMemory, SkillOutcomeStore, or the causal model state. - -Review focus: - -- Confirm no path traversal is possible through event/proposal fields. -- Confirm commit mode stages only the new ledger file. -- Decide whether `.dimos/evolution` should be ignored, committed, or exported - elsewhere depending on privacy policy. - -### 9. Task Feasibility Preflight - -Files: - -- `dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/task_feasibility.py` -- `dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/test_task_feasibility.py` - -Implementation logic: - -- Adds MCP skill `evaluate_task_feasibility(task, context_json)`. -- Reads skill contracts, world/robot context, and optional JSON context. -- Returns a deterministic result with feasibility status, missing context, - required skills, available skills, safety risks, recommended next action, - optional clarifying question, and evidence sources. -- May write a ledger event when an evolution ledger is wired. - -Self-evolution meaning: - -- This is not model self-training. -- It is agent self-assessment: before acting, the agent can ask a deterministic - evaluator whether the task has enough context and capability support. - -Components: - -- `_Go2TaskFeasibility`: MCP skill surface for deterministic preflight. -- Skill contract reader: consumes Layer 5 skill metadata such as required - arguments, context requirements, risk class, and motion sensitivity. -- Context parser: merges explicit `context_json` with wired world/robot - provider summaries. -- Ledger integration: optional event writer for review traces. - -Inputs: - -- `task`: natural-language user request or subtask. -- `context_json`: optional structured context supplied by the caller. -- Layer 5 skill contracts and availability metadata. -- Layer 4 robot/world state when wired. -- Optional outcome/context feedback summaries when available through the - context bundle. - -Decision/data flow: - -1. Parse the task and optional `context_json`. -2. Match the task against known skill contracts and expert domains. -3. Check required arguments and context requirements from Layer 5 contracts. -4. Check robot/world-state availability for motion-sensitive or risky skills. -5. Produce one of three broad outcomes: - `feasible`, `uncertain`, or not feasible. -6. If data is missing, recommend a clarifying question or context-gathering - action. -7. If capability is missing, report it as missing capability evidence. That is - the input that can later justify a skill-interface proposal. - -State and write boundary: - -- It is read-only with respect to robot state, memory, and skill contracts. -- It does not call the target skill. -- It does not create a new skill. -- If the ledger is wired, it may write an audit event describing the preflight - result, but not a code change. - -Decision owner: - -- Feasibility classification is deterministic code. -- The LLM may call this tool and use its result, but the LLM is not the - classifier inside the tool. -- Human review decides whether the deterministic rules are conservative enough - for real hardware. - -Failure mode: - -- Missing required arguments should produce clarification, not a new skill - proposal. -- Missing context should produce context-gathering or uncertainty, not a false - feasible result. -- Missing capability evidence should be specific enough for SkillProposal to - distinguish it from transient runtime failure. - -Review focus: - -- Check false positives for motion-sensitive tasks. -- Check that missing context leads to clarification instead of skill proposal. - -### 10. Context Feedback Store - -Files: - -- `dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/context_feedback.py` -- `dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/test_context_feedback.py` - -Implementation logic: - -- Adds MCP skill `record_context_feedback(...)`. -- Keeps a bounded in-memory deque of the most recent 100 feedback entries. -- Uses schema `go2_context_feedback.v1`. -- Optionally writes feedback events through the evolution ledger. -- ContextProvider includes a compact feedback summary in future context. - -Review focus: - -- Feedback is session memory unless ledger is wired. -- It should not become unbounded or silently override RAG/temporal memory. - -Components: - -- `_Go2ContextFeedbackStore`: in-memory bounded store and MCP skill surface. -- Feedback entry schema: task/context identifiers, rating or usefulness signal, - source labels, optional explanation, and timestamp. -- Optional evolution ledger connection: durable audit trail when wired. -- ContextProvider integration: compact aggregate feedback summary. - -Data flow: - -1. The agent or a human-facing process calls `record_context_feedback(...)` - after a context bundle is helpful, incomplete, stale, or misleading. -2. The store validates and normalizes feedback fields. -3. The entry is appended to a deque capped at 100 records. -4. If the ledger is wired, the feedback is also written as an evolution event. -5. ContextProvider reads a summary, not the full raw feedback list, to avoid - bloating prompts. - -State and persistence: - -- Without the ledger, feedback is process-local and lost on restart. -- With the ledger, feedback becomes reviewable JSON but still does not train a - ranker automatically. - -Decision owner: - -- Feedback recording is an input signal. -- The evidence policy and LLM decide how to treat the signal. There is no - automatic prompt rewrite or memory deletion. - -Failure mode: - -- Malformed feedback should fail validation. -- Excess feedback should evict oldest entries, not grow unbounded. - -### 11. Skill Outcome Store And Predictor - -Files: - -- `dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/skill_outcome_store.py` -- `dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/skill_outcome_predictor.py` -- `dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/test_skill_outcomes.py` - -Implementation logic: - -- Records recent skill outcomes with bounded storage. -- Summarizes success/failure patterns by skill. -- Predictor estimates risk or likely outcome based on recent history and skill - contract metadata. -- ContextProvider can use these summaries as task evidence. - -Review focus: - -- The predictor is heuristic, not a learned model. -- Make sure stale failures do not permanently block useful skills. - -Components: - -- `_Go2SkillOutcomeStore`: bounded recent outcome history. -- `_Go2SkillOutcomePredictor`: heuristic scorer over recent history and skill - contract metadata. -- Outcome summary methods: aggregate successes, failures, repeated errors, and - recent messages by skill. - -Data flow: - -1. After a skill call, an outcome can be recorded with skill name, success, - error code, message, and metadata. -2. The outcome store appends it to bounded memory. -3. Summaries group recent records by skill and error patterns. -4. The predictor combines recent outcomes with skill contract risk/context - metadata to return risk and rationale. -5. ContextProvider can include the summary as evidence for future calls. - -State and persistence: - -- Current store is bounded runtime memory unless a future integration writes it - elsewhere. -- It is separate from the Git ledger. The ledger records review/audit events; - the outcome store supports immediate runtime adaptation. - -Decision owner: - -- The predictor gives a heuristic risk hint. -- It should not block a skill by itself; TaskFeasibility or the LLM should use - it as one evidence source. - -Failure mode: - -- Unknown skill should return low-confidence or no-history output. -- Repeated failure should increase caution but still allow recovery/stop skills. - -### 12. Causal World Model And Dashboard Contract - -Files: - -- `dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/causal_world_model.py` -- `dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/causal_effect_estimator.py` -- `dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/online_transition_model.py` -- `dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/structural_causal_model.py` -- `dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/intervention_log.py` -- `dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/world_model_contract.py` -- `dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/test_causal_world_model.py` -- `dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/test_world_model_contract.py` - -Implementation logic: - -- Records causal transitions and interventions. -- Maintains an online transition model for recent world-state changes. -- Estimates causal effects from recorded events. -- Produces versioned contract outputs: - `dimos.world_model_prediction.v1`, - `dimos.world_model_provider.v1`, and - `dimos.world_model_dashboard.v1`. -- Supports save/load of model state. - -Is it a trained model? - -- Strict answer: there is a very small online model, but there is no offline - training pipeline and no neural world model. -- `OnlineTransitionOutcomeModel` is a lightweight logistic-style linear model. - It stores feature weights in a dictionary and updates them incrementally when - `record_causal_transition()` receives an observed success/failure outcome. -- If no transitions have been recorded, its `sample_count` is zero and its - prediction should be treated as low-confidence prior behavior, not learned - robot knowledge. -- The symbolic SCM is hand-authored. It is not learned from data. -- The causal effect estimator is observational statistics. It compares smoothed - success rates for active features and does not control hidden confounders. - -Components: - -- `OnlineTransitionOutcomeModel`: online success-probability scorer. Features - include bias, skill name, domain, odometry presence, navigation state, spatial - memory availability, spatial match count, temporal availability, and selected - action argument presence/value. The update is one-step gradient adjustment - with L2 shrinkage. -- `CausalEffectEstimator`: bounded observation store with treated/control - success-rate differences for symbolic features. It can produce risk factors, - supporting factors, and intervention suggestions, but its own output declares - the assumption that unobserved confounders are not controlled. -- `StructuralCausalModel`: hand-authored causal graph for variables such as - `spatial_has_matches`, `target_resolvability`, `odom_ready`, - `motion_safety`, and `navigation_success`. It also emits counterfactual - suggestions such as "tag or perceive target before semantic navigation". -- `InterventionLog`: records explicit interventions and lets prediction surface - prior intervention evidence. -- `_WorldTransition`: the compact event record tying task, skill, before/after - state, predicted risk, outcome, inferred cause, and recovery suggestion - together. - -Data flow for recording: - -1. `record_causal_transition()` receives task, skill name, optional args, - before/after Layer 4 snapshots, prediction metadata, and outcome metadata. -2. Inputs are parsed as JSON objects and validated. Missing outcome can fall - back to the latest same-skill outcome if SkillOutcomeStore is wired. -3. The module infers a coarse cause and recovery suggestion from context, - prediction reasons, outcome success, error code, and message. -4. It computes a symbolic state delta from before/after snapshots. -5. It calls `_outcome_model.update(...)` and `_causal_estimator.update(...)`. -6. It appends the transition to an in-memory deque capped at 200 transitions. -7. It autosaves to `DIMOS_GO2_WORLD_MODEL_STATE` only if that env path is set. -8. It publishes dashboard state if WebsocketVis is wired. - -Data flow for prediction: - -1. `predict_next_state()` parses a snapshot and candidate action. -2. It gets recent same-skill transitions and derives repeated failure modes. -3. It computes rule-based risk reasons from the current snapshot and action. -4. It asks the online model for success probability, score, risk, confidence, - and top weighted feature contributions. -5. It asks the causal estimator for observational risk/support factors. -6. It asks the hand-authored SCM for variables, edges, and counterfactuals. -7. It asks the intervention log for matching intervention evidence. -8. It combines rule risk and model risk into a final risk and score. -9. It returns `dimos.world_model_prediction.v1` and optionally publishes the - dashboard contract. - -Trust boundary: - -- The output is advisory. It should help the LLM choose preflight, perception, - clarification, or a safer skill path. -- It should not directly command robot motion. -- High confidence currently requires enough recent samples for the same skill; - otherwise the model should be treated as a low-confidence heuristic. - -Review focus: - -- Treat current causal estimates as advisory evidence. -- Validate that prediction schemas remain stable for dashboard and LLM prompt - consumers. -- Check that persistence does not mix data across runs unexpectedly. -- Check that the UI/prompt labels do not imply a fully trained physical world - model. -- Check that replay or simulation evaluation is added before relying on this - for autonomous action selection. - -### 13. Skill Interface Proposal Generator - -Files: - -- `dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/skill_proposal.py` -- `dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/test_skill_proposal.py` - -Implementation logic: - -- Adds MCP skill `propose_skill_interface(task, failure_context_json)`. -- Generates `dimos.skill_proposal.v1` review artifacts only when there is - explicit missing capability evidence. -- Suppresses proposals for missing arguments or missing context. -- Suppresses duplicate proposals when an existing contract already covers the - capability. -- Writes proposal files through the evolution ledger when wired. -- Does not modify Python code, skill decorators, or blueprint wiring. - -Decision tree: - -1. Parse `failure_context_json`. -2. Look for explicit missing capability evidence, not just failed execution. -3. Check existing Layer 5 contracts to avoid proposing a duplicate interface. -4. Reject cases that are only missing arguments, missing context, or temporary - provider unavailability. -5. Build a proposal artifact with task, evidence, suggested interface shape, - expected inputs, expected output, and review rationale. -6. Write it through the evolution ledger if a ledger is wired. - -Human boundary: - -- This tool is intentionally proposal-only. A developer still writes code, - chooses the container, adds `@skill`, updates contracts/prompts, and tests the - behavior. - -Review focus: - -- This is the safest self-evolution boundary: proposal-only, human-reviewed. -- Check evidence requirements carefully so ordinary user ambiguity does not - generate noisy skill proposals. - -Components: - -- `_Go2SkillProposalGenerator`: MCP skill surface for proposal generation. -- Existing Layer 5 contracts: duplicate-suppression source of truth. -- Evolution ledger: optional proposal writer. -- Proposal schema: review artifact describing the missing capability and - proposed interface, not executable code. - -Inputs: - -- `task`: the user/subtask text that failed. -- `failure_context_json`: structured evidence about why existing skills failed - or were insufficient. -- Existing contracts and known MCP tool names. - -Accepted evidence: - -- Explicit missing capability. -- Repeated failure that indicates no available skill can perform the required - operation. -- A domain/action gap not covered by existing contracts. - -Rejected evidence: - -- Missing required arguments for an existing skill. -- Missing context or unavailable memory/provider. -- Temporary runtime failure. -- Capability already covered by an existing contract. - -Output boundary: - -- The output is a JSON proposal and optional ledger file. -- It does not import, write, or generate Python code. -- It does not update the prompt or registry automatically. - -### 14. MCP Runtime Plumbing - -Files: - -- `dimos/agents/mcp/mcp_client.py` -- `dimos/agents/mcp/mcp_server.py` -- `dimos/agents/mcp/tool_stream.py` -- `dimos/agents/mcp/test_mcp_client_unit.py` -- `dimos/agents/mcp/test_mcp_server.py` -- `dimos/agents/mcp/test_tool_stream.py` - -Implementation logic: - -- `McpServer.on_system_modules()` avoids querying its own proxy over RPC. For - its own skills and actor-class backed proxies, it collects skill schemas - locally. -- `McpClient` defers direct tool registration outside the RPC path so startup - does not block on the server being fully initialized. -- Tool-stream progress frames can be delivered over persistent SSE and include - progress-token metadata. -- Capability release for background tools is tied to stopped frames. -- `agent_send()` now uses `make_transport()` so it respects the active backend. - -Review focus: - -- Validate no startup path waits for a tool that depends on itself. -- Keep MCP test ports isolated from live robot sessions. - -Components: - -- `McpServer`: exposes MCP JSON-RPC endpoints, tool list/call handling, SSE - progress stream, server status tools, capability gating, and agent-send. -- `McpClient`: LLM agent module that discovers tools and exposes them to the - agent runtime. -- `tool_stream.py`: progress/log notification path for long-running skills. -- Capability registry: shared server-side lock manager for mutually exclusive - tool capabilities. - -Startup data flow: - -1. ModuleCoordinator deploys workers and starts modules. -2. `McpServer.start()` starts HTTP/SSE serving and subscribes to tool-stream - frames. -3. `on_system_modules()` registers tool schemas. For the server itself and - actor-class backed modules, schemas are collected locally to avoid RPC - self-deadlock. -4. `McpClient` initializes the agent without blocking startup on direct tool - registration outside the RPC path. - -Tool call flow: - -1. JSON-RPC `tools/call` arrives. -2. Server resolves `SkillInfo` and RPC call target. -3. Capability locks are acquired if the skill declares `uses`. -4. Progress token and capability token are injected through reserved - `_mcp_context`. -5. RPC call runs in an executor. -6. Instant skills release capability after return; background skills hand - release to tool-stream stopped frames. - -State and failure boundary: - -- Server state is process-local: skills, rpc calls, SSE queues, capability - registry. -- Port conflicts are external environment failures. Tests should use isolated - MCP ports when a live server is running. -- Tool not found should return a structured MCP text result, not crash the - server. - -### 15. Runtime Hardening - -Files: - -- `dimos/core/coordination/module_coordinator.py` -- `dimos/core/coordination/worker_manager_python.py` -- `dimos/simulation/mujoco/mujoco_process.py` -- `dimos/robot/unitree/mujoco_connection.py` -- `dimos/robot/unitree/go2/connection.py` -- `dimos/robot/unitree/go2/test_connection.py` - -Implementation logic: - -- Coordinator system-module notifications avoid waiting on known agent clients. -- Worker manager/coordinator changes preserve module restart, reload, and stream - rewiring behavior. -- MuJoCo helper code can read available stderr without blocking. -- macOS headless MuJoCo uses the current Python executable where needed. -- Go2 WebRTC connection forwards AES key configuration from `GlobalConfig`. - -Review focus: - -- These are operational fixes. Review them for process lifecycle correctness, - not agent reasoning behavior. - -Components: - -- ModuleCoordinator notification path: controls build/start/system-module - notification order. -- WorkerManagerPython: deploys modules into workers and handles worker pool - lifecycle. -- MuJoCo process helpers: subprocess executable selection and stderr draining. -- Go2 connection config: replay stop behavior and WebRTC AES key forwarding. - -Data flow: - -1. Build/deploy happens through ModuleCoordinator and worker managers. -2. Stream transports and RPC references are wired before modules receive - system-module notifications. -3. Some notifications use nowait behavior to avoid waiting on an agent client - that is itself initializing from MCP. -4. Restart/reload paths must preserve transports and rewired module refs. -5. Simulator helper code isolates platform-specific process quirks from the - rest of the robot stack. - -State and write boundary: - -- Coordinator owns deployed module proxy maps, transport registry, module - transports, and reload aliases. -- These changes do not alter skill semantics. -- They do affect whether modules start, stop, reload, and reconnect cleanly. - -Failure mode: - -- A stuck build/start should stop managers and surface the original exception. -- Restart should not orphan old transports or leave consumers attached to dead - proxies. -- ReplayConnection stop should be a no-op, not an exception. - -### 16. Optional Rerun/WebsocketVis World-Model Panel - -Files: - -- `dimos/web/websocket_vis/websocket_vis_module.py` -- `dimos/web/websocket_vis_spec.py` -- `dimos/web/templates/rerun_dashboard.html` -- `dimos/web/websocket_vis/test_websocket_vis_module.py` - -Implementation logic: - -- Websocket visualization can carry world-model/dashboard state in addition to - existing visualization payloads. -- Dashboard payload shape is aligned with `world_model_contract.py`. -- This is optional observability. The current `unitree_go2_agentic` blueprint - does not include `WebsocketVisModule`, so this path is inactive unless a - visualization blueprint wires `WebsocketVisSpec`. - -Review focus: - -- Dashboard state should remain display-only. -- Avoid making dashboard consumers depend on unversioned internal Python - objects. -- Do not treat this as a required dependency of the Go2 self-evolution runtime. - -Components: - -- `websocket_vis_spec.py`: typed surface for visualization updates. -- `websocket_vis_module.py`: runtime module that stores/publishes dashboard - state. -- `rerun_dashboard.html`: browser-facing display surface. -- `world_model_contract.py`: schema source for world-model dashboard payloads. - -Data flow: - -1. CausalWorldModel produces a dashboard payload using - `dimos.world_model_dashboard.v1`. -2. If WebsocketVis is wired, CausalWorldModel calls `set_world_model_state(...)`. -3. WebsocketVis stores and serves the latest state to dashboard clients. -4. The dashboard renders the state as inspection data for humans. - -State and persistence: - -- Dashboard state is the latest display snapshot, not the source of truth. -- Durable state, if configured, belongs to CausalWorldModel save/load JSON. -- Browser/dashboard consumers should rely on versioned payload fields only. - -Safety boundary: - -- Dashboard updates must not trigger robot actions. -- UI should not present advisory predictions as guaranteed outcomes. -- If `WebsocketVisModule` is not wired, CausalWorldModel still works; the - dashboard publication path is skipped. - -## Review Order I Recommend - -1. Start with Go2 blueprint wiring: - `unitree_go2_agentic.py`, Layer 3/4/5/6 `__init__.py`, and - `test_world_state.py`. -2. Review MCP runtime plumbing: - `mcp_server.py`, `mcp_client.py`, `tool_stream.py`, and their tests. -3. Review self-evolution write boundaries: - `evolution_ledger.py`, `evolution_proposal.py`, `skill_proposal.py`. -4. Review context quality: - `context_provider.py`, `context_evidence.py`, `context_feedback.py`. -5. Review action safety: - `skill_interface_registry.py`, `task_feasibility.py`, - `skill_outcome_predictor.py`. -6. Review advisory world-model logic: - `causal_world_model.py` and `world_model_contract.py`. -7. Review upstream transport changes separately, especially Zenoh defaults on - macOS. - -## Validation Run During Upstream Sync - -Passed: - -- `.venv/bin/python -m pytest dimos/agents/mcp/test_mcp_server.py -q` - - `10 passed, 1 warning` -- `.venv/bin/python -m pytest dimos/robot/unitree/go2/test_connection.py -q` - - `5 passed` -- `.venv/bin/python -m pytest dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain -q` - - `62 passed` -- `.venv/bin/python -m pytest dimos/robot/unitree/go2/blueprints/layers/layer_4_world_state/test_world_state.py -q` - - `3 passed, 1 warning` -- `.venv/bin/python -m pytest dimos/perception/test_spatial_perception_config.py -q` - - `2 passed` -- `DIMOS_TRANSPORT=lcm .venv/bin/python -m pytest dimos/core/coordination/test_module_coordinator.py -q` - - `35 passed` -- `DIMOS_TRANSPORT=lcm MCP_PORT=9991 .venv/bin/python -m pytest dimos/agents/mcp/test_mcp_client_unit.py dimos/agents/mcp/test_tool_stream.py -q` - - `44 passed, 1 warning` -- `.venv/bin/python -m pytest dimos/robot/test_all_blueprints_generation.py -q` - - `1 passed` -- `git diff --cached --check -- ':(exclude)*.patch'` - - passed - -Known caveats: - -- Running MCP tests on default port `9990` fails on this machine because a live - `dimos-live` process already owns that port. -- Running some tests with upstream's new macOS default `zenoh` backend exposed - native pyo3 thread lifetime issues in pytest. The same coordinator and MCP - suites pass under `DIMOS_TRANSPORT=lcm`. Reviewers should decide whether to - treat Zenoh-on-macOS test behavior as an upstream follow-up or to pin local - tests to LCM until Zenoh cleanup semantics are stable. diff --git a/docs/reviews/2026-07-05-go2-agent-architecture-review.zh.md b/docs/reviews/2026-07-05-go2-agent-architecture-review.zh.md deleted file mode 100644 index 61cac7239e..0000000000 --- a/docs/reviews/2026-07-05-go2-agent-architecture-review.zh.md +++ /dev/null @@ -1,1232 +0,0 @@ -# Go2 Agent 架构 Review 说明 - -日期: 2026-07-05 -分支: `refactor/go2-architecture-layers` -合并上游前的本地 HEAD: `646e7ec5` -已合并的上游目标: `upstream/main` at `6e813a72` -共同祖先: `1f544d05` - -这份文档是当前分支完整 review 的中文入口。它把上游同步内容和本地新增功能分开说明，并重点解释每个新增功能的实现逻辑、数据流、存储位置、判断主体、风险点和验证情况。 - -## 当前 Git 状态 - -合并前，本分支有 16 个本地提交不在 upstream，上游 `main` 有 160 个提交不在本分支。用 merge base 精确比较后，真正与本地改动重叠的文件有 15 个，实际产生冲突的文件有 7 个: - -- `dimos/agents/mcp/mcp_server.py` -- `dimos/agents/mcp/test_mcp_server.py` -- `dimos/perception/spatial_perception.py` -- `dimos/robot/unitree/go2/blueprints/agentic/unitree_go2_agentic.py` -- `dimos/robot/unitree/go2/blueprints/agentic/unitree_go2_temporal_memory.py` -- `dimos/robot/unitree/go2/blueprints/smart/unitree_go2_spatial.py` -- `dimos/robot/unitree/go2/test_connection.py` - -冲突解析原则: - -- 保留上游基础设施更新，尤其是 Zenoh transport、MCP capability gating、Unitree AES key 转发、文档和打包更新。 -- 保留本地 Go2 分层架构作为 Go2 agentic blueprint 的权威 wiring。 -- 使用上游新的 perception 模块路径，同时保留本地基于 `STATE_DIR` 的 spatial memory 默认持久化路径。 -- 两边新增的测试如果覆盖不同风险点，则合并保留，而不是二选一删除。 - -环境注意事项: - -- 上游现在在 macOS 上默认使用 `zenoh` transport。这个后端需要 `eclipse-zenoh`，本地 `.venv` 已安装 `eclipse-zenoh==1.9.0`。 -- 当前机器已经有一个 live `dimos` 进程监听 MCP 端口 `9990`。跑 MCP 测试时要换端口，例如 `MCP_PORT=9991`，否则测试客户端会连到 live 服务。 -- `git diff --cached --check -- ':(exclude)*.patch'` 已通过。完整 `git diff --check` 会报上游 `.patch` 文件内的 trailing whitespace，这些是 patch 文件的 context 行，应单独处理，不能随意改动以免破坏补丁语义。 - -## Review 粒度标准 - -review 每个本地功能时，建议都按下面这些问题看，而不是只看文件名是否合理: - -- 入口: 第一个对外入口是 blueprint、module、MCP skill、RPC method，还是 helper 函数？ -- 输入: 它读取哪些 JSON 字符串、Spec provider、memory provider 或 runtime state？ -- 输出: 它返回哪种 `SkillResult`、dict schema、Spec return value、file artifact 或 dashboard payload？ -- 判断主体: 判断是确定性代码做的，LLM 做的，小型在线统计模型做的，还是人类 reviewer 做的？ -- 状态: 它是无状态、只在内存、写 JSON、写 Git，还是写已有 RAG/vector database？ -- 写边界: 哪个方法会写入？哪些方法是纯 read-only？ -- 失败形态: 缺数据时返回 `unavailable`、`uncertain`、clarifying question、failed `SkillResult`，还是 proposal artifact？ -- 安全边界: 它能执行机器人运动，还是只给上层建议？ -- Review artifact: 哪个 schema 是未来工具要依赖的稳定接口？ -- 测试: 哪个测试证明主数据流？哪个测试证明失败路径？ - -这个分支最重要的区分是三件事: - -- Context selection: LLM 看到上下文前，由确定性代码做过滤和摘要。 -- Agent judgment: LLM 读取已选上下文和工具后决定下一步。 -- Self-evolution artifact: 产出可 review 的 JSON event/proposal，不自动改代码。 - -## 项目整体架构与 Agent 重点展开图 - -![DimOS 项目整体架构与 Go2 Agent 自进化扩展](2026-07-05-dimos-agentic-architecture.svg) - -这是本分支 review 的主图。Panel A 展示 repo-wide DimOS 执行架构: -entry points、blueprint/configuration、core runtime、transports、module -families、robot/simulation targets 和 data assets。Panel B 在同一张图里 -展开 Go2 agentic self-evolution extension，并刻意把 Agent Brain 放在重点位置: -task evaluation、context use、skill validation、feasibility judgment 和 -proposal/artifact emission 都由这个层级发起或裁决。 - -右侧紫色列仍然是可选观测/存储，不是主执行依赖。当前 -`unitree_go2_agentic` blueprint 没有接 `WebsocketVisModule`；只有单独的 -可视化 blueprint 接上 `WebsocketVisSpec` 时，Rerun / WebsocketVis 面板才会 -收到 world-model state。 - -图中的线型语义也是 review contract 的一部分: - -- 黑色实线: control flow、blueprint composition、tool call 或 robot command。 -- 蓝色虚线: agent 读取 context/evidence 的路径。 -- 紫色点线: 写入本地 review artifact、文件或 Git 的路径。 -- 紫色点划线: optional observability path。 - -可编辑源文件: `2026-07-05-dimos-agentic-architecture.svg` -PNG 渲染版: `2026-07-05-dimos-agentic-architecture.png` - -为窄范围 review 保留的辅助细节图: - -- Agent-only detail: `2026-07-05-go2-agent-architecture.svg` / - `2026-07-05-go2-agent-architecture.png` -- Project-only detail: `2026-07-05-dimos-project-architecture.svg` / - `2026-07-05-dimos-project-architecture.png` - -## 接口输入/输出速查 - -review API 边界时优先看这一节。这里列的是本地 Go2 架构改动新增或改动过的稳定 public surface；没有跨 module 边界消费的内部 helper 不列入。 - -| 接口 | 输入 | 输出 | -| --- | --- | --- | -| Go2 blueprint factories (`unitree_go2_agentic`、`unitree_go2_spatial`、`unitree_go2_temporal_memory`) | CLI blueprint name，以及 DimOS 已解析好的 `GlobalConfig`。lazy layer import 本身没有 runtime 入参。 | `autoconnect()` 组合出的 DimOS `Blueprint`。输出的是 module wiring、stream wiring 和 Spec injection path，不是面向用户的 JSON。 | -| `RobotBodyStateSpec.get_robot_body_snapshot()` | 无参数；读取 Layer 6 近期 odom/image/lidar observation，以及 connection/local-policy summary。 | `dict`，包含 connection state、sensor state、local policy state、safety state、freshness/availability markers。 | -| `RobotBodyStateSpec.get_connection_state()` | 无参数。 | 描述 connection availability 和 config-derived mode 的 `dict`。 | -| `RobotBodyStateSpec.get_sensor_state()` | 无参数。 | `dict`，包含 image、lidar、odom 等 sensor counters/freshness。 | -| `RobotBodyStateSpec.get_local_policy_state()` | 无参数。 | `dict`，描述上层 preflight 使用的 local policy/readiness evidence。 | -| `WorldStateSpec.get_world_snapshot(task, spatial_limit)` | `task: str`、`spatial_limit: int`；读取 Layer 6 body state，以及已接线 spatial/temporal memory provider。 | `dict` world snapshot，包含 `robot_state`、`runtime`、`memory_state`、`semantic_temporal_map`、`sources` 和 snapshot-storage metadata。 | -| `WorldStateSpec.get_robot_state()` | 无参数。 | 上层 preflight 使用的 robot/body state `dict`。 | -| `WorldStateSpec.get_runtime_state()` | 无参数。 | `dict` runtime mode/config summary，例如 replay/simulation/hardware、MCP/viewer 设置。 | -| `WorldStateSpec.get_memory_state(task, spatial_limit)` | `task: str`、`spatial_limit: int`。 | `dict` memory section，包含 spatial/temporal availability、matches/summaries，以及 provider probe failure 的 errors。 | -| `WorldStateSpec.get_snapshot_storage_policy()` | 无参数。 | `dict`，说明 snapshot 是 transient、写 memory，还是持久化到其他位置。 | -| `SemanticTemporalMapSpec.query_semantic_temporal_map(query, spatial_limit)` | `query: str`、`spatial_limit: int`；读取 spatial 和 temporal memory provider。 | `dict`，包含 spatial section、temporal section、fused evidence entries、confidence/location/time summaries 和 source errors。 | -| `SkillInterfaceSpec.get_skill_interface_snapshot(domain)` | 可选 `domain: str` filter。 | `dict`，包含 `available`、`version`、`source`、`domain_filter`、`domains`、`skill_count` 和静态 contract 列表 `skills`。 | -| `SkillInterfaceSpec.get_skill_contract(skill_name)` | `skill_name: str`。 | 匹配的 skill-contract `dict`；没有 contract 时返回 `None`。 | -| `SkillInterfaceSpec.validate_skill_request(skill_name, args_json)` | `skill_name: str`、`args_json: str` JSON object。 | `dict`，包含 `valid`、`errors`、`warnings`、匹配的 `contract`；未知 skill 返回 `valid=False`。 | -| `SkillInterfaceSpec.compare_mcp_tools(tools_json)` | `tools_json: str`，MCP `tools/list` 风格 payload 或 list。 | `dict`，包含 `valid`、parser errors、contract/MCP counts、missing contracts、unregistered MCP tools、known internal tools。 | -| `route_task(task, context)` MCP skill | `task: str`，可选 `context: str`。 | `SkillResult` metadata: `domain`、`confidence`、`matched_keywords`、`recommended_tools`、`needs_context`、`reason`、`context_used`。 | -| `get_context(task, focus, spatial_limit)` MCP skill | `task: str`、可选 `focus: str`、`spatial_limit: int`；读取已接线 Layer 4、memory、skill、feedback、outcome、world-model provider。 | `SkillResult` message 加 metadata，包含 `sources`、`runtime`、`robot_state`、`world_state`、`skill_state`、`context_feedback`、`causal_state`、`world_model_state`、`context_evidence` 和 provider `errors`。 | -| `build_context_evidence(metadata, policy)` helper | ContextProvider 生成的 metadata dict，以及 `ContextEvidencePolicy` 阈值。 | `context_evidence.v1` dict，描述 selected evidence、dropped/low-confidence evidence、selected sources 和 policy effect。 | -| `memory_backend_status()` MCP skill | 无参数；对可选 provider 做 read probe。 | `SkillResult` metadata，schema 是 `go2_memory_backend_status.v1`，包含每个 provider 的 wired/available/probe 字段和 warnings。 | -| `record_evolution_event(event_type, task, payload_json, commit)` MCP skill | `event_type: str`、可选 `task: str`、`payload_json: str` JSON object、`commit: bool`。 | `SkillResult` metadata，包含 `schema`、`event_type`、`task`、`ledger_dir`、`event_path`、`commit_requested`、可选 `commit_sha`、`warnings` 和完整 `event`。 | -| `record_skill_proposal(proposal_json, commit)` MCP skill / ledger RPC | `proposal_json: str`，使用 `dimos.skill_proposal.v1`；`commit: bool`。 | `SkillResult` metadata，包含 `schema`、`proposal_id`、`ledger_dir`、`proposal_path`、`commit_requested`、可选 `commit_sha`、`warnings` 和校验后的 `proposal`。 | -| `EvolutionLedgerSpec.write_evolution_event(event_type, task, payload, commit)` | 其他 Layer 3 module 传入的结构化 payload `dict`。 | 和 `record_evolution_event` 相同的 ledger event record dict，但不经过 MCP JSON parsing。 | -| `EvolutionLedgerSpec.write_skill_proposal(proposal, commit)` | 已校验的 proposal `dict`。 | 和 `record_skill_proposal` 相同的 proposal record dict，但不经过 MCP JSON parsing。 | -| `evaluate_task_feasibility(task, context_json)` MCP skill | `task: str`、`context_json: str` JSON object，通常来自 `get_context` metadata；已接线时读取 Layer 5 contracts。 | `SkillResult` metadata: `feasible` (`yes`、`no`、`uncertain`)、`missing_context`、`required_skills`、`available_skills`、`missing_skills`、`safety_risks`、`recommended_next_action`、`clarifying_question`、`evidence_sources` 和 warnings。 | -| `record_context_feedback(task, context_evidence_json, selected_skill, outcome_json, helpful_sources_json, ignored_risks_json)` MCP skill | task text、`context_evidence.v1` JSON、可选 selected skill、outcome JSON object、helpful-source JSON list、ignored-risk JSON list。 | `SkillResult` metadata，包含一条 `go2_context_feedback.v1` feedback record、`total_feedback` 和可选 ledger warnings。 | -| `ContextFeedbackSpec.get_recent_context_feedback(limit, source)` | `limit: int`，可选 source filter。 | newest-first 的 `go2_context_feedback.v1` feedback dict 列表。 | -| `ContextFeedbackSpec.get_context_feedback_summary(limit)` | `limit: int`。 | aggregate `dict`，包含 success/failure/unknown outcome counts，以及 helpful/harmful source counters。 | -| `record_skill_outcome(skill_name, success, domain, error_code, message, risk, recovery)` MCP skill | skill name、success boolean、可选 domain/error/message/risk/recovery 字符串。 | `SkillResult` metadata，包含 recorded outcome dict 和 `total_outcomes`。 | -| `summarize_skill_outcomes(limit, skill_name, domain)` MCP skill | limit，以及可选 skill/domain filters。 | `SkillResult` metadata，包含过滤后的 newest-first `outcomes` list。 | -| `SkillOutcomeStoreSpec.get_recent_outcomes(limit, skill_name, domain)` | limit，以及可选 exact skill/domain filters。 | newest-first outcome dict 列表: timestamp、skill name、success、domain、error code、message、risk、recovery。 | -| `predict_skill_outcome(skill_name, args_json, context)` MCP skill | skill name、planned args JSON object、可选 context text；已接线时读取 SkillOutcomeStore 和 CausalWorldModel。 | `SkillResult` metadata，包含 `risk`、`predicted_success`、`failure_reasons`、`recovery_suggestions`、recent outcomes/transitions、可选 world-model prediction 和 provider availability flags。 | -| `record_causal_transition(...)` MCP skill | task、skill name、args JSON、before/after context text、prediction JSON、outcome JSON、domain、before/after state JSON。 | `SkillResult` metadata，包含 transition dict、`total_transitions`、`autosave_error`、`dashboard_error`。 | -| `predict_world_transition(snapshot_json, action_json, goal, horizon)` MCP skill / `predict_next_state(...)` RPC | Layer 4 snapshot JSON、包含 `skill_name` 和 `args` 的 action JSON、可选 goal、bounded horizon。 | `dimos.world_model_prediction.v1` dict，包含 action、snapshot summary、risk、predicted success、score、confidence、predicted symbolic delta、failure modes、reasons、model output、causal attribution、SCM explanation 和 intervention evidence。 | -| `score_action(snapshot_json, action_json, goal)` RPC | 与 prediction 相同的 snapshot/action/goal 输入。 | compact dict，包含 score、risk、confidence、predicted success、failure modes、reasons、model/causal/SCM/intervention evidence。 | -| `summarize_causal_patterns(skill_name, domain, limit)` MCP skill | 可选 skill/domain filters 和 limit。 | `SkillResult` metadata，包含 repeated causal `patterns` 和过滤后的 `transitions`。 | -| `record_intervention(...)` MCP skill | task、intervention name、target variable、before/after value JSON、action JSON、before/after snapshot JSON、outcome JSON、causal hypothesis。 | `SkillResult` metadata，包含 intervention record、`total_interventions`、`autosave_error`、`dashboard_error`。 | -| `save_world_model_state(path)` / `load_world_model_state(path)` MCP skills | 可选 path；如果未设置 `DIMOS_GO2_WORLD_MODEL_STATE`，则必须传 path。 | `SkillResult` summary，包含保存/加载的 model-state counts，以及可选 dashboard error。 | -| `CausalWorldModelSpec.get_recent_transitions(limit, skill_name, domain, cause)` | limit，以及可选 exact filters。 | newest-first transition dict 列表。 | -| `CausalWorldModelSpec.get_intervention_log(limit, target_variable, intervention_name)` | limit，以及可选 exact filters。 | newest-first intervention dict 列表。 | -| `CausalWorldModelSpec.get_model_state()` | 无参数。 | `dict`，包含 online model sample/weights summary、provider contract、causal estimator snapshot、intervention log snapshot、SCM snapshot 和 persistence config。 | -| `CausalWorldModelSpec.get_provider_contract()` | 无参数。 | `dimos.world_model_provider.v1` dict，声明 provider、model type、capabilities 和 output schemas。 | -| `propose_skill_interface(task, failure_context_json)` MCP skill | task text，以及带 missing-capability evidence 的 failure-context JSON object；已接线时读取 Layer 5 contracts/outcomes。 | `SkillResult` metadata: `proposal_created`、创建时的 `dimos.skill_proposal.v1` `proposal`、`existing_skill_matches`、`recommended_next_action`、可选 `proposal_path` 或 warnings。 | -| MCP `tools/list` / `tools/call` runtime surface | JSON-RPC requests；`tools/call` 携带 MCP arguments 和可选 `_mcp_context` progress/capability token。 | 带 capability metadata 的 tool schema；tool-call text/content result；background tool 的 SSE progress frames；instant return 或 stopped frame 触发 capability release。 | -| 可选 `WebsocketVisSpec.set_world_model_state(state)` | `state: dict`，预期使用 `dimos.world_model_dashboard_state.v1`。只有 blueprint 接入 `WebsocketVisModule` 时才生效；当前 `unitree_go2_agentic` 没有接。 | `dict` acknowledgement/current display state，供可选 Rerun/WebsocketVis clients 读取。 | - -## 需要 Review 的上游更新 - -这些不是本地自进化功能，但会影响本分支。 - -### 1. Zenoh Transport 集成 - -相关文件: - -- `dimos/core/transport_factory.py` -- `dimos/core/transport.py` -- `dimos/protocol/pubsub/impl/zenohpubsub.py` -- `dimos/protocol/service/zenohservice.py` -- `dimos/protocol/rpc/pubsubrpc.py` -- `dimos/protocol/tf/tf.py` -- `dimos/core/global_config.py` - -实现逻辑: - -- `GlobalConfig.transport` 现在可以是 `lcm` 或 `zenoh`。 -- macOS 上游默认值改为 `zenoh`，其他平台默认仍是 `lcm`。 -- `make_transport()` 根据当前后端把逻辑 channel name 映射成 LCM topic 或 Zenoh key expression。 -- Zenoh key expression 会放在 `dimos/` namespace 下。 -- RPC 和 TF 后端分别通过 `rpc_backend()`、`tf_backend()` 选择。 -- 高频 sensor channel 使用 latest-wins/best-effort QoS，human/agent 低频消息使用 reliable/blocking QoS。 - -Review 风险: - -- macOS 默认行为变了。如果本地运行仍假设 LCM，需要设置 `DIMOS_TRANSPORT=lcm`。 -- Zenoh 会创建 native pyo3 callback 线程，现有 pytest thread leak 检测可能在 cleanup 延迟时误报或暴露真实清理问题。 -- 长生命周期服务测试必须避开已经运行的 MCP server 端口。 - -### 2. MCP Tool Capability Gating - -相关文件: - -- `dimos/agents/capabilities.py` -- `dimos/agents/mcp/mcp_server.py` -- `dimos/agents/mcp/tool_stream.py` -- `dimos/agents/annotation.py` -- `dimos/agents/test_capabilities.py` - -实现逻辑: - -- `SkillInfo` 增加 capability 元数据，例如 `uses` 和 `lifecycle`。 -- `tools/list` 在相关工具上暴露 `_meta.dimos/uses` 和 `_meta.dimos/lifecycle`。 -- `tools/call` 在真正调用 skill 前先申请 capability lock。 -- instant skill 的占用可以等待；background skill 的占用需要先调用对应 stop tool。 -- background tool 的 capability release 绑定到 tool-stream stopped frame。 - -Review 风险: - -- 长时间运动或控制类 skill 必须正确声明 capability，否则 MCP server 无法串行化冲突行为。 -- background tool 必须稳定发出 stopped frame，否则 capability 可能被永久占用。 - -### 3. Coordinator RPC 和动态 Blueprint 加载 - -相关文件: - -- `dimos/core/coordination/coordinator_rpc.py` -- `dimos/core/coordination/module_coordinator.py` -- `dimos/core/coordination/worker_manager_python.py` - -实现逻辑: - -- `ModuleCoordinator` 可以暴露一个 singleton coordinator RPC service。 -- 客户端可以通过 coordinator RPC 查询模块、加载 blueprint。 -- 动态加载和 restart 期间，已部署 module proxy 由 lock 保护。 - -和本地改动的关系: - -- 本地 runtime plumbing 保留了 `McpClient` 的 non-blocking system-module notification，避免 agent 启动被直接 tool registration 阻塞。 -- 在 `DIMOS_TRANSPORT=lcm` 下，`ModuleCoordinator` 测试全部通过。 - -### 4. Memory 和 Perception 路径迁移 - -相关文件: - -- `dimos/perception/image_embedding.py` -- `dimos/perception/spatial_vector_db.py` -- `dimos/perception/visual_memory.py` -- `dimos/perception/spatial_perception.py` -- `dimos/memory2/*` - -实现逻辑: - -- 上游把 visual/spatial memory helper 从 `dimos.agents_deprecated.memory` 迁移到 `dimos.perception`。 -- 旧的 deprecated agent memory 目录被删除。 -- 上游新增了 memory2 TF service、replay 和 tooling。 - -本地解析: - -- `spatial_perception.py` 使用上游新的 `dimos.perception.*` 模块路径。 -- 仍保留本地默认持久化路径 `STATE_DIR / "spatial_memory"`，并支持 `DIMOS_SPATIAL_MEMORY_DIR` 覆盖，而不是改成上游 project assets output 路径。 - -### 5. Control、Manipulation、Learning、Docs、LFS 更新 - -概要: - -- control tasks 被拆成带 registry 的 package 目录。 -- 新增 roboplan、learning、data-prep 相关模块。 -- navigation 文档按 Mintlify 结构重组。 -- 新增 LFS 数据包和 native build 脚本。 - -Review 风险: - -- 这些是广泛的上游变更，不属于 Go2 自进化核心逻辑。主要审查 dependency、packaging、blueprint registry、CI 和运行环境影响。 - -## 本地新增功能清单 - -### 1. Go2 分层 Blueprint 架构 - -相关文件: - -- `dimos/robot/unitree/go2/blueprints/agentic/unitree_go2_agentic.py` -- `dimos/robot/unitree/go2/blueprints/smart/unitree_go2_spatial.py` -- `dimos/robot/unitree/go2/blueprints/agentic/unitree_go2_temporal_memory.py` -- `dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/__init__.py` -- `dimos/robot/unitree/go2/blueprints/layers/layer_4_world_state/__init__.py` -- `dimos/robot/unitree/go2/blueprints/layers/layer_5_skill_interface/__init__.py` -- `dimos/robot/unitree/go2/blueprints/layers/layer_6_robot_body/__init__.py` - -实现逻辑: - -- `unitree_go2_agentic` 现在由四层明确组成: `_go2_robot_body`、`_go2_spatial_world_state`、`_go2_skill_interface`、`_go2_agent_brain`。 -- 各 layer package 使用 lazy `__getattr__` 构建，避免 import 某一层时提前拉起重型 perception、VLM 或 robot stack。 -- `unitree_go2_spatial` 是非 agentic 的空间栈，由 Layer 6、Layer 4 spatial world state、Layer 5 spatial skills 组成。 -- `unitree_go2_temporal_memory` 通过 Layer 4 的 `_go2_temporal_memory_world_state()` 懒加载 temporal memory，确保 CLI config 已经生效后再创建 TemporalMemory blueprint。 - -为什么重要: - -- reviewer 可以分别审查 robot body、world state、skill interface、agent reasoning。 -- 防止 agent brain 变成无结构的大型 import hub。 -- 仍保留 DimOS blueprint 语义: 通过 `autoconnect()` 组合模块，通过 Spec 注入 RPC refs。 - -主要风险: - -- 分层不能掩盖漏接模块。`test_all_blueprints_generation.py` 和 Layer 4 wiring test 是主要保护。 - -组成部分: - -- `unitree_go2_agentic`: 完整 Go2 agentic stack，组合 physical body、structured world state、skill interface registry/containers、Layer 3 MCP/LLM brain。 -- `unitree_go2_spatial`: 没有 LLM brain 的 perception/world-state stack，用来单独验证 Layer 4 和 Layer 5。 -- `unitree_go2_temporal_memory`: 包住 agentic stack，并通过 Layer 4 factory 追加 temporal memory。 -- 各 layer package 的 `__init__.py`: lazy blueprint factories。它们不是普通 import 便利层，而是架构边界的一部分，避免过早 import 重型 perception/model 依赖。 - -构建数据流: - -1. CLI 通过 registry 懒加载命名 blueprint。 -2. blueprint 只 import 需要的 layer handle。 -3. 每个 layer handle 构建自己的 `autoconnect()` blueprint。 -4. 最终仍是普通 DimOS blueprint 组合，stream wiring 和 Spec-based RPC injection 仍由 `ModuleCoordinator` 完成。 -5. 测试验证 Layer 3 能通过预期 contract 访问 Layer 4/5/6，并且 blueprint registry 是最新的。 - -状态和持久化: - -- blueprint layer 本身不保存 robot 数据。 -- 状态由各层内部 module 持有: Layer 6 body state、Layer 4 memory/world snapshot、Layer 5 skill contracts、Layer 3 context/outcome/causal state。 -- 持久化属于具体 module，不属于 blueprint 文件。 - -安全边界: - -- blueprint 决定哪些 module 部署在一起。 -- blueprint 不决定调用哪个 skill，也不执行 motion。 -- wiring 错误应该按安全风险看待，因为 LLM 可能看到错误工具或错误上下文。 - -### 2. Layer 6 Robot Body State - -相关文件: - -- `dimos/robot/unitree/go2/blueprints/layers/layer_6_robot_body/robot_body_state.py` -- `dimos/robot/unitree/go2/blueprints/layers/layer_6_robot_body/robot_body_spec.py` -- `dimos/robot/unitree/go2/blueprints/layers/layer_6_robot_body/test_robot_body_state.py` - -实现逻辑: - -- `_Go2RobotBodyState` 是一个小模块，通过 spec-facing API 暴露 robot body 状态。 -- 它贴近真实 Go2 connection stack，但不替代真实连接模块。 -- 上层通过 typed spec 消费 robot state，而不是直接访问硬件 connection module。 - -Review 重点: - -- 是否提供了 agent preflight 足够使用的安全相关状态。 -- 是否避免重复持有或篡改真实控制权。 - -组成部分: - -- `_Go2RobotBodyState`: 把低层 robot status 适配成紧凑 Layer 6 provider 的 module。 -- `robot_body_spec.py`: 上层消费的 typed RPC contract。 -- Layer 6 package factory: 把底层 Go2 robot blueprint 和 body-state module 接在一起。 - -数据流: - -1. 底层 Go2 connection 和 robot modules 持有真实硬件/仿真通信。 -2. `_Go2RobotBodyState` 通过 spec 暴露 read-oriented body-state summary。 -3. Layer 4/Layer 3 把这些状态用于 safety preflight 和 context evidence。 -4. agent brain 看到的是摘要状态，不是直接硬件 handle。 - -判断主体: - -- body-state module 不做任务决策。 -- 它是 evidence provider。Layer 3 确定性 preflight 和 LLM 会使用它的输出判断等待、澄清或调用 skill。 - -状态和写边界: - -- 它应该被视为 current-state adapter。 -- 不应该持久化 memory，也不应该发控制命令。 -- 未来如果增加 body-state cache，必须有明确 bound 和 timestamp。 - -失败形态: - -- 如果底层 robot state 缺失，应明确返回 unavailable/unknown，不应伪造 ready。 -- motion-sensitive preflight 应把缺失 robot body state 视为风险。 - -### 3. Layer 4 Structured World State - -相关文件: - -- `dimos/robot/unitree/go2/blueprints/layers/layer_4_world_state/structured_world_state.py` -- `dimos/robot/unitree/go2/blueprints/layers/layer_4_world_state/semantic_temporal_map.py` -- `dimos/robot/unitree/go2/blueprints/layers/layer_4_world_state/world_state_spec.py` -- `dimos/robot/unitree/go2/blueprints/layers/layer_4_world_state/test_world_state.py` - -实现逻辑: - -- `_Go2StructuredWorldState` 通过 typed spec 暴露紧凑的 robot/world snapshot。 -- `_Go2SemanticTemporalMap` 把 spatial memory 和 temporal memory 摘要合并成更适合 agent 使用的 world-state surface。 -- Layer 3 通过 RPC spec 注入读取这些信息，不直接 import 具体 memory 实现。 - -这里的 "有效上下文" 是什么: - -- Layer 4 不让 LLM 直接判断 memory 相关性。 -- 它把可用 world/memory source 归一化成结构化 provider surface。 -- Layer 3 再负责打分、过滤、组合 evidence，最后交给 LLM 使用。 - -Review 重点: - -- memory backend 缺失时是否明确返回 unavailable，而不是静默空值。 -- snapshot schema 是否足够稳定，能被 prompt 和 dashboard 长期消费。 - -组成部分: - -- `_Go2StructuredWorldState`: 把 robot/world 信息规范化成 compact snapshot provider。 -- `_Go2SemanticTemporalMap`: 融合 spatial memory 和 temporal memory summary，产出 agent context 使用的 semantic evidence。 -- `world_state_spec.py`: Layer 3 依赖的 RPC contract。 -- `_go2_temporal_memory_world_state()`: lazy factory，延迟 TemporalMemory 构建，确保 `--new-memory` 等 CLI flags 已经生效。 - -数据流: - -1. SpatialMemory 和 TemporalMemory 仍是 storage/search 系统。 -2. Layer 4 查询或接收这些系统的 summary。 -3. Layer 4 把 summary 整形成稳定 world-state object，并明确 source availability。 -4. Layer 3 通过 spec 请求 world state，然后根据当前 task 选择 evidence。 - -状态和持久化: - -- Layer 4 可以读取持久化 memory backend，但它自身的 structured snapshot 是 derived view。 -- Temporal memory 持久化由 TemporalMemory module 和 CLI config 控制。 -- Spatial memory 持久化由 `SpatialConfig` 和 `spatial_perception.py` 的 state-dir 路径逻辑控制。 - -判断主体: - -- Layer 4 决定如何规范化 provider output。 -- Layer 4 不判断上下文是否足够行动。这个判断属于 Layer 3 preflight policy 和 LLM。 - -失败形态: - -- provider unavailable 应在 snapshot 里明确表示。 -- search result empty 不等于 backend failure，review 时要确认二者有区别。 - -### 4. Layer 5 Skill Interface Registry - -相关文件: - -- `dimos/robot/unitree/go2/blueprints/layers/layer_5_skill_interface/skill_interface_registry.py` -- `dimos/robot/unitree/go2/blueprints/layers/layer_5_skill_interface/skill_interface_spec.py` -- `dimos/robot/unitree/go2/blueprints/layers/layer_5_skill_interface/test_skill_interface_registry.py` - -实现逻辑: - -- skill contract 是显式静态记录，包含 skill name、domain、module、required args、optional args、是否 motion sensitive、context requirement、推荐 preflight、risk class、outcome shape。 -- registry 把这些 contract 作为数据暴露给 Layer 3。 -- 已知非 Layer 5 MCP tool 被排除在 contract mismatch 检查之外，避免 status/preflight 内部工具污染 skill coverage。 - -和自进化的关系: - -- 机器人不会自动修改 skill 代码。 -- 当任务失败且有明确 missing capability 证据时，可以生成待 review 的 skill interface proposal。 -- 现有 contract 是 duplicate suppression 的判断基准。 - -Review 重点: - -- 每个 MCP action skill 是否都有准确 contract。 -- `risk_class`、`requires_context`、`requires_robot_state` 应按 safety-critical metadata 审查。 - -组成部分: - -- 静态 `_SkillContract` 记录: 每个 action skill 的人工维护接口描述。 -- `_Go2SkillInterfaceRegistry`: 通过 RPC/MCP-facing 方法把 contracts 暴露给 Layer 3。 -- `skill_interface_spec.py`: 消费 skill metadata 的 typed contract。 -- Layer 5 blueprint factory: 接入 action skill containers、perception/security skills 和 registry。 - -数据流: - -1. action modules 通过 MCP 暴露 `@skill` 方法。 -2. registry 提供并行 contract view，包含比 MCP JSON schema 更丰富的 safety/context metadata。 -3. ContextProvider 和 TaskFeasibility 读取 contracts，判断需要哪些 context、arguments、preflight checks。 -4. SkillProposal 读取同一份 contracts，避免提重复 skill。 - -状态和写边界: - -- contracts 是代码/静态数据，不是 learned state。 -- registry 运行时只读。 -- self-evolution proposal 不会直接修改这个文件。开发者 review 后才编辑 contract。 - -判断主体: - -- 人类维护 source-of-truth contract 文案和风险元数据。 -- Layer 3 确定性代码消费这些数据。 -- LLM 可以使用 contract 信息，但不应发明 registry/MCP list 之外的隐藏能力。 - -失败形态: - -- 暴露的 skill 没有 contract 是 review failure，因为 preflight 无法知道 risk/context 需求。 -- contract 存在但 MCP tool 缺失也是 review failure，除非该 blueprint 有意禁用它。 - -### 5. Layer 3 Expert Router - -相关文件: - -- `dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/expert_router.py` -- `dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/prompt_policy.py` -- `dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/test_expert_router.py` -- `dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/test_prompt_policy.py` - -实现逻辑: - -- router 根据 task domain 推荐应该使用的 expert/tool family。 -- prompt policy 把 Layer 3 自进化约束注入 MCP client system prompt，但不完全替代机器人原有 prompt。 -- 输出是确定性的 routing metadata，不是额外 LLM call。 - -判断主体: - -- 第一层 routing 和 evidence packaging 由确定性代码完成。 -- LLM 仍是读取最终上下文并决定如何执行、是否澄清、是否调用工具的主体。 - -Review 重点: - -- routing label 是否和实际 skill contract 对齐。 -- prompt 文案是否过度承诺自治或自动改代码能力。 - -组成部分: - -- `_Go2ExpertRouter`: 确定性 task/domain router。 -- prompt policy helpers: 把 layer-specific instructions 合并进 agent system prompt。 -- router tests: 固定预期 label，避免行为无意扩大。 - -数据流: - -1. user task 或 subtask 进入 router。 -2. router 把任务分类到少量 domain，例如 navigation、perception、person follow、类似 manipulation 但当前不支持的任务、general conversation。 -3. 返回 routing metadata 和 recommended next tools/preflight。 -4. ContextProvider 可以把 routing output 放进 context。 -5. LLM 把 routing 当作提示，而不是不可逆 planner。 - -判断主体: - -- domain classification 是确定性的、可检查的。 -- tool choice 仍是 LLM/tool-calling decision，除非 deterministic preflight 明确拒绝或要求澄清。 - -状态和持久化: - -- router 无状态。 -- 它自己不写 ledger event 或 feedback。 - -失败形态: - -- 未知任务应该 route 到 uncertainty/general handling，不应伪造能力。 -- safety-sensitive domain 应偏向 preflight 和 clarification。 - -### 6. Context Provider 和 Evidence Selection - -相关文件: - -- `dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/context_provider.py` -- `dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/context_evidence.py` -- `dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/test_context_provider.py` -- `dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/test_context_evidence.py` - -实现逻辑: - -- `get_context()` 汇总 task text、runtime metadata、robot state、RAG/spatial memory、temporal memory、skill contracts、skill outcomes、causal world model、context feedback。 -- `context_evidence.py` 提供纯函数策略 `build_context_evidence(metadata, ContextEvidencePolicy)`。 -- policy 字段包括 `min_relevance_score`、`max_entries`、`include_low_confidence`、`require_robot_state_for_motion`。 -- 返回结果包含 compact context 和 `context_evidence.v1`，说明选了哪些 evidence、为什么选。 - -现在 "有效" 的标准: - -- 有效表示被确定性 policy 从可用 provider 里选中。 -- 主要依据包括 relevance score、confidence、source availability、task type、safety requirement。 -- 这还不是基于 replay metric 学出来的策略。 -- LLM 会消费这些上下文，并且仍可以判断上下文不足。 - -组成部分: - -- `_Go2ContextProvider`: MCP-facing context bundle provider。它是聚合器，读取可选 Layer 4/5/3 providers，并把结果整理成 agent 可用的 compact context。 -- `context_evidence.py`: 纯 evidence selector。它负责确定性的排序/过滤策略，可以在没有 robot、MCP server、memory backend 或 LLM 的情况下单测。 -- 可选 provider refs: world state、spatial/RAG memory、temporal memory、skill interface registry、skill outcome store、causal world model、context feedback store。 -- `context_evidence.v1`: 嵌在返回结果里的 review artifact。它记录哪些 evidence 被选中、丢弃或标成 low-confidence。 - -输入: - -- 调用方传入的 task text 和 runtime metadata。 -- 已接线 DimOS modules 提供的 provider snapshot/summary。 -- policy 参数，例如 relevance 阈值、最大条数、low-confidence 处理方式、motion task 是否必须有 robot state。 -- 已接线时的历史 feedback、outcome、world-model summary。 - -数据流: - -1. 调用方带着 task/runtime metadata 调用 `get_context()`。 -2. ContextProvider 向已接线 provider 请求 world state、memory summary、skill contract、outcome summary、causal-world-model state、context feedback。provider 没接线时，应贡献明确的 unavailable 状态，而不是静默丢掉该 source。 -3. metadata 被传给 `build_context_evidence()`。这个 helper 是纯函数: 给定 metadata 和 `ContextEvidencePolicy`，返回被选 evidence，不发 RPC、不写 memory、不调用 LLM。 -4. ContextProvider 格式化 compact context，并附加 `context_evidence.v1`，让 reviewer 能看到为什么选了这些 evidence。 -5. LLM 仍然是最终消费上下文、做用户可见推理和工具选择的主体。ContextProvider 本身不决定执行 skill。 - -状态和写边界: - -- ContextProvider 本身基本是 read-mostly，不写 RAG/vector memory、temporal memory、skill contracts 或 robot state。 -- 唯一的 durable side channel 是间接的: 它可能包含来自 feedback/outcome/ledger-aware modules 的摘要，但写入由那些 module 自己负责。 -- 被选中的 context 是临时 prompt artifact，除非另一个调用方显式记录 feedback 或 evolution event。 - -判断主体: - -- 确定性代码决定哪些 evidence 进入 context bundle。 -- LLM 决定如何消费这个 bundle、是否追问、是否调用已暴露 tool。 -- 人类 reviewer 判断确定性 policy 对当前机器人安全姿态是否足够保守。 - -失败形态: - -- 缺失 provider 应显示为明确 unavailable metadata。 -- provider payload 格式错误时，应降级该 source，而不是清空整个 context bundle。 -- 低相关或低置信 evidence 应按 policy 丢弃或标注，不能静默升级为可信 evidence。 - -没有实现的边界: - -- 还没有用 replay log 训练 learned ranker。 -- ContextProvider 不写 RAG/vector database。 -- context feedback 只是一个摘要信号，不会单独覆盖 provider data。 - -Review 重点: - -- relevance/confidence 默认阈值是否过松。 -- motion task 是否必须要求 robot state。 -- low-confidence evidence 应该隐藏，还是保留但显式标注。 - -### 7. Memory Backend Status - -相关文件: - -- `dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/memory_backend_status.py` -- `dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/test_memory_backend_status.py` - -实现逻辑: - -- 新增 MCP skill `memory_backend_status()`。 -- 报告 world state、spatial memory、temporal memory、skill outcomes、causal world model、skill interface 是否已接线。 -- 能 probe 的 backend 会做小型 read probe。 -- 只返回诊断元数据，不创建新的 memory database。 - -Review 重点: - -- 这是诊断工具，不是新的 memory scheduler。 -- 它不应该写入 RAG、spatial memory、temporal memory 或 Git。 - -组成部分: - -- `_Go2MemoryBackendStatus`: 作为 MCP skill 暴露的诊断 module。 -- 可选 provider specs: world state、spatial memory、temporal memory、outcome store、causal world model、skill interface registry。 -- probe helpers: 小型 read attempt，用来报告 availability 和 error。 - -数据流: - -1. skill 检查哪些可选 provider reference 已注入。 -2. 对每个 provider 记录 `wired`、`available` 和 probe result。 -3. probe failure 被捕获成 status metadata，而不是作为 hard crash 抛给 agent。 -4. 结果以 JSON/text 返回给 LLM 或 human operator。 - -状态和写边界: - -- 它是 read-only。 -- 不持久化 status snapshot。 -- 不初始化缺失 database。如果 database 没运行或没接线，status 应明确说明。 - -判断主体: - -- 这个 skill 不决定使用哪个 memory。 -- 它只回答“现在接了什么、能读什么”。ContextProvider 和 LLM 决定如何响应。 - -失败形态: - -- probe failure 应标明 provider 和 error string。 -- missing provider 应足够明确地区分 blueprint wiring 问题和 memory result empty。 - -### 8. Git-backed Evolution Ledger - -相关文件: - -- `dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/evolution_event.py` -- `dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/evolution_ledger.py` -- `dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/evolution_proposal.py` -- `dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/test_evolution_ledger.py` -- `dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/test_evolution_proposal.py` - -实现逻辑: - -- event 使用 schema `dimos.evolution_event.v1`。 -- proposal 由 `evolution_proposal.py` 做 schema 验证。 -- 默认写入本地仓库路径: - `.dimos/evolution/events/YYYY/MM/DD/*.json` - 和 `.dimos/evolution/proposals/*.json`。 -- `DIMOS_EVOLUTION_LEDGER_DIR` 可以覆盖存储目录。 -- 写入内容是 JSON 文件，目标是低频、可 review、可 diff 的控制面记录。 -- 默认 `commit=False`。 -- 如果 `commit=True`，只把该 event/proposal 文件提交到本地 Git。 - -组成部分: - -- `evolution_event.py`: 低层自进化 observation 的 schema 和校验，例如 feasibility assessment、context feedback、runtime review note。 -- `evolution_proposal.py`: 更高层 proposed change 的 schema 和校验。这些 proposal 必须先经人工 review，才可能变成代码或配置。 -- `_EvolutionLedger`: filesystem 和可选 Git writer。它负责路径选择、JSON serialization、commit mode 下的 staging/commit 行为。 -- 测试: 覆盖 schema validation、路径形状、commit 隔离和 proposal 持久化。 - -输入: - -- Layer 3 MCP tool 传入的 event/proposal payload。 -- 可选的 caller-controlled commit flag。 -- 可选的 `DIMOS_EVOLUTION_LEDGER_DIR` 覆盖。 -- commit mode 打开时的当前 Git repository context。 - -Git 内容会提交到哪里: - -- 文件先进入当前本地仓库工作区。 -- commit 只进入本地 `.git` 历史。 -- 不会自动 push 到 GitHub。 -- 只有人类后续运行 `git push` 或开 PR，GitHub 才会收到这些记录。 - -写入数据流: - -1. 调用方调用 ledger-facing MCP skill，例如 `record_evolution_event()` 或 `record_skill_proposal()`。 -2. 模块验证 schema，并规范化 payload。 -3. 模块选择存储 root。默认是 repo-local `.dimos/evolution`，`DIMOS_EVOLUTION_LEDGER_DIR` 可以覆盖。 -4. 写入 JSON 文件，作为可 review artifact。 -5. 如果 `commit=False`，除了工作区文件外不碰 Git。 -6. 如果 `commit=True`，只 stage 并提交该 event/proposal 文件到本地 Git，不 push。 - -状态和写边界: - -- ledger 是 durable filesystem state，不是内存学习模型。 -- 它只在配置的 ledger root 下面写 JSON artifact。 -- 它不修改 skill 代码、prompt、contracts、memory database 或 model weights。 -- commit mode 有意保持 local-only 且范围很窄: stage 该 artifact，创建本地 commit，然后停止。 - -判断主体: - -- 调用方 module 决定何时值得记录 event/proposal。 -- ledger 只判断 artifact 是否符合 schema，以及应该存到哪里。 -- 人类 reviewer 决定这些 ledger artifact 后续是否变成代码改动、PR、被 ignore 的本地轨迹，或训练/评估数据。 - -失败形态: - -- schema 无效应在写入前失败。 -- 不安全路径或路径穿越应在 filesystem access 前被拒绝。 -- Git commit 失败时，应让 JSON artifact 仍然在 worktree 可见，而不是假装已经提交。 - -隐私和 review 边界: - -- 这些文件可能包含 task text、outcome message、context summary、proposal rationale。真实机器人运行前，要先决定 `.dimos/evolution` 是否应该进入正常 Git 历史。 -- ledger 是控制面审计轨迹，不替代 SpatialMemory、TemporalMemory、SkillOutcomeStore 或 causal model state。 - -Review 重点: - -- event/proposal 字段是否可能造成路径穿越。 -- commit mode 是否只 stage 新 ledger 文件。 -- 根据隐私策略决定 `.dimos/evolution` 应该 ignore、commit，还是导出到别的存储。 - -### 9. Task Feasibility Preflight - -相关文件: - -- `dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/task_feasibility.py` -- `dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/test_task_feasibility.py` - -实现逻辑: - -- 新增 MCP skill `evaluate_task_feasibility(task, context_json)`。 -- 读取 skill contracts、world/robot context 和可选 JSON context。 -- 返回确定性结果: feasibility status、missing context、required skills、available skills、safety risks、recommended next action、clarifying question、evidence sources。 -- 如果 evolution ledger 已接线，可以写入 ledger event。 - -自进化含义: - -- 这不是模型自训练。 -- 它是 agent self-assessment: 在行动前，agent 可以调用确定性 evaluator 判断任务是否具备上下文和能力支持。 - -组成部分: - -- `_Go2TaskFeasibility`: deterministic preflight 的 MCP skill surface。 -- Skill contract reader: 消费 Layer 5 skill metadata，例如 required arguments、context requirements、risk class、motion sensitivity。 -- Context parser: 把显式 `context_json` 和已接线 world/robot provider summary 合并。 -- Ledger integration: 可选的 review trace event writer。 - -输入: - -- `task`: 用户请求或 subtask 的自然语言文本。 -- `context_json`: 调用方可选传入的结构化 context。 -- Layer 5 skill contracts 和 availability metadata。 -- 已接线时的 Layer 4 robot/world state。 -- context bundle 中可用的 outcome/context feedback summary。 - -决策/数据流: - -1. 解析 task 和可选 `context_json`。 -2. 把 task 匹配到已知 skill contract 和 expert domain。 -3. 根据 Layer 5 contract 检查 required args 和 context requirements。 -4. 对 motion-sensitive 或高风险 skill 检查 robot/world-state 是否可用。 -5. 输出三类粗粒度结果之一: `feasible`、`uncertain`、not feasible。 -6. 如果缺数据，推荐澄清问题或 context-gathering action。 -7. 如果缺能力，报告 missing capability evidence。这个 evidence 后续可以成为 skill-interface proposal 的依据。 - -状态和写边界: - -- 它对 robot state、memory 和 skill contracts 是 read-only。 -- 它不会调用目标 skill。 -- 它不会创建新 skill。 -- 如果 ledger 已接线，它可以写一条描述 preflight result 的 audit event，但不是代码改动。 - -判断主体: - -- feasibility classification 是确定性代码。 -- LLM 可以调用这个工具并消费结果，但 LLM 不是这个工具内部的 classifier。 -- 人类 review 判断这些确定性规则对真实硬件是否足够保守。 - -失败形态: - -- 缺少 required arguments 应产生 clarification，而不是新 skill proposal。 -- 缺少 context 应产生 context-gathering 或 uncertainty，而不是误报 feasible。 -- missing capability evidence 要足够具体，方便 SkillProposal 把它和 transient runtime failure 区分开。 - -Review 重点: - -- motion-sensitive task 是否会误判为可行。 -- 缺少 context 时是否优先澄清，而不是生成 skill proposal。 - -### 10. Context Feedback Store - -相关文件: - -- `dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/context_feedback.py` -- `dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/test_context_feedback.py` - -实现逻辑: - -- 新增 MCP skill `record_context_feedback(...)`。 -- 内存里维护最多 100 条 recent feedback 的 bounded deque。 -- schema 为 `go2_context_feedback.v1`。 -- 如果 evolution ledger 已接线，会把反馈写成 ledger event。 -- ContextProvider 会把 compact feedback summary 纳入后续 context。 - -Review 重点: - -- feedback 默认是 session memory，除非 ledger 已接线。 -- 它不能无界增长，也不应该静默覆盖 RAG 或 temporal memory。 - -组成部分: - -- `_Go2ContextFeedbackStore`: bounded in-memory store 和 MCP skill surface。 -- feedback entry schema: task/context identifiers、rating 或 usefulness signal、source labels、optional explanation、timestamp。 -- optional evolution ledger connection: 已接线时提供 durable audit trail。 -- ContextProvider integration: compact aggregate feedback summary。 - -数据流: - -1. agent 或 human-facing process 在 context bundle 有帮助、不完整、过期或误导后调用 `record_context_feedback(...)`。 -2. store 验证并规范化 feedback fields。 -3. entry append 到最多 100 条的 deque。 -4. 如果 ledger 已接线，feedback 也写成 evolution event。 -5. ContextProvider 读取 summary，而不是完整 raw feedback list，避免 prompt 膨胀。 - -状态和持久化: - -- 没有 ledger 时，feedback 是 process-local，重启丢失。 -- 有 ledger 时，feedback 成为可 review JSON，但不会自动训练 ranker。 - -判断主体: - -- feedback recording 是输入信号。 -- evidence policy 和 LLM 决定如何使用该信号。没有自动 prompt rewrite 或 memory deletion。 - -失败形态: - -- malformed feedback 应 validation fail。 -- feedback 超量时淘汰最旧 entry，不应无界增长。 - -### 11. Skill Outcome Store 和 Predictor - -相关文件: - -- `dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/skill_outcome_store.py` -- `dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/skill_outcome_predictor.py` -- `dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/test_skill_outcomes.py` - -实现逻辑: - -- 记录近期 skill outcome，存储是 bounded 的。 -- 按 skill 汇总成功、失败和模式。 -- predictor 根据近期历史和 skill contract metadata 估计风险或可能结果。 -- ContextProvider 可以把这些 summary 作为 task evidence。 - -Review 重点: - -- predictor 是启发式，不是学习模型。 -- 旧失败不应永久阻止可用 skill。 - -组成部分: - -- `_Go2SkillOutcomeStore`: bounded recent outcome history。 -- `_Go2SkillOutcomePredictor`: 基于近期 history 和 skill contract metadata 的 heuristic scorer。 -- outcome summary methods: 按 skill 聚合 successes、failures、repeated errors、recent messages。 - -数据流: - -1. skill call 后可以记录 skill name、success、error code、message、metadata。 -2. outcome store 把记录 append 到 bounded memory。 -3. summary 按 skill 和 error pattern 分组。 -4. predictor 把近期 outcome 和 skill contract risk/context metadata 合并，返回 risk 和 rationale。 -5. ContextProvider 可以把 summary 作为后续调用的 evidence。 - -状态和持久化: - -- 当前 store 是 bounded runtime memory，除非未来接入其他持久化。 -- 它和 Git ledger 分离。ledger 记录 review/audit event；outcome store 支持即时 runtime adaptation。 - -判断主体: - -- predictor 给出 heuristic risk hint。 -- 它不应单独 block 一个 skill；TaskFeasibility 或 LLM 应把它作为 evidence source 之一。 - -失败形态: - -- unknown skill 应返回 low-confidence 或 no-history output。 -- repeated failure 应提高谨慎度，但不应阻止 recovery/stop skills。 - -### 12. Causal World Model 和 Dashboard Contract - -相关文件: - -- `dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/causal_world_model.py` -- `dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/causal_effect_estimator.py` -- `dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/online_transition_model.py` -- `dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/structural_causal_model.py` -- `dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/intervention_log.py` -- `dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/world_model_contract.py` -- `dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/test_causal_world_model.py` -- `dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/test_world_model_contract.py` - -实现逻辑: - -- 记录 causal transition 和 intervention。 -- 维护近期 world-state change 的 online transition model。 -- 基于记录事件估计 causal effect。 -- 输出 versioned contract: - `dimos.world_model_prediction.v1`、 - `dimos.world_model_provider.v1`、 - `dimos.world_model_dashboard.v1`。 -- 支持 save/load model state。 - -它是不是训练了模型: - -- 严格说: 有一个非常小的在线模型，但没有离线训练流程，也没有神经网络世界模型。 -- `OnlineTransitionOutcomeModel` 是 lightweight logistic-style linear model。它把 feature weight 存在 dict 里，每次 `record_causal_transition()` 收到一次已观测 success/failure outcome 时增量更新。 -- 如果还没有记录 transition，它的 `sample_count` 是 0，此时预测只能视为低置信先验行为，不是已学习的机器人知识。 -- symbolic SCM 是手写的，不是从数据学出来的。 -- causal effect estimator 是观测统计估计，只比较 feature active/inactive 的平滑成功率差异，不控制 hidden confounders。 - -组成部分: - -- `OnlineTransitionOutcomeModel`: 在线 success-probability scorer。feature 包括 bias、skill name、domain、是否有 odom、navigation state、spatial memory 是否可用、spatial match count、temporal 是否可用、action argument 是否存在和部分 argument value。更新方式是一阶 gradient adjustment，带 L2 shrinkage。 -- `CausalEffectEstimator`: bounded observation store，对 symbolic feature 做 treated/control success-rate difference。它能给出 risk factor、supporting factor 和 intervention suggestion，但输出里明确声明 unobserved confounders are not controlled。 -- `StructuralCausalModel`: 手写因果图，变量包括 `spatial_has_matches`、`target_resolvability`、`odom_ready`、`motion_safety`、`navigation_success`。它还会生成 counterfactual suggestion，例如语义导航前先 tag 或 perceive target。 -- `InterventionLog`: 记录显式 intervention，并让 prediction 能引用历史 intervention evidence。 -- `_WorldTransition`: compact event record，把 task、skill、before/after state、predicted risk、outcome、inferred cause、recovery suggestion 绑定起来。 - -记录数据流: - -1. `record_causal_transition()` 接收 task、skill name、可选 args、before/after Layer 4 snapshot、prediction metadata、outcome metadata。 -2. 输入会按 JSON object 解析和验证。缺少 outcome 时，如果 SkillOutcomeStore 已接线，可以回退到同 skill 的最新 outcome。 -3. 模块根据 context、prediction reasons、outcome success、error code、message 推断粗粒度 cause 和 recovery suggestion。 -4. 模块计算 before/after snapshot 的 symbolic state delta。 -5. 调用 `_outcome_model.update(...)` 和 `_causal_estimator.update(...)`。 -6. 把 transition append 到内存 deque，最多保留 200 条。 -7. 只有设置了 `DIMOS_GO2_WORLD_MODEL_STATE` 时才 autosave。 -8. 如果 WebsocketVis 已接线，会 publish dashboard state。 - -预测数据流: - -1. `predict_next_state()` 解析 snapshot 和 candidate action。 -2. 读取同 skill 的 recent transitions，提取 repeated failure modes。 -3. 根据当前 snapshot 和 action 计算 rule-based risk reasons。 -4. 调用在线模型得到 success probability、score、risk、confidence、top weighted feature contribution。 -5. 调用 causal estimator 得到 observational risk/support factors。 -6. 调用手写 SCM 得到 variables、edges、counterfactuals。 -7. 调用 intervention log 查找匹配的 intervention evidence。 -8. 合并 rule risk 和 model risk，得到最终 risk 和 score。 -9. 返回 `dimos.world_model_prediction.v1`，并可选 publish dashboard contract。 - -可信边界: - -- 输出只是 advisory evidence，用来帮助 LLM 选择 preflight、perception、clarification 或更安全的 skill path。 -- 它不能直接命令机器人运动。 -- 当前 high confidence 需要同 skill 足够样本；否则应当视为低置信启发式。 - -Review 重点: - -- 当前 causal estimate 只能作为 advisory evidence。 -- prediction schema 要对 dashboard 和 LLM prompt consumer 保持稳定。 -- persistence 不应意外混合不同 run 的数据。 -- UI/prompt 文案不能暗示这是完整训练好的 physical world model。 -- 在依赖它做自主 action selection 前，应补 replay 或 simulation evaluation。 - -### 13. Skill Interface Proposal Generator - -相关文件: - -- `dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/skill_proposal.py` -- `dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/test_skill_proposal.py` - -实现逻辑: - -- 新增 MCP skill `propose_skill_interface(task, failure_context_json)`。 -- 只有在 failure context 里有明确 missing capability evidence 时，才生成 `dimos.skill_proposal.v1` review artifact。 -- 对 missing arguments、missing context 不生成 proposal。 -- 如果现有 contract 已覆盖能力，则抑制重复 proposal。 -- 如果 ledger 已接线，通过 evolution ledger 写入 proposal 文件。 -- 不修改 Python 代码、不新增 `@skill`、不改 blueprint wiring。 - -决策树: - -1. 解析 `failure_context_json`。 -2. 查找明确 missing capability evidence，而不是只看执行失败。 -3. 查询现有 Layer 5 contracts，避免 proposal 重复。 -4. 如果只是缺 argument、缺 context、provider 暂时 unavailable，则拒绝生成 proposal。 -5. 构造 proposal artifact，包含 task、evidence、suggested interface shape、expected inputs、expected output、review rationale。 -6. 如果 ledger 已接线，通过 evolution ledger 写入 proposal。 - -人工边界: - -- 这个工具故意只做 proposal。开发者仍然需要写代码、选择 container、加 `@skill`、更新 contract/prompt，并测试行为。 - -Review 重点: - -- 这是最安全的自进化边界: 只生成 proposal，必须人工 review。 -- evidence requirement 要足够严格，避免用户表达不清时产生大量噪声 proposal。 - -组成部分: - -- `_Go2SkillProposalGenerator`: proposal generation 的 MCP skill surface。 -- 现有 Layer 5 contracts: duplicate suppression 的 source of truth。 -- Evolution ledger: 可选 proposal writer。 -- Proposal schema: 描述 missing capability 和 proposed interface 的 review artifact，不是可执行代码。 - -输入: - -- `task`: 失败的 user/subtask text。 -- `failure_context_json`: 描述为什么现有 skills 失败或不足的结构化 evidence。 -- 现有 contracts 和已知 MCP tool names。 - -接受的 evidence: - -- 明确 missing capability。 -- repeated failure 显示没有可用 skill 能完成目标操作。 -- 现有 contracts 没覆盖的 domain/action gap。 - -拒绝的 evidence: - -- 只是缺少现有 skill 的 required argument。 -- 只是缺 context 或 memory/provider unavailable。 -- 临时 runtime failure。 -- 能力已被现有 contract 覆盖。 - -输出边界: - -- 输出是 JSON proposal 和可选 ledger 文件。 -- 不 import、不写入、不生成 Python code。 -- 不自动更新 prompt 或 registry。 - -### 14. MCP Runtime Plumbing - -相关文件: - -- `dimos/agents/mcp/mcp_client.py` -- `dimos/agents/mcp/mcp_server.py` -- `dimos/agents/mcp/tool_stream.py` -- `dimos/agents/mcp/test_mcp_client_unit.py` -- `dimos/agents/mcp/test_mcp_server.py` -- `dimos/agents/mcp/test_tool_stream.py` - -实现逻辑: - -- `McpServer.on_system_modules()` 避免通过 RPC 查询自己的 proxy。 -- 对自身 skill 和带 actor class 的 proxy，server 本地收集 schema。 -- `McpClient` 在 RPC path 外延迟 direct tool registration，避免启动阶段互相等待。 -- tool-stream progress frame 可以通过 persistent SSE 发送，并带 progress-token metadata。 -- background capability release 绑定 stopped frame。 -- `agent_send()` 使用 `make_transport()`，因此会尊重当前 backend。 - -Review 重点: - -- 启动路径不能等待依赖自身初始化的工具。 -- MCP 测试端口必须和 live robot session 隔离。 - -组成部分: - -- `McpServer`: 暴露 MCP JSON-RPC endpoints、tool list/call handling、SSE progress stream、server status tools、capability gating、agent-send。 -- `McpClient`: LLM agent module，负责发现 tools 并暴露给 agent runtime。 -- `tool_stream.py`: 长运行 skill 的 progress/log notification path。 -- capability registry: server-side lock manager，用于 mutually exclusive tool capabilities。 - -启动数据流: - -1. ModuleCoordinator 部署 worker 并启动 modules。 -2. `McpServer.start()` 启动 HTTP/SSE serving，并订阅 tool-stream frames。 -3. `on_system_modules()` 注册 tool schemas。对 server 自身和 actor-class backed modules，本地收集 schema，避免 RPC self-deadlock。 -4. `McpClient` 初始化 agent，但不在 RPC path 外阻塞等待 direct tool registration。 - -Tool call 数据流: - -1. JSON-RPC `tools/call` 到达。 -2. server 解析 `SkillInfo` 和 RPC call target。 -3. 如果 skill 声明 `uses`，先申请 capability lock。 -4. progress token 和 capability token 通过保留 `_mcp_context` 注入。 -5. RPC call 在 executor 里运行。 -6. instant skill 返回后释放 capability；background skill 把 release 交给 tool-stream stopped frame。 - -状态和失败边界: - -- server state 是 process-local: skills、rpc calls、SSE queues、capability registry。 -- port conflict 是外部环境失败。live server 存在时，测试必须用隔离 MCP port。 -- tool not found 应返回结构化 MCP text result，而不是 crash server。 - -### 15. Runtime Hardening - -相关文件: - -- `dimos/core/coordination/module_coordinator.py` -- `dimos/core/coordination/worker_manager_python.py` -- `dimos/simulation/mujoco/mujoco_process.py` -- `dimos/robot/unitree/mujoco_connection.py` -- `dimos/robot/unitree/go2/connection.py` -- `dimos/robot/unitree/go2/test_connection.py` - -实现逻辑: - -- coordinator 的 system-module notification 避免等待已知 agent client。 -- worker manager/coordinator 变更保留 module restart、reload、stream rewiring 行为。 -- MuJoCo helper 可以读取当前可用 stderr，避免阻塞。 -- macOS headless MuJoCo 在需要时使用当前 Python executable。 -- Go2 WebRTC connection 会从 `GlobalConfig` 转发 AES key。 - -Review 重点: - -- 这些是运行时稳定性修复，重点审查 process lifecycle 和 stream rewiring，而不是 agent reasoning。 - -组成部分: - -- ModuleCoordinator notification path: 控制 build/start/system-module notification 顺序。 -- WorkerManagerPython: 在 workers 中部署 modules，并管理 worker pool lifecycle。 -- MuJoCo process helpers: 处理 subprocess executable 选择和 stderr draining。 -- Go2 connection config: replay stop 行为和 WebRTC AES key forwarding。 - -数据流: - -1. build/deploy 通过 ModuleCoordinator 和 worker managers 完成。 -2. modules 收到 system-module notification 前，stream transports 和 RPC refs 已经 wiring。 -3. 部分 notification 使用 nowait，避免等待正在通过 MCP 初始化的 agent client。 -4. restart/reload paths 必须保留 transports 并重新 wiring module refs。 -5. simulator helper code 把平台特定 process 问题隔离在机器人栈外层。 - -状态和写边界: - -- Coordinator 持有 deployed module proxy maps、transport registry、module transports、reload aliases。 -- 这些改动不改变 skill semantics。 -- 它们影响 module 是否能干净 start、stop、reload、reconnect。 - -失败形态: - -- build/start 卡住时应 stop managers 并暴露原始异常。 -- restart 不应遗留 old transports，也不应让 consumer 连在 dead proxy 上。 -- ReplayConnection stop 应该是 no-op，而不是 exception。 - -### 16. 可选 Rerun/WebsocketVis World-model Panel - -相关文件: - -- `dimos/web/websocket_vis/websocket_vis_module.py` -- `dimos/web/websocket_vis_spec.py` -- `dimos/web/templates/rerun_dashboard.html` -- `dimos/web/websocket_vis/test_websocket_vis_module.py` - -实现逻辑: - -- websocket visualization 除已有 payload 外，还可以携带 world-model/dashboard state。 -- dashboard payload shape 与 `world_model_contract.py` 对齐。 -- 这是可选观测能力。当前 `unitree_go2_agentic` blueprint 不包含 - `WebsocketVisModule`，因此除非单独的可视化 blueprint 接入 - `WebsocketVisSpec`，这条路径不会启动。 - -Review 重点: - -- dashboard state 应保持展示用途。 -- dashboard consumer 不应依赖未 version 的内部 Python object。 -- 不要把它当成 Go2 自进化 runtime 的必需依赖。 - -组成部分: - -- `websocket_vis_spec.py`: visualization update 的 typed surface。 -- `websocket_vis_module.py`: 存储/publish dashboard state 的 runtime module。 -- `rerun_dashboard.html`: browser-facing display surface。 -- `world_model_contract.py`: world-model dashboard payload 的 schema source。 - -数据流: - -1. CausalWorldModel 使用 `dimos.world_model_dashboard.v1` 生成 dashboard payload。 -2. 如果 WebsocketVis 已接线，CausalWorldModel 调用 `set_world_model_state(...)`。 -3. WebsocketVis 保存并向 dashboard clients 提供 latest state。 -4. dashboard 把 state 渲染成人类 inspection data。 - -状态和持久化: - -- dashboard state 是 latest display snapshot，不是 source of truth。 -- 如果配置了 durable state，它属于 CausalWorldModel save/load JSON。 -- browser/dashboard consumer 只应依赖 versioned payload fields。 - -安全边界: - -- dashboard update 不能触发机器人动作。 -- UI 不应把 advisory prediction 展示成 guaranteed outcome。 -- 如果没有接 `WebsocketVisModule`，CausalWorldModel 仍然正常工作；dashboard - publish path 会被跳过。 - -## 推荐 Review 顺序 - -1. 先看 Go2 blueprint wiring: - `unitree_go2_agentic.py`、Layer 3/4/5/6 `__init__.py`、`test_world_state.py`。 -2. 再看 MCP runtime plumbing: - `mcp_server.py`、`mcp_client.py`、`tool_stream.py` 和对应测试。 -3. 再看自进化写入边界: - `evolution_ledger.py`、`evolution_proposal.py`、`skill_proposal.py`。 -4. 再看 context 质量: - `context_provider.py`、`context_evidence.py`、`context_feedback.py`。 -5. 再看 action safety: - `skill_interface_registry.py`、`task_feasibility.py`、`skill_outcome_predictor.py`。 -6. 再看 advisory world-model: - `causal_world_model.py` 和 `world_model_contract.py`。 -7. 最后单独 review 上游 transport 变化，尤其是 macOS 默认 Zenoh。 - -## 本次同步后的验证结果 - -已通过: - -- `.venv/bin/python -m pytest dimos/agents/mcp/test_mcp_server.py -q` - - `10 passed, 1 warning` -- `.venv/bin/python -m pytest dimos/robot/unitree/go2/test_connection.py -q` - - `5 passed` -- `.venv/bin/python -m pytest dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain -q` - - `62 passed` -- `.venv/bin/python -m pytest dimos/robot/unitree/go2/blueprints/layers/layer_4_world_state/test_world_state.py -q` - - `3 passed, 1 warning` -- `.venv/bin/python -m pytest dimos/perception/test_spatial_perception_config.py -q` - - `2 passed` -- `DIMOS_TRANSPORT=lcm .venv/bin/python -m pytest dimos/core/coordination/test_module_coordinator.py -q` - - `35 passed` -- `DIMOS_TRANSPORT=lcm MCP_PORT=9991 .venv/bin/python -m pytest dimos/agents/mcp/test_mcp_client_unit.py dimos/agents/mcp/test_tool_stream.py -q` - - `44 passed, 1 warning` -- `.venv/bin/python -m pytest dimos/robot/test_all_blueprints_generation.py -q` - - `1 passed` -- `git diff --cached --check -- ':(exclude)*.patch'` - - passed - -已知 caveat: - -- 本机有 live `dimos-live` 进程占用默认 MCP 端口 `9990`，所以默认端口跑 MCP 测试会失败。 -- 上游在 macOS 默认 `zenoh` backend 后，部分测试暴露 native pyo3 thread 生命周期问题。同样 coordinator 和 MCP 测试在 `DIMOS_TRANSPORT=lcm` 下通过。 -- 上游 `.patch` 文件包含会触发完整 `git diff --check` 的 whitespace；当前验证排除了 `.patch`，避免破坏补丁语义。 diff --git a/docs/reviews/2026-07-05-go2-agent-architecture.png b/docs/reviews/2026-07-05-go2-agent-architecture.png deleted file mode 100644 index 6bb86c1dd4..0000000000 --- a/docs/reviews/2026-07-05-go2-agent-architecture.png +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:1e80abb6d7371795e672e5652db20754b8fb21bb1a74ec8b88e45c5df372cc0a -size 195112 diff --git a/docs/reviews/2026-07-05-go2-agent-architecture.svg b/docs/reviews/2026-07-05-go2-agent-architecture.svg deleted file mode 100644 index a20e385bd0..0000000000 --- a/docs/reviews/2026-07-05-go2-agent-architecture.svg +++ /dev/null @@ -1,175 +0,0 @@ - - DimOS Go2 Agent Self-Evolution Architecture - Layered Go2 agent architecture with agent brain, world state, skill interface, robot body, local review artifacts, and optional Rerun/WebsocketVis observability. - - - - - - - - - - - - - - - DimOS Go2 Agent 自进化架构 - - - - Layer 3 Agent Brain - - - Layer 4 World State - - - Layer 5 Skill Interface - - - Layer 6 Robot Body - - - - Local Review - Artifacts - local files / optional commit / no auto push - - 本地 Git Ledger - .dimos/evolution - - JSON Events / Proposals - review artifacts - - - - Optional Observability - not wired by unitree_go2_agentic - - 可选观测 - Rerun / WebsocketVis - - - - 用户 Prompt - - - Expert Router - - - Context Provider - - - Task Feasibility - - - MCP Client / Server - - - Skill Outcome - Store - - - Causal World - Model - - - Skill Proposal - - - Evolution - Ledger - - - - Structured World State - - - Semantic Temporal Map - - - RAG / Spatial Memory - - - Temporal Memory - - - - Skill Contracts - - - Preflight Metadata - - - Go2 Skills - - - - Go2 Connection - - - Sensors / Odom - - - Motion Control - - - - - - - - - - - - - - - - - - - - - - - - - - 技能执行结果 / Skill Outcome - - - - - - - - - MCP 调用 Go2 skills；不直接改代码 - observability only; no robot control - From ecef3e133c300f8a03dbbdd08b997b3d281d83de Mon Sep 17 00:00:00 2001 From: ZhangZheMing <1050297972@qq.com> Date: Sun, 5 Jul 2026 10:00:02 +0800 Subject: [PATCH 25/36] Restore Chinese Go2 architecture review guide --- ...-07-05-go2-agent-architecture-review.zh.md | 1212 +++++++++++++++++ 1 file changed, 1212 insertions(+) create mode 100644 docs/reviews/2026-07-05-go2-agent-architecture-review.zh.md diff --git a/docs/reviews/2026-07-05-go2-agent-architecture-review.zh.md b/docs/reviews/2026-07-05-go2-agent-architecture-review.zh.md new file mode 100644 index 0000000000..8c630654ac --- /dev/null +++ b/docs/reviews/2026-07-05-go2-agent-architecture-review.zh.md @@ -0,0 +1,1212 @@ +# Go2 Agent 架构 Review 说明 + +日期: 2026-07-05 +分支: `refactor/go2-architecture-layers` +合并上游前的本地 HEAD: `646e7ec5` +已合并的上游目标: `upstream/main` at `6e813a72` +共同祖先: `1f544d05` + +这份文档是当前分支完整 review 的中文入口。它把上游同步内容和本地新增功能分开说明，并重点解释每个新增功能的实现逻辑、数据流、存储位置、判断主体、风险点和验证情况。 + +## 当前 Git 状态 + +合并前，本分支有 16 个本地提交不在 upstream，上游 `main` 有 160 个提交不在本分支。用 merge base 精确比较后，真正与本地改动重叠的文件有 15 个，实际产生冲突的文件有 7 个: + +- `dimos/agents/mcp/mcp_server.py` +- `dimos/agents/mcp/test_mcp_server.py` +- `dimos/perception/spatial_perception.py` +- `dimos/robot/unitree/go2/blueprints/agentic/unitree_go2_agentic.py` +- `dimos/robot/unitree/go2/blueprints/agentic/unitree_go2_temporal_memory.py` +- `dimos/robot/unitree/go2/blueprints/smart/unitree_go2_spatial.py` +- `dimos/robot/unitree/go2/test_connection.py` + +冲突解析原则: + +- 保留上游基础设施更新，尤其是 Zenoh transport、MCP capability gating、Unitree AES key 转发、文档和打包更新。 +- 保留本地 Go2 分层架构作为 Go2 agentic blueprint 的权威 wiring。 +- 使用上游新的 perception 模块路径，同时保留本地基于 `STATE_DIR` 的 spatial memory 默认持久化路径。 +- 两边新增的测试如果覆盖不同风险点，则合并保留，而不是二选一删除。 + +环境注意事项: + +- 上游现在在 macOS 上默认使用 `zenoh` transport。这个后端需要 `eclipse-zenoh`，本地 `.venv` 已安装 `eclipse-zenoh==1.9.0`。 +- 当前机器已经有一个 live `dimos` 进程监听 MCP 端口 `9990`。跑 MCP 测试时要换端口，例如 `MCP_PORT=9991`，否则测试客户端会连到 live 服务。 +- `git diff --cached --check -- ':(exclude)*.patch'` 已通过。完整 `git diff --check` 会报上游 `.patch` 文件内的 trailing whitespace，这些是 patch 文件的 context 行，应单独处理，不能随意改动以免破坏补丁语义。 + +## Review 粒度标准 + +review 每个本地功能时，建议都按下面这些问题看，而不是只看文件名是否合理: + +- 入口: 第一个对外入口是 blueprint、module、MCP skill、RPC method，还是 helper 函数？ +- 输入: 它读取哪些 JSON 字符串、Spec provider、memory provider 或 runtime state？ +- 输出: 它返回哪种 `SkillResult`、dict schema、Spec return value、file artifact 或 dashboard payload？ +- 判断主体: 判断是确定性代码做的，LLM 做的，小型在线统计模型做的，还是人类 reviewer 做的？ +- 状态: 它是无状态、只在内存、写 JSON、写 Git，还是写已有 RAG/vector database？ +- 写边界: 哪个方法会写入？哪些方法是纯 read-only？ +- 失败形态: 缺数据时返回 `unavailable`、`uncertain`、clarifying question、failed `SkillResult`，还是 proposal artifact？ +- 安全边界: 它能执行机器人运动，还是只给上层建议？ +- Review artifact: 哪个 schema 是未来工具要依赖的稳定接口？ +- 测试: 哪个测试证明主数据流？哪个测试证明失败路径？ + +这个分支最重要的区分是三件事: + +- Context selection: LLM 看到上下文前，由确定性代码做过滤和摘要。 +- Agent judgment: LLM 读取已选上下文和工具后决定下一步。 +- Self-evolution artifact: 产出可 review 的 JSON event/proposal，不自动改代码。 + +## 项目整体架构与 Agent 重点展开 + +本分支的 review 重点是把 repo-wide DimOS 执行架构和 Go2 agentic +self-evolution extension 分开看。repo-wide 部分包括 entry +points、blueprint/configuration、core runtime、transports、module +families、robot/simulation targets 和 data assets。Go2 agentic 扩展部分重点 +看 Agent Brain: task evaluation、context use、skill validation、feasibility +judgment 和 proposal/artifact emission 都由这个层级发起或裁决。 + +可选观测/存储不是主执行依赖。当前 `unitree_go2_agentic` blueprint 没有接 +`WebsocketVisModule`；只有单独的可视化 blueprint 接上 `WebsocketVisSpec` +时，Rerun / WebsocketVis 面板才会收到 world-model state。 + +## 接口输入/输出速查 + +review API 边界时优先看这一节。这里列的是本地 Go2 架构改动新增或改动过的稳定 public surface；没有跨 module 边界消费的内部 helper 不列入。 + +| 接口 | 输入 | 输出 | +| --- | --- | --- | +| Go2 blueprint factories (`unitree_go2_agentic`、`unitree_go2_spatial`、`unitree_go2_temporal_memory`) | CLI blueprint name，以及 DimOS 已解析好的 `GlobalConfig`。lazy layer import 本身没有 runtime 入参。 | `autoconnect()` 组合出的 DimOS `Blueprint`。输出的是 module wiring、stream wiring 和 Spec injection path，不是面向用户的 JSON。 | +| `RobotBodyStateSpec.get_robot_body_snapshot()` | 无参数；读取 Layer 6 近期 odom/image/lidar observation，以及 connection/local-policy summary。 | `dict`，包含 connection state、sensor state、local policy state、safety state、freshness/availability markers。 | +| `RobotBodyStateSpec.get_connection_state()` | 无参数。 | 描述 connection availability 和 config-derived mode 的 `dict`。 | +| `RobotBodyStateSpec.get_sensor_state()` | 无参数。 | `dict`，包含 image、lidar、odom 等 sensor counters/freshness。 | +| `RobotBodyStateSpec.get_local_policy_state()` | 无参数。 | `dict`，描述上层 preflight 使用的 local policy/readiness evidence。 | +| `WorldStateSpec.get_world_snapshot(task, spatial_limit)` | `task: str`、`spatial_limit: int`；读取 Layer 6 body state，以及已接线 spatial/temporal memory provider。 | `dict` world snapshot，包含 `robot_state`、`runtime`、`memory_state`、`semantic_temporal_map`、`sources` 和 snapshot-storage metadata。 | +| `WorldStateSpec.get_robot_state()` | 无参数。 | 上层 preflight 使用的 robot/body state `dict`。 | +| `WorldStateSpec.get_runtime_state()` | 无参数。 | `dict` runtime mode/config summary，例如 replay/simulation/hardware、MCP/viewer 设置。 | +| `WorldStateSpec.get_memory_state(task, spatial_limit)` | `task: str`、`spatial_limit: int`。 | `dict` memory section，包含 spatial/temporal availability、matches/summaries，以及 provider probe failure 的 errors。 | +| `WorldStateSpec.get_snapshot_storage_policy()` | 无参数。 | `dict`，说明 snapshot 是 transient、写 memory，还是持久化到其他位置。 | +| `SemanticTemporalMapSpec.query_semantic_temporal_map(query, spatial_limit)` | `query: str`、`spatial_limit: int`；读取 spatial 和 temporal memory provider。 | `dict`，包含 spatial section、temporal section、fused evidence entries、confidence/location/time summaries 和 source errors。 | +| `SkillInterfaceSpec.get_skill_interface_snapshot(domain)` | 可选 `domain: str` filter。 | `dict`，包含 `available`、`version`、`source`、`domain_filter`、`domains`、`skill_count` 和静态 contract 列表 `skills`。 | +| `SkillInterfaceSpec.get_skill_contract(skill_name)` | `skill_name: str`。 | 匹配的 skill-contract `dict`；没有 contract 时返回 `None`。 | +| `SkillInterfaceSpec.validate_skill_request(skill_name, args_json)` | `skill_name: str`、`args_json: str` JSON object。 | `dict`，包含 `valid`、`errors`、`warnings`、匹配的 `contract`；未知 skill 返回 `valid=False`。 | +| `SkillInterfaceSpec.compare_mcp_tools(tools_json)` | `tools_json: str`，MCP `tools/list` 风格 payload 或 list。 | `dict`，包含 `valid`、parser errors、contract/MCP counts、missing contracts、unregistered MCP tools、known internal tools。 | +| `route_task(task, context)` MCP skill | `task: str`，可选 `context: str`。 | `SkillResult` metadata: `domain`、`confidence`、`matched_keywords`、`recommended_tools`、`needs_context`、`reason`、`context_used`。 | +| `get_context(task, focus, spatial_limit)` MCP skill | `task: str`、可选 `focus: str`、`spatial_limit: int`；读取已接线 Layer 4、memory、skill、feedback、outcome、world-model provider。 | `SkillResult` message 加 metadata，包含 `sources`、`runtime`、`robot_state`、`world_state`、`skill_state`、`context_feedback`、`causal_state`、`world_model_state`、`context_evidence` 和 provider `errors`。 | +| `build_context_evidence(metadata, policy)` helper | ContextProvider 生成的 metadata dict，以及 `ContextEvidencePolicy` 阈值。 | `context_evidence.v1` dict，描述 selected evidence、dropped/low-confidence evidence、selected sources 和 policy effect。 | +| `memory_backend_status()` MCP skill | 无参数；对可选 provider 做 read probe。 | `SkillResult` metadata，schema 是 `go2_memory_backend_status.v1`，包含每个 provider 的 wired/available/probe 字段和 warnings。 | +| `record_evolution_event(event_type, task, payload_json, commit)` MCP skill | `event_type: str`、可选 `task: str`、`payload_json: str` JSON object、`commit: bool`。 | `SkillResult` metadata，包含 `schema`、`event_type`、`task`、`ledger_dir`、`event_path`、`commit_requested`、可选 `commit_sha`、`warnings` 和完整 `event`。 | +| `record_skill_proposal(proposal_json, commit)` MCP skill / ledger RPC | `proposal_json: str`，使用 `dimos.skill_proposal.v1`；`commit: bool`。 | `SkillResult` metadata，包含 `schema`、`proposal_id`、`ledger_dir`、`proposal_path`、`commit_requested`、可选 `commit_sha`、`warnings` 和校验后的 `proposal`。 | +| `EvolutionLedgerSpec.write_evolution_event(event_type, task, payload, commit)` | 其他 Layer 3 module 传入的结构化 payload `dict`。 | 和 `record_evolution_event` 相同的 ledger event record dict，但不经过 MCP JSON parsing。 | +| `EvolutionLedgerSpec.write_skill_proposal(proposal, commit)` | 已校验的 proposal `dict`。 | 和 `record_skill_proposal` 相同的 proposal record dict，但不经过 MCP JSON parsing。 | +| `evaluate_task_feasibility(task, context_json)` MCP skill | `task: str`、`context_json: str` JSON object，通常来自 `get_context` metadata；已接线时读取 Layer 5 contracts。 | `SkillResult` metadata: `feasible` (`yes`、`no`、`uncertain`)、`missing_context`、`required_skills`、`available_skills`、`missing_skills`、`safety_risks`、`recommended_next_action`、`clarifying_question`、`evidence_sources` 和 warnings。 | +| `record_context_feedback(task, context_evidence_json, selected_skill, outcome_json, helpful_sources_json, ignored_risks_json)` MCP skill | task text、`context_evidence.v1` JSON、可选 selected skill、outcome JSON object、helpful-source JSON list、ignored-risk JSON list。 | `SkillResult` metadata，包含一条 `go2_context_feedback.v1` feedback record、`total_feedback` 和可选 ledger warnings。 | +| `ContextFeedbackSpec.get_recent_context_feedback(limit, source)` | `limit: int`，可选 source filter。 | newest-first 的 `go2_context_feedback.v1` feedback dict 列表。 | +| `ContextFeedbackSpec.get_context_feedback_summary(limit)` | `limit: int`。 | aggregate `dict`，包含 success/failure/unknown outcome counts，以及 helpful/harmful source counters。 | +| `record_skill_outcome(skill_name, success, domain, error_code, message, risk, recovery)` MCP skill | skill name、success boolean、可选 domain/error/message/risk/recovery 字符串。 | `SkillResult` metadata，包含 recorded outcome dict 和 `total_outcomes`。 | +| `summarize_skill_outcomes(limit, skill_name, domain)` MCP skill | limit，以及可选 skill/domain filters。 | `SkillResult` metadata，包含过滤后的 newest-first `outcomes` list。 | +| `SkillOutcomeStoreSpec.get_recent_outcomes(limit, skill_name, domain)` | limit，以及可选 exact skill/domain filters。 | newest-first outcome dict 列表: timestamp、skill name、success、domain、error code、message、risk、recovery。 | +| `predict_skill_outcome(skill_name, args_json, context)` MCP skill | skill name、planned args JSON object、可选 context text；已接线时读取 SkillOutcomeStore 和 CausalWorldModel。 | `SkillResult` metadata，包含 `risk`、`predicted_success`、`failure_reasons`、`recovery_suggestions`、recent outcomes/transitions、可选 world-model prediction 和 provider availability flags。 | +| `record_causal_transition(...)` MCP skill | task、skill name、args JSON、before/after context text、prediction JSON、outcome JSON、domain、before/after state JSON。 | `SkillResult` metadata，包含 transition dict、`total_transitions`、`autosave_error`、`dashboard_error`。 | +| `predict_world_transition(snapshot_json, action_json, goal, horizon)` MCP skill / `predict_next_state(...)` RPC | Layer 4 snapshot JSON、包含 `skill_name` 和 `args` 的 action JSON、可选 goal、bounded horizon。 | `dimos.world_model_prediction.v1` dict，包含 action、snapshot summary、risk、predicted success、score、confidence、predicted symbolic delta、failure modes、reasons、model output、causal attribution、SCM explanation 和 intervention evidence。 | +| `score_action(snapshot_json, action_json, goal)` RPC | 与 prediction 相同的 snapshot/action/goal 输入。 | compact dict，包含 score、risk、confidence、predicted success、failure modes、reasons、model/causal/SCM/intervention evidence。 | +| `summarize_causal_patterns(skill_name, domain, limit)` MCP skill | 可选 skill/domain filters 和 limit。 | `SkillResult` metadata，包含 repeated causal `patterns` 和过滤后的 `transitions`。 | +| `record_intervention(...)` MCP skill | task、intervention name、target variable、before/after value JSON、action JSON、before/after snapshot JSON、outcome JSON、causal hypothesis。 | `SkillResult` metadata，包含 intervention record、`total_interventions`、`autosave_error`、`dashboard_error`。 | +| `save_world_model_state(path)` / `load_world_model_state(path)` MCP skills | 可选 path；如果未设置 `DIMOS_GO2_WORLD_MODEL_STATE`，则必须传 path。 | `SkillResult` summary，包含保存/加载的 model-state counts，以及可选 dashboard error。 | +| `CausalWorldModelSpec.get_recent_transitions(limit, skill_name, domain, cause)` | limit，以及可选 exact filters。 | newest-first transition dict 列表。 | +| `CausalWorldModelSpec.get_intervention_log(limit, target_variable, intervention_name)` | limit，以及可选 exact filters。 | newest-first intervention dict 列表。 | +| `CausalWorldModelSpec.get_model_state()` | 无参数。 | `dict`，包含 online model sample/weights summary、provider contract、causal estimator snapshot、intervention log snapshot、SCM snapshot 和 persistence config。 | +| `CausalWorldModelSpec.get_provider_contract()` | 无参数。 | `dimos.world_model_provider.v1` dict，声明 provider、model type、capabilities 和 output schemas。 | +| `propose_skill_interface(task, failure_context_json)` MCP skill | task text，以及带 missing-capability evidence 的 failure-context JSON object；已接线时读取 Layer 5 contracts/outcomes。 | `SkillResult` metadata: `proposal_created`、创建时的 `dimos.skill_proposal.v1` `proposal`、`existing_skill_matches`、`recommended_next_action`、可选 `proposal_path` 或 warnings。 | +| MCP `tools/list` / `tools/call` runtime surface | JSON-RPC requests；`tools/call` 携带 MCP arguments 和可选 `_mcp_context` progress/capability token。 | 带 capability metadata 的 tool schema；tool-call text/content result；background tool 的 SSE progress frames；instant return 或 stopped frame 触发 capability release。 | +| 可选 `WebsocketVisSpec.set_world_model_state(state)` | `state: dict`，预期使用 `dimos.world_model_dashboard_state.v1`。只有 blueprint 接入 `WebsocketVisModule` 时才生效；当前 `unitree_go2_agentic` 没有接。 | `dict` acknowledgement/current display state，供可选 Rerun/WebsocketVis clients 读取。 | + +## 需要 Review 的上游更新 + +这些不是本地自进化功能，但会影响本分支。 + +### 1. Zenoh Transport 集成 + +相关文件: + +- `dimos/core/transport_factory.py` +- `dimos/core/transport.py` +- `dimos/protocol/pubsub/impl/zenohpubsub.py` +- `dimos/protocol/service/zenohservice.py` +- `dimos/protocol/rpc/pubsubrpc.py` +- `dimos/protocol/tf/tf.py` +- `dimos/core/global_config.py` + +实现逻辑: + +- `GlobalConfig.transport` 现在可以是 `lcm` 或 `zenoh`。 +- macOS 上游默认值改为 `zenoh`，其他平台默认仍是 `lcm`。 +- `make_transport()` 根据当前后端把逻辑 channel name 映射成 LCM topic 或 Zenoh key expression。 +- Zenoh key expression 会放在 `dimos/` namespace 下。 +- RPC 和 TF 后端分别通过 `rpc_backend()`、`tf_backend()` 选择。 +- 高频 sensor channel 使用 latest-wins/best-effort QoS，human/agent 低频消息使用 reliable/blocking QoS。 + +Review 风险: + +- macOS 默认行为变了。如果本地运行仍假设 LCM，需要设置 `DIMOS_TRANSPORT=lcm`。 +- Zenoh 会创建 native pyo3 callback 线程，现有 pytest thread leak 检测可能在 cleanup 延迟时误报或暴露真实清理问题。 +- 长生命周期服务测试必须避开已经运行的 MCP server 端口。 + +### 2. MCP Tool Capability Gating + +相关文件: + +- `dimos/agents/capabilities.py` +- `dimos/agents/mcp/mcp_server.py` +- `dimos/agents/mcp/tool_stream.py` +- `dimos/agents/annotation.py` +- `dimos/agents/test_capabilities.py` + +实现逻辑: + +- `SkillInfo` 增加 capability 元数据，例如 `uses` 和 `lifecycle`。 +- `tools/list` 在相关工具上暴露 `_meta.dimos/uses` 和 `_meta.dimos/lifecycle`。 +- `tools/call` 在真正调用 skill 前先申请 capability lock。 +- instant skill 的占用可以等待；background skill 的占用需要先调用对应 stop tool。 +- background tool 的 capability release 绑定到 tool-stream stopped frame。 + +Review 风险: + +- 长时间运动或控制类 skill 必须正确声明 capability，否则 MCP server 无法串行化冲突行为。 +- background tool 必须稳定发出 stopped frame，否则 capability 可能被永久占用。 + +### 3. Coordinator RPC 和动态 Blueprint 加载 + +相关文件: + +- `dimos/core/coordination/coordinator_rpc.py` +- `dimos/core/coordination/module_coordinator.py` +- `dimos/core/coordination/worker_manager_python.py` + +实现逻辑: + +- `ModuleCoordinator` 可以暴露一个 singleton coordinator RPC service。 +- 客户端可以通过 coordinator RPC 查询模块、加载 blueprint。 +- 动态加载和 restart 期间，已部署 module proxy 由 lock 保护。 + +和本地改动的关系: + +- 本地 runtime plumbing 保留了 `McpClient` 的 non-blocking system-module notification，避免 agent 启动被直接 tool registration 阻塞。 +- 在 `DIMOS_TRANSPORT=lcm` 下，`ModuleCoordinator` 测试全部通过。 + +### 4. Memory 和 Perception 路径迁移 + +相关文件: + +- `dimos/perception/image_embedding.py` +- `dimos/perception/spatial_vector_db.py` +- `dimos/perception/visual_memory.py` +- `dimos/perception/spatial_perception.py` +- `dimos/memory2/*` + +实现逻辑: + +- 上游把 visual/spatial memory helper 从 `dimos.agents_deprecated.memory` 迁移到 `dimos.perception`。 +- 旧的 deprecated agent memory 目录被删除。 +- 上游新增了 memory2 TF service、replay 和 tooling。 + +本地解析: + +- `spatial_perception.py` 使用上游新的 `dimos.perception.*` 模块路径。 +- 仍保留本地默认持久化路径 `STATE_DIR / "spatial_memory"`，并支持 `DIMOS_SPATIAL_MEMORY_DIR` 覆盖，而不是改成上游 project assets output 路径。 + +### 5. Control、Manipulation、Learning、Docs、LFS 更新 + +概要: + +- control tasks 被拆成带 registry 的 package 目录。 +- 新增 roboplan、learning、data-prep 相关模块。 +- navigation 文档按 Mintlify 结构重组。 +- 新增 LFS 数据包和 native build 脚本。 + +Review 风险: + +- 这些是广泛的上游变更，不属于 Go2 自进化核心逻辑。主要审查 dependency、packaging、blueprint registry、CI 和运行环境影响。 + +## 本地新增功能清单 + +### 1. Go2 分层 Blueprint 架构 + +相关文件: + +- `dimos/robot/unitree/go2/blueprints/agentic/unitree_go2_agentic.py` +- `dimos/robot/unitree/go2/blueprints/smart/unitree_go2_spatial.py` +- `dimos/robot/unitree/go2/blueprints/agentic/unitree_go2_temporal_memory.py` +- `dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/__init__.py` +- `dimos/robot/unitree/go2/blueprints/layers/layer_4_world_state/__init__.py` +- `dimos/robot/unitree/go2/blueprints/layers/layer_5_skill_interface/__init__.py` +- `dimos/robot/unitree/go2/blueprints/layers/layer_6_robot_body/__init__.py` + +实现逻辑: + +- `unitree_go2_agentic` 现在由四层明确组成: `_go2_robot_body`、`_go2_spatial_world_state`、`_go2_skill_interface`、`_go2_agent_brain`。 +- 各 layer package 使用 lazy `__getattr__` 构建，避免 import 某一层时提前拉起重型 perception、VLM 或 robot stack。 +- `unitree_go2_spatial` 是非 agentic 的空间栈，由 Layer 6、Layer 4 spatial world state、Layer 5 spatial skills 组成。 +- `unitree_go2_temporal_memory` 通过 Layer 4 的 `_go2_temporal_memory_world_state()` 懒加载 temporal memory，确保 CLI config 已经生效后再创建 TemporalMemory blueprint。 + +为什么重要: + +- reviewer 可以分别审查 robot body、world state、skill interface、agent reasoning。 +- 防止 agent brain 变成无结构的大型 import hub。 +- 仍保留 DimOS blueprint 语义: 通过 `autoconnect()` 组合模块，通过 Spec 注入 RPC refs。 + +主要风险: + +- 分层不能掩盖漏接模块。`test_all_blueprints_generation.py` 和 Layer 4 wiring test 是主要保护。 + +组成部分: + +- `unitree_go2_agentic`: 完整 Go2 agentic stack，组合 physical body、structured world state、skill interface registry/containers、Layer 3 MCP/LLM brain。 +- `unitree_go2_spatial`: 没有 LLM brain 的 perception/world-state stack，用来单独验证 Layer 4 和 Layer 5。 +- `unitree_go2_temporal_memory`: 包住 agentic stack，并通过 Layer 4 factory 追加 temporal memory。 +- 各 layer package 的 `__init__.py`: lazy blueprint factories。它们不是普通 import 便利层，而是架构边界的一部分，避免过早 import 重型 perception/model 依赖。 + +构建数据流: + +1. CLI 通过 registry 懒加载命名 blueprint。 +2. blueprint 只 import 需要的 layer handle。 +3. 每个 layer handle 构建自己的 `autoconnect()` blueprint。 +4. 最终仍是普通 DimOS blueprint 组合，stream wiring 和 Spec-based RPC injection 仍由 `ModuleCoordinator` 完成。 +5. 测试验证 Layer 3 能通过预期 contract 访问 Layer 4/5/6，并且 blueprint registry 是最新的。 + +状态和持久化: + +- blueprint layer 本身不保存 robot 数据。 +- 状态由各层内部 module 持有: Layer 6 body state、Layer 4 memory/world snapshot、Layer 5 skill contracts、Layer 3 context/outcome/causal state。 +- 持久化属于具体 module，不属于 blueprint 文件。 + +安全边界: + +- blueprint 决定哪些 module 部署在一起。 +- blueprint 不决定调用哪个 skill，也不执行 motion。 +- wiring 错误应该按安全风险看待，因为 LLM 可能看到错误工具或错误上下文。 + +### 2. Layer 6 Robot Body State + +相关文件: + +- `dimos/robot/unitree/go2/blueprints/layers/layer_6_robot_body/robot_body_state.py` +- `dimos/robot/unitree/go2/blueprints/layers/layer_6_robot_body/robot_body_spec.py` +- `dimos/robot/unitree/go2/blueprints/layers/layer_6_robot_body/test_robot_body_state.py` + +实现逻辑: + +- `_Go2RobotBodyState` 是一个小模块，通过 spec-facing API 暴露 robot body 状态。 +- 它贴近真实 Go2 connection stack，但不替代真实连接模块。 +- 上层通过 typed spec 消费 robot state，而不是直接访问硬件 connection module。 + +Review 重点: + +- 是否提供了 agent preflight 足够使用的安全相关状态。 +- 是否避免重复持有或篡改真实控制权。 + +组成部分: + +- `_Go2RobotBodyState`: 把低层 robot status 适配成紧凑 Layer 6 provider 的 module。 +- `robot_body_spec.py`: 上层消费的 typed RPC contract。 +- Layer 6 package factory: 把底层 Go2 robot blueprint 和 body-state module 接在一起。 + +数据流: + +1. 底层 Go2 connection 和 robot modules 持有真实硬件/仿真通信。 +2. `_Go2RobotBodyState` 通过 spec 暴露 read-oriented body-state summary。 +3. Layer 4/Layer 3 把这些状态用于 safety preflight 和 context evidence。 +4. agent brain 看到的是摘要状态，不是直接硬件 handle。 + +判断主体: + +- body-state module 不做任务决策。 +- 它是 evidence provider。Layer 3 确定性 preflight 和 LLM 会使用它的输出判断等待、澄清或调用 skill。 + +状态和写边界: + +- 它应该被视为 current-state adapter。 +- 不应该持久化 memory，也不应该发控制命令。 +- 未来如果增加 body-state cache，必须有明确 bound 和 timestamp。 + +失败形态: + +- 如果底层 robot state 缺失，应明确返回 unavailable/unknown，不应伪造 ready。 +- motion-sensitive preflight 应把缺失 robot body state 视为风险。 + +### 3. Layer 4 Structured World State + +相关文件: + +- `dimos/robot/unitree/go2/blueprints/layers/layer_4_world_state/structured_world_state.py` +- `dimos/robot/unitree/go2/blueprints/layers/layer_4_world_state/semantic_temporal_map.py` +- `dimos/robot/unitree/go2/blueprints/layers/layer_4_world_state/world_state_spec.py` +- `dimos/robot/unitree/go2/blueprints/layers/layer_4_world_state/test_world_state.py` + +实现逻辑: + +- `_Go2StructuredWorldState` 通过 typed spec 暴露紧凑的 robot/world snapshot。 +- `_Go2SemanticTemporalMap` 把 spatial memory 和 temporal memory 摘要合并成更适合 agent 使用的 world-state surface。 +- Layer 3 通过 RPC spec 注入读取这些信息，不直接 import 具体 memory 实现。 + +这里的 "有效上下文" 是什么: + +- Layer 4 不让 LLM 直接判断 memory 相关性。 +- 它把可用 world/memory source 归一化成结构化 provider surface。 +- Layer 3 再负责打分、过滤、组合 evidence，最后交给 LLM 使用。 + +Review 重点: + +- memory backend 缺失时是否明确返回 unavailable，而不是静默空值。 +- snapshot schema 是否足够稳定，能被 prompt 和 dashboard 长期消费。 + +组成部分: + +- `_Go2StructuredWorldState`: 把 robot/world 信息规范化成 compact snapshot provider。 +- `_Go2SemanticTemporalMap`: 融合 spatial memory 和 temporal memory summary，产出 agent context 使用的 semantic evidence。 +- `world_state_spec.py`: Layer 3 依赖的 RPC contract。 +- `_go2_temporal_memory_world_state()`: lazy factory，延迟 TemporalMemory 构建，确保 `--new-memory` 等 CLI flags 已经生效。 + +数据流: + +1. SpatialMemory 和 TemporalMemory 仍是 storage/search 系统。 +2. Layer 4 查询或接收这些系统的 summary。 +3. Layer 4 把 summary 整形成稳定 world-state object，并明确 source availability。 +4. Layer 3 通过 spec 请求 world state，然后根据当前 task 选择 evidence。 + +状态和持久化: + +- Layer 4 可以读取持久化 memory backend，但它自身的 structured snapshot 是 derived view。 +- Temporal memory 持久化由 TemporalMemory module 和 CLI config 控制。 +- Spatial memory 持久化由 `SpatialConfig` 和 `spatial_perception.py` 的 state-dir 路径逻辑控制。 + +判断主体: + +- Layer 4 决定如何规范化 provider output。 +- Layer 4 不判断上下文是否足够行动。这个判断属于 Layer 3 preflight policy 和 LLM。 + +失败形态: + +- provider unavailable 应在 snapshot 里明确表示。 +- search result empty 不等于 backend failure，review 时要确认二者有区别。 + +### 4. Layer 5 Skill Interface Registry + +相关文件: + +- `dimos/robot/unitree/go2/blueprints/layers/layer_5_skill_interface/skill_interface_registry.py` +- `dimos/robot/unitree/go2/blueprints/layers/layer_5_skill_interface/skill_interface_spec.py` +- `dimos/robot/unitree/go2/blueprints/layers/layer_5_skill_interface/test_skill_interface_registry.py` + +实现逻辑: + +- skill contract 是显式静态记录，包含 skill name、domain、module、required args、optional args、是否 motion sensitive、context requirement、推荐 preflight、risk class、outcome shape。 +- registry 把这些 contract 作为数据暴露给 Layer 3。 +- 已知非 Layer 5 MCP tool 被排除在 contract mismatch 检查之外，避免 status/preflight 内部工具污染 skill coverage。 + +和自进化的关系: + +- 机器人不会自动修改 skill 代码。 +- 当任务失败且有明确 missing capability 证据时，可以生成待 review 的 skill interface proposal。 +- 现有 contract 是 duplicate suppression 的判断基准。 + +Review 重点: + +- 每个 MCP action skill 是否都有准确 contract。 +- `risk_class`、`requires_context`、`requires_robot_state` 应按 safety-critical metadata 审查。 + +组成部分: + +- 静态 `_SkillContract` 记录: 每个 action skill 的人工维护接口描述。 +- `_Go2SkillInterfaceRegistry`: 通过 RPC/MCP-facing 方法把 contracts 暴露给 Layer 3。 +- `skill_interface_spec.py`: 消费 skill metadata 的 typed contract。 +- Layer 5 blueprint factory: 接入 action skill containers、perception/security skills 和 registry。 + +数据流: + +1. action modules 通过 MCP 暴露 `@skill` 方法。 +2. registry 提供并行 contract view，包含比 MCP JSON schema 更丰富的 safety/context metadata。 +3. ContextProvider 和 TaskFeasibility 读取 contracts，判断需要哪些 context、arguments、preflight checks。 +4. SkillProposal 读取同一份 contracts，避免提重复 skill。 + +状态和写边界: + +- contracts 是代码/静态数据，不是 learned state。 +- registry 运行时只读。 +- self-evolution proposal 不会直接修改这个文件。开发者 review 后才编辑 contract。 + +判断主体: + +- 人类维护 source-of-truth contract 文案和风险元数据。 +- Layer 3 确定性代码消费这些数据。 +- LLM 可以使用 contract 信息，但不应发明 registry/MCP list 之外的隐藏能力。 + +失败形态: + +- 暴露的 skill 没有 contract 是 review failure，因为 preflight 无法知道 risk/context 需求。 +- contract 存在但 MCP tool 缺失也是 review failure，除非该 blueprint 有意禁用它。 + +### 5. Layer 3 Expert Router + +相关文件: + +- `dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/expert_router.py` +- `dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/prompt_policy.py` +- `dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/test_expert_router.py` +- `dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/test_prompt_policy.py` + +实现逻辑: + +- router 根据 task domain 推荐应该使用的 expert/tool family。 +- prompt policy 把 Layer 3 自进化约束注入 MCP client system prompt，但不完全替代机器人原有 prompt。 +- 输出是确定性的 routing metadata，不是额外 LLM call。 + +判断主体: + +- 第一层 routing 和 evidence packaging 由确定性代码完成。 +- LLM 仍是读取最终上下文并决定如何执行、是否澄清、是否调用工具的主体。 + +Review 重点: + +- routing label 是否和实际 skill contract 对齐。 +- prompt 文案是否过度承诺自治或自动改代码能力。 + +组成部分: + +- `_Go2ExpertRouter`: 确定性 task/domain router。 +- prompt policy helpers: 把 layer-specific instructions 合并进 agent system prompt。 +- router tests: 固定预期 label，避免行为无意扩大。 + +数据流: + +1. user task 或 subtask 进入 router。 +2. router 把任务分类到少量 domain，例如 navigation、perception、person follow、类似 manipulation 但当前不支持的任务、general conversation。 +3. 返回 routing metadata 和 recommended next tools/preflight。 +4. ContextProvider 可以把 routing output 放进 context。 +5. LLM 把 routing 当作提示，而不是不可逆 planner。 + +判断主体: + +- domain classification 是确定性的、可检查的。 +- tool choice 仍是 LLM/tool-calling decision，除非 deterministic preflight 明确拒绝或要求澄清。 + +状态和持久化: + +- router 无状态。 +- 它自己不写 ledger event 或 feedback。 + +失败形态: + +- 未知任务应该 route 到 uncertainty/general handling，不应伪造能力。 +- safety-sensitive domain 应偏向 preflight 和 clarification。 + +### 6. Context Provider 和 Evidence Selection + +相关文件: + +- `dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/context_provider.py` +- `dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/context_evidence.py` +- `dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/test_context_provider.py` +- `dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/test_context_evidence.py` + +实现逻辑: + +- `get_context()` 汇总 task text、runtime metadata、robot state、RAG/spatial memory、temporal memory、skill contracts、skill outcomes、causal world model、context feedback。 +- `context_evidence.py` 提供纯函数策略 `build_context_evidence(metadata, ContextEvidencePolicy)`。 +- policy 字段包括 `min_relevance_score`、`max_entries`、`include_low_confidence`、`require_robot_state_for_motion`。 +- 返回结果包含 compact context 和 `context_evidence.v1`，说明选了哪些 evidence、为什么选。 + +现在 "有效" 的标准: + +- 有效表示被确定性 policy 从可用 provider 里选中。 +- 主要依据包括 relevance score、confidence、source availability、task type、safety requirement。 +- 这还不是基于 replay metric 学出来的策略。 +- LLM 会消费这些上下文，并且仍可以判断上下文不足。 + +组成部分: + +- `_Go2ContextProvider`: MCP-facing context bundle provider。它是聚合器，读取可选 Layer 4/5/3 providers，并把结果整理成 agent 可用的 compact context。 +- `context_evidence.py`: 纯 evidence selector。它负责确定性的排序/过滤策略，可以在没有 robot、MCP server、memory backend 或 LLM 的情况下单测。 +- 可选 provider refs: world state、spatial/RAG memory、temporal memory、skill interface registry、skill outcome store、causal world model、context feedback store。 +- `context_evidence.v1`: 嵌在返回结果里的 review artifact。它记录哪些 evidence 被选中、丢弃或标成 low-confidence。 + +输入: + +- 调用方传入的 task text 和 runtime metadata。 +- 已接线 DimOS modules 提供的 provider snapshot/summary。 +- policy 参数，例如 relevance 阈值、最大条数、low-confidence 处理方式、motion task 是否必须有 robot state。 +- 已接线时的历史 feedback、outcome、world-model summary。 + +数据流: + +1. 调用方带着 task/runtime metadata 调用 `get_context()`。 +2. ContextProvider 向已接线 provider 请求 world state、memory summary、skill contract、outcome summary、causal-world-model state、context feedback。provider 没接线时，应贡献明确的 unavailable 状态，而不是静默丢掉该 source。 +3. metadata 被传给 `build_context_evidence()`。这个 helper 是纯函数: 给定 metadata 和 `ContextEvidencePolicy`，返回被选 evidence，不发 RPC、不写 memory、不调用 LLM。 +4. ContextProvider 格式化 compact context，并附加 `context_evidence.v1`，让 reviewer 能看到为什么选了这些 evidence。 +5. LLM 仍然是最终消费上下文、做用户可见推理和工具选择的主体。ContextProvider 本身不决定执行 skill。 + +状态和写边界: + +- ContextProvider 本身基本是 read-mostly，不写 RAG/vector memory、temporal memory、skill contracts 或 robot state。 +- 唯一的 durable side channel 是间接的: 它可能包含来自 feedback/outcome/ledger-aware modules 的摘要，但写入由那些 module 自己负责。 +- 被选中的 context 是临时 prompt artifact，除非另一个调用方显式记录 feedback 或 evolution event。 + +判断主体: + +- 确定性代码决定哪些 evidence 进入 context bundle。 +- LLM 决定如何消费这个 bundle、是否追问、是否调用已暴露 tool。 +- 人类 reviewer 判断确定性 policy 对当前机器人安全姿态是否足够保守。 + +失败形态: + +- 缺失 provider 应显示为明确 unavailable metadata。 +- provider payload 格式错误时，应降级该 source，而不是清空整个 context bundle。 +- 低相关或低置信 evidence 应按 policy 丢弃或标注，不能静默升级为可信 evidence。 + +没有实现的边界: + +- 还没有用 replay log 训练 learned ranker。 +- ContextProvider 不写 RAG/vector database。 +- context feedback 只是一个摘要信号，不会单独覆盖 provider data。 + +Review 重点: + +- relevance/confidence 默认阈值是否过松。 +- motion task 是否必须要求 robot state。 +- low-confidence evidence 应该隐藏，还是保留但显式标注。 + +### 7. Memory Backend Status + +相关文件: + +- `dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/memory_backend_status.py` +- `dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/test_memory_backend_status.py` + +实现逻辑: + +- 新增 MCP skill `memory_backend_status()`。 +- 报告 world state、spatial memory、temporal memory、skill outcomes、causal world model、skill interface 是否已接线。 +- 能 probe 的 backend 会做小型 read probe。 +- 只返回诊断元数据，不创建新的 memory database。 + +Review 重点: + +- 这是诊断工具，不是新的 memory scheduler。 +- 它不应该写入 RAG、spatial memory、temporal memory 或 Git。 + +组成部分: + +- `_Go2MemoryBackendStatus`: 作为 MCP skill 暴露的诊断 module。 +- 可选 provider specs: world state、spatial memory、temporal memory、outcome store、causal world model、skill interface registry。 +- probe helpers: 小型 read attempt，用来报告 availability 和 error。 + +数据流: + +1. skill 检查哪些可选 provider reference 已注入。 +2. 对每个 provider 记录 `wired`、`available` 和 probe result。 +3. probe failure 被捕获成 status metadata，而不是作为 hard crash 抛给 agent。 +4. 结果以 JSON/text 返回给 LLM 或 human operator。 + +状态和写边界: + +- 它是 read-only。 +- 不持久化 status snapshot。 +- 不初始化缺失 database。如果 database 没运行或没接线，status 应明确说明。 + +判断主体: + +- 这个 skill 不决定使用哪个 memory。 +- 它只回答“现在接了什么、能读什么”。ContextProvider 和 LLM 决定如何响应。 + +失败形态: + +- probe failure 应标明 provider 和 error string。 +- missing provider 应足够明确地区分 blueprint wiring 问题和 memory result empty。 + +### 8. Git-backed Evolution Ledger + +相关文件: + +- `dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/evolution_event.py` +- `dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/evolution_ledger.py` +- `dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/evolution_proposal.py` +- `dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/test_evolution_ledger.py` +- `dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/test_evolution_proposal.py` + +实现逻辑: + +- event 使用 schema `dimos.evolution_event.v1`。 +- proposal 由 `evolution_proposal.py` 做 schema 验证。 +- 默认写入本地仓库路径: + `.dimos/evolution/events/YYYY/MM/DD/*.json` + 和 `.dimos/evolution/proposals/*.json`。 +- `DIMOS_EVOLUTION_LEDGER_DIR` 可以覆盖存储目录。 +- 写入内容是 JSON 文件，目标是低频、可 review、可 diff 的控制面记录。 +- 默认 `commit=False`。 +- 如果 `commit=True`，只把该 event/proposal 文件提交到本地 Git。 + +组成部分: + +- `evolution_event.py`: 低层自进化 observation 的 schema 和校验，例如 feasibility assessment、context feedback、runtime review note。 +- `evolution_proposal.py`: 更高层 proposed change 的 schema 和校验。这些 proposal 必须先经人工 review，才可能变成代码或配置。 +- `_EvolutionLedger`: filesystem 和可选 Git writer。它负责路径选择、JSON serialization、commit mode 下的 staging/commit 行为。 +- 测试: 覆盖 schema validation、路径形状、commit 隔离和 proposal 持久化。 + +输入: + +- Layer 3 MCP tool 传入的 event/proposal payload。 +- 可选的 caller-controlled commit flag。 +- 可选的 `DIMOS_EVOLUTION_LEDGER_DIR` 覆盖。 +- commit mode 打开时的当前 Git repository context。 + +Git 内容会提交到哪里: + +- 文件先进入当前本地仓库工作区。 +- commit 只进入本地 `.git` 历史。 +- 不会自动 push 到 GitHub。 +- 只有人类后续运行 `git push` 或开 PR，GitHub 才会收到这些记录。 + +写入数据流: + +1. 调用方调用 ledger-facing MCP skill，例如 `record_evolution_event()` 或 `record_skill_proposal()`。 +2. 模块验证 schema，并规范化 payload。 +3. 模块选择存储 root。默认是 repo-local `.dimos/evolution`，`DIMOS_EVOLUTION_LEDGER_DIR` 可以覆盖。 +4. 写入 JSON 文件，作为可 review artifact。 +5. 如果 `commit=False`，除了工作区文件外不碰 Git。 +6. 如果 `commit=True`，只 stage 并提交该 event/proposal 文件到本地 Git，不 push。 + +状态和写边界: + +- ledger 是 durable filesystem state，不是内存学习模型。 +- 它只在配置的 ledger root 下面写 JSON artifact。 +- 它不修改 skill 代码、prompt、contracts、memory database 或 model weights。 +- commit mode 有意保持 local-only 且范围很窄: stage 该 artifact，创建本地 commit，然后停止。 + +判断主体: + +- 调用方 module 决定何时值得记录 event/proposal。 +- ledger 只判断 artifact 是否符合 schema，以及应该存到哪里。 +- 人类 reviewer 决定这些 ledger artifact 后续是否变成代码改动、PR、被 ignore 的本地轨迹，或训练/评估数据。 + +失败形态: + +- schema 无效应在写入前失败。 +- 不安全路径或路径穿越应在 filesystem access 前被拒绝。 +- Git commit 失败时，应让 JSON artifact 仍然在 worktree 可见，而不是假装已经提交。 + +隐私和 review 边界: + +- 这些文件可能包含 task text、outcome message、context summary、proposal rationale。真实机器人运行前，要先决定 `.dimos/evolution` 是否应该进入正常 Git 历史。 +- ledger 是控制面审计轨迹，不替代 SpatialMemory、TemporalMemory、SkillOutcomeStore 或 causal model state。 + +Review 重点: + +- event/proposal 字段是否可能造成路径穿越。 +- commit mode 是否只 stage 新 ledger 文件。 +- 根据隐私策略决定 `.dimos/evolution` 应该 ignore、commit，还是导出到别的存储。 + +### 9. Task Feasibility Preflight + +相关文件: + +- `dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/task_feasibility.py` +- `dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/test_task_feasibility.py` + +实现逻辑: + +- 新增 MCP skill `evaluate_task_feasibility(task, context_json)`。 +- 读取 skill contracts、world/robot context 和可选 JSON context。 +- 返回确定性结果: feasibility status、missing context、required skills、available skills、safety risks、recommended next action、clarifying question、evidence sources。 +- 如果 evolution ledger 已接线，可以写入 ledger event。 + +自进化含义: + +- 这不是模型自训练。 +- 它是 agent self-assessment: 在行动前，agent 可以调用确定性 evaluator 判断任务是否具备上下文和能力支持。 + +组成部分: + +- `_Go2TaskFeasibility`: deterministic preflight 的 MCP skill surface。 +- Skill contract reader: 消费 Layer 5 skill metadata，例如 required arguments、context requirements、risk class、motion sensitivity。 +- Context parser: 把显式 `context_json` 和已接线 world/robot provider summary 合并。 +- Ledger integration: 可选的 review trace event writer。 + +输入: + +- `task`: 用户请求或 subtask 的自然语言文本。 +- `context_json`: 调用方可选传入的结构化 context。 +- Layer 5 skill contracts 和 availability metadata。 +- 已接线时的 Layer 4 robot/world state。 +- context bundle 中可用的 outcome/context feedback summary。 + +决策/数据流: + +1. 解析 task 和可选 `context_json`。 +2. 把 task 匹配到已知 skill contract 和 expert domain。 +3. 根据 Layer 5 contract 检查 required args 和 context requirements。 +4. 对 motion-sensitive 或高风险 skill 检查 robot/world-state 是否可用。 +5. 输出三类粗粒度结果之一: `feasible`、`uncertain`、not feasible。 +6. 如果缺数据，推荐澄清问题或 context-gathering action。 +7. 如果缺能力，报告 missing capability evidence。这个 evidence 后续可以成为 skill-interface proposal 的依据。 + +状态和写边界: + +- 它对 robot state、memory 和 skill contracts 是 read-only。 +- 它不会调用目标 skill。 +- 它不会创建新 skill。 +- 如果 ledger 已接线，它可以写一条描述 preflight result 的 audit event，但不是代码改动。 + +判断主体: + +- feasibility classification 是确定性代码。 +- LLM 可以调用这个工具并消费结果，但 LLM 不是这个工具内部的 classifier。 +- 人类 review 判断这些确定性规则对真实硬件是否足够保守。 + +失败形态: + +- 缺少 required arguments 应产生 clarification，而不是新 skill proposal。 +- 缺少 context 应产生 context-gathering 或 uncertainty，而不是误报 feasible。 +- missing capability evidence 要足够具体，方便 SkillProposal 把它和 transient runtime failure 区分开。 + +Review 重点: + +- motion-sensitive task 是否会误判为可行。 +- 缺少 context 时是否优先澄清，而不是生成 skill proposal。 + +### 10. Context Feedback Store + +相关文件: + +- `dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/context_feedback.py` +- `dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/test_context_feedback.py` + +实现逻辑: + +- 新增 MCP skill `record_context_feedback(...)`。 +- 内存里维护最多 100 条 recent feedback 的 bounded deque。 +- schema 为 `go2_context_feedback.v1`。 +- 如果 evolution ledger 已接线，会把反馈写成 ledger event。 +- ContextProvider 会把 compact feedback summary 纳入后续 context。 + +Review 重点: + +- feedback 默认是 session memory，除非 ledger 已接线。 +- 它不能无界增长，也不应该静默覆盖 RAG 或 temporal memory。 + +组成部分: + +- `_Go2ContextFeedbackStore`: bounded in-memory store 和 MCP skill surface。 +- feedback entry schema: task/context identifiers、rating 或 usefulness signal、source labels、optional explanation、timestamp。 +- optional evolution ledger connection: 已接线时提供 durable audit trail。 +- ContextProvider integration: compact aggregate feedback summary。 + +数据流: + +1. agent 或 human-facing process 在 context bundle 有帮助、不完整、过期或误导后调用 `record_context_feedback(...)`。 +2. store 验证并规范化 feedback fields。 +3. entry append 到最多 100 条的 deque。 +4. 如果 ledger 已接线，feedback 也写成 evolution event。 +5. ContextProvider 读取 summary，而不是完整 raw feedback list，避免 prompt 膨胀。 + +状态和持久化: + +- 没有 ledger 时，feedback 是 process-local，重启丢失。 +- 有 ledger 时，feedback 成为可 review JSON，但不会自动训练 ranker。 + +判断主体: + +- feedback recording 是输入信号。 +- evidence policy 和 LLM 决定如何使用该信号。没有自动 prompt rewrite 或 memory deletion。 + +失败形态: + +- malformed feedback 应 validation fail。 +- feedback 超量时淘汰最旧 entry，不应无界增长。 + +### 11. Skill Outcome Store 和 Predictor + +相关文件: + +- `dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/skill_outcome_store.py` +- `dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/skill_outcome_predictor.py` +- `dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/test_skill_outcomes.py` + +实现逻辑: + +- 记录近期 skill outcome，存储是 bounded 的。 +- 按 skill 汇总成功、失败和模式。 +- predictor 根据近期历史和 skill contract metadata 估计风险或可能结果。 +- ContextProvider 可以把这些 summary 作为 task evidence。 + +Review 重点: + +- predictor 是启发式，不是学习模型。 +- 旧失败不应永久阻止可用 skill。 + +组成部分: + +- `_Go2SkillOutcomeStore`: bounded recent outcome history。 +- `_Go2SkillOutcomePredictor`: 基于近期 history 和 skill contract metadata 的 heuristic scorer。 +- outcome summary methods: 按 skill 聚合 successes、failures、repeated errors、recent messages。 + +数据流: + +1. skill call 后可以记录 skill name、success、error code、message、metadata。 +2. outcome store 把记录 append 到 bounded memory。 +3. summary 按 skill 和 error pattern 分组。 +4. predictor 把近期 outcome 和 skill contract risk/context metadata 合并，返回 risk 和 rationale。 +5. ContextProvider 可以把 summary 作为后续调用的 evidence。 + +状态和持久化: + +- 当前 store 是 bounded runtime memory，除非未来接入其他持久化。 +- 它和 Git ledger 分离。ledger 记录 review/audit event；outcome store 支持即时 runtime adaptation。 + +判断主体: + +- predictor 给出 heuristic risk hint。 +- 它不应单独 block 一个 skill；TaskFeasibility 或 LLM 应把它作为 evidence source 之一。 + +失败形态: + +- unknown skill 应返回 low-confidence 或 no-history output。 +- repeated failure 应提高谨慎度，但不应阻止 recovery/stop skills。 + +### 12. Causal World Model 和 Dashboard Contract + +相关文件: + +- `dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/causal_world_model.py` +- `dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/causal_effect_estimator.py` +- `dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/online_transition_model.py` +- `dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/structural_causal_model.py` +- `dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/intervention_log.py` +- `dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/world_model_contract.py` +- `dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/test_causal_world_model.py` +- `dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/test_world_model_contract.py` + +实现逻辑: + +- 记录 causal transition 和 intervention。 +- 维护近期 world-state change 的 online transition model。 +- 基于记录事件估计 causal effect。 +- 输出 versioned contract: + `dimos.world_model_prediction.v1`、 + `dimos.world_model_provider.v1`、 + `dimos.world_model_dashboard.v1`。 +- 支持 save/load model state。 + +它是不是训练了模型: + +- 严格说: 有一个非常小的在线模型，但没有离线训练流程，也没有神经网络世界模型。 +- `OnlineTransitionOutcomeModel` 是 lightweight logistic-style linear model。它把 feature weight 存在 dict 里，每次 `record_causal_transition()` 收到一次已观测 success/failure outcome 时增量更新。 +- 如果还没有记录 transition，它的 `sample_count` 是 0，此时预测只能视为低置信先验行为，不是已学习的机器人知识。 +- symbolic SCM 是手写的，不是从数据学出来的。 +- causal effect estimator 是观测统计估计，只比较 feature active/inactive 的平滑成功率差异，不控制 hidden confounders。 + +组成部分: + +- `OnlineTransitionOutcomeModel`: 在线 success-probability scorer。feature 包括 bias、skill name、domain、是否有 odom、navigation state、spatial memory 是否可用、spatial match count、temporal 是否可用、action argument 是否存在和部分 argument value。更新方式是一阶 gradient adjustment，带 L2 shrinkage。 +- `CausalEffectEstimator`: bounded observation store，对 symbolic feature 做 treated/control success-rate difference。它能给出 risk factor、supporting factor 和 intervention suggestion，但输出里明确声明 unobserved confounders are not controlled。 +- `StructuralCausalModel`: 手写因果图，变量包括 `spatial_has_matches`、`target_resolvability`、`odom_ready`、`motion_safety`、`navigation_success`。它还会生成 counterfactual suggestion，例如语义导航前先 tag 或 perceive target。 +- `InterventionLog`: 记录显式 intervention，并让 prediction 能引用历史 intervention evidence。 +- `_WorldTransition`: compact event record，把 task、skill、before/after state、predicted risk、outcome、inferred cause、recovery suggestion 绑定起来。 + +记录数据流: + +1. `record_causal_transition()` 接收 task、skill name、可选 args、before/after Layer 4 snapshot、prediction metadata、outcome metadata。 +2. 输入会按 JSON object 解析和验证。缺少 outcome 时，如果 SkillOutcomeStore 已接线，可以回退到同 skill 的最新 outcome。 +3. 模块根据 context、prediction reasons、outcome success、error code、message 推断粗粒度 cause 和 recovery suggestion。 +4. 模块计算 before/after snapshot 的 symbolic state delta。 +5. 调用 `_outcome_model.update(...)` 和 `_causal_estimator.update(...)`。 +6. 把 transition append 到内存 deque，最多保留 200 条。 +7. 只有设置了 `DIMOS_GO2_WORLD_MODEL_STATE` 时才 autosave。 +8. 如果 WebsocketVis 已接线，会 publish dashboard state。 + +预测数据流: + +1. `predict_next_state()` 解析 snapshot 和 candidate action。 +2. 读取同 skill 的 recent transitions，提取 repeated failure modes。 +3. 根据当前 snapshot 和 action 计算 rule-based risk reasons。 +4. 调用在线模型得到 success probability、score、risk、confidence、top weighted feature contribution。 +5. 调用 causal estimator 得到 observational risk/support factors。 +6. 调用手写 SCM 得到 variables、edges、counterfactuals。 +7. 调用 intervention log 查找匹配的 intervention evidence。 +8. 合并 rule risk 和 model risk，得到最终 risk 和 score。 +9. 返回 `dimos.world_model_prediction.v1`，并可选 publish dashboard contract。 + +可信边界: + +- 输出只是 advisory evidence，用来帮助 LLM 选择 preflight、perception、clarification 或更安全的 skill path。 +- 它不能直接命令机器人运动。 +- 当前 high confidence 需要同 skill 足够样本；否则应当视为低置信启发式。 + +Review 重点: + +- 当前 causal estimate 只能作为 advisory evidence。 +- prediction schema 要对 dashboard 和 LLM prompt consumer 保持稳定。 +- persistence 不应意外混合不同 run 的数据。 +- UI/prompt 文案不能暗示这是完整训练好的 physical world model。 +- 在依赖它做自主 action selection 前，应补 replay 或 simulation evaluation。 + +### 13. Skill Interface Proposal Generator + +相关文件: + +- `dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/skill_proposal.py` +- `dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/test_skill_proposal.py` + +实现逻辑: + +- 新增 MCP skill `propose_skill_interface(task, failure_context_json)`。 +- 只有在 failure context 里有明确 missing capability evidence 时，才生成 `dimos.skill_proposal.v1` review artifact。 +- 对 missing arguments、missing context 不生成 proposal。 +- 如果现有 contract 已覆盖能力，则抑制重复 proposal。 +- 如果 ledger 已接线，通过 evolution ledger 写入 proposal 文件。 +- 不修改 Python 代码、不新增 `@skill`、不改 blueprint wiring。 + +决策树: + +1. 解析 `failure_context_json`。 +2. 查找明确 missing capability evidence，而不是只看执行失败。 +3. 查询现有 Layer 5 contracts，避免 proposal 重复。 +4. 如果只是缺 argument、缺 context、provider 暂时 unavailable，则拒绝生成 proposal。 +5. 构造 proposal artifact，包含 task、evidence、suggested interface shape、expected inputs、expected output、review rationale。 +6. 如果 ledger 已接线，通过 evolution ledger 写入 proposal。 + +人工边界: + +- 这个工具故意只做 proposal。开发者仍然需要写代码、选择 container、加 `@skill`、更新 contract/prompt，并测试行为。 + +Review 重点: + +- 这是最安全的自进化边界: 只生成 proposal，必须人工 review。 +- evidence requirement 要足够严格，避免用户表达不清时产生大量噪声 proposal。 + +组成部分: + +- `_Go2SkillProposalGenerator`: proposal generation 的 MCP skill surface。 +- 现有 Layer 5 contracts: duplicate suppression 的 source of truth。 +- Evolution ledger: 可选 proposal writer。 +- Proposal schema: 描述 missing capability 和 proposed interface 的 review artifact，不是可执行代码。 + +输入: + +- `task`: 失败的 user/subtask text。 +- `failure_context_json`: 描述为什么现有 skills 失败或不足的结构化 evidence。 +- 现有 contracts 和已知 MCP tool names。 + +接受的 evidence: + +- 明确 missing capability。 +- repeated failure 显示没有可用 skill 能完成目标操作。 +- 现有 contracts 没覆盖的 domain/action gap。 + +拒绝的 evidence: + +- 只是缺少现有 skill 的 required argument。 +- 只是缺 context 或 memory/provider unavailable。 +- 临时 runtime failure。 +- 能力已被现有 contract 覆盖。 + +输出边界: + +- 输出是 JSON proposal 和可选 ledger 文件。 +- 不 import、不写入、不生成 Python code。 +- 不自动更新 prompt 或 registry。 + +### 14. MCP Runtime Plumbing + +相关文件: + +- `dimos/agents/mcp/mcp_client.py` +- `dimos/agents/mcp/mcp_server.py` +- `dimos/agents/mcp/tool_stream.py` +- `dimos/agents/mcp/test_mcp_client_unit.py` +- `dimos/agents/mcp/test_mcp_server.py` +- `dimos/agents/mcp/test_tool_stream.py` + +实现逻辑: + +- `McpServer.on_system_modules()` 避免通过 RPC 查询自己的 proxy。 +- 对自身 skill 和带 actor class 的 proxy，server 本地收集 schema。 +- `McpClient` 在 RPC path 外延迟 direct tool registration，避免启动阶段互相等待。 +- tool-stream progress frame 可以通过 persistent SSE 发送，并带 progress-token metadata。 +- background capability release 绑定 stopped frame。 +- `agent_send()` 使用 `make_transport()`，因此会尊重当前 backend。 + +Review 重点: + +- 启动路径不能等待依赖自身初始化的工具。 +- MCP 测试端口必须和 live robot session 隔离。 + +组成部分: + +- `McpServer`: 暴露 MCP JSON-RPC endpoints、tool list/call handling、SSE progress stream、server status tools、capability gating、agent-send。 +- `McpClient`: LLM agent module，负责发现 tools 并暴露给 agent runtime。 +- `tool_stream.py`: 长运行 skill 的 progress/log notification path。 +- capability registry: server-side lock manager，用于 mutually exclusive tool capabilities。 + +启动数据流: + +1. ModuleCoordinator 部署 worker 并启动 modules。 +2. `McpServer.start()` 启动 HTTP/SSE serving，并订阅 tool-stream frames。 +3. `on_system_modules()` 注册 tool schemas。对 server 自身和 actor-class backed modules，本地收集 schema，避免 RPC self-deadlock。 +4. `McpClient` 初始化 agent，但不在 RPC path 外阻塞等待 direct tool registration。 + +Tool call 数据流: + +1. JSON-RPC `tools/call` 到达。 +2. server 解析 `SkillInfo` 和 RPC call target。 +3. 如果 skill 声明 `uses`，先申请 capability lock。 +4. progress token 和 capability token 通过保留 `_mcp_context` 注入。 +5. RPC call 在 executor 里运行。 +6. instant skill 返回后释放 capability；background skill 把 release 交给 tool-stream stopped frame。 + +状态和失败边界: + +- server state 是 process-local: skills、rpc calls、SSE queues、capability registry。 +- port conflict 是外部环境失败。live server 存在时，测试必须用隔离 MCP port。 +- tool not found 应返回结构化 MCP text result，而不是 crash server。 + +### 15. Runtime Hardening + +相关文件: + +- `dimos/core/coordination/module_coordinator.py` +- `dimos/core/coordination/worker_manager_python.py` +- `dimos/simulation/mujoco/mujoco_process.py` +- `dimos/robot/unitree/mujoco_connection.py` +- `dimos/robot/unitree/go2/connection.py` +- `dimos/robot/unitree/go2/test_connection.py` + +实现逻辑: + +- coordinator 的 system-module notification 避免等待已知 agent client。 +- worker manager/coordinator 变更保留 module restart、reload、stream rewiring 行为。 +- MuJoCo helper 可以读取当前可用 stderr，避免阻塞。 +- macOS headless MuJoCo 在需要时使用当前 Python executable。 +- Go2 WebRTC connection 会从 `GlobalConfig` 转发 AES key。 + +Review 重点: + +- 这些是运行时稳定性修复，重点审查 process lifecycle 和 stream rewiring，而不是 agent reasoning。 + +组成部分: + +- ModuleCoordinator notification path: 控制 build/start/system-module notification 顺序。 +- WorkerManagerPython: 在 workers 中部署 modules，并管理 worker pool lifecycle。 +- MuJoCo process helpers: 处理 subprocess executable 选择和 stderr draining。 +- Go2 connection config: replay stop 行为和 WebRTC AES key forwarding。 + +数据流: + +1. build/deploy 通过 ModuleCoordinator 和 worker managers 完成。 +2. modules 收到 system-module notification 前，stream transports 和 RPC refs 已经 wiring。 +3. 部分 notification 使用 nowait，避免等待正在通过 MCP 初始化的 agent client。 +4. restart/reload paths 必须保留 transports 并重新 wiring module refs。 +5. simulator helper code 把平台特定 process 问题隔离在机器人栈外层。 + +状态和写边界: + +- Coordinator 持有 deployed module proxy maps、transport registry、module transports、reload aliases。 +- 这些改动不改变 skill semantics。 +- 它们影响 module 是否能干净 start、stop、reload、reconnect。 + +失败形态: + +- build/start 卡住时应 stop managers 并暴露原始异常。 +- restart 不应遗留 old transports，也不应让 consumer 连在 dead proxy 上。 +- ReplayConnection stop 应该是 no-op，而不是 exception。 + +### 16. 可选 Rerun/WebsocketVis World-model Panel + +相关文件: + +- `dimos/web/websocket_vis/websocket_vis_module.py` +- `dimos/web/websocket_vis_spec.py` +- `dimos/web/templates/rerun_dashboard.html` +- `dimos/web/websocket_vis/test_websocket_vis_module.py` + +实现逻辑: + +- websocket visualization 除已有 payload 外，还可以携带 world-model/dashboard state。 +- dashboard payload shape 与 `world_model_contract.py` 对齐。 +- 这是可选观测能力。当前 `unitree_go2_agentic` blueprint 不包含 + `WebsocketVisModule`，因此除非单独的可视化 blueprint 接入 + `WebsocketVisSpec`，这条路径不会启动。 + +Review 重点: + +- dashboard state 应保持展示用途。 +- dashboard consumer 不应依赖未 version 的内部 Python object。 +- 不要把它当成 Go2 自进化 runtime 的必需依赖。 + +组成部分: + +- `websocket_vis_spec.py`: visualization update 的 typed surface。 +- `websocket_vis_module.py`: 存储/publish dashboard state 的 runtime module。 +- `rerun_dashboard.html`: browser-facing display surface。 +- `world_model_contract.py`: world-model dashboard payload 的 schema source。 + +数据流: + +1. CausalWorldModel 使用 `dimos.world_model_dashboard.v1` 生成 dashboard payload。 +2. 如果 WebsocketVis 已接线，CausalWorldModel 调用 `set_world_model_state(...)`。 +3. WebsocketVis 保存并向 dashboard clients 提供 latest state。 +4. dashboard 把 state 渲染成人类 inspection data。 + +状态和持久化: + +- dashboard state 是 latest display snapshot，不是 source of truth。 +- 如果配置了 durable state，它属于 CausalWorldModel save/load JSON。 +- browser/dashboard consumer 只应依赖 versioned payload fields。 + +安全边界: + +- dashboard update 不能触发机器人动作。 +- UI 不应把 advisory prediction 展示成 guaranteed outcome。 +- 如果没有接 `WebsocketVisModule`，CausalWorldModel 仍然正常工作；dashboard + publish path 会被跳过。 + +## 推荐 Review 顺序 + +1. 先看 Go2 blueprint wiring: + `unitree_go2_agentic.py`、Layer 3/4/5/6 `__init__.py`、`test_world_state.py`。 +2. 再看 MCP runtime plumbing: + `mcp_server.py`、`mcp_client.py`、`tool_stream.py` 和对应测试。 +3. 再看自进化写入边界: + `evolution_ledger.py`、`evolution_proposal.py`、`skill_proposal.py`。 +4. 再看 context 质量: + `context_provider.py`、`context_evidence.py`、`context_feedback.py`。 +5. 再看 action safety: + `skill_interface_registry.py`、`task_feasibility.py`、`skill_outcome_predictor.py`。 +6. 再看 advisory world-model: + `causal_world_model.py` 和 `world_model_contract.py`。 +7. 最后单独 review 上游 transport 变化，尤其是 macOS 默认 Zenoh。 + +## 本次同步后的验证结果 + +已通过: + +- `.venv/bin/python -m pytest dimos/agents/mcp/test_mcp_server.py -q` + - `10 passed, 1 warning` +- `.venv/bin/python -m pytest dimos/robot/unitree/go2/test_connection.py -q` + - `5 passed` +- `.venv/bin/python -m pytest dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain -q` + - `62 passed` +- `.venv/bin/python -m pytest dimos/robot/unitree/go2/blueprints/layers/layer_4_world_state/test_world_state.py -q` + - `3 passed, 1 warning` +- `.venv/bin/python -m pytest dimos/perception/test_spatial_perception_config.py -q` + - `2 passed` +- `DIMOS_TRANSPORT=lcm .venv/bin/python -m pytest dimos/core/coordination/test_module_coordinator.py -q` + - `35 passed` +- `DIMOS_TRANSPORT=lcm MCP_PORT=9991 .venv/bin/python -m pytest dimos/agents/mcp/test_mcp_client_unit.py dimos/agents/mcp/test_tool_stream.py -q` + - `44 passed, 1 warning` +- `.venv/bin/python -m pytest dimos/robot/test_all_blueprints_generation.py -q` + - `1 passed` +- `git diff --cached --check -- ':(exclude)*.patch'` + - passed + +已知 caveat: + +- 本机有 live `dimos-live` 进程占用默认 MCP 端口 `9990`，所以默认端口跑 MCP 测试会失败。 +- 上游在 macOS 默认 `zenoh` backend 后，部分测试暴露 native pyo3 thread 生命周期问题。同样 coordinator 和 MCP 测试在 `DIMOS_TRANSPORT=lcm` 下通过。 +- 上游 `.patch` 文件包含会触发完整 `git diff --check` 的 whitespace；当前验证排除了 `.patch`，避免破坏补丁语义。 From 86526e41c7e393b36d95a06f33c4c8cdbaf165c0 Mon Sep 17 00:00:00 2001 From: ZhangZheMing <1050297972@qq.com> Date: Sun, 5 Jul 2026 10:01:50 +0800 Subject: [PATCH 26/36] Restore Go2 review docs and diagrams --- .../2026-07-05-dimos-project-architecture.png | 3 + ...026-07-05-go2-agent-architecture-review.md | 1476 +++++++++++++++++ ...-07-05-go2-agent-architecture-review.zh.md | 28 +- .../2026-07-05-go2-agent-architecture.png | 3 + 4 files changed, 1498 insertions(+), 12 deletions(-) create mode 100644 docs/reviews/2026-07-05-dimos-project-architecture.png create mode 100644 docs/reviews/2026-07-05-go2-agent-architecture-review.md create mode 100644 docs/reviews/2026-07-05-go2-agent-architecture.png diff --git a/docs/reviews/2026-07-05-dimos-project-architecture.png b/docs/reviews/2026-07-05-dimos-project-architecture.png new file mode 100644 index 0000000000..6574655451 --- /dev/null +++ b/docs/reviews/2026-07-05-dimos-project-architecture.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bdb38a4089ea44c83e826a198ea18bb19aae5db857ccb7dc02c10dd5e1288660 +size 221143 diff --git a/docs/reviews/2026-07-05-go2-agent-architecture-review.md b/docs/reviews/2026-07-05-go2-agent-architecture-review.md new file mode 100644 index 0000000000..c3f75cdc17 --- /dev/null +++ b/docs/reviews/2026-07-05-go2-agent-architecture-review.md @@ -0,0 +1,1476 @@ +# Go2 Agent Architecture Review Notes + +Date: 2026-07-05 +Branch: `refactor/go2-architecture-layers` +Local HEAD before upstream merge: `646e7ec5` +Upstream merged target: `upstream/main` at `6e813a72` +Merge base: `1f544d05` + +This document is the entry point for a full review of the current fork +changes. It separates upstream changes from local feature work, then describes +each local feature in enough detail to review implementation logic rather than +only file names. + +## Current Git State + +The branch had 16 local commits not in upstream, and upstream had 160 commits +not in the branch. The accurate conflict surface from the merge base was 15 +overlapping files; the actual merge conflicts were 7 files: + +- `dimos/agents/mcp/mcp_server.py` +- `dimos/agents/mcp/test_mcp_server.py` +- `dimos/perception/spatial_perception.py` +- `dimos/robot/unitree/go2/blueprints/agentic/unitree_go2_agentic.py` +- `dimos/robot/unitree/go2/blueprints/agentic/unitree_go2_temporal_memory.py` +- `dimos/robot/unitree/go2/blueprints/smart/unitree_go2_spatial.py` +- `dimos/robot/unitree/go2/test_connection.py` + +Resolution strategy: + +- Keep upstream infrastructure changes, especially Zenoh transport, capability + gating, AES key forwarding, docs and packaging updates. +- Keep the local Go2 layer architecture as the authoritative Go2 agentic wiring. +- Use upstream's new perception module paths, while preserving the local + state-dir based spatial-memory default. +- Combine non-overlapping tests rather than dropping either side. + +Important environment note: + +- Upstream now defaults macOS transport to `zenoh`. That requires + `eclipse-zenoh`; I installed `eclipse-zenoh==1.9.0` in the local `.venv`. +- The machine already has a live `dimos` process listening on MCP port `9990`. + MCP tests must use another port, for example `MCP_PORT=9991`, or they will + talk to the live server instead of the test server. +- `git diff --cached --check` passes when excluding upstream `.patch` files. + Full `diff --check` reports trailing whitespace inside upstream patch files; + those are patch context lines and should be handled separately if we want a + repository-wide whitespace cleanup. + +## Review Granularity + +Use this structure when reviewing each local feature. The goal is to avoid +approving a feature just because the file names sound plausible. + +- Entry point: Which blueprint, module, MCP skill, RPC method, or helper is the + first public surface? +- Inputs: Which JSON strings, spec providers, memory providers, or runtime + states feed it? +- Outputs: Which `SkillResult`, dict schema, Spec return value, file artifact, + or dashboard payload does the interface return? +- Decision owner: Is the decision made by deterministic code, the LLM reading + context, a small online statistical model, or a human reviewer? +- State: Is the feature stateless, in-memory only, persisted to JSON, persisted + to Git, or stored in an existing RAG/vector database? +- Write boundary: Which exact method writes, and which methods are read-only? +- Failure mode: Does missing data produce `unavailable`, `uncertain`, a + clarifying question, a failed `SkillResult`, or a proposal artifact? +- Safety boundary: Can it execute robot motion, or only advise another layer? +- Review artifact: Which schema should be stable enough for future tooling? +- Tests: Which test proves the data path, and which test proves the failure path? + +The most important distinction in this branch is between three very different +things: + +- Context selection: deterministic filtering and summarization before the LLM + sees context. +- Agent judgment: the LLM deciding what to do with selected context and tools. +- Self-evolution artifact: reviewable JSON events/proposals, not automatic code + mutation. + +## Architecture Diagram + +![DimOS Go2 agent self-evolution architecture](2026-07-05-go2-agent-architecture.png) + +The purple right-side column is optional observability/storage. The current +`unitree_go2_agentic` blueprint does not wire `WebsocketVisModule`; the Rerun / +WebsocketVis panel only receives world-model state if a separate visualization +blueprint wires `WebsocketVisSpec`. + +## Project Architecture Diagram + +![DimOS project architecture](2026-07-05-dimos-project-architecture.png) + +This diagram is repo-wide, not Go2-specific. It shows the main DimOS flow: +entry points, blueprint/configuration, core runtime, communication transports, +functional module families, robots/simulation/hardware, and data/model/ +observability assets. + +## Interface Input/Output Quick Reference + +Use this section when reviewing API boundaries. It lists the stable public +surfaces added or changed by the local Go2 architecture work; helper functions +that are not consumed across module boundaries are intentionally omitted. + +| Surface | Inputs | Outputs | +| --- | --- | --- | +| Go2 blueprint factories (`unitree_go2_agentic`, `unitree_go2_spatial`, `unitree_go2_temporal_memory`) | CLI blueprint name plus `GlobalConfig` values already resolved by DimOS. Lazy layer imports have no runtime arguments. | DimOS `Blueprint` objects composed with `autoconnect()`. They output module wiring, streams, and Spec injection paths, not user-facing JSON. | +| `RobotBodyStateSpec.get_robot_body_snapshot()` | No arguments; reads recent odom/image/lidar observations and connection/local-policy summaries from Layer 6. | `dict` with connection state, sensor state, local policy state, safety state, and freshness/availability markers. | +| `RobotBodyStateSpec.get_connection_state()` | No arguments. | `dict` describing connection availability/config-derived mode. | +| `RobotBodyStateSpec.get_sensor_state()` | No arguments. | `dict` with observed sensor counters/freshness such as image, lidar, and odom availability. | +| `RobotBodyStateSpec.get_local_policy_state()` | No arguments. | `dict` describing local policy/readiness state used as evidence by upper layers. | +| `WorldStateSpec.get_world_snapshot(task, spatial_limit)` | `task: str`, `spatial_limit: int`; reads Layer 6 body state plus spatial/temporal memory providers when wired. | `dict` world snapshot with `robot_state`, `runtime`, `memory_state`, `semantic_temporal_map`, `sources`, and snapshot-storage metadata. | +| `WorldStateSpec.get_robot_state()` | No arguments. | `dict` robot/body state view for upper-layer preflight. | +| `WorldStateSpec.get_runtime_state()` | No arguments. | `dict` runtime mode/config summary such as replay/simulation/hardware and MCP/viewer settings. | +| `WorldStateSpec.get_memory_state(task, spatial_limit)` | `task: str`, `spatial_limit: int`. | `dict` memory section with spatial and temporal availability, matches/summaries, and errors when a provider probe fails. | +| `WorldStateSpec.get_snapshot_storage_policy()` | No arguments. | `dict` explaining whether snapshots are transient, written to memory, or persisted elsewhere. | +| `SemanticTemporalMapSpec.query_semantic_temporal_map(query, spatial_limit)` | `query: str`, `spatial_limit: int`; reads spatial and temporal memory providers. | `dict` with spatial section, temporal section, fused evidence entries, confidence/location/time summaries, and source errors. | +| `SkillInterfaceSpec.get_skill_interface_snapshot(domain)` | Optional `domain: str` filter. | `dict` with `available`, `version`, `source`, `domain_filter`, `domains`, `skill_count`, and a `skills` list of static contracts. | +| `SkillInterfaceSpec.get_skill_contract(skill_name)` | `skill_name: str`. | Matching skill-contract `dict`, or `None` when no contract exists. | +| `SkillInterfaceSpec.validate_skill_request(skill_name, args_json)` | `skill_name: str`, `args_json: str` JSON object. | `dict` with `valid`, `errors`, `warnings`, and the matched `contract`; unknown skills return `valid=False`. | +| `SkillInterfaceSpec.compare_mcp_tools(tools_json)` | `tools_json: str` containing an MCP `tools/list` style payload or list. | `dict` with `valid`, parser errors, contract/MCP counts, missing contracts, unregistered MCP tools, and known internal tools. | +| `route_task(task, context)` MCP skill | `task: str`, optional `context: str`. | `SkillResult` metadata: `domain`, `confidence`, `matched_keywords`, `recommended_tools`, `needs_context`, `reason`, and `context_used`. | +| `get_context(task, focus, spatial_limit)` MCP skill | `task: str`, optional `focus: str`, `spatial_limit: int`; reads Layer 4, memory, skill, feedback, outcome, and world-model providers when wired. | `SkillResult` message plus metadata containing `sources`, `runtime`, `robot_state`, `world_state`, `skill_state`, `context_feedback`, `causal_state`, `world_model_state`, `context_evidence`, and provider `errors`. | +| `build_context_evidence(metadata, policy)` helper | Metadata dict from ContextProvider and `ContextEvidencePolicy` thresholds. | `context_evidence.v1` dict describing selected evidence, dropped/low-confidence evidence, selected sources, and policy effects. | +| `memory_backend_status()` MCP skill | No arguments; read probes optional providers. | `SkillResult` metadata with schema `go2_memory_backend_status.v1`, per-provider wired/available/probe fields, and warnings. | +| `record_evolution_event(event_type, task, payload_json, commit)` MCP skill | `event_type: str`, optional `task: str`, `payload_json: str` JSON object, `commit: bool`. | `SkillResult` metadata with `schema`, `event_type`, `task`, `ledger_dir`, `event_path`, `commit_requested`, optional `commit_sha`, `warnings`, and full `event`. | +| `record_skill_proposal(proposal_json, commit)` MCP skill / ledger RPC | `proposal_json: str` using `dimos.skill_proposal.v1`, `commit: bool`. | `SkillResult` metadata with `schema`, `proposal_id`, `ledger_dir`, `proposal_path`, `commit_requested`, optional `commit_sha`, `warnings`, and validated `proposal`. | +| `EvolutionLedgerSpec.write_evolution_event(event_type, task, payload, commit)` | Structured payload `dict` from another Layer 3 module. | Same ledger event record dict as `record_evolution_event`, without MCP JSON parsing. | +| `EvolutionLedgerSpec.write_skill_proposal(proposal, commit)` | Validated proposal `dict`. | Same proposal record dict as `record_skill_proposal`, without MCP JSON parsing. | +| `evaluate_task_feasibility(task, context_json)` MCP skill | `task: str`, `context_json: str` JSON object, usually from `get_context` metadata. Reads Layer 5 contracts when wired. | `SkillResult` metadata: `feasible` (`yes`, `no`, `uncertain`), `missing_context`, `required_skills`, `available_skills`, `missing_skills`, `safety_risks`, `recommended_next_action`, `clarifying_question`, `evidence_sources`, and warnings. | +| `record_context_feedback(task, context_evidence_json, selected_skill, outcome_json, helpful_sources_json, ignored_risks_json)` MCP skill | Task text, `context_evidence.v1` JSON, optional selected skill, outcome JSON object, helpful-source JSON list, ignored-risk JSON list. | `SkillResult` metadata with one `go2_context_feedback.v1` feedback record, `total_feedback`, and optional ledger warnings. | +| `ContextFeedbackSpec.get_recent_context_feedback(limit, source)` | `limit: int`, optional source filter. | Newest-first list of `go2_context_feedback.v1` feedback dicts. | +| `ContextFeedbackSpec.get_context_feedback_summary(limit)` | `limit: int`. | Aggregate `dict` with counts for success/failure/unknown outcomes plus helpful/harmful source counters. | +| `record_skill_outcome(skill_name, success, domain, error_code, message, risk, recovery)` MCP skill | Skill name, success boolean, optional domain/error/message/risk/recovery strings. | `SkillResult` metadata with recorded outcome dict and `total_outcomes`. | +| `summarize_skill_outcomes(limit, skill_name, domain)` MCP skill | Limit plus optional skill/domain filters. | `SkillResult` metadata with filtered newest-first `outcomes` list. | +| `SkillOutcomeStoreSpec.get_recent_outcomes(limit, skill_name, domain)` | Limit plus optional exact skill/domain filters. | Newest-first list of outcome dicts: timestamp, skill name, success, domain, error code, message, risk, recovery. | +| `predict_skill_outcome(skill_name, args_json, context)` MCP skill | Skill name, planned args JSON object, optional context text. Reads SkillOutcomeStore and CausalWorldModel when wired. | `SkillResult` metadata with `risk`, `predicted_success`, `failure_reasons`, `recovery_suggestions`, recent outcomes/transitions, optional world-model prediction, and provider availability flags. | +| `record_causal_transition(...)` MCP skill | Task, skill name, args JSON, before/after context text, prediction JSON, outcome JSON, domain, before/after state JSON. | `SkillResult` metadata with transition dict, `total_transitions`, `autosave_error`, and `dashboard_error`. | +| `predict_world_transition(snapshot_json, action_json, goal, horizon)` MCP skill / `predict_next_state(...)` RPC | Layer 4 snapshot JSON, action JSON with `skill_name` and `args`, optional goal, bounded horizon. | `dimos.world_model_prediction.v1` dict with action, snapshot summary, risk, predicted success, score, confidence, predicted symbolic delta, failure modes, reasons, model output, causal attribution, SCM explanation, and intervention evidence. | +| `score_action(snapshot_json, action_json, goal)` RPC | Same snapshot/action/goal inputs as prediction. | Compact dict with score, risk, confidence, predicted success, failure modes, reasons, model/causal/SCM/intervention evidence. | +| `summarize_causal_patterns(skill_name, domain, limit)` MCP skill | Optional skill/domain filters and limit. | `SkillResult` metadata with repeated causal `patterns` and filtered `transitions`. | +| `record_intervention(...)` MCP skill | Task, intervention name, target variable, before/after value JSON, action JSON, before/after snapshot JSON, outcome JSON, causal hypothesis. | `SkillResult` metadata with intervention record, `total_interventions`, `autosave_error`, and `dashboard_error`. | +| `save_world_model_state(path)` / `load_world_model_state(path)` MCP skills | Optional path; required unless `DIMOS_GO2_WORLD_MODEL_STATE` is set. | `SkillResult` summary with saved/loaded model-state counts plus optional dashboard error. | +| `CausalWorldModelSpec.get_recent_transitions(limit, skill_name, domain, cause)` | Limit plus optional exact filters. | Newest-first list of transition dicts. | +| `CausalWorldModelSpec.get_intervention_log(limit, target_variable, intervention_name)` | Limit plus optional exact filters. | Newest-first list of intervention dicts. | +| `CausalWorldModelSpec.get_model_state()` | No arguments. | Dict with online model sample/weights summary, provider contract, causal estimator snapshot, intervention log snapshot, SCM snapshot, and persistence config. | +| `CausalWorldModelSpec.get_provider_contract()` | No arguments. | `dimos.world_model_provider.v1` dict naming provider, model type, capabilities, and output schemas. | +| `propose_skill_interface(task, failure_context_json)` MCP skill | Task text and failure-context JSON object with missing-capability evidence. Reads Layer 5 contracts/outcomes when wired. | `SkillResult` metadata: `proposal_created`, `proposal` using `dimos.skill_proposal.v1` when created, `existing_skill_matches`, `recommended_next_action`, optional `proposal_path` or warnings. | +| MCP `tools/list` / `tools/call` runtime surface | JSON-RPC requests; `tools/call` also carries MCP arguments and optional `_mcp_context` progress/capability token. | Tool schemas with capability metadata; tool-call text/content results; SSE progress frames for background tools; capability release on instant return or stopped frame. | +| Optional `WebsocketVisSpec.set_world_model_state(state)` | `state: dict` expected to use `dimos.world_model_dashboard_state.v1`. This is only active when a blueprint wires `WebsocketVisModule`; current `unitree_go2_agentic` does not. | `dict` acknowledgement/current display state for optional Rerun/WebsocketVis clients. | + +## Upstream Changes To Review + +These are not local self-evolution features, but they can affect this branch. + +1. Zenoh transport integration + + Files: + + - `dimos/core/transport_factory.py` + - `dimos/core/transport.py` + - `dimos/protocol/pubsub/impl/zenohpubsub.py` + - `dimos/protocol/service/zenohservice.py` + - `dimos/protocol/rpc/pubsubrpc.py` + - `dimos/protocol/tf/tf.py` + - `dimos/core/global_config.py` + + Implementation logic: + + - `GlobalConfig.transport` can be `lcm` or `zenoh`. + - On Darwin, upstream now defaults to `zenoh`; elsewhere it defaults to + `lcm`. + - `make_transport()` maps logical names to either LCM topics or Zenoh key + expressions. Zenoh topics are namespaced under `dimos/`. + - RPC and TF backends are selected through `rpc_backend()` and `tf_backend()`. + - High-rate sensor channels get best-effort/latest-wins QoS; human/agent + channels get reliable/blocking QoS. + + Review risk: + + - macOS default behavior changed. If existing local runs assume LCM, set + `DIMOS_TRANSPORT=lcm`. + - Zenoh creates native pyo3 callback threads. Existing pytest thread-leak + checks can flag these if cleanup is delayed. + - Tests involving long-lived services must avoid collisions with an already + running MCP server. + +2. Capability gating for MCP tools + + Files: + + - `dimos/agents/capabilities.py` + - `dimos/agents/mcp/mcp_server.py` + - `dimos/agents/mcp/tool_stream.py` + - `dimos/agents/annotation.py` + - `dimos/agents/test_capabilities.py` + + Implementation logic: + + - `SkillInfo` now carries capability metadata such as `uses` and + `lifecycle`. + - `tools/list` includes `_meta.dimos/uses` and `_meta.dimos/lifecycle` when + relevant. + - `tools/call` acquires capability locks before invoking a skill. + - Instant holders may be waited on; background holders require a stop tool. + - Background tool capability release is tied to a tool-stream stopped frame. + + Review risk: + + - Skill annotations must accurately declare long-running motion/control + capabilities, otherwise the server cannot serialize conflicting behaviors. + - Tool-stream stop frames must always be emitted for background tools that + hold capabilities. + +3. Coordinator RPC and dynamic blueprint loading + + Files: + + - `dimos/core/coordination/coordinator_rpc.py` + - `dimos/core/coordination/module_coordinator.py` + - `dimos/core/coordination/worker_manager_python.py` + + Implementation logic: + + - `ModuleCoordinator` can expose a singleton coordinator RPC service. + - Clients can list modules and load blueprints through that coordinator RPC. + - Existing deployed module proxies are protected by a lock during dynamic + loading and restart operations. + + Local interaction: + + - Our local runtime plumbing preserved non-blocking system-module + notification for `McpClient`, so agent startup is not blocked by direct + tool registration. + - Validation passed under `DIMOS_TRANSPORT=lcm`. + +4. Memory and perception migration + + Files: + + - `dimos/perception/image_embedding.py` + - `dimos/perception/spatial_vector_db.py` + - `dimos/perception/visual_memory.py` + - `dimos/perception/spatial_perception.py` + - `dimos/memory2/*` + + Implementation logic: + + - Upstream moved visual/spatial memory helpers out of + `dimos.agents_deprecated.memory` into `dimos.perception`. + - Old deprecated agent memory modules were removed. + - A new memory2 TF service/replay/tooling area was added upstream. + + Local interaction: + + - We use upstream's new module paths in `spatial_perception.py`. + - We keep local default persistence under `STATE_DIR / "spatial_memory"`, + with `DIMOS_SPATIAL_MEMORY_DIR` override, instead of upstream's project + assets output path. + +5. Control, manipulation, learning, docs, and LFS updates + + Summary: + + - Control tasks were split into package directories with registries. + - Roboplan/learning/data-prep modules were added. + - Navigation docs were reorganized for Mintlify. + - New LFS data packages and native build scripts were added. + + Review risk: + + - These are broad upstream changes and not directly part of the Go2 + self-evolution logic. They should be reviewed mainly for dependency, + packaging, and blueprint registry side effects. + +## Local Feature Inventory + +### 1. Go2 Layered Blueprint Architecture + +Files: + +- `dimos/robot/unitree/go2/blueprints/agentic/unitree_go2_agentic.py` +- `dimos/robot/unitree/go2/blueprints/smart/unitree_go2_spatial.py` +- `dimos/robot/unitree/go2/blueprints/agentic/unitree_go2_temporal_memory.py` +- `dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/__init__.py` +- `dimos/robot/unitree/go2/blueprints/layers/layer_4_world_state/__init__.py` +- `dimos/robot/unitree/go2/blueprints/layers/layer_5_skill_interface/__init__.py` +- `dimos/robot/unitree/go2/blueprints/layers/layer_6_robot_body/__init__.py` + +Implementation logic: + +- `unitree_go2_agentic` is now composed from four explicit layers: + `_go2_robot_body`, `_go2_spatial_world_state`, `_go2_skill_interface`, and + `_go2_agent_brain`. +- Layer packages use lazy `__getattr__` construction so importing one layer does + not eagerly import heavy perception, VLM, or robot stacks. +- `unitree_go2_spatial` is the non-agentic spatial stack built from Layer 6, + Layer 4 spatial world state, and Layer 5 spatial skills. +- `unitree_go2_temporal_memory` adds temporal memory through Layer 4's lazy + `_go2_temporal_memory_world_state()` helper after CLI config has been applied. + +Why this matters: + +- Reviewers can inspect robot body, world state, skill interface, and agent + reasoning independently. +- It prevents the agent brain from becoming an unstructured import hub. +- It preserves DimOS blueprint semantics: modules are still composed with + `autoconnect()` and RPC refs are injected by specs. + +Primary risk: + +- The layering must not hide missing blueprint modules. The static registry test + and Layer 4 wiring test are the main protections. + +Components: + +- `unitree_go2_agentic`: the full agentic Go2 stack. It composes the physical + body, structured world state, skill interface registry/containers, and Layer + 3 MCP/LLM brain. +- `unitree_go2_spatial`: the perception/world-state stack without the LLM brain. + It is useful for validating Layer 4 and Layer 5 without agent behavior. +- `unitree_go2_temporal_memory`: an additive blueprint that wraps the agentic + stack and adds temporal memory through the Layer 4 factory. +- Layer package `__init__.py` files: lazy blueprint factories. They are part of + the architecture, not just import convenience, because they prevent import + side effects from pulling in heavy perception/model dependencies too early. + +Construction flow: + +1. CLI imports a named blueprint lazily through the registry. +2. The blueprint imports only the layer handles it needs. +3. Each layer handle builds an `autoconnect()` blueprint for its modules. +4. The final blueprint is still ordinary DimOS composition, so stream wiring and + Spec-based RPC injection happen in `ModuleCoordinator`. +5. Tests validate that Layer 3 can reach Layer 4/5/6 through the intended + contracts and that the generated blueprint registry remains current. + +State and persistence: + +- The blueprint layer itself stores no robot data. +- State is owned by modules inside the layers: body state in Layer 6, memory + and world snapshots in Layer 4, skill contracts in Layer 5, context/outcome + and causal state in Layer 3. +- Any persistence belongs to those modules, not the blueprint file. + +Safety boundary: + +- The blueprint decides what modules are deployed together. +- It does not decide which skill to call, and it does not execute motion. +- Reviewers should treat wiring mistakes as safety risks because the LLM may + see missing or wrong tools/context. + +### 2. Layer 6 Robot Body State + +Files: + +- `dimos/robot/unitree/go2/blueprints/layers/layer_6_robot_body/robot_body_state.py` +- `dimos/robot/unitree/go2/blueprints/layers/layer_6_robot_body/robot_body_spec.py` +- `dimos/robot/unitree/go2/blueprints/layers/layer_6_robot_body/test_robot_body_state.py` + +Implementation logic: + +- `_Go2RobotBodyState` is a small module that exposes robot-body status through + a spec-facing API. +- It sits next to the real Go2 connection stack rather than replacing it. +- Higher layers consume robot state through a typed spec rather than directly + reaching into the hardware connection module. + +Review focus: + +- Make sure it reports enough state for safety-sensitive agent preflight. +- Make sure it does not duplicate live control authority. + +Components: + +- `_Go2RobotBodyState`: the module that adapts low-level robot status into a + compact Layer 6 provider. +- `robot_body_spec.py`: the typed RPC contract consumed by upper layers. +- Layer 6 package factory: wires the underlying Go2 robot blueprint with the + body-state module. + +Data flow: + +1. The underlying Go2 connection and robot modules own live hardware/simulation + communication. +2. `_Go2RobotBodyState` exposes read-oriented body-state summaries through its + spec. +3. Layer 4/Layer 3 use the spec for safety preflight and context evidence. +4. The agent brain sees summarized state, not direct hardware handles. + +Decision owner: + +- The body-state module does not make task decisions. +- It is an evidence provider. Layer 3 deterministic preflight and the LLM use + its output to decide whether to wait, clarify, or call a skill. + +State and write boundary: + +- It should be treated as a current-state adapter. +- It should not persist memory and should not issue control commands. +- Any future body-state caching should be explicitly bounded and timestamped. + +Failure mode: + +- If the underlying robot state is missing, the correct behavior is explicit + unavailable/unknown state, not fabricated readiness. +- Motion-sensitive preflight should treat missing robot body state as risk. + +### 3. Layer 4 Structured World State + +Files: + +- `dimos/robot/unitree/go2/blueprints/layers/layer_4_world_state/structured_world_state.py` +- `dimos/robot/unitree/go2/blueprints/layers/layer_4_world_state/semantic_temporal_map.py` +- `dimos/robot/unitree/go2/blueprints/layers/layer_4_world_state/world_state_spec.py` +- `dimos/robot/unitree/go2/blueprints/layers/layer_4_world_state/test_world_state.py` + +Implementation logic: + +- `_Go2StructuredWorldState` exposes a compact robot/world snapshot through a + typed spec. +- `_Go2SemanticTemporalMap` combines spatial and temporal memory summaries into + a more agent-friendly world-state surface. +- Layer 3 consumes this via RPC spec injection instead of importing memory + implementations directly. + +What "effective context" means here: + +- This layer does not ask the LLM to judge memory relevance directly. +- It normalizes available world and memory sources into a structured provider + surface; Layer 3 then scores, filters, and packages evidence for the LLM. + +Review focus: + +- Check that missing memory backends degrade to explicit unavailable statuses. +- Check that snapshot schemas are stable enough for prompt and dashboard + consumers. + +Components: + +- `_Go2StructuredWorldState`: normalizes robot/world information into a compact + snapshot provider. +- `_Go2SemanticTemporalMap`: fuses spatial memory and temporal memory summaries + into semantic evidence for agent context. +- `world_state_spec.py`: the RPC contract Layer 3 relies on. +- `_go2_temporal_memory_world_state()`: lazy factory that delays TemporalMemory + construction until CLI flags such as `--new-memory` are applied. + +Data flow: + +1. SpatialMemory and TemporalMemory remain the storage/search systems. +2. Layer 4 queries or receives summaries from those systems. +3. Layer 4 shapes them into a stable world-state object with explicit source + availability. +4. Layer 3 asks for world state through the spec and then selects evidence for + the current task. + +State and persistence: + +- Layer 4 may read from persistent memory backends, but its own structured + snapshot is a derived view. +- Temporal memory persistence is controlled by the TemporalMemory module and CLI + config. +- Spatial memory persistence is controlled by `SpatialConfig` and the state-dir + path logic in `spatial_perception.py`. + +Decision owner: + +- Layer 4 decides how to normalize provider output. +- Layer 4 does not decide whether the context is sufficient for action. That + decision belongs to Layer 3 preflight policy and the LLM. + +Failure mode: + +- Provider unavailable means the snapshot should include unavailable metadata. +- Empty search results are not the same as backend failure; reviewers should + check that these are represented differently. + +### 4. Layer 5 Skill Interface Registry + +Files: + +- `dimos/robot/unitree/go2/blueprints/layers/layer_5_skill_interface/skill_interface_registry.py` +- `dimos/robot/unitree/go2/blueprints/layers/layer_5_skill_interface/skill_interface_spec.py` +- `dimos/robot/unitree/go2/blueprints/layers/layer_5_skill_interface/test_skill_interface_registry.py` + +Implementation logic: + +- Skill contracts are explicit static records: skill name, domain, module, + required args, optional args, motion sensitivity, context requirements, + preflight recommendations, risk class, and outcome shape. +- The registry exposes contracts to Layer 3 as data. +- Known non-Layer-5 MCP tools are excluded from contract mismatch checks so + internal status/preflight tools do not pollute skill coverage. + +Self-evolution relationship: + +- The robot does not mutate skills automatically. +- Missing capability evidence can create a reviewable proposed skill interface. +- Existing contracts are the ground truth for duplicate suppression. + +Review focus: + +- Verify every exposed action skill has an accurate contract. +- Treat risk class and required context as safety-critical metadata. + +Components: + +- Static `_SkillContract` records: the human-authored interface description for + each expected action skill. +- `_Go2SkillInterfaceRegistry`: exposes contracts through RPC/MCP-facing + methods for Layer 3. +- `skill_interface_spec.py`: typed contract for modules that consume skill + metadata. +- Layer 5 blueprint factory: wires action skill containers, perception/security + skills, and the registry. + +Data flow: + +1. Action modules expose `@skill` methods through MCP. +2. The registry provides a parallel contract view with richer safety/context + metadata than the MCP JSON schema alone. +3. ContextProvider and TaskFeasibility read contracts to decide what context, + arguments, and preflight checks are needed. +4. SkillProposal reads the same contracts to avoid proposing duplicate skills. + +State and write boundary: + +- Contracts are code/static data, not learned state. +- The registry is read-only at runtime. +- Self-evolution proposals never mutate this file directly. A developer edits + contracts after review. + +Decision owner: + +- Humans own the source-of-truth contract text and risk metadata. +- Deterministic Layer 3 code consumes it. +- The LLM may use contract information, but should not invent hidden skill + capabilities outside the registry/MCP list. + +Failure mode: + +- Missing contract for an exposed skill is a review failure because preflight + cannot know risk/context needs. +- Contract exists but MCP tool is absent is also a review failure unless it is + intentionally disabled for a blueprint. + +### 5. Layer 3 Expert Router + +Files: + +- `dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/expert_router.py` +- `dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/prompt_policy.py` +- `dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/test_expert_router.py` +- `dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/test_prompt_policy.py` + +Implementation logic: + +- The router classifies task domains and recommends which expert/tool family + should be used. +- Prompt policy injects Layer 3 self-evolution guidance into the MCP client + system prompt without replacing robot-specific prompts entirely. +- The output is deterministic routing metadata, not an LLM call. + +Judgment subject: + +- Deterministic code performs the first-pass routing and evidence packaging. +- The LLM remains the actor that reads the final context and decides how to + execute or ask clarification, unless a deterministic MCP skill is explicitly + called. + +Review focus: + +- Verify routing labels line up with actual skill contracts. +- Watch for prompt text that could overclaim autonomy or code mutation. + +Components: + +- `_Go2ExpertRouter`: deterministic task/domain router. +- Prompt policy helpers: merge layer-specific instructions into the agent system + prompt. +- Router tests: pin expected labels and guard against accidental broadening. + +Data flow: + +1. A user task or subtask is passed to the router. +2. The router classifies the task into a small set of domains such as + navigation, perception, person follow, manipulation-like unsupported work, + or general conversation. +3. It returns routing metadata and recommended next tools/preflight. +4. ContextProvider can include routing output in context. +5. The LLM uses routing as a hint, not as an irreversible planner. + +Decision owner: + +- Domain classification is deterministic and inspectable. +- Tool choice remains an LLM/tool-calling decision unless a deterministic + preflight skill refuses or asks for clarification. + +State and persistence: + +- The router is stateless. +- It does not write ledger events or feedback by itself. + +Failure mode: + +- Unknown tasks should route to uncertainty/general handling, not fabricate a + capability. +- Safety-sensitive domains should bias toward preflight and clarification. + +### 6. Context Provider And Evidence Selection + +Files: + +- `dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/context_provider.py` +- `dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/context_evidence.py` +- `dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/test_context_provider.py` +- `dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/test_context_evidence.py` + +Implementation logic: + +- `get_context()` gathers available data from task text, runtime metadata, + robot state, RAG/spatial memory, temporal memory, skill contracts, skill + outcomes, causal world model, and context feedback. +- `context_evidence.py` contains the pure policy helper + `build_context_evidence(metadata, ContextEvidencePolicy)`. +- The policy supports thresholds such as `min_relevance_score`, `max_entries`, + `include_low_confidence`, and `require_robot_state_for_motion`. +- The returned context contains a compact answer plus `context_evidence.v1`, + which explains which evidence was selected and why. + +What "effective" means today: + +- Effective means selected by deterministic policy from available providers: + relevance score, confidence, source availability, task type, and safety needs. +- It is not yet learned from replay metrics. +- The LLM consumes the selected context and may still judge it insufficient. + +Components: + +- `_Go2ContextProvider`: MCP-facing context bundle provider. It is the + aggregator that reads optional Layer 4/5/3 providers and formats the compact + context returned to the agent. +- `context_evidence.py`: pure evidence selector. It owns the deterministic + ranking/filtering policy and can be unit-tested without a running robot, + MCP server, memory backend, or LLM. +- Optional provider refs: world state, spatial/RAG memory, temporal memory, + skill interface registry, skill outcome store, causal world model, and + context feedback store. +- `context_evidence.v1`: review artifact embedded in the response. It records + which evidence entries were selected, dropped, or marked low-confidence. + +Inputs: + +- Task text and runtime metadata supplied by the caller. +- Provider snapshots/summaries from wired DimOS modules. +- Policy values such as relevance threshold, entry cap, low-confidence + handling, and whether motion tasks require robot state. +- Prior feedback/outcome/world-model summaries when those providers are wired. + +Data flow: + +1. The caller invokes `get_context()` with task/runtime metadata. +2. ContextProvider asks wired providers for world state, memory summaries, + skill contracts, outcome summaries, causal-world-model state, and context + feedback. If a provider is not wired, the provider contributes an explicit + unavailable status rather than throwing away the source silently. +3. Metadata is passed to `build_context_evidence()`. That helper is pure: + given metadata and a `ContextEvidencePolicy`, it returns selected evidence + without calling RPC, writing memory, or consulting an LLM. +4. ContextProvider formats the compact context and attaches + `context_evidence.v1` so reviewers can see why the context contained those + entries. +5. The LLM is still the consumer and final user-facing reasoning actor. The + provider does not decide to execute a skill. + +State and write boundary: + +- ContextProvider itself is read-mostly. It does not write RAG/vector memory, + temporal memory, skill contracts, or robot state. +- The only durable side channel is indirect: it may include summaries from + feedback/outcome/ledger-aware modules, but those modules own their writes. +- The selected context is a transient prompt artifact unless another caller + explicitly records feedback or an evolution event. + +Decision owner: + +- Deterministic code decides what evidence enters the context bundle. +- The LLM decides how to reason over that bundle, whether to ask a clarifying + question, and whether to call an exposed tool. +- Human reviewers decide whether the deterministic policy is acceptable for a + given robot safety posture. + +Failure mode: + +- Missing providers should appear as explicit unavailable metadata. +- Malformed provider payloads should degrade that source, not erase the whole + context bundle. +- Low relevance or low confidence evidence should be dropped or marked + according to policy, never silently upgraded to trusted evidence. + +What is not implemented: + +- No learned ranker is trained from replay logs yet. +- No RAG/vector database entries are written by ContextProvider. +- Context feedback is summarized as a signal, but it does not override provider + data by itself. + +Review focus: + +- Check whether relevance/confidence defaults are too permissive. +- Check whether motion tasks always require robot state. +- Check whether low-confidence evidence should be visible but marked, or hidden. + +### 7. Memory Backend Status + +Files: + +- `dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/memory_backend_status.py` +- `dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/test_memory_backend_status.py` + +Implementation logic: + +- Adds MCP skill `memory_backend_status()`. +- Reports whether providers are wired for world state, spatial memory, temporal + memory, skill outcomes, causal world model, and skill interface. +- Performs small read probes where possible. +- Returns status metadata only; it does not create another memory database. + +Review focus: + +- This is diagnostic, not a memory scheduler. +- It should never write to RAG, spatial memory, temporal memory, or Git. + +Components: + +- `_Go2MemoryBackendStatus`: diagnostic module exposed as an MCP skill. +- Optional provider specs: world state, spatial memory, temporal memory, + outcome store, causal world model, and skill interface registry. +- Probe helpers: small read attempts that report availability and errors. + +Data flow: + +1. The skill checks which optional provider references are injected. +2. For each provider, it records `wired`, `available`, and any probe result. +3. Probe failures are captured as status metadata rather than raised to the + agent as hard crashes. +4. The result is returned as JSON/text for the LLM or human operator. + +State and write boundary: + +- It is read-only. +- It does not persist a status snapshot. +- It does not initialize missing databases. If a database is not running or not + wired, the status should say so. + +Decision owner: + +- The skill does not decide which memory to use. +- It answers the operational question "what is connected and readable right + now?" ContextProvider and the LLM decide how to respond to that. + +Failure mode: + +- A probe failure should identify the provider and error string. +- A missing provider should be explicit enough to distinguish blueprint wiring + problems from empty memory results. + +### 8. Git-Backed Evolution Ledger + +Files: + +- `dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/evolution_event.py` +- `dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/evolution_ledger.py` +- `dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/evolution_proposal.py` +- `dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/test_evolution_ledger.py` +- `dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/test_evolution_proposal.py` + +Implementation logic: + +- Events use schema `dimos.evolution_event.v1`. +- Proposals use schema records validated by `evolution_proposal.py`. +- Default storage is local repository path: + `.dimos/evolution/events/YYYY/MM/DD/*.json` and + `.dimos/evolution/proposals/*.json`. +- `DIMOS_EVOLUTION_LEDGER_DIR` can override that location. +- Writes are JSON files with deterministic schema and reviewable payloads. +- `commit=False` by default. +- If `commit=True`, the module creates a local Git commit containing only the + event/proposal file. + +Components: + +- `evolution_event.py`: schema and validation for low-level self-evolution + observations such as feasibility assessments, context feedback, and runtime + review notes. +- `evolution_proposal.py`: schema and validation for higher-level proposed + changes that need human review before becoming code or configuration. +- `_EvolutionLedger`: filesystem and optional Git writer. It owns path + selection, JSON serialization, and commit-mode staging/commit behavior. +- Tests: verify schema validation, path shape, commit isolation, and proposal + persistence. + +Inputs: + +- Event/proposal payload supplied by a Layer 3 MCP tool. +- Optional caller-controlled commit flag. +- Optional `DIMOS_EVOLUTION_LEDGER_DIR` override. +- Current Git repository context when commit mode is enabled. + +Where does Git data go: + +- It goes into this local repository's working tree and local `.git` history. +- It does not push to GitHub. +- GitHub only receives it if a human later runs `git push` or opens a PR. + +Write data flow: + +1. A caller invokes a ledger-facing MCP skill such as + `record_evolution_event()` or `record_skill_proposal()`. +2. The module validates the schema and normalizes the payload. +3. The module chooses the storage root. The default is the repo-local + `.dimos/evolution` tree; `DIMOS_EVOLUTION_LEDGER_DIR` overrides it. +4. The JSON file is written as the review artifact. +5. If `commit=False`, Git is not touched beyond the working tree file. +6. If `commit=True`, only that new event/proposal file is staged and committed + locally. There is no push. + +State and write boundary: + +- The ledger is durable filesystem state, not an in-memory learning model. +- It writes only JSON artifacts under the configured ledger root. +- It does not mutate skill code, prompts, contracts, memory databases, or model + weights. +- Commit mode is intentionally local-only and narrow: stage the artifact, make + a local commit, stop. + +Decision owner: + +- Caller modules decide when an event or proposal is worth recording. +- The ledger decides only whether the artifact is schema-valid and where it is + stored. +- Human reviewers decide whether ledger artifacts should become code changes, + PRs, ignored local traces, or training/evaluation data later. + +Failure mode: + +- Invalid schema should fail before writing. +- Unsafe paths or path traversal should be rejected before filesystem access. +- Git commit failure should leave the JSON artifact visible in the worktree + rather than pretending the record was committed. + +Privacy and review boundary: + +- These files can contain task text, outcome messages, context summaries, and + proposal rationale. Decide whether `.dimos/evolution` belongs in normal Git + history before enabling commit mode on real robot runs. +- The ledger is a control-plane audit trail. It is not a replacement for + SpatialMemory, TemporalMemory, SkillOutcomeStore, or the causal model state. + +Review focus: + +- Confirm no path traversal is possible through event/proposal fields. +- Confirm commit mode stages only the new ledger file. +- Decide whether `.dimos/evolution` should be ignored, committed, or exported + elsewhere depending on privacy policy. + +### 9. Task Feasibility Preflight + +Files: + +- `dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/task_feasibility.py` +- `dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/test_task_feasibility.py` + +Implementation logic: + +- Adds MCP skill `evaluate_task_feasibility(task, context_json)`. +- Reads skill contracts, world/robot context, and optional JSON context. +- Returns a deterministic result with feasibility status, missing context, + required skills, available skills, safety risks, recommended next action, + optional clarifying question, and evidence sources. +- May write a ledger event when an evolution ledger is wired. + +Self-evolution meaning: + +- This is not model self-training. +- It is agent self-assessment: before acting, the agent can ask a deterministic + evaluator whether the task has enough context and capability support. + +Components: + +- `_Go2TaskFeasibility`: MCP skill surface for deterministic preflight. +- Skill contract reader: consumes Layer 5 skill metadata such as required + arguments, context requirements, risk class, and motion sensitivity. +- Context parser: merges explicit `context_json` with wired world/robot + provider summaries. +- Ledger integration: optional event writer for review traces. + +Inputs: + +- `task`: natural-language user request or subtask. +- `context_json`: optional structured context supplied by the caller. +- Layer 5 skill contracts and availability metadata. +- Layer 4 robot/world state when wired. +- Optional outcome/context feedback summaries when available through the + context bundle. + +Decision/data flow: + +1. Parse the task and optional `context_json`. +2. Match the task against known skill contracts and expert domains. +3. Check required arguments and context requirements from Layer 5 contracts. +4. Check robot/world-state availability for motion-sensitive or risky skills. +5. Produce one of three broad outcomes: + `feasible`, `uncertain`, or not feasible. +6. If data is missing, recommend a clarifying question or context-gathering + action. +7. If capability is missing, report it as missing capability evidence. That is + the input that can later justify a skill-interface proposal. + +State and write boundary: + +- It is read-only with respect to robot state, memory, and skill contracts. +- It does not call the target skill. +- It does not create a new skill. +- If the ledger is wired, it may write an audit event describing the preflight + result, but not a code change. + +Decision owner: + +- Feasibility classification is deterministic code. +- The LLM may call this tool and use its result, but the LLM is not the + classifier inside the tool. +- Human review decides whether the deterministic rules are conservative enough + for real hardware. + +Failure mode: + +- Missing required arguments should produce clarification, not a new skill + proposal. +- Missing context should produce context-gathering or uncertainty, not a false + feasible result. +- Missing capability evidence should be specific enough for SkillProposal to + distinguish it from transient runtime failure. + +Review focus: + +- Check false positives for motion-sensitive tasks. +- Check that missing context leads to clarification instead of skill proposal. + +### 10. Context Feedback Store + +Files: + +- `dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/context_feedback.py` +- `dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/test_context_feedback.py` + +Implementation logic: + +- Adds MCP skill `record_context_feedback(...)`. +- Keeps a bounded in-memory deque of the most recent 100 feedback entries. +- Uses schema `go2_context_feedback.v1`. +- Optionally writes feedback events through the evolution ledger. +- ContextProvider includes a compact feedback summary in future context. + +Review focus: + +- Feedback is session memory unless ledger is wired. +- It should not become unbounded or silently override RAG/temporal memory. + +Components: + +- `_Go2ContextFeedbackStore`: in-memory bounded store and MCP skill surface. +- Feedback entry schema: task/context identifiers, rating or usefulness signal, + source labels, optional explanation, and timestamp. +- Optional evolution ledger connection: durable audit trail when wired. +- ContextProvider integration: compact aggregate feedback summary. + +Data flow: + +1. The agent or a human-facing process calls `record_context_feedback(...)` + after a context bundle is helpful, incomplete, stale, or misleading. +2. The store validates and normalizes feedback fields. +3. The entry is appended to a deque capped at 100 records. +4. If the ledger is wired, the feedback is also written as an evolution event. +5. ContextProvider reads a summary, not the full raw feedback list, to avoid + bloating prompts. + +State and persistence: + +- Without the ledger, feedback is process-local and lost on restart. +- With the ledger, feedback becomes reviewable JSON but still does not train a + ranker automatically. + +Decision owner: + +- Feedback recording is an input signal. +- The evidence policy and LLM decide how to treat the signal. There is no + automatic prompt rewrite or memory deletion. + +Failure mode: + +- Malformed feedback should fail validation. +- Excess feedback should evict oldest entries, not grow unbounded. + +### 11. Skill Outcome Store And Predictor + +Files: + +- `dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/skill_outcome_store.py` +- `dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/skill_outcome_predictor.py` +- `dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/test_skill_outcomes.py` + +Implementation logic: + +- Records recent skill outcomes with bounded storage. +- Summarizes success/failure patterns by skill. +- Predictor estimates risk or likely outcome based on recent history and skill + contract metadata. +- ContextProvider can use these summaries as task evidence. + +Review focus: + +- The predictor is heuristic, not a learned model. +- Make sure stale failures do not permanently block useful skills. + +Components: + +- `_Go2SkillOutcomeStore`: bounded recent outcome history. +- `_Go2SkillOutcomePredictor`: heuristic scorer over recent history and skill + contract metadata. +- Outcome summary methods: aggregate successes, failures, repeated errors, and + recent messages by skill. + +Data flow: + +1. After a skill call, an outcome can be recorded with skill name, success, + error code, message, and metadata. +2. The outcome store appends it to bounded memory. +3. Summaries group recent records by skill and error patterns. +4. The predictor combines recent outcomes with skill contract risk/context + metadata to return risk and rationale. +5. ContextProvider can include the summary as evidence for future calls. + +State and persistence: + +- Current store is bounded runtime memory unless a future integration writes it + elsewhere. +- It is separate from the Git ledger. The ledger records review/audit events; + the outcome store supports immediate runtime adaptation. + +Decision owner: + +- The predictor gives a heuristic risk hint. +- It should not block a skill by itself; TaskFeasibility or the LLM should use + it as one evidence source. + +Failure mode: + +- Unknown skill should return low-confidence or no-history output. +- Repeated failure should increase caution but still allow recovery/stop skills. + +### 12. Causal World Model And Dashboard Contract + +Files: + +- `dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/causal_world_model.py` +- `dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/causal_effect_estimator.py` +- `dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/online_transition_model.py` +- `dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/structural_causal_model.py` +- `dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/intervention_log.py` +- `dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/world_model_contract.py` +- `dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/test_causal_world_model.py` +- `dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/test_world_model_contract.py` + +Implementation logic: + +- Records causal transitions and interventions. +- Maintains an online transition model for recent world-state changes. +- Estimates causal effects from recorded events. +- Produces versioned contract outputs: + `dimos.world_model_prediction.v1`, + `dimos.world_model_provider.v1`, and + `dimos.world_model_dashboard.v1`. +- Supports save/load of model state. + +Is it a trained model? + +- Strict answer: there is a very small online model, but there is no offline + training pipeline and no neural world model. +- `OnlineTransitionOutcomeModel` is a lightweight logistic-style linear model. + It stores feature weights in a dictionary and updates them incrementally when + `record_causal_transition()` receives an observed success/failure outcome. +- If no transitions have been recorded, its `sample_count` is zero and its + prediction should be treated as low-confidence prior behavior, not learned + robot knowledge. +- The symbolic SCM is hand-authored. It is not learned from data. +- The causal effect estimator is observational statistics. It compares smoothed + success rates for active features and does not control hidden confounders. + +Components: + +- `OnlineTransitionOutcomeModel`: online success-probability scorer. Features + include bias, skill name, domain, odometry presence, navigation state, spatial + memory availability, spatial match count, temporal availability, and selected + action argument presence/value. The update is one-step gradient adjustment + with L2 shrinkage. +- `CausalEffectEstimator`: bounded observation store with treated/control + success-rate differences for symbolic features. It can produce risk factors, + supporting factors, and intervention suggestions, but its own output declares + the assumption that unobserved confounders are not controlled. +- `StructuralCausalModel`: hand-authored causal graph for variables such as + `spatial_has_matches`, `target_resolvability`, `odom_ready`, + `motion_safety`, and `navigation_success`. It also emits counterfactual + suggestions such as "tag or perceive target before semantic navigation". +- `InterventionLog`: records explicit interventions and lets prediction surface + prior intervention evidence. +- `_WorldTransition`: the compact event record tying task, skill, before/after + state, predicted risk, outcome, inferred cause, and recovery suggestion + together. + +Data flow for recording: + +1. `record_causal_transition()` receives task, skill name, optional args, + before/after Layer 4 snapshots, prediction metadata, and outcome metadata. +2. Inputs are parsed as JSON objects and validated. Missing outcome can fall + back to the latest same-skill outcome if SkillOutcomeStore is wired. +3. The module infers a coarse cause and recovery suggestion from context, + prediction reasons, outcome success, error code, and message. +4. It computes a symbolic state delta from before/after snapshots. +5. It calls `_outcome_model.update(...)` and `_causal_estimator.update(...)`. +6. It appends the transition to an in-memory deque capped at 200 transitions. +7. It autosaves to `DIMOS_GO2_WORLD_MODEL_STATE` only if that env path is set. +8. It publishes dashboard state if WebsocketVis is wired. + +Data flow for prediction: + +1. `predict_next_state()` parses a snapshot and candidate action. +2. It gets recent same-skill transitions and derives repeated failure modes. +3. It computes rule-based risk reasons from the current snapshot and action. +4. It asks the online model for success probability, score, risk, confidence, + and top weighted feature contributions. +5. It asks the causal estimator for observational risk/support factors. +6. It asks the hand-authored SCM for variables, edges, and counterfactuals. +7. It asks the intervention log for matching intervention evidence. +8. It combines rule risk and model risk into a final risk and score. +9. It returns `dimos.world_model_prediction.v1` and optionally publishes the + dashboard contract. + +Trust boundary: + +- The output is advisory. It should help the LLM choose preflight, perception, + clarification, or a safer skill path. +- It should not directly command robot motion. +- High confidence currently requires enough recent samples for the same skill; + otherwise the model should be treated as a low-confidence heuristic. + +Review focus: + +- Treat current causal estimates as advisory evidence. +- Validate that prediction schemas remain stable for dashboard and LLM prompt + consumers. +- Check that persistence does not mix data across runs unexpectedly. +- Check that the UI/prompt labels do not imply a fully trained physical world + model. +- Check that replay or simulation evaluation is added before relying on this + for autonomous action selection. + +### 13. Skill Interface Proposal Generator + +Files: + +- `dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/skill_proposal.py` +- `dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/test_skill_proposal.py` + +Implementation logic: + +- Adds MCP skill `propose_skill_interface(task, failure_context_json)`. +- Generates `dimos.skill_proposal.v1` review artifacts only when there is + explicit missing capability evidence. +- Suppresses proposals for missing arguments or missing context. +- Suppresses duplicate proposals when an existing contract already covers the + capability. +- Writes proposal files through the evolution ledger when wired. +- Does not modify Python code, skill decorators, or blueprint wiring. + +Decision tree: + +1. Parse `failure_context_json`. +2. Look for explicit missing capability evidence, not just failed execution. +3. Check existing Layer 5 contracts to avoid proposing a duplicate interface. +4. Reject cases that are only missing arguments, missing context, or temporary + provider unavailability. +5. Build a proposal artifact with task, evidence, suggested interface shape, + expected inputs, expected output, and review rationale. +6. Write it through the evolution ledger if a ledger is wired. + +Human boundary: + +- This tool is intentionally proposal-only. A developer still writes code, + chooses the container, adds `@skill`, updates contracts/prompts, and tests the + behavior. + +Review focus: + +- This is the safest self-evolution boundary: proposal-only, human-reviewed. +- Check evidence requirements carefully so ordinary user ambiguity does not + generate noisy skill proposals. + +Components: + +- `_Go2SkillProposalGenerator`: MCP skill surface for proposal generation. +- Existing Layer 5 contracts: duplicate-suppression source of truth. +- Evolution ledger: optional proposal writer. +- Proposal schema: review artifact describing the missing capability and + proposed interface, not executable code. + +Inputs: + +- `task`: the user/subtask text that failed. +- `failure_context_json`: structured evidence about why existing skills failed + or were insufficient. +- Existing contracts and known MCP tool names. + +Accepted evidence: + +- Explicit missing capability. +- Repeated failure that indicates no available skill can perform the required + operation. +- A domain/action gap not covered by existing contracts. + +Rejected evidence: + +- Missing required arguments for an existing skill. +- Missing context or unavailable memory/provider. +- Temporary runtime failure. +- Capability already covered by an existing contract. + +Output boundary: + +- The output is a JSON proposal and optional ledger file. +- It does not import, write, or generate Python code. +- It does not update the prompt or registry automatically. + +### 14. MCP Runtime Plumbing + +Files: + +- `dimos/agents/mcp/mcp_client.py` +- `dimos/agents/mcp/mcp_server.py` +- `dimos/agents/mcp/tool_stream.py` +- `dimos/agents/mcp/test_mcp_client_unit.py` +- `dimos/agents/mcp/test_mcp_server.py` +- `dimos/agents/mcp/test_tool_stream.py` + +Implementation logic: + +- `McpServer.on_system_modules()` avoids querying its own proxy over RPC. For + its own skills and actor-class backed proxies, it collects skill schemas + locally. +- `McpClient` defers direct tool registration outside the RPC path so startup + does not block on the server being fully initialized. +- Tool-stream progress frames can be delivered over persistent SSE and include + progress-token metadata. +- Capability release for background tools is tied to stopped frames. +- `agent_send()` now uses `make_transport()` so it respects the active backend. + +Review focus: + +- Validate no startup path waits for a tool that depends on itself. +- Keep MCP test ports isolated from live robot sessions. + +Components: + +- `McpServer`: exposes MCP JSON-RPC endpoints, tool list/call handling, SSE + progress stream, server status tools, capability gating, and agent-send. +- `McpClient`: LLM agent module that discovers tools and exposes them to the + agent runtime. +- `tool_stream.py`: progress/log notification path for long-running skills. +- Capability registry: shared server-side lock manager for mutually exclusive + tool capabilities. + +Startup data flow: + +1. ModuleCoordinator deploys workers and starts modules. +2. `McpServer.start()` starts HTTP/SSE serving and subscribes to tool-stream + frames. +3. `on_system_modules()` registers tool schemas. For the server itself and + actor-class backed modules, schemas are collected locally to avoid RPC + self-deadlock. +4. `McpClient` initializes the agent without blocking startup on direct tool + registration outside the RPC path. + +Tool call flow: + +1. JSON-RPC `tools/call` arrives. +2. Server resolves `SkillInfo` and RPC call target. +3. Capability locks are acquired if the skill declares `uses`. +4. Progress token and capability token are injected through reserved + `_mcp_context`. +5. RPC call runs in an executor. +6. Instant skills release capability after return; background skills hand + release to tool-stream stopped frames. + +State and failure boundary: + +- Server state is process-local: skills, rpc calls, SSE queues, capability + registry. +- Port conflicts are external environment failures. Tests should use isolated + MCP ports when a live server is running. +- Tool not found should return a structured MCP text result, not crash the + server. + +### 15. Runtime Hardening + +Files: + +- `dimos/core/coordination/module_coordinator.py` +- `dimos/core/coordination/worker_manager_python.py` +- `dimos/simulation/mujoco/mujoco_process.py` +- `dimos/robot/unitree/mujoco_connection.py` +- `dimos/robot/unitree/go2/connection.py` +- `dimos/robot/unitree/go2/test_connection.py` + +Implementation logic: + +- Coordinator system-module notifications avoid waiting on known agent clients. +- Worker manager/coordinator changes preserve module restart, reload, and stream + rewiring behavior. +- MuJoCo helper code can read available stderr without blocking. +- macOS headless MuJoCo uses the current Python executable where needed. +- Go2 WebRTC connection forwards AES key configuration from `GlobalConfig`. + +Review focus: + +- These are operational fixes. Review them for process lifecycle correctness, + not agent reasoning behavior. + +Components: + +- ModuleCoordinator notification path: controls build/start/system-module + notification order. +- WorkerManagerPython: deploys modules into workers and handles worker pool + lifecycle. +- MuJoCo process helpers: subprocess executable selection and stderr draining. +- Go2 connection config: replay stop behavior and WebRTC AES key forwarding. + +Data flow: + +1. Build/deploy happens through ModuleCoordinator and worker managers. +2. Stream transports and RPC references are wired before modules receive + system-module notifications. +3. Some notifications use nowait behavior to avoid waiting on an agent client + that is itself initializing from MCP. +4. Restart/reload paths must preserve transports and rewired module refs. +5. Simulator helper code isolates platform-specific process quirks from the + rest of the robot stack. + +State and write boundary: + +- Coordinator owns deployed module proxy maps, transport registry, module + transports, and reload aliases. +- These changes do not alter skill semantics. +- They do affect whether modules start, stop, reload, and reconnect cleanly. + +Failure mode: + +- A stuck build/start should stop managers and surface the original exception. +- Restart should not orphan old transports or leave consumers attached to dead + proxies. +- ReplayConnection stop should be a no-op, not an exception. + +### 16. Optional Rerun/WebsocketVis World-Model Panel + +Files: + +- `dimos/web/websocket_vis/websocket_vis_module.py` +- `dimos/web/websocket_vis_spec.py` +- `dimos/web/templates/rerun_dashboard.html` +- `dimos/web/websocket_vis/test_websocket_vis_module.py` + +Implementation logic: + +- Websocket visualization can carry world-model/dashboard state in addition to + existing visualization payloads. +- Dashboard payload shape is aligned with `world_model_contract.py`. +- This is optional observability. The current `unitree_go2_agentic` blueprint + does not include `WebsocketVisModule`, so this path is inactive unless a + visualization blueprint wires `WebsocketVisSpec`. + +Review focus: + +- Dashboard state should remain display-only. +- Avoid making dashboard consumers depend on unversioned internal Python + objects. +- Do not treat this as a required dependency of the Go2 self-evolution runtime. + +Components: + +- `websocket_vis_spec.py`: typed surface for visualization updates. +- `websocket_vis_module.py`: runtime module that stores/publishes dashboard + state. +- `rerun_dashboard.html`: browser-facing display surface. +- `world_model_contract.py`: schema source for world-model dashboard payloads. + +Data flow: + +1. CausalWorldModel produces a dashboard payload using + `dimos.world_model_dashboard.v1`. +2. If WebsocketVis is wired, CausalWorldModel calls `set_world_model_state(...)`. +3. WebsocketVis stores and serves the latest state to dashboard clients. +4. The dashboard renders the state as inspection data for humans. + +State and persistence: + +- Dashboard state is the latest display snapshot, not the source of truth. +- Durable state, if configured, belongs to CausalWorldModel save/load JSON. +- Browser/dashboard consumers should rely on versioned payload fields only. + +Safety boundary: + +- Dashboard updates must not trigger robot actions. +- UI should not present advisory predictions as guaranteed outcomes. +- If `WebsocketVisModule` is not wired, CausalWorldModel still works; the + dashboard publication path is skipped. + +## Review Order I Recommend + +1. Start with Go2 blueprint wiring: + `unitree_go2_agentic.py`, Layer 3/4/5/6 `__init__.py`, and + `test_world_state.py`. +2. Review MCP runtime plumbing: + `mcp_server.py`, `mcp_client.py`, `tool_stream.py`, and their tests. +3. Review self-evolution write boundaries: + `evolution_ledger.py`, `evolution_proposal.py`, `skill_proposal.py`. +4. Review context quality: + `context_provider.py`, `context_evidence.py`, `context_feedback.py`. +5. Review action safety: + `skill_interface_registry.py`, `task_feasibility.py`, + `skill_outcome_predictor.py`. +6. Review advisory world-model logic: + `causal_world_model.py` and `world_model_contract.py`. +7. Review upstream transport changes separately, especially Zenoh defaults on + macOS. + +## Validation Run During Upstream Sync + +Passed: + +- `.venv/bin/python -m pytest dimos/agents/mcp/test_mcp_server.py -q` + - `10 passed, 1 warning` +- `.venv/bin/python -m pytest dimos/robot/unitree/go2/test_connection.py -q` + - `5 passed` +- `.venv/bin/python -m pytest dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain -q` + - `62 passed` +- `.venv/bin/python -m pytest dimos/robot/unitree/go2/blueprints/layers/layer_4_world_state/test_world_state.py -q` + - `3 passed, 1 warning` +- `.venv/bin/python -m pytest dimos/perception/test_spatial_perception_config.py -q` + - `2 passed` +- `DIMOS_TRANSPORT=lcm .venv/bin/python -m pytest dimos/core/coordination/test_module_coordinator.py -q` + - `35 passed` +- `DIMOS_TRANSPORT=lcm MCP_PORT=9991 .venv/bin/python -m pytest dimos/agents/mcp/test_mcp_client_unit.py dimos/agents/mcp/test_tool_stream.py -q` + - `44 passed, 1 warning` +- `.venv/bin/python -m pytest dimos/robot/test_all_blueprints_generation.py -q` + - `1 passed` +- `git diff --cached --check -- ':(exclude)*.patch'` + - passed + +Known caveats: + +- Running MCP tests on default port `9990` fails on this machine because a live + `dimos-live` process already owns that port. +- Running some tests with upstream's new macOS default `zenoh` backend exposed + native pyo3 thread lifetime issues in pytest. The same coordinator and MCP + suites pass under `DIMOS_TRANSPORT=lcm`. Reviewers should decide whether to + treat Zenoh-on-macOS test behavior as an upstream follow-up or to pin local + tests to LCM until Zenoh cleanup semantics are stable. diff --git a/docs/reviews/2026-07-05-go2-agent-architecture-review.zh.md b/docs/reviews/2026-07-05-go2-agent-architecture-review.zh.md index 8c630654ac..05a2e15089 100644 --- a/docs/reviews/2026-07-05-go2-agent-architecture-review.zh.md +++ b/docs/reviews/2026-07-05-go2-agent-architecture-review.zh.md @@ -54,18 +54,22 @@ review 每个本地功能时，建议都按下面这些问题看，而不是只 - Agent judgment: LLM 读取已选上下文和工具后决定下一步。 - Self-evolution artifact: 产出可 review 的 JSON event/proposal，不自动改代码。 -## 项目整体架构与 Agent 重点展开 - -本分支的 review 重点是把 repo-wide DimOS 执行架构和 Go2 agentic -self-evolution extension 分开看。repo-wide 部分包括 entry -points、blueprint/configuration、core runtime、transports、module -families、robot/simulation targets 和 data assets。Go2 agentic 扩展部分重点 -看 Agent Brain: task evaluation、context use、skill validation、feasibility -judgment 和 proposal/artifact emission 都由这个层级发起或裁决。 - -可选观测/存储不是主执行依赖。当前 `unitree_go2_agentic` blueprint 没有接 -`WebsocketVisModule`；只有单独的可视化 blueprint 接上 `WebsocketVisSpec` -时，Rerun / WebsocketVis 面板才会收到 world-model state。 +## 架构图 + +![DimOS Go2 Agent 自进化架构](2026-07-05-go2-agent-architecture.png) + +右侧紫色列是可选观测/存储，不是主执行依赖。当前 `unitree_go2_agentic` +blueprint 没有接 `WebsocketVisModule`；只有单独的可视化 blueprint 接上 +`WebsocketVisSpec` 时，Rerun / WebsocketVis 面板才会收到 world-model state。 + +## 项目整体架构图 + +![DimOS 项目整体架构](2026-07-05-dimos-project-architecture.png) + +这张图是 repo-wide，不是 Go2 专属。它展示 DimOS 的主流程: entry +points、blueprint/configuration、core runtime、communication transports、 +functional module families、robots/simulation/hardware，以及 +data/model/observability assets。 ## 接口输入/输出速查 diff --git a/docs/reviews/2026-07-05-go2-agent-architecture.png b/docs/reviews/2026-07-05-go2-agent-architecture.png new file mode 100644 index 0000000000..6bb86c1dd4 --- /dev/null +++ b/docs/reviews/2026-07-05-go2-agent-architecture.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1e80abb6d7371795e672e5652db20754b8fb21bb1a74ec8b88e45c5df372cc0a +size 195112 From 72590e0d529394f12ff03ba0ba24722743e6f123 Mon Sep 17 00:00:00 2001 From: ZhangZheMing <1050297972@qq.com> Date: Sun, 5 Jul 2026 10:04:59 +0800 Subject: [PATCH 27/36] Use composite Go2 review architecture figure --- .../2026-07-05-dimos-agentic-architecture.png | 3 + .../2026-07-05-dimos-agentic-architecture.svg | 227 ++++++++++++++++++ .../2026-07-05-dimos-project-architecture.png | 3 - ...026-07-05-go2-agent-architecture-review.md | 34 ++- ...-07-05-go2-agent-architecture-review.zh.md | 31 ++- .../2026-07-05-go2-agent-architecture.png | 3 - 6 files changed, 272 insertions(+), 29 deletions(-) create mode 100644 docs/reviews/2026-07-05-dimos-agentic-architecture.png create mode 100644 docs/reviews/2026-07-05-dimos-agentic-architecture.svg delete mode 100644 docs/reviews/2026-07-05-dimos-project-architecture.png delete mode 100644 docs/reviews/2026-07-05-go2-agent-architecture.png diff --git a/docs/reviews/2026-07-05-dimos-agentic-architecture.png b/docs/reviews/2026-07-05-dimos-agentic-architecture.png new file mode 100644 index 0000000000..43c3c2c3ce --- /dev/null +++ b/docs/reviews/2026-07-05-dimos-agentic-architecture.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:42111678889fbfc0fd1926390a2ef9e09fa3f4ae2a440d708c7f99306eeac953 +size 352055 diff --git a/docs/reviews/2026-07-05-dimos-agentic-architecture.svg b/docs/reviews/2026-07-05-dimos-agentic-architecture.svg new file mode 100644 index 0000000000..4e360c089b --- /dev/null +++ b/docs/reviews/2026-07-05-dimos-agentic-architecture.svg @@ -0,0 +1,227 @@ + + DimOS Project Architecture With Go2 Agentic Extension + A scientific-style system architecture diagram showing the DimOS project runtime and a zoomed Go2 agentic self-evolution extension in one figure. + + + + + + + + + + + + + + + DimOS 项目整体架构与 Go2 Agent 自进化扩展 + Scientific-style composite figure: project runtime context plus the current Go2 agentic extension boundary. + + + + A + DimOS project architecture + + + + Entry points + + CLI + + Python API + + + Blueprint layer + + Blueprints + + Registry + + + Configuration + + GlobalConfig + + .env + + + External interfaces + + MCP + + Rerun / Web + + + Targets + Robots / Simulation + Go2 · G1 · xArm · Drones · MuJoCo + + + + Core runtime + + Module In[T]/Out[T] + + ModuleCoordinator + + Workers + + RPC / Specs + + Messages + + Run registry/logs + + Lifecycle · deployment · system wiring + + + + Communication layer + LCM · Zenoh · SHM/pSHM · ROS · DDS · Tool streams · HTTP/WebSocket + + + Functional module families + Agents · Skills · Perception · Mapping · Navigation · Control · Manipulation · Learning + Data/model stores: LFS assets · Memory2 · vector/blob stores · VLM/embedding/segmentation models + + + + + + + + + + Panel A abstracts the full repository. The Go2 agentic branch is one blueprint-specific extension within the Agents/Skills/Robot target path. + + + + zoom into Go2 agentic blueprint + + + + B + Go2 agentic self-evolution extension inside DimOS + Current `unitree_go2_agentic`: robot body + world state + skill interface + agent brain. WebsocketVis is optional and not wired here. + + + + Layer 3 Agent Brain + + Prompt + + Router + + Context Provider + + Feasibility + + MCP + + Outcome Store + + Causal Model + + Skill Proposal + + + Layer 4 World State + + Structured State + + Semantic Temporal Map + + Spatial/RAG + + Temporal + + + Layer 5 Skill Interface + + Skill Contracts + + Preflight Metadata + + Go2 Skills + + + Layer 6 Robot Body + Go2 Connection · Sensors/Odom · Motion Control + + + + Local review artifacts + local JSON files; optional local Git commit; no automatic GitHub push + + .dimos/evolution/events + + .dimos/evolution/proposals + + + Optional observability + Rerun / WebsocketVis + not wired by current `unitree_go2_agentic` + + + + Legend and rigor notes + + control/tool call + + context read / evidence + + audit artifact write + + optional observability + No arrow means no runtime dependency. + Causal model output is advisory only. + Skill proposal creates review artifacts, not code. + + + + + + + + + + + + + + + + + + + Figure: DimOS as a modular runtime, with the Go2 agentic extension shown as a blueprint-specific subsystem. Solid arrows are calls/control, dashed arrows are context/evidence reads, dotted arrows are local audit writes, and dash-dot arrows are optional visualization. + diff --git a/docs/reviews/2026-07-05-dimos-project-architecture.png b/docs/reviews/2026-07-05-dimos-project-architecture.png deleted file mode 100644 index 6574655451..0000000000 --- a/docs/reviews/2026-07-05-dimos-project-architecture.png +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:bdb38a4089ea44c83e826a198ea18bb19aae5db857ccb7dc02c10dd5e1288660 -size 221143 diff --git a/docs/reviews/2026-07-05-go2-agent-architecture-review.md b/docs/reviews/2026-07-05-go2-agent-architecture-review.md index c3f75cdc17..7e68299101 100644 --- a/docs/reviews/2026-07-05-go2-agent-architecture-review.md +++ b/docs/reviews/2026-07-05-go2-agent-architecture-review.md @@ -77,23 +77,33 @@ things: - Self-evolution artifact: reviewable JSON events/proposals, not automatic code mutation. -## Architecture Diagram +## Combined Project and Agent Architecture Diagram -![DimOS Go2 agent self-evolution architecture](2026-07-05-go2-agent-architecture.png) +![DimOS project architecture with Go2 agentic self-evolution extension](2026-07-05-dimos-agentic-architecture.png) -The purple right-side column is optional observability/storage. The current -`unitree_go2_agentic` blueprint does not wire `WebsocketVisModule`; the Rerun / -WebsocketVis panel only receives world-model state if a separate visualization -blueprint wires `WebsocketVisSpec`. +This is the canonical review figure for this branch. Panel A shows the +repo-wide DimOS execution architecture: entry points, blueprint/configuration, +core runtime, transports, module families, robot/simulation targets, and data +assets. Panel B expands the Go2 agentic self-evolution extension inside that +project architecture, with the agent brain deliberately emphasized because it +owns task evaluation, context use, skill validation, feasibility judgment, and +proposal/artifact emission. -## Project Architecture Diagram +The purple right-side column remains optional observability/storage. The +current `unitree_go2_agentic` blueprint does not wire `WebsocketVisModule`; the +Rerun / WebsocketVis panel only receives world-model state if a separate +visualization blueprint wires `WebsocketVisSpec`. -![DimOS project architecture](2026-07-05-dimos-project-architecture.png) +Line semantics are part of the figure contract: -This diagram is repo-wide, not Go2-specific. It shows the main DimOS flow: -entry points, blueprint/configuration, core runtime, communication transports, -functional module families, robots/simulation/hardware, and data/model/ -observability assets. +- Solid black arrows: control flow, blueprint composition, tool calls, or robot + commands. +- Dashed blue arrows: context/evidence reads used by the agent. +- Dotted purple arrows: local review artifacts written to files/Git. +- Dash-dot purple arrows: optional observability paths. + +Editable source: `2026-07-05-dimos-agentic-architecture.svg` +PNG render: `2026-07-05-dimos-agentic-architecture.png` ## Interface Input/Output Quick Reference diff --git a/docs/reviews/2026-07-05-go2-agent-architecture-review.zh.md b/docs/reviews/2026-07-05-go2-agent-architecture-review.zh.md index 05a2e15089..897ead18b8 100644 --- a/docs/reviews/2026-07-05-go2-agent-architecture-review.zh.md +++ b/docs/reviews/2026-07-05-go2-agent-architecture-review.zh.md @@ -54,22 +54,31 @@ review 每个本地功能时，建议都按下面这些问题看，而不是只 - Agent judgment: LLM 读取已选上下文和工具后决定下一步。 - Self-evolution artifact: 产出可 review 的 JSON event/proposal，不自动改代码。 -## 架构图 +## 项目整体架构与 Agent 重点展开图 -![DimOS Go2 Agent 自进化架构](2026-07-05-go2-agent-architecture.png) +![DimOS 项目整体架构与 Go2 Agent 自进化扩展](2026-07-05-dimos-agentic-architecture.png) -右侧紫色列是可选观测/存储，不是主执行依赖。当前 `unitree_go2_agentic` -blueprint 没有接 `WebsocketVisModule`；只有单独的可视化 blueprint 接上 -`WebsocketVisSpec` 时，Rerun / WebsocketVis 面板才会收到 world-model state。 +这是本分支 review 的主图。Panel A 展示 repo-wide DimOS 执行架构: +entry points、blueprint/configuration、core runtime、transports、module +families、robot/simulation targets 和 data assets。Panel B 在同一张图里 +展开 Go2 agentic self-evolution extension，并刻意把 Agent Brain 放在重点位置: +task evaluation、context use、skill validation、feasibility judgment 和 +proposal/artifact emission 都由这个层级发起或裁决。 -## 项目整体架构图 +右侧紫色列仍然是可选观测/存储，不是主执行依赖。当前 +`unitree_go2_agentic` blueprint 没有接 `WebsocketVisModule`；只有单独的 +可视化 blueprint 接上 `WebsocketVisSpec` 时，Rerun / WebsocketVis 面板才会 +收到 world-model state。 -![DimOS 项目整体架构](2026-07-05-dimos-project-architecture.png) +图中的线型语义也是 review contract 的一部分: -这张图是 repo-wide，不是 Go2 专属。它展示 DimOS 的主流程: entry -points、blueprint/configuration、core runtime、communication transports、 -functional module families、robots/simulation/hardware，以及 -data/model/observability assets。 +- 黑色实线: control flow、blueprint composition、tool call 或 robot command。 +- 蓝色虚线: agent 读取 context/evidence 的路径。 +- 紫色点线: 写入本地 review artifact、文件或 Git 的路径。 +- 紫色点划线: optional observability path。 + +可编辑源文件: `2026-07-05-dimos-agentic-architecture.svg` +PNG 渲染版: `2026-07-05-dimos-agentic-architecture.png` ## 接口输入/输出速查 diff --git a/docs/reviews/2026-07-05-go2-agent-architecture.png b/docs/reviews/2026-07-05-go2-agent-architecture.png deleted file mode 100644 index 6bb86c1dd4..0000000000 --- a/docs/reviews/2026-07-05-go2-agent-architecture.png +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:1e80abb6d7371795e672e5652db20754b8fb21bb1a74ec8b88e45c5df372cc0a -size 195112 From 080786f2ef581bf59a9f80d49ee5a4491f50ce91 Mon Sep 17 00:00:00 2001 From: ZhangZheMing <1050297972@qq.com> Date: Mon, 6 Jul 2026 12:21:41 +0800 Subject: [PATCH 28/36] Disambiguate reused proposal_id filenames in evolution ledger write_skill_proposal_file derived the artifact path solely from proposal_id via _safe_filename, then atomically replaced it. A reused stable proposal_id (e.g. improve_navigation) would silently overwrite the earlier review artifact, losing the proposal reviewers were meant to inspect. Mirror the events path's _unique_event_path approach: append -1, -2, ... when the path exists so both artifacts are kept. The author's proposal_id stays verbatim in the JSON; only the filename is disambiguated. --- .../layer_3_agent_brain/evolution_proposal.py | 18 +++++++++- .../test_evolution_proposal.py | 34 +++++++++++++++++++ 2 files changed, 51 insertions(+), 1 deletion(-) diff --git a/dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/evolution_proposal.py b/dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/evolution_proposal.py index c6ee0b5cdc..d399232598 100644 --- a/dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/evolution_proposal.py +++ b/dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/evolution_proposal.py @@ -122,7 +122,7 @@ def write_skill_proposal_file( proposal_dir.mkdir(parents=True, exist_ok=True) _ensure_readme(ledger_dir) - path = proposal_dir / f"{_safe_filename(normalized['proposal_id'])}.json" + path = _unique_proposal_path(proposal_dir, normalized["proposal_id"]) payload = json.dumps(normalized, indent=2, sort_keys=True) + "\n" tmp_path = path.with_name(f".{path.name}.tmp-{os.getpid()}") tmp_path.write_text(payload, encoding="utf-8") @@ -167,6 +167,22 @@ def _safe_filename(value: str) -> str: return re.sub(r"[^A-Za-z0-9_.-]+", "_", value).strip("._-") or "proposal" +def _unique_proposal_path(proposal_dir: Path, proposal_id: str) -> Path: + """Return a non-colliding proposal path, appending -1, -2, ... when needed. + + A reused stable ``proposal_id`` (e.g. ``improve_navigation``) must not + silently overwrite an earlier review artifact. The JSON keeps the + author's ``proposal_id`` verbatim; only the filename is disambiguated. + """ + stem = _safe_filename(str(proposal_id)) + path = proposal_dir / f"{stem}.json" + suffix = 1 + while path.exists(): + path = proposal_dir / f"{stem}-{suffix}.json" + suffix += 1 + return path + + __all__ = [ "SKILL_PROPOSAL_SCHEMA", "normalize_skill_proposal", diff --git a/dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/test_evolution_proposal.py b/dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/test_evolution_proposal.py index 030d3ac93b..1ddcd0cd1a 100644 --- a/dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/test_evolution_proposal.py +++ b/dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/test_evolution_proposal.py @@ -98,6 +98,40 @@ def test_record_skill_proposal_writes_validated_proposal(monkeypatch, tmp_path: _stop_modules(ledger) +def test_record_skill_proposal_preserves_reused_proposal_id( + monkeypatch, tmp_path: Path +) -> None: + ledger_dir = tmp_path / ".dimos" / "evolution" + monkeypatch.setenv("DIMOS_EVOLUTION_LEDGER_DIR", str(ledger_dir)) + ledger = _Go2EvolutionLedger() + try: + first = ledger.record_skill_proposal( + proposal_json=json.dumps(_proposal(title="First iteration of the proposal")) + ) + second = ledger.record_skill_proposal( + proposal_json=json.dumps(_proposal(title="Second iteration of the proposal")) + ) + + assert first.success is True + assert second.success is True + + first_path = Path(first.metadata["proposal_path"]) + second_path = Path(second.metadata["proposal_path"]) + assert first_path != second_path + assert first_path.exists() + assert second_path.exists() + + first_data = json.loads(first_path.read_text()) + second_data = json.loads(second_path.read_text()) + # The author's proposal_id is preserved verbatim in both artifacts; + # only the filename is disambiguated. + assert first_data["proposal_id"] == second_data["proposal_id"] + assert first_data["title"] == "First iteration of the proposal" + assert second_data["title"] == "Second iteration of the proposal" + finally: + _stop_modules(ledger) + + def test_record_skill_proposal_rejects_invalid_target(monkeypatch, tmp_path: Path) -> None: monkeypatch.setenv("DIMOS_EVOLUTION_LEDGER_DIR", str(tmp_path / "ledger")) ledger = _Go2EvolutionLedger() From ea16db2902be5a7f7b980b21acd209277cef4bf8 Mon Sep 17 00:00:00 2001 From: ZhangZheMing <1050297972@qq.com> Date: Mon, 6 Jul 2026 12:21:55 +0800 Subject: [PATCH 29/36] Anchor proposal filenames on digest to avoid sanitized-id collisions _safe_filename maps every non-alphanumeric character to '_', so distinct proposal_ids such as skill-v2 and skill.v2 both reduce to skill_v2 and land on the same path. The prior -N suffix guard only treated genuine same-id reuses; it would file the second distinct proposal as skill_v2-1.json, conflating unrelated proposals as iterations and risking overwrite of the first when ordering changes. Anchor the stem on an injective digest of the original (un-sanitized) proposal_id: {slug}-{sha1(id)[:8]}. Distinct ids now always map to distinct files; the -1/-2 suffix still handles true same-id reuse. JSON proposal_id stays verbatim. This mirrors how the events path uses a timestamp prefix to sidestep lossy sanitization. --- .../layer_3_agent_brain/evolution_proposal.py | 25 ++++++++--- .../test_evolution_proposal.py | 44 +++++++++++++++++++ 2 files changed, 63 insertions(+), 6 deletions(-) diff --git a/dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/evolution_proposal.py b/dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/evolution_proposal.py index d399232598..fc775a44ef 100644 --- a/dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/evolution_proposal.py +++ b/dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/evolution_proposal.py @@ -16,6 +16,7 @@ from __future__ import annotations from datetime import UTC, datetime +import hashlib import json import os from pathlib import Path @@ -168,13 +169,18 @@ def _safe_filename(value: str) -> str: def _unique_proposal_path(proposal_dir: Path, proposal_id: str) -> Path: - """Return a non-colliding proposal path, appending -1, -2, ... when needed. - - A reused stable ``proposal_id`` (e.g. ``improve_navigation``) must not - silently overwrite an earlier review artifact. The JSON keeps the - author's ``proposal_id`` verbatim; only the filename is disambiguated. + """Return a non-colliding proposal path for ``proposal_id``. + + The stem is anchored on a short digest of the *original* (un-sanitized) + ``proposal_id`` so that distinct ids never collapse onto the same file: + ``_safe_filename`` maps every non-alphanumeric character to ``_``, which + would otherwise make ``skill-v2`` and ``skill.v2`` share ``skill_v2.json`` + and let a later proposal overwrite an earlier review artifact. With the + digest anchor, distinct ids land on distinct files; a genuine reuse of the + *same* id still disambiguates via ``-1``, ``-2``, ... The JSON keeps the + author's ``proposal_id`` verbatim; only the filename carries the digest. """ - stem = _safe_filename(str(proposal_id)) + stem = _proposal_stem(str(proposal_id)) path = proposal_dir / f"{stem}.json" suffix = 1 while path.exists(): @@ -183,6 +189,13 @@ def _unique_proposal_path(proposal_dir: Path, proposal_id: str) -> Path: return path +def _proposal_stem(proposal_id: str) -> str: + """Human-readable slug plus an injective digest of the original id.""" + slug = _safe_filename(proposal_id) + digest = hashlib.sha1(proposal_id.strip().encode("utf-8")).hexdigest()[:8] + return f"{slug}-{digest}" + + __all__ = [ "SKILL_PROPOSAL_SCHEMA", "normalize_skill_proposal", diff --git a/dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/test_evolution_proposal.py b/dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/test_evolution_proposal.py index 1ddcd0cd1a..fe996691bb 100644 --- a/dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/test_evolution_proposal.py +++ b/dimos/robot/unitree/go2/blueprints/layers/layer_3_agent_brain/test_evolution_proposal.py @@ -132,6 +132,50 @@ def test_record_skill_proposal_preserves_reused_proposal_id( _stop_modules(ledger) +def test_record_skill_proposal_keeps_distinct_sanitized_ids_separate( + monkeypatch, tmp_path: Path +) -> None: + """Distinct proposal_ids that sanitize to the same slug must not collide. + + ``_safe_filename`` maps every non-alphanumeric character to ``_``, so + ``skill-v2`` and ``skill.v2`` both reduce to ``skill_v2``. They are + unrelated proposals and must land on distinct files — neither overwriting + the other, nor conflated as iterations (``-1``) of a single proposal. + """ + ledger_dir = tmp_path / ".dimos" / "evolution" + monkeypatch.setenv("DIMOS_EVOLUTION_LEDGER_DIR", str(ledger_dir)) + ledger = _Go2EvolutionLedger() + try: + dash = ledger.record_skill_proposal( + proposal_json=json.dumps(_proposal(proposal_id="skill-v2", title="Dash variant")) + ) + dot = ledger.record_skill_proposal( + proposal_json=json.dumps(_proposal(proposal_id="skill.v2", title="Dot variant")) + ) + + assert dash.success is True + assert dot.success is True + + dash_path = Path(dash.metadata["proposal_path"]) + dot_path = Path(dot.metadata["proposal_path"]) + assert dash_path != dot_path + assert dash_path.exists() + assert dot_path.exists() + # Neither is a ``-1`` iteration of the other: the digests anchored on + # the original ids differ, so both keep their unsuffixed base name. + assert dash_path.name != "skill_v2-1.json" + assert dot_path.name != "skill_v2-1.json" + + dash_data = json.loads(dash_path.read_text()) + dot_data = json.loads(dot_path.read_text()) + assert dash_data["proposal_id"] == "skill-v2" + assert dot_data["proposal_id"] == "skill.v2" + assert dash_data["title"] == "Dash variant" + assert dot_data["title"] == "Dot variant" + finally: + _stop_modules(ledger) + + def test_record_skill_proposal_rejects_invalid_target(monkeypatch, tmp_path: Path) -> None: monkeypatch.setenv("DIMOS_EVOLUTION_LEDGER_DIR", str(tmp_path / "ledger")) ledger = _Go2EvolutionLedger() From 0311ada6385b39796e3cc12a474a8accc73a8fb0 Mon Sep 17 00:00:00 2001 From: ZhangZheMing <1050297972@qq.com> Date: Mon, 6 Jul 2026 13:28:35 +0800 Subject: [PATCH 30/36] fix(mcp): use module.remote_name instead of skill.class_name for direct RPC routing Direct tool registration was routing calls to skill.class_name/skill.func_name instead of the module's actual RPCClient.remote_name. When workers register with aliased or remapped names the direct path sends tool calls to a route that does not match the running module, causing failures or empty results. --- dimos/agents/mcp/mcp_client.py | 16 ++++++++++++---- dimos/agents/mcp/mcp_server.py | 21 ++++++++++++++++----- 2 files changed, 28 insertions(+), 9 deletions(-) diff --git a/dimos/agents/mcp/mcp_client.py b/dimos/agents/mcp/mcp_client.py index 7e1eb72f7c..756262ac70 100644 --- a/dimos/agents/mcp/mcp_client.py +++ b/dimos/agents/mcp/mcp_client.py @@ -337,14 +337,22 @@ def _register_direct_tools(self, modules: list[RPCClient]) -> None: from dimos.agents.mcp.mcp_server import _skill_infos_from_class skills: list[SkillInfo] = [] + func_to_remote: dict[str, str] = {} for module in modules: - if getattr(module, "remote_name", None) == self.__class__.__name__: + remote_name = getattr(module, "remote_name", None) + if remote_name == self.__class__.__name__: continue if actor_class := getattr(module, "actor_class", None): - skills.extend(_skill_infos_from_class(actor_class)) + module_skills = _skill_infos_from_class(actor_class) + skills.extend(module_skills) + for s in module_skills: + func_to_remote[s.func_name] = remote_name continue try: - skills.extend(module.get_skills() or []) + module_skills = module.get_skills() or [] + skills.extend(module_skills) + for s in module_skills: + func_to_remote[s.func_name] = remote_name except Exception: logger.debug( "Skipping direct MCP tool registration for module.", @@ -362,7 +370,7 @@ def _register_direct_tools(self, modules: list[RPCClient]) -> None: None, rpc_client, skill.func_name, - skill.class_name, + func_to_remote[skill.func_name], [], ) for skill in skills diff --git a/dimos/agents/mcp/mcp_server.py b/dimos/agents/mcp/mcp_server.py index c0053f7afd..c689ca194e 100644 --- a/dimos/agents/mcp/mcp_server.py +++ b/dimos/agents/mcp/mcp_server.py @@ -399,19 +399,30 @@ def on_system_modules(self, modules: list[RPCClient]) -> None: # TODO: this is a bit hacky, also not thread-safe assert self.rpc is not None skills: list[SkillInfo] = [] + func_to_remote: dict[str, str] = {} for module in modules: - if getattr(module, "remote_name", None) == self.__class__.__name__: - skills.extend(self.get_skills()) + remote_name = getattr(module, "remote_name", None) + if remote_name == self.__class__.__name__: + own_skills = self.get_skills() + skills.extend(own_skills) + for s in own_skills: + func_to_remote[s.func_name] = remote_name elif actor_class := getattr(module, "actor_class", None): - skills.extend(_skill_infos_from_class(actor_class)) + module_skills = _skill_infos_from_class(actor_class) + skills.extend(module_skills) + for s in module_skills: + func_to_remote[s.func_name] = remote_name else: - skills.extend(module.get_skills() or []) + module_skills = module.get_skills() or [] + skills.extend(module_skills) + for s in module_skills: + func_to_remote[s.func_name] = remote_name app.state.skills = skills app.state.skills_by_name = {s.func_name: s for s in app.state.skills} app.state.rpc_calls = { skill_info.func_name: RpcCall( - None, self.rpc, skill_info.func_name, skill_info.class_name, [] + None, self.rpc, skill_info.func_name, func_to_remote[skill_info.func_name], [] ) for skill_info in app.state.skills } From 98f99e069a0900b973eee416f1cd9aa7f9c6a49e Mon Sep 17 00:00:00 2001 From: ZhangZheMing <1050297972@qq.com> Date: Mon, 6 Jul 2026 13:29:19 +0800 Subject: [PATCH 31/36] fix(dashboard): resolve Rerun iframe host/port from config instead of hardcoding localhost The iframe src was hardcoded to localhost:9090 and localhost:9876, ignoring DIMOS_RERUN_HOST / --rerun-host and listen_host. When the dashboard is served from a remote machine the embedded Rerun viewer connects to the browser user's localhost and the stream never appears. Read the template at serve time, substitute the correct host and ports (RERUN_WEB_VIEWER_PORT=9878, RERUN_GRPC_PORT=9877). --- dimos/web/templates/rerun_dashboard.html | 2 +- dimos/web/websocket_vis/websocket_vis_module.py | 14 +++++++++++++- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/dimos/web/templates/rerun_dashboard.html b/dimos/web/templates/rerun_dashboard.html index 393ae98837..d0a9cd035d 100644 --- a/dimos/web/templates/rerun_dashboard.html +++ b/dimos/web/templates/rerun_dashboard.html @@ -51,7 +51,7 @@ - +