Initial xArm6 WorldBelief perception stack

dimos/hardware/manipulators/xarm/adapter.py

@@ -60,11 +60,20 @@ class XArmAdapter(ManipulatorAdapter):
     No inheritance required - just matching method signatures.
     """
 
-    def __init__(self, address: str, dof: int = 6, **_: object) -> None:
+    def __init__(
+        self,
+        address: str,
+        dof: int = 6,
+        initial_positions: list[float] | None = None,
+        **_: object,
+    ) -> None:
         if not address:
             raise ValueError("address (IP) is required for XArmAdapter")
         self._ip = address
         self._dof = dof
+        self._initial_positions = (
+            list(initial_positions) if initial_positions is not None else None
+        )
         self._arm: XArmAPI | None = None
         self._control_mode: ControlMode = ControlMode.POSITION
         self._gripper_enabled: bool = False
@@ -267,6 +276,8 @@ def _move_to_initial_pose(self) -> bool:
         return code == 0
 
     def _initial_joints_degrees(self) -> list[float] | None:
+        if self._initial_positions is not None:
+            return [math.degrees(position) for position in self._initial_positions]
         if self._dof == 6:
             return _XARM6_INITIAL_JOINTS_DEG
         if self._dof == 7:

dimos/manipulation/blueprints.py

@@ -31,3 +31,6 @@
 from dimos.robot.manipulators.xarm.blueprints.simulation import (
     xarm_perception_sim as xarm_perception_sim,
 )
+from dimos.robot.manipulators.xarm.blueprints.worldbelief import (
+    xarm6_worldbelief as xarm6_worldbelief,
+)

dimos/memory2/module.py

@@ -359,7 +359,9 @@ def start(self) -> None:
 
         for name, port in self.inputs.items():
             stream_name = self.config.stream_remapping.get(name, name)
-            stream: Stream[Any] = self.store.stream(stream_name, port.type)
+            stream: Stream[Any] = self.store.stream(
+                stream_name, port.type, **self._stream_kwargs(name)
+            )
             self._port_to_stream(name, port, stream)
             logger.info("Recording %s -> %s (%s)", name, stream_name, port.type.__name__)
 
@@ -389,10 +391,18 @@ async def on_msg(msg: Any) -> None:
                     ts,
                     getattr(msg, "ts", None),
                 )
-            stream.append(msg, ts=ts, pose=pose)
+            try:
+                stream.append(msg, ts=ts, pose=pose)
+            except sqlite3.ProgrammingError as exc:
+                if "closed database" in str(exc).lower():
+                    return
+                raise
 
         self.process_observable(input_topic.pure_observable(), on_msg)
 
+    def _stream_kwargs(self, name: str) -> dict[str, Any]:
+        return {}
+
     def _prepare_streams(self) -> None:
         """On APPEND, drop the streams this recorder is about to (re)write — the
         remapped In-port streams plus ``tf`` — so a re-run replaces them instead

dimos/models/embedding/clip.py

@@ -57,7 +57,7 @@ def embed(self, *images: Image) -> Embedding | list[Embedding]:
         Returns embeddings as torch.Tensor on device for efficient GPU comparisons.
         """
         # Convert to PIL images
-        pil_images = [PILImage.fromarray(img.to_opencv()) for img in images]
+        pil_images = [PILImage.fromarray(img.to_rgb().data) for img in images]
 
         # Process images
         with torch.inference_mode():

dimos/models/embedding/dino.py

@@ -0,0 +1,84 @@
+# Copyright 2025-2026 Dimensional Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import annotations
+
+from functools import cached_property
+from typing import Any, overload
+
+from PIL import Image as PILImage
+import torch
+import torch.nn.functional as functional
+from transformers import AutoImageProcessor, AutoModel
+
+from dimos.models.base import HuggingFaceModel
+from dimos.models.embedding.base import Embedding, EmbeddingModel, HuggingFaceEmbeddingModelConfig
+from dimos.msgs.sensor_msgs.Image import Image
+
+
+class DINOModelConfig(HuggingFaceEmbeddingModelConfig):
+    model_name: str = "facebook/dinov2-base"
+    dtype: torch.dtype = torch.float32
+
+
+class DINOModel(EmbeddingModel, HuggingFaceModel):
+    """DINOv2 image embedding model for visual instance identity."""
+
+    config: DINOModelConfig
+    _model_class = AutoModel
+
+    @cached_property
+    def _model(self) -> Any:
+        return super()._model.eval()
+
+    @cached_property
+    def _processor(self) -> AutoImageProcessor:
+        return AutoImageProcessor.from_pretrained(self.config.model_name)
+
+    @overload
+    def embed(self, image: Image, /) -> Embedding: ...
+    @overload
+    def embed(self, *images: Image) -> list[Embedding]: ...
+    def embed(self, *images: Image) -> Embedding | list[Embedding]:
+        pil_images = [PILImage.fromarray(img.to_rgb().data) for img in images]
+        with torch.inference_mode():
+            inputs = self._processor(images=pil_images, return_tensors="pt").to(self.config.device)
+            outputs = self._model(**inputs)
+            feats = getattr(outputs, "pooler_output", None)
+            if feats is None:
+                last_hidden = getattr(outputs, "last_hidden_state", None)
+                if last_hidden is None:
+                    raise RuntimeError(
+                        "DINO model did not return pooler_output or last_hidden_state"
+                    )
+                feats = last_hidden[:, 0]
+            if self.config.normalize:
+                feats = functional.normalize(feats, dim=-1)
+
+        embeddings: list[Embedding] = []
+        for i, feat in enumerate(feats):
+            embeddings.append(Embedding(vector=feat, timestamp=images[i].ts))
+        return embeddings[0] if len(images) == 1 else embeddings
+
+    @overload
+    def embed_text(self, text: str, /) -> Embedding: ...
+    @overload
+    def embed_text(self, *texts: str) -> list[Embedding]: ...
+    def embed_text(self, *texts: str) -> Embedding | list[Embedding]:
+        raise NotImplementedError("DINOModel does not support text embeddings")
+
+    def stop(self) -> None:
+        if "_processor" in self.__dict__:
+            del self.__dict__["_processor"]
+        super().stop()

dimos/models/embedding/mobileclip.py

@@ -66,7 +66,7 @@ def embed(self, *images: Image) -> Embedding | list[Embedding]:
         Returns embeddings as torch.Tensor on device for efficient GPU comparisons.
         """
         # Convert to PIL images
-        pil_images = [PILImage.fromarray(img.to_opencv()) for img in images]
+        pil_images = [PILImage.fromarray(img.to_rgb().data) for img in images]
 
         # Preprocess and batch
         with torch.inference_mode():

dimos/perception/detection/detectors/yoloe.py

@@ -25,6 +25,9 @@
 from dimos.perception.detection.type.detection2d.imageDetections2D import ImageDetections2D
 from dimos.utils.data import get_data
 from dimos.utils.gpu_utils import is_cuda_available
+from dimos.utils.logging_config import setup_logger
+
+logger = setup_logger()
 
 
 class YoloePromptMode(Enum):
@@ -43,6 +46,9 @@ def __init__(
         prompt_mode: YoloePromptMode = YoloePromptMode.LRPC,
         exclude_class_ids: list[int] | None = None,
         max_area_ratio: float | None = 0.3,
+        conf: float = 0.6,
+        iou: float = 0.6,
+        max_det: int | None = None,
     ) -> None:
         """
         Initialize YOLO-E 2D detector.
@@ -54,6 +60,9 @@ def __init__(
             prompt_mode: LRPC for prompt-free detection, PROMPT for text/visual prompting.
             exclude_class_ids: Class IDs to filter out from results (pass [] to disable).
             max_area_ratio: Maximum bbox area ratio (0-1) relative to image.
+            conf: Confidence threshold passed to Ultralytics track().
+            iou: NMS IoU threshold passed to Ultralytics track().
+            max_det: Optional maximum detections per frame.
         """
         if model_name is None:
             if prompt_mode == YoloePromptMode.LRPC:
@@ -62,9 +71,15 @@ def __init__(
                 model_name = "yoloe-11s-seg.pt"
 
         self.model = YOLOE(get_data(model_path) / model_name)
+        self.detector_id = "yoloe"
+        self.detector_model_name = model_name
+        self.tracker_source = "ultralytics.track"
         self.prompt_mode = prompt_mode
         self._visual_prompts: dict[str, NDArray[Any]] | None = None
         self.max_area_ratio = max_area_ratio
+        self.conf = conf
+        self.iou = iou
+        self.max_det = max_det
         self._lock = threading.Lock()
 
         if prompt_mode == YoloePromptMode.PROMPT:
@@ -80,6 +95,10 @@ def __init__(
             self.device = "cuda"
         else:
             self.device = "cpu"
+        logger.info(
+            f"YOLO-E detector loaded model={model_name} prompt_mode={prompt_mode.value} "
+            f"device={self.device} conf={self.conf} iou={self.iou} max_det={self.max_det}"
+        )
 
     def set_prompts(
         self,
@@ -120,11 +139,13 @@ def process_image(self, image: Image) -> "ImageDetections2D[Any]":
         track_kwargs = {
             "source": image.to_opencv(),
             "device": self.device,
-            "conf": 0.6,
-            "iou": 0.6,
+            "conf": self.conf,
+            "iou": self.iou,
             "persist": True,
             "verbose": False,
         }
+        if self.max_det is not None:
+            track_kwargs["max_det"] = self.max_det
 
         with self._lock:
             if self._visual_prompts is not None: