diff --git a/dimos/constants.py b/dimos/constants.py index d849f4aaf3..448f90ca38 100644 --- a/dimos/constants.py +++ b/dimos/constants.py @@ -21,9 +21,11 @@ except ImportError: CONFIG_DIR = Path(os.environ.get("XDG_CONFIG_HOME", Path.home() / ".config")) STATE_DIR = Path(os.environ.get("XDG_STATE_HOME", Path.home() / ".local" / "state")) / "dimos" + CACHE_DIR = Path(os.environ.get("XDG_CACHE_HOME", Path.home() / ".cache")) / "dimos" else: CONFIG_DIR = Path(GLib.get_user_config_dir()) STATE_DIR = Path(GLib.get_user_state_dir()) / "dimos" + CACHE_DIR = Path(GLib.get_user_cache_dir()) / "dimos" DIMOS_PROJECT_ROOT = Path(__file__).parent.parent diff --git a/dimos/experimental/scene_cooking/browser/collision.py b/dimos/experimental/scene_cooking/browser/collision.py index ff8711b177..2544a9beda 100644 --- a/dimos/experimental/scene_cooking/browser/collision.py +++ b/dimos/experimental/scene_cooking/browser/collision.py @@ -22,7 +22,9 @@ from typing import Any import numpy as np +from numpy.typing import NDArray import open3d as o3d # type: ignore[import-untyped] +import trimesh # type: ignore[import-untyped] from dimos.experimental.scene_cooking.mujoco.collision_policy import CollisionSpec from dimos.experimental.scene_cooking.package_config import BrowserCollisionSpec @@ -115,8 +117,6 @@ def cook_browser_collision( def _write_glb(mesh: o3d.geometry.TriangleMesh, path: Path) -> None: - import trimesh - # Quadric decimation collapses triangles but leaves the original input # vertices in the buffer (referenced by nothing). Drop them before export, # else the GLB carries millions of orphan vertices (~25x larger on a 100k @@ -184,8 +184,8 @@ def _build_fused_collision_mesh( prims: list[ScenePrimMesh], spec: CollisionSpec, ) -> o3d.geometry.TriangleMesh: - vertices: list[np.ndarray] = [] - faces: list[np.ndarray] = [] + vertices: list[NDArray[np.float64]] = [] + faces: list[NDArray[np.int64]] = [] vertex_offset = 0 for prim in prims: mesh = _mesh_for_prim(prim, spec) @@ -257,7 +257,9 @@ def _mesh_for_prim( return mesh -def _mesh_from_arrays(vertices: np.ndarray, faces: np.ndarray) -> o3d.geometry.TriangleMesh: +def _mesh_from_arrays( + vertices: NDArray[np.float64], faces: NDArray[np.int64] +) -> o3d.geometry.TriangleMesh: mesh = o3d.geometry.TriangleMesh() mesh.vertices = o3d.utility.Vector3dVector(vertices) mesh.triangles = o3d.utility.Vector3iVector(faces.astype(np.int32)) diff --git a/dimos/experimental/scene_cooking/browser/visuals.py b/dimos/experimental/scene_cooking/browser/visuals.py index 8bc267738e..d12475c6c4 100644 --- a/dimos/experimental/scene_cooking/browser/visuals.py +++ b/dimos/experimental/scene_cooking/browser/visuals.py @@ -49,6 +49,7 @@ ".ply", } _GLTFPACK_INPUT_SUFFIXES = {".gltf", ".glb", ".obj"} +_GLTFPACK_WARNING_TAIL_LINES = 30 _BLENDER_SCRIPT = r""" import pathlib @@ -340,7 +341,7 @@ def _export_with_gltfpack( ) from exc raise if output and "Warning:" in output: - logger.warning("gltfpack output:\n%s", _tail(output, 30)) + logger.warning("gltfpack output:\n%s", _tail(output, _GLTFPACK_WARNING_TAIL_LINES)) report = _read_json(report_path) return ("gltfpack", report) diff --git a/dimos/experimental/scene_cooking/coacd_util.py b/dimos/experimental/scene_cooking/coacd_util.py new file mode 100644 index 0000000000..c616c21e8e --- /dev/null +++ b/dimos/experimental/scene_cooking/coacd_util.py @@ -0,0 +1,29 @@ +# Copyright 2025-2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Shared CoACD setup for scene-cooking convex decomposition.""" + +from __future__ import annotations + + +def silence_coacd_logging() -> None: + """Quiet CoACD's per-invocation C-library log spam. + + ``set_log_level`` is a cheap call, so this is safe to call before + every ``coacd.run_coacd`` invocation instead of caching "already + silenced" state on a function attribute. + """ + import coacd # type: ignore[import-not-found, import-untyped] + + coacd.set_log_level("error") diff --git a/dimos/experimental/scene_cooking/cook.py b/dimos/experimental/scene_cooking/cook.py index b6bebd6832..b9462bab50 100644 --- a/dimos/experimental/scene_cooking/cook.py +++ b/dimos/experimental/scene_cooking/cook.py @@ -61,6 +61,9 @@ SCENE_PACKAGE_DIR = get_data_dir("scene_packages") _PACKAGE_KEY_LEN = 12 _COOK_VERSION = 4 +#: Cap on entity id samples recorded in cook stats -- diagnostics only, not +#: the full entity list (that lives in ``scene.meta.json``). +_ENTITY_ID_SAMPLE_CAP = 100 def cook_scene_package( @@ -140,7 +143,7 @@ def cook_scene_package( if entities: stats["interactables"] = { "count": len(entities), - "id_samples": [entity["id"] for entity in entities[:100]], + "id_samples": [entity["id"] for entity in entities[:_ENTITY_ID_SAMPLE_CAP]], "static_visual_filter": "plan/blender", } @@ -237,7 +240,7 @@ def cook_scene_package( stats=stats, ) package.write_metadata() - logger.info("scene package cooked: %s", package.metadata_path) + logger.info("scene package cooked", metadata_path=package.metadata_path) return package @@ -262,9 +265,9 @@ def _cook_entity_collision( visual_path = entity.get("visual_path") if not visual_path or not Path(visual_path).exists(): logger.warning( - "entity %s: mesh entity has no cooked visual GLB; " + "mesh entity has no cooked visual GLB; " "no collision hulls (runtime falls back to AABB box)", - entity.get("id"), + entity_id=entity.get("id"), ) continue hull_paths = cook_entity_collision_hulls( @@ -330,6 +333,8 @@ def _compile_mujoco_binary(scene_xml_path: Path, *, rebake: bool) -> tuple[Path, requires the XML wrapper unless a robot-specific composed binary is produced separately. """ + # Lazy: mujoco is a `sim` extra, not a `scene` one -- browser/rerun-only + # cooks (mujoco.compile_binary=False) shouldn't require it installed. import mujoco binary_path = scene_xml_path.with_suffix(".mjb") diff --git a/dimos/experimental/scene_cooking/entities/collision.py b/dimos/experimental/scene_cooking/entities/collision.py index 8d53a0bc0a..fdac014320 100644 --- a/dimos/experimental/scene_cooking/entities/collision.py +++ b/dimos/experimental/scene_cooking/entities/collision.py @@ -36,7 +36,11 @@ from __future__ import annotations from pathlib import Path +from typing import Any +import numpy as np + +from dimos.experimental.scene_cooking.coacd_util import silence_coacd_logging from dimos.utils.logging_config import setup_logger logger = setup_logger() @@ -59,9 +63,9 @@ def cook_entity_collision_hulls( """Decompose one entity mesh into convex hulls under ``out_dir``. Idempotent: existing ``hull_*.obj`` files are reused unless ``rebake``. - Falls back to a single convex hull when CoACD is unavailable or fails - on the mesh. Returns ``[]`` (with a warning) when the mesh can't be - read at all — the runtime composer then uses an AABB box. + Falls back to a single convex hull when CoACD fails on the mesh. + Returns ``[]`` (with a warning) when the mesh can't be read at all — + the runtime composer then uses an AABB box. """ mesh_path = Path(visual_mesh_path) out_dir = Path(out_dir) @@ -88,8 +92,6 @@ def cook_entity_collision_hulls( for stale in out_dir.glob("hull_*.obj"): stale.unlink() - import numpy as np - out_paths: list[Path] = [] if parts: for i, (vertices, triangles) in enumerate(parts): @@ -108,19 +110,16 @@ def cook_entity_collision_hulls( return out_paths -def _run_coacd(mesh: object, mesh_path: Path) -> list[tuple[object, object]]: +def _run_coacd(mesh: Any, mesh_path: Path) -> list[tuple[Any, Any]]: """CoACD parts for an open3d mesh; ``[]`` means fall back to one hull.""" - try: - import coacd # type: ignore[import-not-found, import-untyped] - import numpy as np + import coacd # type: ignore[import-not-found, import-untyped] - if not getattr(_run_coacd, "_coacd_silenced", False): - coacd.set_log_level("error") - _run_coacd._coacd_silenced = True # type: ignore[attr-defined] + silence_coacd_logging() + try: cm = coacd.Mesh( - np.asarray(mesh.vertices, dtype=np.float64), # type: ignore[attr-defined] - np.asarray(mesh.triangles, dtype=np.int32), # type: ignore[attr-defined] + np.asarray(mesh.vertices, dtype=np.float64), + np.asarray(mesh.triangles, dtype=np.int32), ) parts = coacd.run_coacd( cm, @@ -130,9 +129,9 @@ def _run_coacd(mesh: object, mesh_path: Path) -> list[tuple[object, object]]: mcts_iterations=_COACD_MCTS_ITERATIONS, mcts_nodes=_COACD_MCTS_NODES, ) - return list(parts) - except Exception as exc: + except (AssertionError, RuntimeError, ValueError) as exc: logger.warning( "entity hulls: CoACD failed for %s (%s); using single convex hull", mesh_path, exc ) return [] + return list(parts) diff --git a/dimos/experimental/scene_cooking/entities/test_collision.py b/dimos/experimental/scene_cooking/entities/test_collision.py index 29a82ed962..30aa16ed00 100644 --- a/dimos/experimental/scene_cooking/entities/test_collision.py +++ b/dimos/experimental/scene_cooking/entities/test_collision.py @@ -16,13 +16,13 @@ from pathlib import Path -import pytest +import coacd # noqa: F401 ensures collection fails loudly if coacd is missing +import open3d as o3d from dimos.experimental.scene_cooking.entities.collision import cook_entity_collision_hulls def _write_box_mesh(path: Path) -> None: - o3d = pytest.importorskip("open3d") mesh = o3d.geometry.TriangleMesh.create_box(0.1, 0.1, 0.1) o3d.io.write_triangle_mesh(str(path), mesh) @@ -30,7 +30,6 @@ def _write_box_mesh(path: Path) -> None: def test_cook_entity_collision_hulls(tmp_path: Path) -> None: source = tmp_path / "visual.obj" _write_box_mesh(source) - pytest.importorskip("coacd") out_dir = tmp_path / "mujoco_collision" hulls = cook_entity_collision_hulls(source, out_dir) @@ -44,7 +43,6 @@ def test_cook_entity_collision_hulls(tmp_path: Path) -> None: def test_cook_entity_collision_hulls_is_idempotent(tmp_path: Path) -> None: source = tmp_path / "visual.obj" _write_box_mesh(source) - pytest.importorskip("coacd") out_dir = tmp_path / "mujoco_collision" first = cook_entity_collision_hulls(source, out_dir) @@ -60,7 +58,6 @@ def test_cook_entity_collision_hulls_is_idempotent(tmp_path: Path) -> None: def test_cook_entity_collision_hulls_unreadable_mesh(tmp_path: Path) -> None: - pytest.importorskip("open3d") source = tmp_path / "visual.obj" source.write_text("not a mesh\n") diff --git a/dimos/experimental/scene_cooking/entities/visuals.py b/dimos/experimental/scene_cooking/entities/visuals.py index 6183d7f08a..72fd07df20 100644 --- a/dimos/experimental/scene_cooking/entities/visuals.py +++ b/dimos/experimental/scene_cooking/entities/visuals.py @@ -20,6 +20,7 @@ from pathlib import Path import shutil import tempfile +from typing import Any from dimos.experimental.scene_cooking.command import ( blender_output_line_is_interesting, @@ -280,7 +281,7 @@ def cook_plan_visual_assets( return static_visual_source -def _blender_plan_json(plan: SceneCookPlan, static_visual_source: Path) -> dict[str, object]: +def _blender_plan_json(plan: SceneCookPlan, static_visual_source: Path) -> dict[str, Any]: return { "visual_plan_version": _VISUAL_PLAN_VERSION, "alignment": { @@ -307,7 +308,7 @@ def _blender_plan_json(plan: SceneCookPlan, static_visual_source: Path) -> dict[ } -def _manifest_matches(path: Path, expected: dict[str, object]) -> bool: +def _manifest_matches(path: Path, expected: dict[str, Any]) -> bool: if not path.exists(): return False try: diff --git a/dimos/experimental/scene_cooking/mujoco/collision_export.py b/dimos/experimental/scene_cooking/mujoco/collision_export.py index 8362e57190..87f5f8c48c 100644 --- a/dimos/experimental/scene_cooking/mujoco/collision_export.py +++ b/dimos/experimental/scene_cooking/mujoco/collision_export.py @@ -59,6 +59,7 @@ from concurrent.futures import ProcessPoolExecutor, as_completed from dataclasses import asdict, dataclass, replace import hashlib +import json import multiprocessing import os from pathlib import Path @@ -66,8 +67,12 @@ from typing import Any import numpy as np +from numpy.typing import NDArray import open3d as o3d # type: ignore[import-untyped] +from scipy.spatial import ConvexHull, QhullError # type: ignore[import-untyped] +from scipy.spatial.transform import Rotation # type: ignore[import-untyped] +from dimos.constants import CACHE_DIR as _DIMOS_CACHE_DIR from dimos.experimental.scene_cooking.mujoco.collision_policy import ( CollisionSpec, decide_for_prim, @@ -83,7 +88,7 @@ logger = setup_logger() -CACHE_DIR = Path.home() / ".cache" / "dimos" / "scene_meshes" +CACHE_DIR = _DIMOS_CACHE_DIR / "scene_meshes" # Scene-only wrapper -- no robot include. Robots are attached at runtime @@ -171,6 +176,23 @@ # contract and old cache directories can safely stay on disk. _CACHE_SCHEMA_VERSION = "scene-only-v11" +# Cap on worker processes when the parent has multiple native threads +# (see ``_native_thread_count``) -- ``forkserver``/``spawn`` startup gets +# expensive past this many workers. +_MAX_THREADED_WORKERS = 8 +# Log progress roughly this many times across a bake, regardless of scene size. +_PROGRESS_LOG_EVERY_SMALL_SCENE = 25 +_PROGRESS_LOG_EVERY_LARGE_SCENE = 250 +_PROGRESS_LOG_LARGE_SCENE_THRESHOLD = 500 +# Pad the viewer-framing extent beyond the tight scene bounding sphere so +# geometry right at the edge isn't clipped by the camera's far plane. +_SCENE_EXTENT_PADDING = 1.1 +# MuJoCo visual zfar: scale with scene extent, with a floor high enough for +# small scenes to still render distant geometry (e.g. a skybox) without +# clipping. +_VISUAL_ZFAR_EXTENT_MULTIPLIER = 20.0 +_MIN_VISUAL_ZFAR_M = 10000.0 + @dataclass class _BakeArtifacts: @@ -215,8 +237,8 @@ def bake_scene_mjcf( etc. Anything ``source_assets.mesh.load_scene_prims`` accepts. alignment: scale / translation / rotation / y-up swap to bake into world frame before any geom is emitted. - cache_root: override the cache root (defaults to - ``~/.cache/dimos/scene_meshes``). + cache_root: override the cache root (defaults to the XDG cache dir + under ``dimos/scene_meshes``). collision_spec: per-prim policy. ``None`` auto-discovers a sidecar ``.collision.json`` next to the source, or falls back to ``CollisionSpec()`` defaults (auto-fit @@ -381,7 +403,6 @@ def _cache_key( Robot-agnostic: the cooked scene wrapper is the same regardless of which robot will eventually be attached at runtime via ``MjSpec``. """ - import json def _file_signature(path: Path) -> str: st = path.stat() @@ -461,8 +482,8 @@ def _process_one_prim( asset_lines.append(_ASSET_LINE.format(name=vis_name, file=vis_path.name)) geom_lines.append(_VISUAL_GEOM_LINE.format(name=f"{vis_name}_geom", mesh=vis_name)) counters["visuals"] = 1 - except Exception: - pass + except (ValueError, RuntimeError, ZeroDivisionError, OSError) as exc: + logger.warning(f"visual OBJ write failed for {prim.name}: {exc}") friction_attr = "" if decision.friction is not None: @@ -567,7 +588,7 @@ def _bake_prims( n_workers = max(1, (os.cpu_count() or 4) - 1) if _native_thread_count() > 1: - n_workers = min(n_workers, 8) + n_workers = min(n_workers, _MAX_THREADED_WORKERS) start_method = ( "forkserver" if "forkserver" in multiprocessing.get_all_start_methods() else "spawn" ) @@ -585,7 +606,11 @@ def _bake_prims( n_skipped += n_pre_skipped reasons.update(pre_skip_reasons) total_work = len(work_items) - progress_every = 25 if total_work <= 500 else 250 + progress_every = ( + _PROGRESS_LOG_EVERY_SMALL_SCENE + if total_work <= _PROGRESS_LOG_LARGE_SCENE_THRESHOLD + else _PROGRESS_LOG_EVERY_LARGE_SCENE + ) with executor as ex: futures = [ex.submit(_process_one_prim, item) for item in work_items] done = 0 @@ -686,7 +711,7 @@ def _emit_primitive_geom( # --------------------------------------------------------------------------- # -def _valid_hull(v: np.ndarray, f: np.ndarray) -> bool: +def _valid_hull(v: NDArray[np.float32], f: NDArray[np.int32]) -> bool: """Reject hulls that MuJoCo's qhull would choke on at compile time. Four layers: @@ -710,15 +735,15 @@ def _valid_hull(v: np.ndarray, f: np.ndarray) -> bool: if np.linalg.matrix_rank(centered, tol=_DEGENERATE_EPS) < 3: return False try: - from scipy.spatial import ConvexHull, QhullError # type: ignore[import-untyped] - ConvexHull(v, qhull_options="Qt") except (QhullError, ValueError): return False return True -def _fallback_box_geom(name: str, vertices: np.ndarray, friction_attr: str = "") -> str | None: +def _fallback_box_geom( + name: str, vertices: NDArray[np.float32], friction_attr: str = "" +) -> str | None: """Emit a thin OBB box geom for vertices that can't form a valid hull. The thickness floor (``_FALLBACK_BOX_THICKNESS_M = 3 cm``) keeps the @@ -751,16 +776,16 @@ def _fallback_box_geom(name: str, vertices: np.ndarray, friction_attr: str = "") def _oriented_box( - vertices: np.ndarray, -) -> tuple[np.ndarray, np.ndarray, np.ndarray]: + vertices: NDArray[np.float64], +) -> tuple[NDArray[np.float64], NDArray[np.float64], NDArray[np.float64]]: """OBB via trimesh's ``bounding_box_oriented``. Falls back to AABB if trimesh's OBB fitter produces non-finite output or the prim has < 3 vertices. """ - try: - import trimesh # type: ignore[import-untyped] + import trimesh # type: ignore[import-untyped] + try: tm = trimesh.Trimesh(vertices=vertices, faces=np.empty((0, 3), dtype=np.int32)) obb = tm.bounding_box_oriented transform = np.asarray(obb.primitive.transform, dtype=np.float64) @@ -771,27 +796,25 @@ def _oriented_box( rotation[:, 0] *= -1.0 if np.isfinite(center).all() and np.isfinite(rotation).all() and np.isfinite(extent).all(): return center, rotation, np.abs(extent) - except Exception: - pass + except (ValueError, RuntimeError, ZeroDivisionError) as exc: + logger.debug(f"oriented-box fit failed; falling back to AABB: {exc}") lo = vertices.min(axis=0) hi = vertices.max(axis=0) return (lo + hi) * 0.5, np.eye(3), hi - lo -def _rotation_matrix_to_wxyz(rotation: np.ndarray) -> np.ndarray: +def _rotation_matrix_to_wxyz(rotation: NDArray[np.float64]) -> NDArray[np.float64]: """3x3 rotation -> ``(w, x, y, z)`` quaternion.""" - from scipy.spatial.transform import Rotation # type: ignore[import-untyped] - xyzw = Rotation.from_matrix(rotation).as_quat() return np.array([xyzw[3], xyzw[0], xyzw[1], xyzw[2]], dtype=np.float64) -def _fmt_vec(values: np.ndarray) -> str: +def _fmt_vec(values: NDArray[np.float64]) -> str: return " ".join(f"{float(v):.9g}" for v in values) -def _scene_bounds(prims: list[ScenePrimMesh]) -> tuple[np.ndarray, float]: +def _scene_bounds(prims: list[ScenePrimMesh]) -> tuple[NDArray[np.float64], float]: """Return a viewer-friendly center and extent for the aligned scene. MuJoCo's viewer uses ``statistic.center`` / ``statistic.extent`` for @@ -799,8 +822,8 @@ def _scene_bounds(prims: list[ScenePrimMesh]) -> tuple[np.ndarray, float]: much too small for baked building-scale scenes, so wrappers need to advertise the scene bounds explicitly. """ - mins: list[np.ndarray] = [] - maxs: list[np.ndarray] = [] + mins: list[NDArray[np.float64]] = [] + maxs: list[NDArray[np.float64]] = [] for prim in prims: vertices = np.asarray(prim.vertices, dtype=np.float64) if vertices.ndim != 2 or vertices.shape[1] != 3 or len(vertices) == 0: @@ -818,7 +841,7 @@ def _scene_bounds(prims: list[ScenePrimMesh]) -> tuple[np.ndarray, float]: scene_max = np.max(np.vstack(maxs), axis=0) center = (scene_min + scene_max) * 0.5 diagonal = scene_max - scene_min - extent = max(float(np.linalg.norm(diagonal) * 0.5 * 1.1), 1.0) + extent = max(float(np.linalg.norm(diagonal) * 0.5 * _SCENE_EXTENT_PADDING), 1.0) return center, extent @@ -827,17 +850,19 @@ def _scene_bounds(prims: list[ScenePrimMesh]) -> tuple[np.ndarray, float]: # --------------------------------------------------------------------------- # -def _write_hull_obj(obj_file: Path, vertices: np.ndarray, faces: np.ndarray) -> None: +def _write_hull_obj( + obj_file: Path, vertices: NDArray[np.float32], faces: NDArray[np.int32] +) -> None: """Write a CoACD/single-hull mesh. No watertight check -- hulls are closed by construction.""" _write_mesh_obj(obj_file, vertices, faces) def _simplify_mesh_geom( - vertices: np.ndarray, - faces: np.ndarray, + vertices: NDArray[np.float32], + faces: NDArray[np.int32], target_faces: int, -) -> tuple[np.ndarray, np.ndarray]: +) -> tuple[NDArray[np.float32], NDArray[np.int32]]: if target_faces <= 0 or len(faces) <= target_faces: return vertices, faces @@ -863,17 +888,17 @@ def _simplify_mesh_geom( out_faces = np.asarray(simplified.triangles, dtype=np.int32) if len(out_vertices) >= 4 and 4 <= len(out_faces) <= target_faces: return out_vertices, out_faces - except Exception: - logger.debug("mesh simplification failed; falling back to convex hull", exc_info=True) + except RuntimeError: + logger.warning("mesh simplification failed; falling back to convex hull", exc_info=True) hull = _convex_hull_mesh(vertices) return hull if hull is not None else (vertices, faces) -def _convex_hull_mesh(vertices: np.ndarray) -> tuple[np.ndarray, np.ndarray] | None: +def _convex_hull_mesh( + vertices: NDArray[np.float32], +) -> tuple[NDArray[np.float32], NDArray[np.int32]] | None: try: - from scipy.spatial import ConvexHull, QhullError # type: ignore[import-untyped] - hull = ConvexHull(vertices.astype(np.float64)) except (QhullError, ValueError): return None @@ -885,7 +910,9 @@ def _convex_hull_mesh(vertices: np.ndarray) -> tuple[np.ndarray, np.ndarray] | N return vertices[used].astype(np.float32), remapped_faces.astype(np.int32) -def _write_visual_obj(obj_file: Path, vertices: np.ndarray, faces: np.ndarray) -> None: +def _write_visual_obj( + obj_file: Path, vertices: NDArray[np.float64], faces: NDArray[np.int32] +) -> None: """Write a *renderable* OBJ -- closed under all viewing angles. UE's static-mesh exporter culls hidden faces (a floor slab ships @@ -914,12 +941,15 @@ def _write_visual_obj(obj_file: Path, vertices: np.ndarray, faces: np.ndarray) - if len(hull.vertices) >= 4 and len(hull.faces) >= 4: vertices = np.asarray(hull.vertices, dtype=np.float64) faces = np.asarray(hull.faces, dtype=np.int32) - except Exception: - pass # fall back to original; visual may look hollow + except (ValueError, RuntimeError, ZeroDivisionError) as exc: + # Fall back to the original (possibly hollow-looking) mesh. + logger.warning(f"convex hull fallback failed for {obj_file.name}: {exc}") _write_mesh_obj(obj_file, vertices, faces) -def _write_mesh_obj(obj_file: Path, vertices: np.ndarray, faces: np.ndarray) -> None: +def _write_mesh_obj( + obj_file: Path, vertices: NDArray[np.floating[Any]], faces: NDArray[np.int32] +) -> None: o3d_mesh = o3d.geometry.TriangleMesh() o3d_mesh.vertices = o3d.utility.Vector3dVector(vertices.astype(np.float64)) o3d_mesh.triangles = o3d.utility.Vector3iVector(faces) @@ -944,14 +974,14 @@ def _write_wrapper( cache_key: str, asset_lines: list[str], geom_lines: list[str], - statistic_center: np.ndarray, + statistic_center: NDArray[np.float64], statistic_extent: float, ) -> None: """Emit the scene-only wrapper.xml. Robots attach at runtime via ``MjSpec``; the wrapper directory holds only this file plus the cooked scene OBJs that it references with relative paths. """ - visual_zfar = max(float(statistic_extent) * 20.0, 10000.0) + visual_zfar = max(float(statistic_extent) * _VISUAL_ZFAR_EXTENT_MULTIPLIER, _MIN_VISUAL_ZFAR_M) wrapper_xml = _WRAPPER_TEMPLATE.format( model_name=f"scene_{cache_key}", statistic_center=_fmt_vec(statistic_center), diff --git a/dimos/experimental/scene_cooking/mujoco/collision_policy.py b/dimos/experimental/scene_cooking/mujoco/collision_policy.py index 1ba69abaf5..beab34d89d 100644 --- a/dimos/experimental/scene_cooking/mujoco/collision_policy.py +++ b/dimos/experimental/scene_cooking/mujoco/collision_policy.py @@ -33,10 +33,10 @@ future UE-side extractor, an LLM…) doesn't matter to the bake — the sidecar is the contract. -The dispatcher ``emit_for_prim()`` walks: sidecar override → generic -heuristics → primitive auto-fit → CoACD fallback, and returns a list -of ``GeomEmission`` describing every ```` the wrapper should -include for the prim. +The dispatcher ``decide_for_prim()`` walks: sidecar override → generic +heuristics → primitive auto-fit → CoACD fallback, and returns a +``PrimDecision`` describing the ````(s) the wrapper should emit +for the prim. """ from __future__ import annotations @@ -48,8 +48,10 @@ from typing import Any, Literal import numpy as np +from numpy.typing import NDArray from scipy.spatial import ConvexHull, QhullError # type: ignore[import-untyped] +from dimos.experimental.scene_cooking.coacd_util import silence_coacd_logging from dimos.utils.logging_config import setup_logger logger = setup_logger() @@ -246,53 +248,12 @@ def resolve(self, prim_path: str) -> OverrideConfig: return {"type": self.default} -# --------------------------------------------------------------------------- # -# Emission record # -# --------------------------------------------------------------------------- # - - -@dataclass -class GeomEmission: - """One MuJoCo ```` to emit, in dimos world frame. - - Either ``mesh_path`` is set (mesh-type geom — also emits a - ```` asset) or the primitive parameters (``size``, ``pos``, - ``quat``) are set. - """ - - name: str - purpose: Literal["collision", "visual"] - kind: Literal["mesh", "box", "sphere", "cylinder", "capsule", "plane"] - - #: For ``kind="mesh"``: absolute path to the OBJ. - mesh_path: Path | None = None - - #: Primitive size (semantics depend on ``kind``): - #: * box → (hx, hy, hz) (half-extents) - #: * sphere → (r,) - #: * cylinder→ (r, half_height) - #: * capsule → (r, half_height) (caps extend beyond half_height) - #: * plane → (hx, hy, _grid_spacing) — last is cosmetic only - size: tuple[float, ...] | None = None - - #: World-frame position of the primitive centre. ``None`` for meshes - #: (their geometry is already in world frame). - pos: tuple[float, float, float] | None = None - - #: World-frame orientation (wxyz, MuJoCo convention). ``None`` → - #: identity. Not used for meshes. - quat: tuple[float, float, float, float] | None = None - - #: Optional friction override (slide, spin, roll). - friction: tuple[float, float, float] | None = None - - # --------------------------------------------------------------------------- # # Math helpers # # --------------------------------------------------------------------------- # -def _matrix_to_quat_wxyz(R: np.ndarray) -> tuple[float, float, float, float]: +def _matrix_to_quat_wxyz(R: NDArray[np.float64]) -> tuple[float, float, float, float]: """3x3 right-handed rotation → quaternion ``(w, x, y, z)``. Standard Shepperd's method; avoids the singularity at ``trace == -1``. @@ -326,7 +287,7 @@ def _matrix_to_quat_wxyz(R: np.ndarray) -> tuple[float, float, float, float]: return (float(w), float(x), float(y), float(z)) -def _quat_z_to(axis: np.ndarray) -> tuple[float, float, float, float]: +def _quat_z_to(axis: NDArray[np.float64]) -> tuple[float, float, float, float]: """Quaternion that rotates ``+Z`` onto ``axis`` (unit vector). Used for cylinder/capsule placement — MuJoCo's primitive long-axis @@ -367,7 +328,7 @@ def _quat_z_to(axis: np.ndarray) -> tuple[float, float, float, float]: _SHEET_PRISM_MAX_TRIANGLES = 1024 -def _fit_aabb_box(vertices: np.ndarray) -> PrimitiveFit: +def _fit_aabb_box(vertices: NDArray[np.float64]) -> PrimitiveFit: """Axis-aligned bounding box. Identity quat.""" mn, mx = vertices.min(0), vertices.max(0) half_ext = np.maximum((mx - mn) / 2.0, _MIN_SIZE_M) @@ -381,7 +342,7 @@ def _fit_aabb_box(vertices: np.ndarray) -> PrimitiveFit: } -def _fit_obb_box(vertices: np.ndarray) -> PrimitiveFit: +def _fit_obb_box(vertices: NDArray[np.float64]) -> PrimitiveFit: """Oriented bounding box via PCA. Tighter than AABB when the prim is rotated relative to world axes (most UE props are world-aligned, so OBB ≈ AABB, but rotated assets benefit).""" @@ -406,7 +367,7 @@ def _fit_obb_box(vertices: np.ndarray) -> PrimitiveFit: } -def _fit_sphere(vertices: np.ndarray) -> PrimitiveFit: +def _fit_sphere(vertices: NDArray[np.float64]) -> PrimitiveFit: """Centroid + farthest-vertex. Looser than Welzl/Ritter but fine for fill-ratio comparison.""" centroid = vertices.mean(0) @@ -420,7 +381,7 @@ def _fit_sphere(vertices: np.ndarray) -> PrimitiveFit: } -def _fit_cylinder(vertices: np.ndarray) -> PrimitiveFit: +def _fit_cylinder(vertices: NDArray[np.float64]) -> PrimitiveFit: """Cylinder along PCA principal axis.""" centroid = vertices.mean(0) centered = vertices - centroid @@ -443,7 +404,7 @@ def _fit_cylinder(vertices: np.ndarray) -> PrimitiveFit: } -def _fit_capsule(vertices: np.ndarray) -> PrimitiveFit: +def _fit_capsule(vertices: NDArray[np.float64]) -> PrimitiveFit: """Capsule along PCA principal axis. MuJoCo capsule half-height is the *cylindrical* portion only; total length = 2*(half_h + r).""" cyl = _fit_cylinder(vertices) @@ -459,7 +420,7 @@ def _fit_capsule(vertices: np.ndarray) -> PrimitiveFit: } -def _hull_volume(vertices: np.ndarray) -> float | None: +def _hull_volume(vertices: NDArray[np.float64]) -> float | None: """Convex-hull volume in m³, or ``None`` if qhull rejects the points.""" try: return float(ConvexHull(vertices).volume) @@ -467,7 +428,7 @@ def _hull_volume(vertices: np.ndarray) -> float | None: return None -def _mesh_volume(vertices: np.ndarray, triangles: np.ndarray) -> float: +def _mesh_volume(vertices: NDArray[np.float64], triangles: NDArray[np.int32]) -> float: """Signed mesh volume (Gauss / divergence theorem on triangle fans). Closed meshes return a positive number; for non-closed inputs the @@ -479,7 +440,7 @@ def _mesh_volume(vertices: np.ndarray, triangles: np.ndarray) -> float: def _best_primitive_fit( - vertices: np.ndarray, + vertices: NDArray[np.float64], hull_vol: float, candidates: tuple[str, ...] = ("box", "cylinder", "sphere", "capsule"), ) -> PrimitiveFit | None: @@ -500,8 +461,8 @@ def _best_primitive_fit( continue f["fill_ratio"] = hull_vol / f["volume"] fits.append(f) - except Exception as e: - logger.debug(f" primitive fit {kind} failed: {e}") + except (np.linalg.LinAlgError, ValueError, ZeroDivisionError) as e: + logger.warning(f" primitive fit {kind} failed: {e}") if not fits: return None return max(fits, key=lambda f: f["fill_ratio"]) @@ -512,11 +473,11 @@ def _best_primitive_fit( # --------------------------------------------------------------------------- # -def _is_tiny(extent: np.ndarray, threshold_m: float) -> bool: +def _is_tiny(extent: NDArray[np.float64], threshold_m: float) -> bool: return bool(extent.max() < threshold_m) -def _is_slab(extent: np.ndarray, aspect_ratio: float) -> bool: +def _is_slab(extent: NDArray[np.float64], aspect_ratio: float) -> bool: """Wall / floor / door / panel — one axis is much smaller than the other two (or one much larger than the other two — covers beams).""" sorted_ext = np.sort(extent) @@ -526,8 +487,8 @@ def _is_slab(extent: np.ndarray, aspect_ratio: float) -> bool: def _sheet_footprint_stats( - vertices: np.ndarray, - triangles: np.ndarray, + vertices: NDArray[np.float64], + triangles: NDArray[np.int32], thin_axis: int, ) -> tuple[float, float] | None: """Return ``(projected_aabb_area, projected_triangle_fill)`` for a sheet.""" @@ -547,8 +508,8 @@ def _sheet_footprint_stats( def _is_boxlike_sheet( - vertices: np.ndarray, - triangles: np.ndarray, + vertices: NDArray[np.float64], + triangles: NDArray[np.int32], thin_axis: int, ) -> bool: """Whether a thin mesh roughly fills its projected bounding rectangle. @@ -566,8 +527,8 @@ def _is_boxlike_sheet( def _should_emit_triangle_prisms( - vertices: np.ndarray, - triangles: np.ndarray, + vertices: NDArray[np.float64], + triangles: NDArray[np.int32], thin_axis: int, ) -> bool: """Use exact-ish triangle prisms only for large horizontal sheets. @@ -587,12 +548,12 @@ def _should_emit_triangle_prisms( def _thin_sheet_hulls( - vertices: np.ndarray, - triangles: np.ndarray, + vertices: NDArray[np.float64], + triangles: NDArray[np.int32], thickness: float = _SHEET_PRISM_THICKNESS_M, -) -> list[tuple[np.ndarray, np.ndarray]]: +) -> list[tuple[NDArray[np.float32], NDArray[np.int32]]]: """Represent a thin non-rectangular sheet as convex triangle prisms.""" - hulls: list[tuple[np.ndarray, np.ndarray]] = [] + hulls: list[tuple[NDArray[np.float32], NDArray[np.int32]]] = [] faces = np.asarray( [ [0, 1, 2], @@ -622,7 +583,7 @@ def _thin_sheet_hulls( return hulls -def _is_flat_horizontal_box(extent: np.ndarray, thin_axis: int) -> bool: +def _is_flat_horizontal_box(extent: NDArray[np.float64], thin_axis: int) -> bool: """Thin in world Z, broad in world X/Y, and flat enough to box safely. PCA boxes are unstable for nearly flat floors/ceilings: any small @@ -647,7 +608,7 @@ def _is_flat_horizontal_box(extent: np.ndarray, thin_axis: int) -> bool: @dataclass class PrimDecision: """What the dispatcher decided for one prim. Consumed by the bake - which materialises ``GeomEmission`` records and writes OBJs.""" + which materialises MJCF ```` lines and writes OBJs.""" #: ``"skip"`` (no collision), ``"primitive"`` (one ```` with #: kind ∈ {box, sphere, cylinder, capsule, plane}), or ``"hulls"`` @@ -659,7 +620,9 @@ class PrimDecision: primitive: PrimitiveFit | None = None #: For ``"hulls"``: list of ``(vertices, triangles)`` ready to write. - hulls: list[tuple[np.ndarray, np.ndarray]] = field(default_factory=list) + #: Vertex precision varies by source (CoACD/sheet-prisms emit float32, + #: single-hull pass-through keeps the input prim's float64). + hulls: list[tuple[NDArray[np.floating[Any]], NDArray[np.int32]]] = field(default_factory=list) #: For diagnostics: which rule fired. reason: str = "" @@ -673,8 +636,8 @@ class PrimDecision: def decide_for_prim( - vertices: np.ndarray, - triangles: np.ndarray, + vertices: NDArray[np.float64], + triangles: NDArray[np.int32], prim_path: str, spec: CollisionSpec, ) -> PrimDecision: @@ -827,7 +790,7 @@ def decide_for_prim( def _resolve_explicit_primitive( - vertices: np.ndarray, + vertices: NDArray[np.float64], kind: str, override: OverrideConfig, ) -> PrimitiveFit: @@ -875,7 +838,7 @@ def _resolve_explicit_primitive( def _apply_box_min_thickness( fit: PrimitiveFit, - vertices: np.ndarray, + vertices: NDArray[np.float64], override: OverrideConfig, ) -> None: raw_min_thickness = override.get("min_thickness") @@ -926,11 +889,11 @@ def _target_faces(override: OverrideConfig) -> int | None: def _coacd_decompose( - vertices: np.ndarray, - triangles: np.ndarray, + vertices: NDArray[np.float64], + triangles: NDArray[np.int32], threshold: float, max_hulls: int, -) -> list[tuple[np.ndarray, np.ndarray]]: +) -> list[tuple[NDArray[np.float32], NDArray[np.int32]]]: """Run CoACD on a single prim, return list of ``(verts, tris)`` hulls. CoACD is imported lazily — it ships its own C library and we don't @@ -938,10 +901,7 @@ def _coacd_decompose( """ import coacd # type: ignore[import-not-found, import-untyped] - # CoACD's C lib prints a lot per invocation; quiet it once per process. - if not getattr(_coacd_decompose, "_silenced", False): - coacd.set_log_level("error") - _coacd_decompose._silenced = True # type: ignore[attr-defined] + silence_coacd_logging() mesh = coacd.Mesh(vertices.astype(np.float64), triangles.astype(np.int32)) # CoACD's MCTS defaults (mcts_iterations=150, resolution=2000) are tuned @@ -958,7 +918,7 @@ def _coacd_decompose( mcts_iterations=30, mcts_nodes=10, ) - out: list[tuple[np.ndarray, np.ndarray]] = [] + out: list[tuple[NDArray[np.float32], NDArray[np.int32]]] = [] for v, t in parts: v = np.asarray(v, dtype=np.float32) t = np.asarray(t, dtype=np.int32) diff --git a/dimos/experimental/scene_cooking/package_config.py b/dimos/experimental/scene_cooking/package_config.py index bb0a007380..0956083b61 100644 --- a/dimos/experimental/scene_cooking/package_config.py +++ b/dimos/experimental/scene_cooking/package_config.py @@ -54,23 +54,9 @@ def artifact_name(self) -> str: return self.output_name or f"visual.{self.target_key}.glb" +#: Per-target overrides layered on top of ``BrowserVisualSpec``'s defaults. +#: "rerun" has no entry here -- its values *are* the dataclass defaults. _BROWSER_VISUAL_PROFILES: dict[str, dict[str, Any]] = { - "rerun": { - "optimizer": "gltfpack", - "simplify_ratio": 0.3, - "simplify_error": 0.02, - "texture_format": None, - "max_texture_size": None, - "normalize_textures": True, - "quantize": False, - "use_gpu_instancing": False, - "demote_required_extensions": ("KHR_texture_transform",), - "max_meshes": 200, - "max_materials": 50, - "max_textures": 750, - "max_vertices": 750_000, - "max_vertex_growth_ratio": 1.25, - }, "babylon": { "optimizer": "gltfpack", "simplify_ratio": 0.3, @@ -104,7 +90,7 @@ def artifact_name(self) -> str: "max_vertex_growth_ratio": 1.5, }, } -BROWSER_VISUAL_TARGETS = tuple(sorted(_BROWSER_VISUAL_PROFILES)) +BROWSER_VISUAL_TARGETS = tuple(sorted({"rerun", *_BROWSER_VISUAL_PROFILES})) def browser_visual_spec_for_target( @@ -113,13 +99,10 @@ def browser_visual_spec_for_target( ) -> BrowserVisualSpec: """Build a visual cook spec for a named browser/viewer target.""" target_key = target.strip().lower() - try: - values = dict(_BROWSER_VISUAL_PROFILES[target_key]) - except KeyError as exc: + if target_key not in BROWSER_VISUAL_TARGETS: known = ", ".join(BROWSER_VISUAL_TARGETS) - raise ValueError( - f"unknown browser visual target {target!r}; expected one of: {known}" - ) from exc + raise ValueError(f"unknown browser visual target {target!r}; expected one of: {known}") + values = dict(_BROWSER_VISUAL_PROFILES.get(target_key, {})) values.update(overrides) return BrowserVisualSpec(target=target_key, **values) diff --git a/dimos/experimental/scene_cooking/planning.py b/dimos/experimental/scene_cooking/planning.py index 312f8ffc65..57846c49f6 100644 --- a/dimos/experimental/scene_cooking/planning.py +++ b/dimos/experimental/scene_cooking/planning.py @@ -28,6 +28,7 @@ from typing import Any import numpy as np +from numpy.typing import NDArray from scipy.spatial.transform import Rotation as R from dimos.experimental.scene_cooking.mujoco.collision_policy import CollisionSpec @@ -44,6 +45,19 @@ _HASH_SUFFIX_RE = re.compile(r"_[0-9a-fA-F]{6,}$") +#: Floor for a prim/entity's AABB extent (metres). Keeps zero-thickness +#: sheets (a floor tile authored with no depth) from producing a zero-size +#: physics extent. +_MIN_EXTENT_M = 1e-4 + + +def _safe_extents( + aabb_min: NDArray[np.float64], aabb_max: NDArray[np.float64] +) -> NDArray[np.float64]: + """AABB extents clamped to ``_MIN_EXTENT_M`` so no axis is zero-size.""" + extents: NDArray[np.float64] = np.maximum(aabb_max - aabb_min, _MIN_EXTENT_M).astype(float) + return extents + @dataclass(frozen=True) class EntityCookPlan: @@ -118,8 +132,8 @@ class EntityPrototypePlan: id: str safe_id: str source_prim_path: str - vertices: np.ndarray - triangles: np.ndarray + vertices: NDArray[np.float32] + triangles: NDArray[np.int32] collision_dir: Path def to_json_dict(self) -> dict[str, Any]: @@ -243,7 +257,7 @@ def _build_matched_entity_plan( aabb_min_np = vertices.min(axis=0).astype(float) aabb_max_np = vertices.max(axis=0).astype(float) center_np = ((aabb_min_np + aabb_max_np) * 0.5).astype(float) - extents = np.maximum(aabb_max_np - aabb_min_np, 1e-4).astype(float) + extents = _safe_extents(aabb_min_np, aabb_max_np) safe_id = _safe_entity_id(spec.id) visual_path = entities_dir / safe_id / "visual.glb" @@ -375,7 +389,7 @@ def _build_group_entity_plan( vertices = np.asarray(prim.vertices, dtype=np.float64) aabb_min_np = vertices.min(axis=0).astype(float) aabb_max_np = vertices.max(axis=0).astype(float) - extents = np.maximum(aabb_max_np - aabb_min_np, 1e-4).astype(float) + extents = _safe_extents(aabb_min_np, aabb_max_np) local_vertices, center_np, quat = _localize_prim_mesh(vertices) shape_hint, shape_extents = _resolve_shape(spec, extents) descriptor = _make_descriptor(spec, shape_hint, shape_extents, visual_path=None) @@ -407,7 +421,7 @@ def _build_group_entity_plan( def _resolve_shape( spec: InteractableSpec, - extents_np: np.ndarray, + extents_np: NDArray[np.float64], ) -> tuple[str, list[float]]: shape_hint = str(spec.physics.get("shape", "box")) shape_extents = spec.physics.get("extents") @@ -469,8 +483,8 @@ def _entity_group_prototype_key(group: EntityGroupSpec, prim: ScenePrimMesh) -> def _localize_prim_mesh( - vertices: np.ndarray, -) -> tuple[np.ndarray, tuple[float, float, float], tuple[float, float, float, float]]: + vertices: NDArray[np.float64], +) -> tuple[NDArray[np.float64], tuple[float, float, float], tuple[float, float, float, float]]: aabb_min = vertices.min(axis=0) aabb_max = vertices.max(axis=0) center = (aabb_min + aabb_max) * 0.5 diff --git a/dimos/experimental/scene_cooking/sidecar.py b/dimos/experimental/scene_cooking/sidecar.py index 84644fb401..8cf2473e81 100644 --- a/dimos/experimental/scene_cooking/sidecar.py +++ b/dimos/experimental/scene_cooking/sidecar.py @@ -225,12 +225,12 @@ def auto_discover(cls, scene_path: str | Path) -> SceneCookSidecar: for suffix in _COOK_SIDECAR_SUFFIXES: sidecar = source.with_suffix(suffix) if sidecar.exists(): - logger.info("loading scene cook sidecar: %s", sidecar) + logger.info("loading scene cook sidecar", path=sidecar) return cls.from_json(sidecar) legacy_collision = source.with_suffix(".collision.json") if legacy_collision.exists(): - logger.info("loading legacy collision sidecar: %s", legacy_collision) + logger.info("loading legacy collision sidecar", path=legacy_collision) return cls(path=legacy_collision, collision=CollisionSpec.from_json(legacy_collision)) return cls() diff --git a/dimos/experimental/scene_cooking/source_assets/glb.py b/dimos/experimental/scene_cooking/source_assets/glb.py index e84733970c..657146e4f4 100644 --- a/dimos/experimental/scene_cooking/source_assets/glb.py +++ b/dimos/experimental/scene_cooking/source_assets/glb.py @@ -32,10 +32,16 @@ GLB_CHUNK_HEADER_SIZE = 8 GLB_JSON_CHUNK_TYPE = 0x4E4F534A GLB_BIN_CHUNK_TYPE = 0x004E4942 +GLB_ALIGNMENT = 4 PNG_SIGNATURE = b"\x89PNG\r\n\x1a\n" STANDARD_TEXTURE_MIME_TYPES = {"image/png", "image/jpeg"} STANDARD_TEXTURE_MODES = {"RGB", "RGBA"} +#: Byte offset of the IHDR chunk's bit-depth field: 8-byte PNG signature + +#: 4-byte chunk length + 4-byte "IHDR" type + 4-byte width + 4-byte height. +_PNG_IHDR_BIT_DEPTH_OFFSET = 24 +_PNG_STANDARD_BIT_DEPTH = 8 + def read_glb(path: Path) -> tuple[dict[str, Any], bytes]: """Read a GLB v2 file as a JSON chunk plus one BIN chunk.""" @@ -95,15 +101,15 @@ def write_glb( payload = buffer_view_replacements.get(index) if payload is None: payload = buffer_view_bytes(bin_chunk, view) - _pad_bytearray(new_bin, alignment=4, pad=0) + _pad_bytearray(new_bin, alignment=GLB_ALIGNMENT, pad=0) view["byteOffset"] = len(new_bin) view["byteLength"] = len(payload) new_bin.extend(payload) - _pad_bytearray(new_bin, alignment=4, pad=0) + _pad_bytearray(new_bin, alignment=GLB_ALIGNMENT, pad=0) buffers[0]["byteLength"] = len(new_bin) json_chunk = json.dumps(gltf, separators=(",", ":"), sort_keys=True).encode("utf-8") - json_chunk = _padded_bytes(json_chunk, alignment=4, pad=b" ") + json_chunk = _padded_bytes(json_chunk, alignment=GLB_ALIGNMENT, pad=b" ") bin_bytes = bytes(new_bin) total_length = ( GLB_HEADER_SIZE @@ -212,19 +218,23 @@ def normalized_texture_bytes( ) -> bytes | None: """Return an 8-bit PNG replacement, or None when the texture is standard.""" try: - with Image.open(BytesIO(texture_bytes)) as image: - image.load() - if _is_standard_embedded_texture(image, texture_bytes, mime_type): - return None - has_alpha = image.mode in {"RGBA", "LA"} or "transparency" in image.info - mode = "RGBA" if has_alpha else "RGB" - converted = image.convert(mode) - out = BytesIO() - converted.save(out, format="PNG", compress_level=1) - return out.getvalue() - except Exception as exc: + image = Image.open(BytesIO(texture_bytes)) + image.load() + except OSError as exc: + # OSError covers PIL.UnidentifiedImageError (a subclass) along with + # truncated/corrupt image payloads. raise RuntimeError("failed to normalize embedded GLB texture to 8-bit PNG") from exc + with image: + if _is_standard_embedded_texture(image, texture_bytes, mime_type): + return None + has_alpha = image.mode in {"RGBA", "LA"} or "transparency" in image.info + mode = "RGBA" if has_alpha else "RGB" + converted = image.convert(mode) + out = BytesIO() + converted.save(out, format="PNG", compress_level=1) + return out.getvalue() + def _is_standard_embedded_texture( image: Image.Image, @@ -239,9 +249,10 @@ def _is_standard_embedded_texture( def _is_high_bit_depth_png(texture_bytes: bytes) -> bool: - if not texture_bytes.startswith(PNG_SIGNATURE) or len(texture_bytes) < 25: + min_length = _PNG_IHDR_BIT_DEPTH_OFFSET + 1 + if not texture_bytes.startswith(PNG_SIGNATURE) or len(texture_bytes) < min_length: return False - return texture_bytes[24] > 8 + return texture_bytes[_PNG_IHDR_BIT_DEPTH_OFFSET] > _PNG_STANDARD_BIT_DEPTH def _pad_bytearray(data: bytearray, *, alignment: int, pad: int) -> None: diff --git a/dimos/experimental/scene_cooking/source_assets/mesh.py b/dimos/experimental/scene_cooking/source_assets/mesh.py index 20790a2ccb..320f07dbad 100644 --- a/dimos/experimental/scene_cooking/source_assets/mesh.py +++ b/dimos/experimental/scene_cooking/source_assets/mesh.py @@ -37,14 +37,52 @@ from typing import Any import numpy as np +from numpy.typing import NDArray import open3d as o3d # type: ignore[import-untyped] from dimos.simulation.scene_assets.spec import SceneMeshAlignment _TRIMESH_DUPLICATE_SUFFIX_RE = re.compile(r"_[0-9a-f]{6}$", re.IGNORECASE) +#: Height (metres) the downward probe ray in ``floor_z_under_origin`` starts +#: from. Must clear any plausible scene ceiling; the hit depth is measured +#: from this same height, so the two uses must stay equal. +_FLOOR_PROBE_RAY_HEIGHT_M = 1000.0 -def _world_rotation(alignment: SceneMeshAlignment) -> np.ndarray: + +def _fan_triangulate( + face_counts: NDArray[np.int32], face_verts: NDArray[np.int32] +) -> NDArray[np.int32]: + """Vectorized fan triangulation of USD polygonal faces. + + For a face with local vertex indices ``v0..v_{n-1}`` (``n = + face_counts[i]``), emits ``(v0, vk, vk+1)`` for ``k = 1..n-2`` -- the + same result as nested Python loops over faces and ``k``, without the + per-triangle Python overhead. Returns ``(T, 3)`` int32 indices into + ``face_verts``' referent (empty when every face is a degenerate + <3-gon). + """ + counts = np.asarray(face_counts, dtype=np.int64) + if len(counts) == 0: + return np.empty((0, 3), dtype=np.int32) + face_starts = np.concatenate(([0], np.cumsum(counts)[:-1])) + tri_counts = np.maximum(counts - 2, 0) + total_tris = int(tri_counts.sum()) + if total_tris == 0: + return np.empty((0, 3), dtype=np.int32) + + face_id = np.repeat(np.arange(len(counts)), tri_counts) + tri_start_in_face = np.repeat(np.cumsum(tri_counts) - tri_counts, tri_counts) + k = np.arange(total_tris) - tri_start_in_face + 1 # 1-based fan index + + base = face_starts[face_id] + v0 = face_verts[base] + v1 = face_verts[base + k] + v2 = face_verts[base + k + 1] + return np.stack([v0, v1, v2], axis=1).astype(np.int32) + + +def _world_rotation(alignment: SceneMeshAlignment) -> NDArray[np.float64]: """Compose the y-up swap + ZYX Euler into one 3x3.""" rad = np.radians(alignment.rotation_zyx_deg) cz, sz = np.cos(rad[0]), np.sin(rad[0]) @@ -64,8 +102,8 @@ def _world_rotation(alignment: SceneMeshAlignment) -> np.ndarray: def _average_per_face_vertex( - per_fv: np.ndarray, face_verts: np.ndarray, n_verts: int -) -> np.ndarray: + per_fv: NDArray[np.floating[Any]], face_verts: NDArray[np.int32], n_verts: int +) -> NDArray[np.float64]: """Scatter-average ``(n_face_verts, 3)`` values onto ``(n_verts, 3)`` indices.""" out = np.zeros((n_verts, 3), dtype=np.float32) counts = np.zeros(n_verts, dtype=np.int32) @@ -78,9 +116,9 @@ def _average_per_face_vertex( def _color_from_displaycolor( mesh: Any, n_verts: int, - face_counts: np.ndarray, - face_verts: np.ndarray, -) -> np.ndarray | None: + face_counts: NDArray[np.int32], + face_verts: NDArray[np.int32], +) -> NDArray[np.floating[Any]] | None: """Per-vertex RGB from ``primvars:displayColor`` if present and valued. Handles the four standard interpolations: ``constant`` / ``vertex`` / @@ -118,8 +156,8 @@ def _color_from_displaycolor( def _color_from_material( - prim: Any, material_color_cache: dict[str, np.ndarray | None] -) -> np.ndarray | None: + prim: Any, material_color_cache: dict[str, NDArray[np.float32] | None] +) -> NDArray[np.float32] | None: """Per-prim RGB from the bound material's ``inputs:diffuseColor``. Walks ``UsdShadeMaterialBindingAPI`` → surface shader → ``inputs:diffuseColor``, @@ -146,7 +184,7 @@ def _color_from_material( return color -def _resolve_diffuse_color(material: Any) -> np.ndarray | None: +def _resolve_diffuse_color(material: Any) -> NDArray[np.float32] | None: """Pull a literal ``diffuseColor`` out of a UsdShade material's surface shader.""" from pxr import UsdShade # type: ignore[import-not-found, import-untyped] @@ -188,12 +226,12 @@ def _load_usd_mesh(path: Path) -> o3d.geometry.TriangleMesh: if stage is None: raise RuntimeError(f"could not open USD stage: {path}") - all_pts: list[np.ndarray] = [] - all_tris: list[np.ndarray] = [] - all_colors: list[np.ndarray] = [] + all_pts: list[NDArray[np.float32]] = [] + all_tris: list[NDArray[np.int32]] = [] + all_colors: list[NDArray[np.floating[Any]]] = [] any_color = False vtx_offset = 0 - material_color_cache: dict[str, np.ndarray | None] = {} + material_color_cache: dict[str, NDArray[np.float32] | None] = {} for prim in stage.Traverse(): if not prim.IsA(UsdGeom.Mesh): @@ -229,23 +267,11 @@ def _load_usd_mesh(path: Path) -> o3d.geometry.TriangleMesh: prim_colors = np.full((len(pts), 3), 0.7, dtype=np.float32) # USD allows quads / n-gons; fan-triangulate so o3d gets pure tris. - tris: list[tuple[int, int, int]] = [] - cursor = 0 - for n in face_counts: - for k in range(1, n - 1): - tris.append( - ( - int(face_verts[cursor]) + vtx_offset, - int(face_verts[cursor + k]) + vtx_offset, - int(face_verts[cursor + k + 1]) + vtx_offset, - ) - ) - cursor += n - - if not tris: + tris = _fan_triangulate(face_counts, face_verts) + if len(tris) == 0: continue all_pts.append(pts_world) - all_tris.append(np.asarray(tris, dtype=np.int32)) + all_tris.append((tris + vtx_offset).astype(np.int32)) all_colors.append(prim_colors) vtx_offset += len(pts_world) @@ -302,8 +328,8 @@ def load_scene_mesh( faces_world = np.asarray(scene_or_mesh.faces, dtype=np.int64) else: scene = scene_or_mesh - verts_chunks: list[np.ndarray] = [] - faces_chunks: list[np.ndarray] = [] + verts_chunks: list[NDArray[np.float64]] = [] + faces_chunks: list[NDArray[np.int64]] = [] v_off = 0 for node_name in scene.graph.nodes_geometry: xform, geom_name = scene.graph[node_name] @@ -357,12 +383,12 @@ def floor_z_under_origin( mesh = load_scene_mesh(scene_mesh_path, alignment=alignment) scene = make_raycasting_scene(mesh) rays = o3c.Tensor( - np.array([[0.0, 0.0, 1000.0, 0.0, 0.0, -1.0]], dtype=np.float32), + np.array([[0.0, 0.0, _FLOOR_PROBE_RAY_HEIGHT_M, 0.0, 0.0, -1.0]], dtype=np.float32), dtype=o3c.Dtype.Float32, ) t_hit = float(scene.cast_rays(rays)["t_hit"].numpy()[0]) if np.isfinite(t_hit): - return 1000.0 - t_hit + return _FLOOR_PROBE_RAY_HEIGHT_M - t_hit bbox = mesh.get_axis_aligned_bounding_box() return float(bbox.min_bound[2]) @@ -390,10 +416,10 @@ class ScenePrimMesh: """Sanitized identifier (safe for MJCF asset names) — typically the USD prim path with non-alphanumerics replaced.""" - vertices: np.ndarray + vertices: NDArray[np.float32] """``(N, 3)`` float32, in world frame after alignment.""" - triangles: np.ndarray + triangles: NDArray[np.int32] """``(M, 3)`` int32 vertex indices.""" prim_path: str | None = None @@ -670,19 +696,8 @@ def load_scene_prims( pts_world = (R @ (s * pts_stage).T).T + T # Triangulate any quads / n-gons (vertex indices are local to this prim now). - tris: list[tuple[int, int, int]] = [] - cursor = 0 - for n in face_counts: - for k in range(1, n - 1): - tris.append( - ( - int(face_verts[cursor]), - int(face_verts[cursor + k]), - int(face_verts[cursor + k + 1]), - ) - ) - cursor += n - if not tris: + tris = _fan_triangulate(face_counts, face_verts) + if len(tris) == 0: continue # MJCF asset names: strip the leading slash, swap remaining diff --git a/dimos/experimental/scene_cooking/source_assets/normalize.py b/dimos/experimental/scene_cooking/source_assets/normalize.py index 24a32a9f4f..3362565651 100644 --- a/dimos/experimental/scene_cooking/source_assets/normalize.py +++ b/dimos/experimental/scene_cooking/source_assets/normalize.py @@ -24,6 +24,7 @@ import tempfile from typing import Any +from dimos.constants import CACHE_DIR from dimos.experimental.scene_cooking.command import ( blender_output_line_is_interesting, run_logged_command, @@ -44,7 +45,7 @@ ".usdz", } -SOURCE_CACHE_DIR = Path.home() / ".cache" / "dimos" / "scene_sources" +SOURCE_CACHE_DIR = CACHE_DIR / "scene_sources" _BLENDER_NORMALIZER_VERSION = "blend-evaluated-depsgraph-v1" @@ -145,7 +146,7 @@ def _normalize_blend_source( if not target.exists(): raise RuntimeError(f"Blender source normalization did not write {target}") - logger.info("normalized Blender scene source: %s -> %s", source, target) + logger.info("normalized Blender scene source", source=source, target=target) return PreparedSceneSource( original_path=source, cook_path=target, diff --git a/dimos/experimental/scene_cooking/source_assets/test_glb.py b/dimos/experimental/scene_cooking/source_assets/test_glb.py index 07929cd35f..d7b868e755 100644 --- a/dimos/experimental/scene_cooking/source_assets/test_glb.py +++ b/dimos/experimental/scene_cooking/source_assets/test_glb.py @@ -22,6 +22,12 @@ from PIL import Image from dimos.experimental.scene_cooking.source_assets.glb import ( + GLB_BIN_CHUNK_TYPE, + GLB_CHUNK_HEADER_SIZE, + GLB_HEADER_SIZE, + GLB_JSON_CHUNK_TYPE, + GLB_MAGIC, + GLB_VERSION, buffer_view_bytes, demote_required_extensions, normalize_embedded_textures, @@ -115,12 +121,18 @@ def _write_test_glb( json_chunk = json.dumps(gltf, separators=(",", ":")).encode("utf-8") json_chunk = _padded(json_chunk, b" ") bin_bytes = bytes(bin_chunk) - total_length = 12 + 8 + len(json_chunk) + 8 + len(bin_bytes) + total_length = ( + GLB_HEADER_SIZE + + GLB_CHUNK_HEADER_SIZE + + len(json_chunk) + + GLB_CHUNK_HEADER_SIZE + + len(bin_bytes) + ) with path.open("wb") as file: - file.write(struct.pack("<4sII", b"glTF", 2, total_length)) - file.write(struct.pack(" dict[str, object]: +def _metadata(tmp_path: Path) -> dict[str, Any]: return { "source_path": str(tmp_path / "source.glb"), "package_dir": str(tmp_path), @@ -176,7 +177,7 @@ def test_scene_package_metadata_uses_package_relative_paths(tmp_path: Path) -> N def test_load_scene_package_tolerates_missing_objects_sidecar(tmp_path: Path) -> None: raw = _metadata(tmp_path) # Older cooked packages without the semantic sidecar should still load. - raw["artifacts"].pop("objects") # type: ignore[union-attr] + raw["artifacts"].pop("objects") metadata_path = tmp_path / "scene.meta.json" metadata_path.write_text(json.dumps(raw)) @@ -201,6 +202,8 @@ def test_browser_visual_profiles_are_backend_specific() -> None: def test_extract_scene_objects_emits_per_prim_aabb() -> None: + # Inlined so importing this test module doesn't pull heavy open3d/trimesh + # for the many tests here that don't need browser.collision. from dimos.experimental.scene_cooking.browser.collision import extract_scene_objects triangles = np.array([[0, 1, 2]], dtype=np.int32) diff --git a/pyproject.toml b/pyproject.toml index 5bf7ed71e9..3babf9c551 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -323,12 +323,11 @@ apriltag = [ "reportlab>=4.5.0", ] -# Offline scene-package cooking + loading: mesh/USD import (open3d, usd-core), -# convex decomposition for collision (coacd), and mesh ops (trimesh). The -# Blender visual bake is an optional external `blender` binary, not a wheel. +# Offline scene-package cooking + loading: USD import (usd-core), convex +# decomposition for collision (coacd), and mesh ops (trimesh). open3d is +# already a core dependency. The Blender visual bake is an optional external +# `blender` binary, not a wheel. scene = [ - "open3d-unofficial-arm>=0.19.0.post9; platform_system == 'Linux' and platform_machine == 'aarch64'", - "open3d>=0.18.0; platform_system != 'Linux' or platform_machine != 'aarch64'", "trimesh>=4.0.0", "coacd>=1.0.0", "usd-core>=23.11", diff --git a/uv.lock b/uv.lock index 0d2e7bf2b2..f80fc50d5b 100644 --- a/uv.lock +++ b/uv.lock @@ -1605,8 +1605,6 @@ all = [ { name = "onnxruntime" }, { name = "onnxruntime-gpu", marker = "platform_machine == 'x86_64'" }, { name = "open-clip-torch" }, - { name = "open3d", marker = "platform_machine != 'aarch64' or sys_platform != 'linux'" }, - { name = "open3d-unofficial-arm", marker = "platform_machine == 'aarch64' and sys_platform == 'linux'" }, { name = "openai" }, { name = "opencv-contrib-python" }, { name = "pillow" }, @@ -1739,8 +1737,6 @@ perception = [ ] scene = [ { name = "coacd" }, - { name = "open3d", marker = "platform_machine != 'aarch64' or sys_platform != 'linux'" }, - { name = "open3d-unofficial-arm", marker = "platform_machine == 'aarch64' and sys_platform == 'linux'" }, { name = "trimesh" }, { name = "usd-core" }, ] @@ -2035,9 +2031,7 @@ requires-dist = [ { name = "onnxruntime-gpu", marker = "platform_machine == 'x86_64' and extra == 'cuda'", specifier = ">=1.17.1" }, { name = "open-clip-torch", marker = "extra == 'misc'", specifier = "==3.2.0" }, { name = "open3d", marker = "platform_machine != 'aarch64' or sys_platform != 'linux'", specifier = ">=0.18.0" }, - { name = "open3d", marker = "(platform_machine != 'aarch64' and extra == 'scene') or (sys_platform != 'linux' and extra == 'scene')", specifier = ">=0.18.0" }, { name = "open3d-unofficial-arm", marker = "platform_machine == 'aarch64' and sys_platform == 'linux'", specifier = ">=0.19.0.post9" }, - { name = "open3d-unofficial-arm", marker = "platform_machine == 'aarch64' and sys_platform == 'linux' and extra == 'scene'", specifier = ">=0.19.0.post9" }, { name = "openai", marker = "extra == 'agents'" }, { name = "opencv-contrib-python", marker = "extra == 'apriltag'", specifier = "==4.10.0.84" }, { name = "opencv-python" },