GARfVDB Training Visualization Fixes

.gitignore

@@ -1,3 +1,6 @@
+# Data
+*.ply
+
 # Generated version file
 fvdb/version.py
 

instance_segmentation/garfvdb/garfvdb/model.py

@@ -576,7 +576,12 @@ def get_encoded_features(self, input: GARfVDBInput) -> torch.Tensor:
             if not self.model_config.enc_feats_one_idx_per_ray:
                 # Weighted sum of the enc_feats and transmittance weights
                 enc_feats.jdata = enc_feats.jdata * weights.jdata.unsqueeze(-1)
-                enc_feats = enc_feats.jsum(dim=0, keepdim=True)
+                # When every pixel has exactly 1 contributor, fvdb returns a
+                # 1-level JaggedTensor [C,[R]] instead of 2-level [C,[R,[K]]].
+                # jsum(dim=0) on a 1-level tensor would collapse pixels within
+                # each camera rather than depth samples within each pixel.
+                if isinstance(enc_feats.lshape[0], list):
+                    enc_feats = enc_feats.jsum(dim=0, keepdim=True)
 
             epsilon = 1e-6
             enc_feats.jdata = enc_feats.jdata / (torch.linalg.norm(enc_feats.jdata, dim=-1, keepdim=True) + epsilon)

instance_segmentation/garfvdb/garfvdb/training/segmentation.py

@@ -292,6 +292,17 @@ def device(self) -> torch.device:
             return torch.device(dev)
         return dev
 
+    @property
+    def viz_callback(self) -> Callable[["GaussianSplatScaleConditionedSegmentation", int], None] | None:
+        """Get or set the visualization callback invoked at epoch boundaries."""
+        return self._viz_callback
+
+    @viz_callback.setter
+    def viz_callback(
+        self, callback: Callable[["GaussianSplatScaleConditionedSegmentation", int], None] | None
+    ) -> None:
+        self._viz_callback = callback
+
     @staticmethod
     def _init_model(
         model_config: GARfVDBModelConfig,

instance_segmentation/garfvdb/garfvdb_environment.yml

@@ -3,10 +3,12 @@ channels:
   - conda-forge
   - nodefaults
 dependencies:
+  - fvdb-core>=0.4.2,<0.5.0
+  - cxx-compiler
   - blas=*=mkl
   - python=3.12
-  - pytorch-gpu=2.8.0
-  - cuda-version=12.9
+  - pytorch-gpu=2.10.0
+  - cuda-version>=12.9
   - pip
   - git
   - gitpython
@@ -16,6 +18,8 @@ dependencies:
   - tyro
   - scikit-learn
   - py-opencv
+  - open-clip-torch
+  - openusd
   - imageio
   - sam2
   - pyproj
@@ -32,3 +36,7 @@ dependencies:
   - rapidsai::cuml
   - cupy
   - matplotlib
+  - pip:
+    - dlnr_lite
+    - fvdb-reality-capture>=0.4.0<0.5.0
+    - nanovdb-editor>=0.0.24

instance_segmentation/garfvdb/pyproject.toml

@@ -17,8 +17,9 @@ dependencies = [
     "tqdm",
     "tyro",
     "tensorboard",
-    "fvdb-core",
-    "fvdb-reality-capture",
+    "fvdb-core>=0.4.2,<0.5.0",
+    "nanovdb-editor>=0.0.24,<0.2.0",
+    "fvdb-reality-capture>=0.4.0<0.5.0",
     "pycocotools",
     "matplotlib",
     "pillow",