feat(canvas) - Image To 3D

invokeai/app/api/dependencies.py

@@ -6,6 +6,7 @@
 import torch
 
 from invokeai.app.services.app_settings import AppSettingsService
+from invokeai.app.services.asset_files.asset_files_disk import DiskAssetFileStorage
 from invokeai.app.services.auth.token_service import set_jwt_secret
 from invokeai.app.services.board_image_records.board_image_records_sqlite import SqliteBoardImageRecordStorage
 from invokeai.app.services.board_images.board_images_default import BoardImagesService
@@ -108,6 +109,7 @@ def initialize(
             raise ValueError("Output folder is not set")
 
         image_files = DiskImageFileStorage(f"{output_folder}/images")
+        asset_files = DiskAssetFileStorage(output_folder / "assets")
 
         model_images_folder = config.models_path
         style_presets_folder = config.style_presets_path
@@ -192,6 +194,7 @@ def initialize(
         users = UserService(db=db)
 
         services = InvocationServices(
+            asset_files=asset_files,
             board_image_records=board_image_records,
             board_images=board_images,
             board_records=board_records,

invokeai/app/api/routers/assets.py

@@ -0,0 +1,35 @@
+from fastapi import APIRouter, HTTPException, Path
+from fastapi.responses import FileResponse
+
+from invokeai.app.api.dependencies import ApiDependencies
+
+ASSET_MAX_AGE = 31536000
+
+assets_router = APIRouter(prefix="/v1/assets", tags=["assets"])
+
+
+@assets_router.get(
+    "/i/{asset_name}",
+    operation_id="get_asset",
+    responses={
+        200: {"description": "The 3D asset was fetched successfully"},
+        404: {"description": "The 3D asset could not be found"},
+    },
+    status_code=200,
+)
+async def get_asset(
+    asset_name: str = Path(description="The name of the 3D asset file to get"),
+) -> FileResponse:
+    """Gets a 3D asset file (e.g. a Gaussian-splat .ply)."""
+    try:
+        path = ApiDependencies.invoker.services.asset_files.get_path(asset_name)
+        response = FileResponse(
+            path,
+            media_type="application/octet-stream",
+            filename=asset_name,
+            content_disposition_type="inline",
+        )
+        response.headers["Cache-Control"] = f"max-age={ASSET_MAX_AGE}"
+        return response
+    except Exception:
+        raise HTTPException(status_code=404)

invokeai/app/api_app.py

@@ -17,6 +17,7 @@
 from invokeai.app.api.no_cache_staticfiles import NoCacheStaticFiles
 from invokeai.app.api.routers import (
     app_info,
+    assets,
     auth,
     board_images,
     boards,
@@ -179,6 +180,7 @@ async def dispatch(self, request: Request, call_next: RequestResponseEndpoint):
 app.include_router(download_queue.download_queue_router, prefix="/api")
 app.include_router(image_moves.image_moves_router, prefix="/api")
 app.include_router(images.images_router, prefix="/api")
+app.include_router(assets.assets_router, prefix="/api")
 app.include_router(boards.boards_router, prefix="/api")
 app.include_router(board_images.board_images_router, prefix="/api")
 app.include_router(virtual_boards.virtual_boards_router, prefix="/api")

invokeai/app/invocations/fields.py

@@ -241,6 +241,12 @@ class ImageField(BaseModel):
     image_name: str = Field(description="The name of the image")
 
 
+class Asset3DField(BaseModel):
+    """A 3D asset primitive field"""
+
+    asset_name: str = Field(description="The name of the 3D asset file")
+
+
 class BoardField(BaseModel):
     """A board primitive field"""
 

invokeai/app/invocations/image_to_3d.py

@@ -0,0 +1,96 @@
+from pathlib import Path
+
+import torch
+
+from invokeai.app.invocations.baseinvocation import BaseInvocation, Classification, invocation
+from invokeai.app.invocations.fields import ImageField, InputField, WithBoard, WithMetadata
+from invokeai.app.invocations.primitives import Asset3DOutput
+from invokeai.app.services.session_processor.session_processor_common import CanceledException
+from invokeai.app.services.shared.invocation_context import InvocationContext
+from invokeai.app.util.misc import uuid_string
+from invokeai.backend.image_util.triposplat.triposplat_model import TripoSplatModel
+
+# HuggingFace repo id. The repo holds five safetensors checkpoints in subfolders (diffusion_models/, vae/,
+# clip_vision/, background_removal/). NOTE: we fetch it with huggingface_hub.snapshot_download rather than
+# InvokeAI's load_remote_model, because the latter's filter_files() only keeps weight files whose name
+# matches `model.*` — which excludes ALL of TripoSplat's checkpoints (it would resolve "0 files").
+# License: MIT (code + weights).
+TRIPOSPLAT_SOURCE = "VAST-AI/TripoSplat"
+
+# Bounds enforced by the upstream pipeline (TripoSplatPipeline._validate_num_gaussians).
+TRIPOSPLAT_MIN_GAUSSIANS = 32768
+TRIPOSPLAT_MAX_GAUSSIANS = 262144
+
+
+@invocation(
+    "image_to_3d",
+    title="Image to 3D (TripoSplat)",
+    tags=["3d", "image", "gaussian", "splat"],
+    category="3d",
+    version="1.0.0",
+    classification=Classification.Prototype,
+)
+class ImageTo3DInvocation(BaseInvocation, WithMetadata, WithBoard):
+    """Generates a 3D Gaussian splat (.ply) from a single image using TripoSplat."""
+
+    image: ImageField = InputField(description="The image to convert to a 3D Gaussian splat.")
+    remove_background: bool = InputField(
+        default=True,
+        description="Isolate the subject by removing the background (BiRefNet). Disable to feed the full image as-is.",
+    )
+    num_gaussians: int = InputField(
+        default=TRIPOSPLAT_MAX_GAUSSIANS,
+        ge=TRIPOSPLAT_MIN_GAUSSIANS,
+        le=TRIPOSPLAT_MAX_GAUSSIANS,
+        description="Number of 3D Gaussians to generate. Higher is more detailed but larger and slower to render.",
+    )
+    steps: int = InputField(default=20, ge=1, le=100, description="Number of flow-matching sampler steps.")
+    guidance_scale: float = InputField(
+        default=3.0, ge=0.0, description="Classifier-free guidance scale. Values <= 1.0 disable guidance."
+    )
+    seed: int = InputField(default=42, ge=0, description="Seed for reproducible generation.")
+
+    @torch.no_grad()
+    def invoke(self, context: InvocationContext) -> Asset3DOutput:
+        if self.remove_background:
+            # RGBA honors an existing alpha cutout; a fully-opaque input falls back to TripoSplat's own
+            # BiRefNet background removal (which isolates the single most salient subject).
+            image = context.images.get_pil(self.image.image_name, mode="RGBA")
+        else:
+            # Force-skip BiRefNet by handing the model a uniformly near-opaque alpha — TripoSplat only runs
+            # background removal when there's no real alpha channel, so this feeds the whole frame as-is.
+            image = context.images.get_pil(self.image.image_name, mode="RGB").convert("RGBA")
+            image.putalpha(254)
+
+        # Fetch the full repo snapshot directly (see TRIPOSPLAT_SOURCE note re: load_remote_model). Cached
+        # by huggingface_hub after the first download; subsequent runs resolve instantly.
+        from huggingface_hub import snapshot_download
+
+        context.util.signal_progress("Downloading TripoSplat weights (first run only)")
+        weights_dir = Path(snapshot_download(repo_id=TRIPOSPLAT_SOURCE))
+
+        loaded_model = context.models.load_local_model(weights_dir, TripoSplatModel.load_model)
+
+        with loaded_model as model:
+            assert isinstance(model, TripoSplatModel)
+
+            def _on_step(step: int, total: int) -> None:
+                if context.util.is_canceled():
+                    raise CanceledException
+                context.util.signal_progress("Generating 3D Gaussian splat", step / total)
+
+            gaussian, _prepared = model._pipe.run(
+                image,
+                seed=self.seed,
+                steps=self.steps,
+                guidance_scale=self.guidance_scale,
+                num_gaussians=self.num_gaussians,
+                show_progress=False,
+                callback=_on_step,
+            )
+            ply_bytes = gaussian.to_ply_bytes()
+
+        asset_name = f"{uuid_string()}.ply"
+        # asset_files is a custom service with no curated InvocationContext accessor, so reach it directly.
+        context._services.asset_files.save(asset_name, ply_bytes)
+        return Asset3DOutput.build(asset_name=asset_name)

invokeai/app/invocations/primitives.py

@@ -13,6 +13,7 @@
 from invokeai.app.invocations.constants import LATENT_SCALE_FACTOR
 from invokeai.app.invocations.fields import (
     AnimaConditioningField,
+    Asset3DField,
     BoundingBoxField,
     CogView4ConditioningField,
     ColorField,
@@ -253,6 +254,17 @@ def build(cls, image_dto: ImageDTO) -> "ImageOutput":
         )
 
 
+@invocation_output("asset_3d_output")
+class Asset3DOutput(BaseInvocationOutput):
+    """Base class for nodes that output a single 3D asset"""
+
+    asset: Asset3DField = OutputField(description="The output 3D asset")
+
+    @classmethod
+    def build(cls, asset_name: str) -> "Asset3DOutput":
+        return cls(asset=Asset3DField(asset_name=asset_name))
+
+
 @invocation_output("image_collection_output")
 class ImageCollectionOutput(BaseInvocationOutput):
     """Base class for nodes that output a collection of images"""

invokeai/app/services/asset_files/asset_files_base.py

@@ -0,0 +1,26 @@
+from abc import ABC, abstractmethod
+from pathlib import Path
+
+
+class AssetFilesServiceBase(ABC):
+    """Stores and serves binary 3D asset files (e.g. Gaussian-splat .ply / .splat)."""
+
+    @abstractmethod
+    def save(self, asset_name: str, data: bytes) -> None:
+        """Saves a 3D asset file."""
+        pass
+
+    @abstractmethod
+    def get_path(self, asset_name: str) -> Path:
+        """Gets the path to a 3D asset file."""
+        pass
+
+    @abstractmethod
+    def get_url(self, asset_name: str) -> str:
+        """Gets the URL of a 3D asset file."""
+        pass
+
+    @abstractmethod
+    def delete(self, asset_name: str) -> None:
+        """Deletes a 3D asset file."""
+        pass

invokeai/app/services/asset_files/asset_files_common.py

@@ -0,0 +1,22 @@
+class AssetFileNotFoundException(Exception):
+    """Raised when a 3D asset file is not found"""
+
+    def __init__(self, message: str = "Asset file not found"):
+        self.message = message
+        super().__init__(self.message)
+
+
+class AssetFileSaveException(Exception):
+    """Raised when a 3D asset file cannot be saved"""
+
+    def __init__(self, message: str = "Asset file cannot be saved"):
+        self.message = message
+        super().__init__(self.message)
+
+
+class AssetFileDeleteException(Exception):
+    """Raised when a 3D asset file cannot be deleted"""
+
+    def __init__(self, message: str = "Asset file cannot be deleted"):
+        self.message = message
+        super().__init__(self.message)

invokeai/app/services/asset_files/asset_files_disk.py

@@ -0,0 +1,62 @@
+from pathlib import Path
+
+from invokeai.app.services.asset_files.asset_files_base import AssetFilesServiceBase
+from invokeai.app.services.asset_files.asset_files_common import (
+    AssetFileDeleteException,
+    AssetFileNotFoundException,
+    AssetFileSaveException,
+)
+from invokeai.app.services.invoker import Invoker
+
+
+class DiskAssetFileStorage(AssetFilesServiceBase):
+    """Stores 3D asset files (Gaussian-splat .ply / .splat) on disk."""
+
+    def __init__(self, asset_files_folder: Path):
+        self._asset_files_folder = asset_files_folder
+        self._validate_storage_folder()
+
+    def start(self, invoker: Invoker) -> None:
+        self._invoker = invoker
+
+    def save(self, asset_name: str, data: bytes) -> None:
+        try:
+            self._validate_storage_folder()
+            path = self._resolve_path(asset_name)
+            path.write_bytes(data)
+        except Exception as e:
+            raise AssetFileSaveException from e
+
+    def get_path(self, asset_name: str) -> Path:
+        path = self._resolve_path(asset_name)
+        if not path.exists():
+            raise AssetFileNotFoundException
+        return path
+
+    def get_url(self, asset_name: str) -> str:
+        return self._invoker.services.urls.get_asset_url(asset_name)
+
+    def delete(self, asset_name: str) -> None:
+        try:
+            path = self._resolve_path(asset_name)
+            if not path.exists():
+                raise AssetFileNotFoundException
+            path.unlink()
+        except AssetFileNotFoundException as e:
+            raise AssetFileNotFoundException from e
+        except Exception as e:
+            raise AssetFileDeleteException from e
+
+    def _resolve_path(self, asset_name: str) -> Path:
+        """Resolves an asset name to a path, guarding against path traversal."""
+        # Only bare filenames are allowed - reject any path separators / traversal.
+        if not asset_name or asset_name != Path(asset_name).name:
+            raise ValueError(f"Invalid asset name: {asset_name}")
+        path = (self._asset_files_folder / asset_name).resolve()
+        if not path.is_relative_to(self._asset_files_folder.resolve()):
+            raise ValueError(f"Invalid asset name: {asset_name}")
+        return path
+
+    def _validate_storage_folder(self) -> None:
+        """Creates the storage folder if it does not exist."""
+        self._asset_files_folder.mkdir(parents=True, exist_ok=True)

invokeai/app/services/invocation_services.py

@@ -12,6 +12,7 @@
 
     import torch
 
+    from invokeai.app.services.asset_files.asset_files_base import AssetFilesServiceBase
     from invokeai.app.services.board_image_records.board_image_records_base import BoardImageRecordStorageBase
     from invokeai.app.services.board_images.board_images_base import BoardImagesServiceABC
     from invokeai.app.services.board_records.board_records_base import BoardRecordStorageBase
@@ -49,6 +50,7 @@ class InvocationServices:
 
     def __init__(
         self,
+        asset_files: "AssetFilesServiceBase",
         board_images: "BoardImagesServiceABC",
         board_image_records: "BoardImageRecordStorageBase",
         boards: "BoardServiceABC",
@@ -82,6 +84,7 @@ def __init__(
         users: "UserServiceBase",
         image_moves: "ImageMoveService | None" = None,
     ):
+        self.asset_files = asset_files
         self.board_images = board_images
         self.board_image_records = board_image_records
         self.boards = boards

invokeai/app/services/urls/urls_base.py

@@ -23,3 +23,8 @@ def get_style_preset_image_url(self, style_preset_id: str) -> str:
     def get_workflow_thumbnail_url(self, workflow_id: str) -> str:
         """Gets the URL for a workflow thumbnail"""
         pass
+
+    @abstractmethod
+    def get_asset_url(self, asset_name: str) -> str:
+        """Gets the URL for a 3D asset file"""
+        pass

invokeai/app/services/urls/urls_default.py

@@ -25,3 +25,10 @@ def get_style_preset_image_url(self, style_preset_id: str) -> str:
 
     def get_workflow_thumbnail_url(self, workflow_id: str) -> str:
         return f"{self._base_url}/workflows/i/{workflow_id}/thumbnail"
+
+    def get_asset_url(self, asset_name: str) -> str:
+        import os
+
+        asset_basename = os.path.basename(asset_name)
+        # This path is determined by the route in invokeai/app/api/routers/assets.py
+        return f"{self._base_url}/assets/i/{asset_basename}"

invokeai/backend/image_util/triposplat/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 VAST
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.