Feat(model support): ideogram4 support

README.md

@@ -75,6 +75,7 @@ Invoke features an organized gallery system for easily storing, accessing, and r
 - Anima
 - Qwen Image
 - Qwen Image Edit
+- Ideogram 4
 - Nano Banana (API Only)
 - GPT Image (API Only)
 - Wan (API Only)

invokeai/app/api/dependencies.py

@@ -60,6 +60,7 @@
     CogView4ConditioningInfo,
     ConditioningFieldData,
     FLUXConditioningInfo,
+    Ideogram4ConditioningInfo,
     QwenImageConditioningInfo,
     SD3ConditioningInfo,
     SDXLConditioningInfo,
@@ -152,6 +153,7 @@ def initialize(
                     SD3ConditioningInfo,
                     CogView4ConditioningInfo,
                     ZImageConditioningInfo,
+                    Ideogram4ConditioningInfo,
                     QwenImageConditioningInfo,
                     AnimaConditioningInfo,
                 ],

invokeai/app/invocations/fields.py

@@ -344,6 +344,12 @@ class ZImageConditioningField(BaseModel):
     )
 
 
+class Ideogram4ConditioningField(BaseModel):
+    """An Ideogram 4 conditioning tensor primitive value"""
+
+    conditioning_name: str = Field(description="The name of conditioning tensor")
+
+
 class QwenImageConditioningField(BaseModel):
     """A Qwen Image Edit conditioning tensor primitive value"""
 

invokeai/app/invocations/ideogram4_denoise.py

@@ -0,0 +1,135 @@
+from typing import Literal, Optional
+
+import torch
+
+from invokeai.app.invocations.baseinvocation import BaseInvocation, Classification, invocation
+from invokeai.app.invocations.fields import (
+    FieldDescriptions,
+    Ideogram4ConditioningField,
+    Input,
+    InputField,
+)
+from invokeai.app.invocations.model import TransformerField
+from invokeai.app.invocations.primitives import LatentsOutput
+from invokeai.app.services.shared.invocation_context import InvocationContext
+from invokeai.backend.ideogram4 import run_ideogram4_denoise
+from invokeai.backend.ideogram4.sampler_configs import PRESETS
+from invokeai.backend.ideogram4.transformer_pair import Ideogram4TransformerPair
+from invokeai.backend.stable_diffusion.diffusion.conditioning_data import Ideogram4ConditioningInfo
+from invokeai.backend.util.devices import TorchDevice
+
+# Named sampler presets bundle step count, guidance schedule (with polish tail), and the
+# logit-normal schedule mean/std. V4_QUALITY_48 is the reference default.
+IDEOGRAM4_SAMPLER_PRESETS = Literal["V4_QUALITY_48", "V4_DEFAULT_20", "V4_TURBO_12"]
+
+
+def _effective_guidance_schedule(
+    base_schedule: tuple[float, ...], preset_num_steps: int, num_steps: int, guidance_scale: Optional[float]
+) -> tuple[float, ...]:
+    """Build the per-step guidance schedule for the (possibly overridden) step count.
+
+    The preset schedule is ``(polish_gw,)*N_polish + (main_gw,)*N_main`` in loop-index order
+    (index 0 = the final/polish step). A ``guidance_scale`` override replaces the main weight while
+    the preset's polish tail is preserved; a changed step count rescales the polish tail
+    proportionally (always keeping at least one polish and one main step).
+    """
+    polish_gw = base_schedule[0]
+    main_gw = float(guidance_scale) if guidance_scale is not None else float(base_schedule[-1])
+    if num_steps == preset_num_steps and guidance_scale is None:
+        return base_schedule
+    n_polish_base = sum(1 for gw in base_schedule if gw == base_schedule[0])
+    polish_count = min(num_steps, max(1, round(n_polish_base * num_steps / preset_num_steps)))
+    main_count = num_steps - polish_count
+    return (polish_gw,) * polish_count + (main_gw,) * main_count
+
+
+@invocation(
+    "ideogram4_denoise",
+    title="Denoise - Ideogram 4",
+    tags=["image", "ideogram4"],
+    category="latents",
+    version="1.0.0",
+    classification=Classification.Prototype,
+)
+class Ideogram4DenoiseInvocation(BaseInvocation):
+    """Runs the Ideogram 4 dual-branch flow-matching denoising loop (text-to-image)."""
+
+    transformer: TransformerField = InputField(
+        description=FieldDescriptions.transformer, input=Input.Connection, title="Transformer"
+    )
+    positive_conditioning: Ideogram4ConditioningField = InputField(
+        description=FieldDescriptions.positive_cond, input=Input.Connection
+    )
+    sampler_preset: IDEOGRAM4_SAMPLER_PRESETS = InputField(
+        default="V4_QUALITY_48",
+        description="Sampler preset (steps + guidance schedule + schedule mean/std).",
+        title="Sampler Preset",
+    )
+    width: int = InputField(default=1024, multiple_of=16, description="Width of the generated image.")
+    height: int = InputField(default=1024, multiple_of=16, description="Height of the generated image.")
+    seed: int = InputField(default=0, description="Randomness seed for reproducibility.")
+    # Optional advanced overrides of the sampler preset. None = use the preset's value.
+    steps: Optional[int] = InputField(
+        default=None,
+        ge=1,
+        le=100,
+        description="Override the preset's step count. Leave empty to use the preset.",
+    )
+    guidance_scale: Optional[float] = InputField(
+        default=None,
+        ge=1.0,
+        le=20.0,
+        description="Override the main guidance weight (the preset's polish tail is preserved). "
+        "Empty = use the preset.",
+    )
+    mu: Optional[float] = InputField(
+        default=None,
+        ge=-4.0,
+        le=4.0,
+        description="Override the logit-normal schedule mean (resolution-adjusted internally). "
+        "Empty = use the preset.",
+    )
+
+    @torch.no_grad()
+    def invoke(self, context: InvocationContext) -> LatentsOutput:
+        device = TorchDevice.choose_torch_device()
+        preset = PRESETS[self.sampler_preset]
+
+        # Apply optional advanced overrides on top of the preset.
+        num_steps = self.steps if self.steps is not None else preset.num_steps
+        mu = self.mu if self.mu is not None else preset.mu
+        guidance_schedule = _effective_guidance_schedule(
+            preset.guidance_schedule, preset.num_steps, num_steps, self.guidance_scale
+        )
+
+        # Load conditioning (the stacked Qwen3-VL features).
+        cond_data = context.conditioning.load(self.positive_conditioning.conditioning_name)
+        assert len(cond_data.conditionings) == 1
+        info = cond_data.conditionings[0]
+        assert isinstance(info, Ideogram4ConditioningInfo)
+        llm_features = info.prompt_embeds.to(device=device, dtype=torch.float32)
+
+        def step_callback(step: int, total: int, _latents: torch.Tensor) -> None:
+            context.util.signal_progress("Running Ideogram 4 denoising", step / total)
+
+        transformer_info = context.models.load(self.transformer.transformer)
+        with transformer_info.model_on_device() as (_, transformers):
+            assert isinstance(transformers, Ideogram4TransformerPair)
+            packed = run_ideogram4_denoise(
+                conditional_transformer=transformers.conditional,
+                unconditional_transformer=transformers.unconditional,
+                llm_features=llm_features,
+                height=self.height,
+                width=self.width,
+                num_steps=num_steps,
+                mu=mu,
+                std=preset.std,
+                guidance_schedule=guidance_schedule,
+                seed=self.seed,
+                device=device,
+                step_callback=step_callback,
+            )
+
+        packed = packed.detach().to("cpu")
+        name = context.tensors.save(tensor=packed)
+        return LatentsOutput.build(latents_name=name, latents=packed, seed=None)

invokeai/app/invocations/ideogram4_latents_to_image.py

@@ -0,0 +1,66 @@
+import torch
+from einops import rearrange
+from PIL import Image
+
+from invokeai.app.invocations.baseinvocation import BaseInvocation, Classification, invocation
+from invokeai.app.invocations.fields import (
+    FieldDescriptions,
+    Input,
+    InputField,
+    LatentsField,
+    WithBoard,
+    WithMetadata,
+)
+from invokeai.app.invocations.model import VAEField
+from invokeai.app.invocations.primitives import ImageOutput
+from invokeai.app.services.shared.invocation_context import InvocationContext
+from invokeai.backend.ideogram4.autoencoder import AutoEncoder
+from invokeai.backend.ideogram4.latent_norm import get_latent_norm
+from invokeai.backend.ideogram4.sampling_utils import unpatchify_and_denormalize
+from invokeai.backend.util.devices import TorchDevice
+
+
+@invocation(
+    "ideogram4_l2i",
+    title="Latents to Image - Ideogram 4",
+    tags=["latents", "image", "vae", "l2i", "ideogram4"],
+    category="latents",
+    version="1.0.0",
+    classification=Classification.Prototype,
+)
+class Ideogram4LatentsToImageInvocation(BaseInvocation, WithMetadata, WithBoard):
+    """Decodes Ideogram 4 packed latents to an image with the FLUX.2-style VAE."""
+
+    latents: LatentsField = InputField(description=FieldDescriptions.latents, input=Input.Connection)
+    vae: VAEField = InputField(description=FieldDescriptions.vae, input=Input.Connection)
+
+    @torch.no_grad()
+    def invoke(self, context: InvocationContext) -> ImageOutput:
+        # Packed latents from denoise: (1, 128, grid_h, grid_w).
+        latents = context.tensors.load(self.latents.latents_name)
+        device = TorchDevice.choose_torch_device()
+
+        vae_info = context.models.load(self.vae.vae)
+        latent_shift, latent_scale = get_latent_norm()
+
+        with vae_info.model_on_device() as (_, vae):
+            assert isinstance(vae, AutoEncoder), (
+                f"Expected Ideogram 4 AutoEncoder, got {type(vae).__name__}."
+            )
+            context.util.signal_progress("Running VAE")
+            vae_dtype = next(vae.parameters()).dtype
+
+            # Denormalize + unpatchify to a standard (1, 32, H/8, W/8) latent.
+            z = unpatchify_and_denormalize(
+                latents.float().to(device), latent_shift.to(device), latent_scale.to(device)
+            )
+            TorchDevice.empty_cache()
+            decoded = vae.decoder(z.to(vae_dtype))
+
+            img = decoded.float().clamp(-1.0, 1.0)
+            img = rearrange(img[0], "c h w -> h w c")
+            img_pil = Image.fromarray((127.5 * (img + 1.0)).byte().cpu().numpy())
+
+        TorchDevice.empty_cache()
+        image_dto = context.images.save(image=img_pil)
+        return ImageOutput.build(image_dto)

invokeai/app/invocations/ideogram4_model_loader.py

@@ -0,0 +1,64 @@
+from invokeai.app.invocations.baseinvocation import (
+    BaseInvocation,
+    BaseInvocationOutput,
+    Classification,
+    invocation,
+    invocation_output,
+)
+from invokeai.app.invocations.fields import FieldDescriptions, Input, InputField, OutputField
+from invokeai.app.invocations.model import (
+    ModelIdentifierField,
+    Qwen3EncoderField,
+    TransformerField,
+    VAEField,
+)
+from invokeai.app.services.shared.invocation_context import InvocationContext
+from invokeai.backend.model_manager.taxonomy import BaseModelType, ModelType, SubModelType
+
+
+@invocation_output("ideogram4_model_loader_output")
+class Ideogram4ModelLoaderOutput(BaseInvocationOutput):
+    """Ideogram 4 model loader output."""
+
+    transformer: TransformerField = OutputField(description=FieldDescriptions.transformer, title="Transformer")
+    qwen3_encoder: Qwen3EncoderField = OutputField(
+        description=FieldDescriptions.qwen3_encoder, title="Qwen3-VL Encoder"
+    )
+    vae: VAEField = OutputField(description=FieldDescriptions.vae, title="VAE")
+
+
+@invocation(
+    "ideogram4_model_loader",
+    title="Main Model - Ideogram 4",
+    tags=["model", "ideogram4"],
+    category="model",
+    version="1.0.0",
+    classification=Classification.Prototype,
+)
+class Ideogram4ModelLoaderInvocation(BaseInvocation):
+    """Loads an Ideogram 4 model, outputting its submodels.
+
+    Ideogram 4 is distributed as a single bundled diffusers folder, so the transformer
+    (both branches), the Qwen3-VL text encoder + tokenizer, and the VAE are all loaded
+    from the one selected model.
+    """
+
+    model: ModelIdentifierField = InputField(
+        description="The Ideogram 4 model to load.",
+        input=Input.Direct,
+        ui_model_base=BaseModelType.Ideogram4,
+        ui_model_type=ModelType.Main,
+        title="Model",
+    )
+
+    def invoke(self, context: InvocationContext) -> Ideogram4ModelLoaderOutput:
+        transformer = self.model.model_copy(update={"submodel_type": SubModelType.Transformer})
+        text_encoder = self.model.model_copy(update={"submodel_type": SubModelType.TextEncoder})
+        tokenizer = self.model.model_copy(update={"submodel_type": SubModelType.Tokenizer})
+        vae = self.model.model_copy(update={"submodel_type": SubModelType.VAE})
+
+        return Ideogram4ModelLoaderOutput(
+            transformer=TransformerField(transformer=transformer, loras=[]),
+            qwen3_encoder=Qwen3EncoderField(tokenizer=tokenizer, text_encoder=text_encoder),
+            vae=VAEField(vae=vae),
+        )

invokeai/app/invocations/ideogram4_text_encoder.py

@@ -0,0 +1,60 @@
+from contextlib import ExitStack
+
+import torch
+
+from invokeai.app.invocations.baseinvocation import BaseInvocation, Classification, invocation
+from invokeai.app.invocations.fields import FieldDescriptions, Input, InputField, UIComponent
+from invokeai.app.invocations.model import Qwen3EncoderField
+from invokeai.app.invocations.primitives import Ideogram4ConditioningOutput
+from invokeai.app.services.shared.invocation_context import InvocationContext
+from invokeai.backend.ideogram4.text_encoding import encode_qwen3vl_prompt
+from invokeai.backend.stable_diffusion.diffusion.conditioning_data import (
+    ConditioningFieldData,
+    Ideogram4ConditioningInfo,
+)
+
+
+@invocation(
+    "ideogram4_text_encoder",
+    title="Prompt - Ideogram 4",
+    tags=["prompt", "conditioning", "ideogram4"],
+    category="conditioning",
+    version="1.0.0",
+    classification=Classification.Prototype,
+)
+class Ideogram4TextEncoderInvocation(BaseInvocation):
+    """Encodes a prompt for Ideogram 4 using the Qwen3-VL encoder.
+
+    The prompt is normally a structured JSON caption (see the Ideogram 4 prompting guide);
+    plain text also works but yields lower-quality results.
+    """
+
+    prompt: str = InputField(
+        description="The prompt to encode. A structured JSON caption is recommended.",
+        ui_component=UIComponent.Textarea,
+    )
+    qwen3_encoder: Qwen3EncoderField = InputField(
+        title="Qwen3-VL Encoder",
+        description=FieldDescriptions.qwen3_encoder,
+        input=Input.Connection,
+    )
+
+    @torch.no_grad()
+    def invoke(self, context: InvocationContext) -> Ideogram4ConditioningOutput:
+        text_encoder_info = context.models.load(self.qwen3_encoder.text_encoder)
+        tokenizer_info = context.models.load(self.qwen3_encoder.tokenizer)
+
+        with ExitStack() as exit_stack:
+            (_, text_encoder) = exit_stack.enter_context(text_encoder_info.model_on_device())
+            (_, tokenizer) = exit_stack.enter_context(tokenizer_info.model_on_device())
+
+            context.util.signal_progress("Running Qwen3-VL text encoder")
+            prompt_embeds = encode_qwen3vl_prompt(self.prompt, tokenizer, text_encoder)
+
+        # Move to CPU for storage to save VRAM.
+        prompt_embeds = prompt_embeds.detach().to("cpu")
+        conditioning_data = ConditioningFieldData(
+            conditionings=[Ideogram4ConditioningInfo(prompt_embeds=prompt_embeds)]
+        )
+        conditioning_name = context.conditioning.save(conditioning_data)
+        return Ideogram4ConditioningOutput.build(conditioning_name)

invokeai/app/invocations/metadata.py

@@ -166,6 +166,7 @@ def invoke(self, context: InvocationContext) -> MetadataOutput:
     "z_image_img2img",
     "z_image_inpaint",
     "z_image_outpaint",
+    "ideogram4_txt2img",
     "qwen_image_txt2img",
     "qwen_image_img2img",
     "qwen_image_inpaint",

invokeai/app/invocations/primitives.py

@@ -1,38 +1,39 @@
 # Copyright (c) 2023 Kyle Schouviller (https://github.com/kyle0654)
 
 from typing import Optional
 
 import torch
 
 from invokeai.app.invocations.baseinvocation import (
     BaseInvocation,
     BaseInvocationOutput,
     invocation,
     invocation_output,
 )
 from invokeai.app.invocations.constants import LATENT_SCALE_FACTOR
 from invokeai.app.invocations.fields import (
     AnimaConditioningField,
     BoundingBoxField,
     CogView4ConditioningField,
     ColorField,
     ConditioningField,
     DenoiseMaskField,
     FieldDescriptions,
     FluxConditioningField,
     ImageField,
     Input,
     InputField,
     LatentsField,
     OutputField,
     QwenImageConditioningField,
     SD3ConditioningField,
     TensorField,
     UIComponent,
+    Ideogram4ConditioningField,
     ZImageConditioningField,
 )
 from invokeai.app.services.images.images_common import ImageDTO
 from invokeai.app.services.shared.invocation_context import InvocationContext
 
 """
 Primitives: Boolean, Integer, Float, String, Image, Latents, Conditioning, Color
@@ -488,6 +489,17 @@
         return cls(conditioning=ZImageConditioningField(conditioning_name=conditioning_name))
 
 
+@invocation_output("ideogram4_conditioning_output")
+class Ideogram4ConditioningOutput(BaseInvocationOutput):
+    """Base class for nodes that output an Ideogram 4 text conditioning tensor."""
+
+    conditioning: Ideogram4ConditioningField = OutputField(description=FieldDescriptions.cond)
+
+    @classmethod
+    def build(cls, conditioning_name: str) -> "Ideogram4ConditioningOutput":
+        return cls(conditioning=Ideogram4ConditioningField(conditioning_name=conditioning_name))
+
+
 @invocation_output("qwen_image_conditioning_output")
 class QwenImageConditioningOutput(BaseInvocationOutput):
     """Base class for nodes that output a Qwen Image Edit conditioning tensor."""