diff --git a/examples/diffusers/Dockerfile.dreamverse b/examples/diffusers/Dockerfile.dreamverse new file mode 100644 index 000000000000..c161352ebb3a --- /dev/null +++ b/examples/diffusers/Dockerfile.dreamverse @@ -0,0 +1,63 @@ +# SPDX-License-Identifier: Apache-2.0 +# +# LTX-2.3 SPEED serving image, built FROM FastVideo's validated dreamverse env +# (fastvideo @ 4f3ad3f6 with its intended prerelease kernel stack) + the Dynamo +# serving layer (ai-dynamo, deepinfra patches, worker app). Rebuild the base: +# git clone https://github.com/hao-ai-lab/FastVideo && cd FastVideo +# git checkout 4f3ad3f6df9327257f08f4c45d24154b52c06616 +# docker build -f apps/dreamverse/docker/Dockerfile -t dreamverse:johan-ltx . +FROM dreamverse:johan-ltx + +# dreamverse venv is /opt/venv (python 3.12); uv is on PATH (/root/.local/bin). +ENV VIRTUAL_ENV=/opt/venv +ENV PATH="/opt/venv/bin:${PATH}" + +# Freeze the proven prerelease FP4-kernel stack + torch. These versions are +# ALREADY installed in dreamverse; the constraint just prevents the ai-dynamo +# resolve below from moving them. --prerelease=allow (below) is required because +# nvidia-cutlass-dsl==4.6.0.dev0 is a prerelease. +RUN printf 'torch==2.11.0\nnvidia-cutlass-dsl==4.6.0.dev0\nquack-kernels==0.5.3\nflashinfer-python==0.6.13\n' > /opt/dynamo-constraint.txt +ENV UV_CONSTRAINT=/opt/dynamo-constraint.txt + +# Dynamo runtime needs libucx0 (already present in dreamverse's CUDA base in +# most cases, but install defensively). +RUN apt-get update \ + && apt-get install -yq libucx0 \ + && apt-get clean + +# Install Dynamo with /v1/videos support. --prerelease=allow so the constraint's +# cutlass-dsl dev pin is honored; the constraint keeps torch/kernels frozen. +RUN . /opt/venv/bin/activate \ + && uv pip install --prerelease=allow ai-dynamo==1.1.1 + +# deepinfra patches, applied to the INSTALLED (non-editable) fastvideo package. +# The patches are git-format (a/fastvideo/... b/fastvideo/...); -p1 from +# site-packages strips a/ -> fastvideo/... . dreamverse's fastvideo is @4f3ad3f6, +# the exact commit these patches were authored against, so they apply clean. +COPY patches/ltx23_gpu_worker_megacache.patch /tmp/megacache.patch +COPY patches/x264-threads-cap.patch /tmp/x264-threads-cap.patch +# NOTE: verify by grepping the patched source, NOT by importing fastvideo -- +# importing pulls in triton, which fails to init its driver in a GPU-less build +# ("0 active drivers"). +RUN SP=/opt/venv/lib/python3.12/site-packages \ + && patch -p1 -d "$SP" --forward --fuzz=0 < /tmp/megacache.patch \ + && patch -p1 -d "$SP" --forward --fuzz=0 < /tmp/x264-threads-cap.patch \ + && grep -q 'torch.compiler.load_cache_artifacts' "$SP/fastvideo/worker/gpu_worker.py" \ + && grep -q 'FASTVIDEO_X264_THREADS' "$SP/fastvideo/entrypoints/video_generator.py" \ + && echo "patches applied + verified" + +ENV FASTVIDEO_VIDEO_CODEC=libx264 +ENV FASTVIDEO_X264_PRESET=ultrafast + +# Worker app tree + entrypoint (page-cache warming; unsets LD_LIBRARY_PATH, +# which is a no-op for refine speed -- Experiment X showed torch loads its +# bundled cuBLAS regardless of LD, refine time is set by the FP4-kernel packages). +WORKDIR /opt/app +COPY . /opt/app/ +ENTRYPOINT ["/opt/app/entrypoint.sh"] + +# LOAD-BEARING: FastVideo picks the DISTILLED preset by string-matching the +# weights path (needs "ltx-2"+"distilled"; miss = silent base preset, ~2.4x +# slower). Prod mounts weights at /data/default, so alias it. factory.py +# hard-fails at boot if the preset does not resolve. +RUN mkdir -p /models && ln -s /data/default /models/ltx-2.3-distilled-diffusers diff --git a/examples/diffusers/ltx23/RUNBOOK.md b/examples/diffusers/ltx23/RUNBOOK.md index fc40a456a2f8..912ace4c979d 100644 --- a/examples/diffusers/ltx23/RUNBOOK.md +++ b/examples/diffusers/ltx23/RUNBOOK.md @@ -161,6 +161,80 @@ docker run --rm localhost:30500/fastvideo-runtime:-ltx2-$HASH \ Should report sizes matching the host's `/cache`. +### Step 4b: profile selection (quality vs speed) — pin it into the IMAGE + +LTX-2.3 has two recipes, selected by the `LTX23_PROFILE` env var, which +`ltx23/factory.py` reads at load time (`quality`=bf16/8-step default, +`speed`=NVFP4/max-autotune/5-step — see `PROFILES.md`). The two recipes +produce **different torch.compile cache keys**, so the warmup bake and the +serving worker MUST agree on the profile, or the serving path misses the +baked cache and cold-compiles 10-15 min per shape. + +**Ship rule: bake `LTX23_PROFILE` into the image ENV; never set it in the +model/deploy config.** Making the image tag the single source of truth means +the recipe travels with the image — a `-speed-` image always serves speed, a +`-quality-` image always serves quality. A `LTX23_PROFILE` set in the redis +model-config env would *override* the image ENV and can silently drift away +from the baked cache (wrong recipe + cache miss, no error). Leave it out of +the deploy config so the image wins. + +To bake the **SPEED** (fast / ~10s) image (flow validated 2026-07-03: dynamo pool ++ prod weights = 10.6–10.9s/clip warm): + +0. **Base image = `Dockerfile.dreamverse`** (not the old `warmupbase`): FastVideo's + validated dreamverse env + the dynamo layer. It creates the load-bearing + `/models/ltx-2.3-distilled-diffusers -> /data/default` symlink: FastVideo picks + the DISTILLED preset by string-matching the weights path; a miss silently + serves the ~2.4x-slower base preset. `factory.py` hard-fails boot if the + preset does not resolve. The recipe is `ltx23/streaming_speed.yaml`, loaded + verbatim (`VideoGenerator.from_file`) -- never re-derive it in code. + ```bash + docker build -f Dockerfile.dreamverse -t localhost:30500/fastvideo-runtime:-ltx23-dreamverse-base . + ``` + +1. Step 3 warmup — two passes on ONE persistent `/cache`, both with + `-e LTX23_PROFILE=speed -e LTX_MEGACACHE_DIR=/cache/megacache + -e VIDEO_POOL_GEN_TIMEOUT_S=5400` and weights at `/data/default`: + ```bash + # pass 1: t2v compile + blob (~25 min cold) + ... localhost:30500/fastvideo-runtime:-ltx23-dreamverse-base \ + python3 ltx23/bake_bench.py t2v + # pass 2: extend the blob with i2v graphs (move the pass-1 blob aside first — + # a present blob suppresses saving — and set LTX_MEGACACHE_SAVE_EVERY=1) + ... -e LTX_MEGACACHE_SAVE_EVERY=1 ... python3 ltx23/bake_bench.py i2v + ``` + The pool runs `enable_optimizations=True`, so SPEED applies NVFP4 (QUALITY + stays bf16 regardless). + + **`LTX_MEGACACHE_DIR` is NOT optional for a ship image.** Without it the bake + produces only the inductor `/cache`, not the per-shape `*.megacache.bin` + blobs — and a fresh pod then cold-compiles the torch.compile front-end (~33 + min/shape SPEED) on *every* boot instead of the ~12 min Mega-Cache boot-warm. + `warmup.py` prints a loud WARNING if it's unset. Putting it **under `/cache`** + means the existing `cache.tar` step below captures the blobs automatically — + no separate `megacache.tar`. Full rationale: `ltx23/CACHING.md`. +2. Step 4 bake — add `ENV LTX23_PROFILE=speed` **and `ENV LTX_MEGACACHE_DIR=/cache/megacache`** + to the bake Dockerfile and use a distinct tag so quality and speed images are + never conflated (`IMAGE_SHAPE_HASH` only encodes the shape menu, not the recipe): + ```bash + printf 'FROM localhost:30500/fastvideo-runtime:-ltx23-dreamverse-base\nADD cache.tar /\nENV IMAGE_SHAPE_HASH=%s\nENV LTX23_PROFILE=speed\nENV LTX_MEGACACHE_DIR=/cache/megacache\n' "$HASH" > Dockerfile + docker build -t localhost:30500/fastvideo-runtime:-ltx23-speed-$HASH . + ``` + Pinning `LTX_MEGACACHE_DIR` in the image ENV (rather than the model's redis + `extra_env`) keeps the serving side from depending on a deploy-config flag — + the image alone knows where its Mega-Cache lives. + +The QUALITY image is the default (no `LTX23_PROFILE` needed at warmup; omit the +`ENV LTX23_PROFILE` line at bake, or set it to `quality` explicitly). It still +needs `LTX_MEGACACHE_DIR` at both bake and serve for the same boot-warm reason. + +The attention backend follows the profile automatically (`config.py` +`profile_attention_backend`: `quality`→`FLASH_ATTN`, `speed`→`TORCH_SDPA`) in +both the warmup bake and the serving worker, so you do **not** set +`FASTVIDEO_ATTENTION_BACKEND` separately — the one `LTX23_PROFILE` env pins it. +Note SPEED ships `TORCH_SDPA` (the config we validated, ~10s), not FastVideo's +`FLASH_ATTN` speed path; override with `worker.py --attention-backend` to revisit. + ### Step 5: validate with `benchmark.py` ```bash diff --git a/examples/diffusers/ltx23/bake_bench.py b/examples/diffusers/ltx23/bake_bench.py new file mode 100644 index 000000000000..b63810b9d175 --- /dev/null +++ b/examples/diffusers/ltx23/bake_bench.py @@ -0,0 +1,74 @@ +"""Bake + bench driver for the LTX-2.3 SPEED image (see RUNBOOK.md). + + python3 ltx23/bake_bench.py t2v # pass 1: t2v compile + Mega-Cache blob + warm bench + python3 ltx23/bake_bench.py i2v # pass 2: extend the blob with i2v graphs + # (move the pass-1 blob aside; LTX_MEGACACHE_SAVE_EVERY=1) + +Exits non-zero if a warm generation exceeds MAX_WARM_S: a silently-slowed bake +(wrong preset, package drift, upstream change) must fail here, not reach prod. +""" +import asyncio, logging, os, sys, time + +logging.basicConfig(level=logging.INFO, format="%(message)s") +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +from lib.pool import SubprocessPool + +OUT = os.environ.get("BENCH_OUT", "/tmp/bench-out") +os.makedirs(OUT, exist_ok=True) +W, H, NF, FPS, STEPS, GS = 1920, 1088, 121, 24, 5, 1.0 +SHAPE = "%dx%d@%df" % (W, H, NF) +MAX_WARM_S = float(os.environ.get("LTX23_MAX_WARM_S", "14")) +I2V_IMAGE = os.environ.get("LTX23_I2V_IMAGE", "/work/joy_nordic_woman_mid.png") +T2V_PROMPTS = [ + ("musician", "A street musician in her thirties sings and strums an acoustic guitar on a sunny city sidewalk, natural human face, photorealistic, sharp focus."), + ("dog", "A close-up tracking shot of a golden retriever sprinting through a sunlit alpine meadow at golden hour, photorealistic"), +] + + +def req(tag: str, prompt: str, **extra) -> dict: + d = { + "request_id": "bake_" + tag, "prompt": prompt, "width": W, "height": H, + "num_frames": NF, "fps": FPS, "num_inference_steps": STEPS, + "guidance_scale": GS, "seed": 42, "negative_prompt": None, + "output_path": os.path.join(OUT, tag + ".mp4"), + } + d.update(extra) + return d + + +async def main(mode: str) -> int: + pool = SubprocessPool( + model_path=os.environ.get("LTX23_MODEL_PATH", "/models/ltx-2.3-distilled-diffusers"), + num_gpus=1, enable_optimizations=True, attention_backend="TORCH_SDPA", + model_factory_dotted="ltx23.factory:load_model", model_label="ltx23-bake", + ) + warm_times: list[tuple[str, float]] = [] + try: + t = time.perf_counter() + r = await pool.route(SHAPE, req("warm", "a calm test warmup clip")) + print("BOOT_WARM_S=%.1f status=%s" % (time.perf_counter() - t, r.get("status")), flush=True) + if mode == "t2v": + gens = [(tag, req(tag, prompt)) for tag, prompt in T2V_PROMPTS] + else: + gens = [("i2v", req( + "i2v", "The scene comes alive with gentle natural motion and a slow cinematic push-in.", + ltx2_images=[(I2V_IMAGE, 0, 1.0)], ltx2_image_crf=0.0))] + for tag, r_ in gens: + t = time.perf_counter() + r = await pool.route(SHAPE, r_) + dt = time.perf_counter() - t + warm_times.append((tag, dt)) + print("GEN[%s]=%.1fs status=%s" % (tag, dt, r.get("status")), flush=True) + print("BAKE_BENCH_DONE mode=%s" % mode, flush=True) + finally: + await pool.shutdown() + slow = [(tag, dt) for tag, dt in warm_times[1:] or warm_times if dt > MAX_WARM_S] + if slow: + print("LATENCY GATE FAILED (> %.0fs warm): %s -- do NOT ship this bake; " + "check the preset boot log and recent env/package changes." % (MAX_WARM_S, slow), flush=True) + return 1 + return 0 + + +if __name__ == "__main__": + sys.exit(asyncio.run(main(sys.argv[1] if len(sys.argv) > 1 else "t2v"))) diff --git a/examples/diffusers/ltx23/config.py b/examples/diffusers/ltx23/config.py index b38b7bc741a0..89638a576fe6 100755 --- a/examples/diffusers/ltx23/config.py +++ b/examples/diffusers/ltx23/config.py @@ -25,9 +25,16 @@ Both profiles share: model FastVideo/LTX-2.3-Distilled-Diffusers, 1920x1088 landscape, 121 frames @ 24fps, guidance 1.0, negative "", refine upsampler = -/spatial_upscaler, FLASH_ATTN, vae_tiling False, the Blackwell Inductor +/spatial_upscaler, vae_tiling False, the Blackwell Inductor knobs (factory.py: shape_padding=False etc.), LD_LIBRARY_PATH unset, cu128 env. +Attention backend is per-profile (see profile_attention_backend): QUALITY uses +FLASH_ATTN (FastVideo's SM100/SM103 fast kernels, what prod runs); SPEED uses +TORCH_SDPA -- the config we validated end-to-end on B200 (~10s, quality OK'd by +Johan). FastVideo's own streaming_demo speed path uses FLASH_ATTN (faster, +~4.5s), but SPEED+FLASH_ATTN is NOT something we have validated, so we ship the +attention we tested. Revisit with worker.py --attention-backend. + Changing any of these requires a coordinated re-bake (the compile cache is keyed on the kwargs + shape). Do NOT hand-edit values away from the FastVideo source. """ @@ -118,3 +125,29 @@ def profile_kwargs(profile: str) -> dict[str, Any]: def profile_uses_nvfp4(profile: str) -> bool: """SPEED uses NVFP4; QUALITY is bf16.""" return (profile or "quality").strip().lower() == "speed" + + +def profile_attention_backend(profile: str) -> str: + """Attention backend per profile. + + QUALITY -> FLASH_ATTN (FastVideo's SM100/SM103 fast kernels; what prod runs). + SPEED -> TORCH_SDPA: the config validated end-to-end on B200 (~10s, + quality OK'd). FastVideo's streaming_demo speed path actually uses FLASH_ATTN + (faster, ~4.5s), but we have NOT validated SPEED+FLASH_ATTN ourselves, so we + ship the attention we tested. Bake (warmup) and serve (worker) both derive + the default from here so the compile cache always matches; override with + worker.py --attention-backend to revisit FLASH_ATTN. + """ + return "TORCH_SDPA" if (profile or "quality").strip().lower() == "speed" else "FLASH_ATTN" + + +def profile_default_optimizations(profile: str) -> bool: + """Whether NVFP4 + torch.compile optimizations default ON for this profile. + + SPEED needs them ON (NVFP4 is the whole point); QUALITY is bf16, OFF. Coupling + this to the profile means a SPEED ship image (LTX23_PROFILE=speed baked in) + serves NVFP4 with no extra launch arg -- the recipe lives in the image, not a + redis deploy-config flag. factory.load_model still hardware-gates NVFP4 to + Blackwell, so a non-Blackwell host safely falls back to bf16 regardless. + """ + return profile_uses_nvfp4(profile) diff --git a/examples/diffusers/ltx23/factory.py b/examples/diffusers/ltx23/factory.py index 4c45162babbd..84ab74ef0422 100644 --- a/examples/diffusers/ltx23/factory.py +++ b/examples/diffusers/ltx23/factory.py @@ -22,6 +22,31 @@ logger = logging.getLogger(__name__) +def _assert_distilled_preset(yaml_path: str) -> None: + """Fail boot if FastVideo would not resolve the DISTILLED preset. + + FastVideo picks the preset by string-matching the weights path (needs + "ltx-2"+"distilled"); a miss silently serves the BASE preset ~2.4x slower. + """ + import yaml as _yaml + + with open(yaml_path) as fh: + model_path = _yaml.safe_load(fh)["generator"]["model_path"] + + from fastvideo.registry import _get_config_info + + info = _get_config_info(model_path, raise_on_missing=False) + preset = getattr(info, "default_preset", None) if info else None + if preset != "ltx2_distilled": + raise RuntimeError( + f"LTX23_PROFILE=speed but FastVideo resolved preset {preset!r} for " + f"model_path={model_path!r}; the path must contain 'ltx-2' and " + "'distilled' (base preset is ~2.4x slower). Fix the weights alias " + "(Dockerfile symlink /models/ltx-2.3-distilled-diffusers)." + ) + logger.info("LTX-2.3 preset check OK: %s -> ltx2_distilled", model_path) + + def load_model( model_path: str, num_gpus: int, @@ -52,6 +77,22 @@ def load_model( # See ltx23/config.py / ltx23/PROFILES.md. The denoise step count (8 vs 5) # lives in the shapes file / per-request num_inference_steps, not here. profile = os.environ.get("LTX23_PROFILE", "quality").strip().lower() + + # SPEED loads the validated recipe verbatim via from_file -- do NOT re-derive + # compile/quant/inductor settings in code (hand-set knobs are where the + # recipe drifted before). Attention backend comes from the env (TORCH_SDPA). + if profile == "speed": + yaml_path = os.path.join( + os.path.dirname(os.path.abspath(__file__)), "streaming_speed.yaml" + ) + _assert_distilled_preset(yaml_path) + logger.info( + "LTX-2.3 profile=speed: VideoGenerator.from_file(%s) " + "[exact 10s recipe, no re-derivation]", + yaml_path, + ) + return VideoGenerator.from_file(yaml_path) + optimization_kwargs = profile_kwargs(profile) # Inductor knobs from FastVideo's LTX-2.3 reference (basic_ltx2_3_distilled). diff --git a/examples/diffusers/ltx23/streaming_speed.yaml b/examples/diffusers/ltx23/streaming_speed.yaml new file mode 100644 index 000000000000..83257254338b --- /dev/null +++ b/examples/diffusers/ltx23/streaming_speed.yaml @@ -0,0 +1,154 @@ +# Dreamverse streaming demo — fastvideo serve --config target. +# +# This config is for testing Dreamverse-like streaming through the lower-level +# `fastvideo serve --config` entrypoint. Use `dreamverse-server` for the full +# Dreamverse web app. +# +# Mirrors ../FastVideo-internal/ui/ltx2-streaming/server/config.py +# defaults so the public typed surface produces the same runtime +# behavior as the internal UI's GPU pool. Sources of each setting are +# annotated inline; deviations are explicit. +# +# Boot: uv run fastvideo serve --config serve_configs/streaming_demo.yaml +# Override at the CLI: +# uv run fastvideo serve --config \ +# --server.port 8010 \ +# --streaming.warmup.enabled false +# +# Required environment (sourced from ~/.env when launched via the +# launch-demo skill): +# * CEREBRAS_API_KEY (prompt enhancement, default provider) +# * CEREBRAS_IFM_API_KEY (only required if you set +# streaming.prompt.provider to cerebras_ifm +# via dotted override; the public typed +# schema currently only accepts cerebras / +# groq, so cerebras_ifm flows via env on +# dreamverse-server, not fastvideo serve) +# * GROQ_API_KEY (only if you switch provider to groq) +# * OPENAI_API_KEY (downstream rewrites in some prompt +# system prompts) +# +# Hosts without flashinfer / NVFP4: comment out the +# `engine.quantization` block below — the loader falls back to bf16 +# automatically when no quant_config is set. + +generator: + # internal: MODEL_REGISTRY["fast-ltx2"] (config.py:14-19) + model_path: /models/ltx-2.3-distilled-diffusers + + engine: + # internal: NUM_GPUS=1 hardcoded in gpu_pool.py worker boot + num_gpus: 1 + + # internal: gpu_pool.py:251-258 — all offload flags False so the + # full DiT, text encoder, and VAE stay resident on GPU. + offload: + dit: false + dit_layerwise: false + text_encoder: false + vae: false + pin_cpu_memory: true + + # internal: gpu_pool.py:251-258 + the just-uncommented "mode" line + # — torch.compile on for transformer + text encoder, inductor + # backend, fullgraph for graph-break debugging, max-autotune- + # no-cudagraphs for the kernel sweep, dynamic off (static shapes). + compile: + enabled: true + text_encoder_enabled: true + backend: inductor + fullgraph: true + mode: max-autotune-no-cudagraphs + dynamic: false + + # internal: pipeline_config.dit_config.quant_config = FP4Config() + # set in gpu_pool.py:280 (via the legacy in-place mutation). The + # public typed surface resolves "NVFP4" to NVFP4Config() and pins + # it on dit_config in FastVideoArgs.__post_init__. Comment this + # block out on hosts without flashinfer / NVFP4 hardware. + quantization: + transformer_quant: NVFP4 + + pipeline: + workload_type: t2v + # internal: pipeline_config.vae_tiling left default (False). + vae_tiling: false + + components: + # Same as model_path; LTX-2 reads its tokenizer / scheduler + # config from the same root. + config_root: /models/ltx-2.3-distilled-diffusers + upsampler_weights: /models/ltx-2.3-distilled-diffusers/spatial_upscaler + + # internal: gpu_pool.py:259-265 ltx2_refine_* kwargs. + # The distilled stage-2 refine runs in 2 inference steps with + # gs=1.0 and add_noise=True (no LoRA — empty path). + preset_overrides: + refine: + enabled: true + num_inference_steps: 2 + guidance_scale: 1.0 + add_noise: true + +server: + # internal: main.py defaults (host 0.0.0.0, port 8009). + host: 0.0.0.0 + port: 8009 + # internal: outputs/ relative to server dir. + output_dir: outputs/ + +# default_request is the operator-pinned baseline merged into every +# /v1/stream session_init. Internal/ui hardcodes these in gpu_pool.py +# (NUM_FRAMES, FRAME_HEIGHT, FRAME_WIDTH, NUM_INFERENCE_STEPS, +# TARGET_FPS); we put them on default_request.sampling so explicit +# client overrides still win. +default_request: + sampling: + num_frames: 121 # internal: config.py:36 NUM_FRAMES + height: 1088 # internal: config.py:37 FRAME_HEIGHT + width: 1920 # internal: config.py:38 FRAME_WIDTH + num_inference_steps: 5 # internal: config.py:39 NUM_INFERENCE_STEPS + fps: 24 # internal: gpu_pool.py:85 TARGET_FPS + +streaming: + # internal: config.py:33 SESSION_TIMEOUT_SECONDS = 300 + session_timeout_seconds: 300 + # internal: config.py:282-284 GENERATION_SEGMENT_CAP default 6 + generation_segment_cap: 6 + # internal: config.py:46 STREAM_MODE default "av_fmp4" + stream_mode: av_fmp4 + + # internal: config.py:288-300 STARTUP_WARMUP_* + warmup: + enabled: true + prompt: "A cinematic drone shot over coastal cliffs at sunrise, golden light, gentle ocean waves, ultra detailed" + timeout_seconds: 2400 + + # internal: gpu_pool.py session-controller defaults — 9 conditioning + # frames, 0 end-offset, audio re-encode on (matches Dreamverse + # session_controller and FastVideo-internal video_generation paths). + pool: + num_workers: 1 # one worker per GPU; gpu_pool spawns based on CUDA_VISIBLE_DEVICES + enable_audio_reencode: true + conditioning_num_frames: 9 + conditioning_end_offset: 0 + + # internal: PROMPT_PROVIDER = "cerebras_ifm" hardcoded in config.py:143. + # The public typed Literal is currently {"cerebras", "groq"} only — + # cerebras_ifm requires the legacy env-driven path on dreamverse-server. + # For the typed fastvideo serve path we default to "cerebras" with the + # same model id; switch to groq via dotted override if cerebras is down. + prompt: + enabled: true + provider: cerebras + model: gpt-oss-120b # internal: config.py:189 PROMPT_MODEL + timeout_ms: 20000 # internal: config.py:216 PROMPT_TIMEOUT_MS + # system_prompt_dir intentionally unset; the public PromptEnhancer + # falls back to its packaged defaults when None. Set to an absolute + # directory if you want to pin operator-edited prompts. + + # internal: PROMPT_SAFETY_ENABLED default False (config.py:115). + # Enable + provide classifier_path on hosts with the fasttext extra + # installed (uv sync --extra safety in Dreamverse). + safety: + enabled: false diff --git a/examples/diffusers/ltx23/warmup.py b/examples/diffusers/ltx23/warmup.py index acdfaa58e653..acf2e099fbe8 100755 --- a/examples/diffusers/ltx23/warmup.py +++ b/examples/diffusers/ltx23/warmup.py @@ -136,6 +136,7 @@ def _run_driver(args: argparse.Namespace) -> int: runtime worker asks for at serve time. """ from lib.pool import SubprocessPool + from ltx23.config import profile_attention_backend cfg = _read_shapes_config(args) @@ -146,6 +147,29 @@ def _run_driver(args: argparse.Namespace) -> int: seed = int(cfg.get("seed", 42)) shapes = cfg["shapes"] + # Bake attention MUST match what the serving worker uses, or the compile + # cache misses. Both derive the default from LTX23_PROFILE (quality -> + # FLASH_ATTN, speed -> TORCH_SDPA) via the same helper. + profile = os.environ.get("LTX23_PROFILE", "quality") + attention_backend = profile_attention_backend(profile) + + # LOUD GUARD: a ship image needs the Mega-Cache blobs, not just the inductor + # /cache. Without LTX_MEGACACHE_DIR set here, this bake produces NO + # .megacache.bin, and a fresh pod from the resulting image cold-compiles the + # torch.compile front-end (~33 min/shape for SPEED) on EVERY boot instead of + # the ~12 min Mega-Cache boot-warm. This is silent unless you look. See + # ltx23/CACHING.md "Production: bake the Mega-Cache blobs into the image". + if not os.environ.get("LTX_MEGACACHE_DIR"): + print( + "[warmup] *** WARNING: LTX_MEGACACHE_DIR is UNSET. This bake will NOT " + "produce Mega-Cache blobs -- a ship image from this cache cold-compiles " + "on every pod boot (no ~12 min boot-warm). For a ship image, set " + "LTX_MEGACACHE_DIR (e.g. -e LTX_MEGACACHE_DIR=/cache/megacache). " + "See ltx23/CACHING.md. ***", + file=sys.stderr, + flush=True, + ) + output_dir = Path(args.output_dir) output_dir.mkdir(parents=True, exist_ok=True) @@ -171,10 +195,12 @@ async def _route_one_shape_with_timeout( # by the LTX23_PROFILE env (quality|speed), which factory.load_model # reads -- the pool subprocess inherits it. enable_optimizations=True # lets the SPEED profile apply NVFP4 (QUALITY stays bf16 regardless). - # Denoise steps come from the shapes file (8 quality / 5 speed). - # See ltx23/config.py + ltx23/PROFILES.md. + # attention_backend follows the profile (quality -> FLASH_ATTN, + # speed -> TORCH_SDPA) via the same helper the worker uses, so the + # baked cache matches what serving asks for. Denoise steps come from + # the shapes file (8 quality / 5 speed). See config.py + PROFILES.md. enable_optimizations=True, - attention_backend="FLASH_ATTN", + attention_backend=attention_backend, model_factory_dotted="ltx23.factory:load_model", model_label="ltx2-3-distilled", ) diff --git a/examples/diffusers/ltx23/worker.py b/examples/diffusers/ltx23/worker.py index 74932484ee5b..b2467c2ea896 100644 --- a/examples/diffusers/ltx23/worker.py +++ b/examples/diffusers/ltx23/worker.py @@ -44,16 +44,17 @@ from dynamo.common.utils.prometheus import register_engine_metrics_callback from dynamo.runtime import DistributedRuntime +from .config import profile_attention_backend, profile_default_optimizations from .factory import load_model logger = logging.getLogger(__name__) DEFAULT_MODEL = "FastVideo/LTX-2.3-Distilled-Diffusers" -# FastVideo's deployed LTX-2.3 1080p recipe uses FLASH_ATTN (their optimized -# SM100/SM103 kernels); streaming_demo launch sets FASTVIDEO_ATTENTION_BACKEND= -# FLASH_ATTN. TORCH_SDPA (the prior default, inherited from LTX-2) is the slow -# generic fallback and does NOT reproduce the ~4.5s/1080p result. -DEFAULT_ATTENTION_BACKEND = "FLASH_ATTN" +# Attention backend and NVFP4-optimization defaults follow LTX23_PROFILE (see +# config.py): QUALITY -> FLASH_ATTN + bf16; SPEED -> TORCH_SDPA + NVFP4. Pinning +# them to the profile means a ship image only needs LTX23_PROFILE baked in -- the +# full recipe lives in the image, not a redis deploy-config flag. Both are still +# overridable via --attention-backend / --[no-]enable-optimizations. DEFAULT_MODEL_LABEL = "ltx2-3-distilled" # Where the LTX-2.3 shapes JSON lives in the production image. Production @@ -77,6 +78,10 @@ def _attention_backend_choices() -> tuple[str, ...]: def _parse_args() -> argparse.Namespace: choices = _attention_backend_choices() + # Defaults follow the baked-in profile so a ship image is self-describing. + profile = os.environ.get("LTX23_PROFILE", "quality") + default_attention = profile_attention_backend(profile) + default_optimizations = profile_default_optimizations(profile) parser = argparse.ArgumentParser( description="FastVideo Worker for Dynamo (non-streaming)" ) @@ -99,19 +104,23 @@ def _parse_args() -> argparse.Namespace: ) parser.add_argument( "--enable-optimizations", - action="store_true", + action=argparse.BooleanOptionalAction, + default=default_optimizations, dest="enable_optimizations", - help="Enable FP4 quantization (if available) and torch.compile", + help=( + "Enable FP4 quantization (if available) and torch.compile. " + f"Defaults from LTX23_PROFILE={profile!r} (default: {default_optimizations})" + ), ) parser.add_argument( "--attention-backend", choices=choices, - default=DEFAULT_ATTENTION_BACKEND, + default=default_attention, dest="attention_backend", help=( "Attention backend to set via FASTVIDEO_ATTENTION_BACKEND " f"(choices: {', '.join(choices)}; " - f"default: {DEFAULT_ATTENTION_BACKEND})" + f"default from LTX23_PROFILE={profile!r}: {default_attention})" ), ) return parser.parse_args()