diff --git a/.github/backend-matrix.yml b/.github/backend-matrix.yml index 4d1bd34fdc43..111c8b32bb68 100644 --- a/.github/backend-matrix.yml +++ b/.github/backend-matrix.yml @@ -478,6 +478,19 @@ include: dockerfile: "./backend/Dockerfile.python" context: "./" ubuntu-version: '2404' + - build-type: 'cublas' + cuda-major-version: "12" + cuda-minor-version: "8" + platforms: 'linux/amd64' + tag-latest: 'auto' + tag-suffix: '-gpu-nvidia-cuda-12-longcat-video' + runs-on: 'ubuntu-latest' + base-image: "ubuntu:24.04" + skip-drivers: 'false' + backend: "longcat-video" + dockerfile: "./backend/Dockerfile.python" + context: "./" + ubuntu-version: '2404' - build-type: 'cublas' cuda-major-version: "12" cuda-minor-version: "8" @@ -1149,6 +1162,19 @@ include: dockerfile: "./backend/Dockerfile.python" context: "./" ubuntu-version: '2404' + - build-type: 'cublas' + cuda-major-version: "13" + cuda-minor-version: "0" + platforms: 'linux/amd64' + tag-latest: 'auto' + tag-suffix: '-gpu-nvidia-cuda-13-longcat-video' + runs-on: 'ubuntu-latest' + base-image: "ubuntu:24.04" + skip-drivers: 'false' + backend: "longcat-video" + dockerfile: "./backend/Dockerfile.python" + context: "./" + ubuntu-version: '2404' - build-type: 'cublas' cuda-major-version: "13" cuda-minor-version: "0" @@ -1357,6 +1383,19 @@ include: backend: "vllm-omni" dockerfile: "./backend/Dockerfile.python" context: "./" + - build-type: 'l4t' + cuda-major-version: "13" + cuda-minor-version: "0" + platforms: 'linux/arm64' + tag-latest: 'auto' + tag-suffix: '-nvidia-l4t-cuda-13-arm64-longcat-video' + runs-on: 'ubuntu-24.04-arm' + base-image: "ubuntu:24.04" + skip-drivers: 'false' + ubuntu-version: '2404' + backend: "longcat-video" + dockerfile: "./backend/Dockerfile.python" + context: "./" - build-type: 'l4t' cuda-major-version: "13" cuda-minor-version: "0" diff --git a/.github/workflows/bump_deps.yaml b/.github/workflows/bump_deps.yaml index cde01ef528db..d9b3b4069943 100644 --- a/.github/workflows/bump_deps.yaml +++ b/.github/workflows/bump_deps.yaml @@ -26,6 +26,10 @@ jobs: variable: "DS4_VERSION" branch: "main" file: "backend/cpp/ds4/Makefile" + - repository: "meituan-longcat/LongCat-Video" + variable: "LONGCAT_VIDEO_VERSION" + branch: "main" + file: "backend/python/longcat-video/Makefile" - repository: "localai-org/privacy-filter.cpp" variable: "PRIVACY_FILTER_VERSION" branch: "master" diff --git a/Makefile b/Makefile index c6f38b1d089c..da2e638b07c4 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,5 @@ # Disable parallel execution for backend builds -.NOTPARALLEL: backends/diffusers backends/llama-cpp backends/turboquant backends/outetts backends/piper backends/stablediffusion-ggml backends/whisper backends/crispasr backends/parakeet-cpp backends/moss-transcribe-cpp backends/faster-whisper backends/silero-vad backends/local-store backends/huggingface backends/rfdetr backends/rfdetr-cpp backends/insightface backends/speaker-recognition backends/kitten-tts backends/kokoro backends/chatterbox backends/llama-cpp-darwin backends/neutts build-darwin-python-backend build-darwin-go-backend backends/mlx backends/diffuser-darwin backends/mlx-vlm backends/mlx-audio backends/mlx-distributed backends/stablediffusion-ggml-darwin backends/vllm backends/vllm-omni backends/sglang backends/moonshine backends/pocket-tts backends/qwen-tts backends/faster-qwen3-tts backends/qwen-asr backends/nemo backends/voxcpm backends/whisperx backends/ace-step backends/acestep-cpp backends/fish-speech backends/voxtral backends/opus backends/trl backends/llama-cpp-quantization backends/kokoros backends/sam3-cpp backends/qwen3-tts-cpp backends/omnivoice-cpp backends/vibevoice-cpp backends/localvqe backends/tinygrad backends/sherpa-onnx backends/ds4 backends/ds4-darwin backends/liquid-audio backends/supertonic backends/depth-anything-cpp backends/privacy-filter backends/privacy-filter-darwin +.NOTPARALLEL: backends/diffusers backends/llama-cpp backends/turboquant backends/outetts backends/piper backends/stablediffusion-ggml backends/whisper backends/crispasr backends/parakeet-cpp backends/moss-transcribe-cpp backends/faster-whisper backends/silero-vad backends/local-store backends/huggingface backends/rfdetr backends/rfdetr-cpp backends/insightface backends/speaker-recognition backends/kitten-tts backends/kokoro backends/chatterbox backends/llama-cpp-darwin backends/neutts build-darwin-python-backend build-darwin-go-backend backends/mlx backends/diffuser-darwin backends/mlx-vlm backends/mlx-audio backends/mlx-distributed backends/stablediffusion-ggml-darwin backends/vllm backends/vllm-omni backends/longcat-video backends/sglang backends/moonshine backends/pocket-tts backends/qwen-tts backends/faster-qwen3-tts backends/qwen-asr backends/nemo backends/voxcpm backends/whisperx backends/ace-step backends/acestep-cpp backends/fish-speech backends/voxtral backends/opus backends/trl backends/llama-cpp-quantization backends/kokoros backends/sam3-cpp backends/qwen3-tts-cpp backends/omnivoice-cpp backends/vibevoice-cpp backends/localvqe backends/tinygrad backends/sherpa-onnx backends/ds4 backends/ds4-darwin backends/liquid-audio backends/supertonic backends/depth-anything-cpp backends/privacy-filter backends/privacy-filter-darwin GOCMD=go GOTEST=$(GOCMD) test @@ -565,6 +565,7 @@ prepare-test-extra: protogen-python $(MAKE) -C backend/python/chatterbox $(MAKE) -C backend/python/vllm $(MAKE) -C backend/python/vllm-omni + $(MAKE) -C backend/python/longcat-video $(MAKE) -C backend/python/sglang $(MAKE) -C backend/python/vibevoice $(MAKE) -C backend/python/liquid-audio @@ -594,6 +595,7 @@ test-extra: prepare-test-extra $(MAKE) -C backend/python/chatterbox test $(MAKE) -C backend/python/vllm test $(MAKE) -C backend/python/vllm-omni test + $(MAKE) -C backend/python/longcat-video test $(MAKE) -C backend/python/vibevoice test $(MAKE) -C backend/python/liquid-audio test $(MAKE) -C backend/python/moonshine test @@ -1254,6 +1256,7 @@ BACKEND_NEUTTS = neutts|python|.|false|true BACKEND_KOKORO = kokoro|python|.|false|true BACKEND_VLLM = vllm|python|.|false|true BACKEND_VLLM_OMNI = vllm-omni|python|.|false|true +BACKEND_LONGCAT_VIDEO = longcat-video|python|.|--progress=plain|true BACKEND_SGLANG = sglang|python|.|false|true BACKEND_DIFFUSERS = diffusers|python|.|--progress=plain|true BACKEND_CHATTERBOX = chatterbox|python|.|false|true @@ -1339,6 +1342,7 @@ $(eval $(call generate-docker-build-target,$(BACKEND_NEUTTS))) $(eval $(call generate-docker-build-target,$(BACKEND_KOKORO))) $(eval $(call generate-docker-build-target,$(BACKEND_VLLM))) $(eval $(call generate-docker-build-target,$(BACKEND_VLLM_OMNI))) +$(eval $(call generate-docker-build-target,$(BACKEND_LONGCAT_VIDEO))) $(eval $(call generate-docker-build-target,$(BACKEND_SGLANG))) $(eval $(call generate-docker-build-target,$(BACKEND_DIFFUSERS))) $(eval $(call generate-docker-build-target,$(BACKEND_CHATTERBOX))) @@ -1375,7 +1379,7 @@ $(eval $(call generate-docker-build-target,$(BACKEND_SUPERTONIC))) docker-save-%: backend-images docker save local-ai-backend:$* -o backend-images/$*.tar -docker-build-backends: docker-build-llama-cpp docker-build-ik-llama-cpp docker-build-turboquant docker-build-ds4 docker-build-rerankers docker-build-vllm docker-build-vllm-omni docker-build-sglang docker-build-transformers docker-build-outetts docker-build-diffusers docker-build-kokoro docker-build-faster-whisper docker-build-crispasr docker-build-coqui docker-build-chatterbox docker-build-vibevoice docker-build-liquid-audio docker-build-moonshine docker-build-pocket-tts docker-build-qwen-tts docker-build-fish-speech docker-build-faster-qwen3-tts docker-build-qwen-asr docker-build-nemo docker-build-voxcpm docker-build-whisperx docker-build-ace-step docker-build-acestep-cpp docker-build-voxtral docker-build-mlx-distributed docker-build-trl docker-build-llama-cpp-quantization docker-build-tinygrad docker-build-kokoros docker-build-sam3-cpp docker-build-rfdetr-cpp docker-build-qwen3-tts-cpp docker-build-omnivoice-cpp docker-build-vibevoice-cpp docker-build-localvqe docker-build-insightface docker-build-speaker-recognition docker-build-sherpa-onnx docker-build-cloud-proxy docker-build-supertonic docker-build-depth-anything-cpp docker-build-moss-transcribe-cpp docker-build-privacy-filter +docker-build-backends: docker-build-llama-cpp docker-build-ik-llama-cpp docker-build-turboquant docker-build-ds4 docker-build-rerankers docker-build-vllm docker-build-vllm-omni docker-build-longcat-video docker-build-sglang docker-build-transformers docker-build-outetts docker-build-diffusers docker-build-kokoro docker-build-faster-whisper docker-build-crispasr docker-build-coqui docker-build-chatterbox docker-build-vibevoice docker-build-liquid-audio docker-build-moonshine docker-build-pocket-tts docker-build-qwen-tts docker-build-fish-speech docker-build-faster-qwen3-tts docker-build-qwen-asr docker-build-nemo docker-build-voxcpm docker-build-whisperx docker-build-ace-step docker-build-acestep-cpp docker-build-voxtral docker-build-mlx-distributed docker-build-trl docker-build-llama-cpp-quantization docker-build-tinygrad docker-build-kokoros docker-build-sam3-cpp docker-build-rfdetr-cpp docker-build-qwen3-tts-cpp docker-build-omnivoice-cpp docker-build-vibevoice-cpp docker-build-localvqe docker-build-insightface docker-build-speaker-recognition docker-build-sherpa-onnx docker-build-cloud-proxy docker-build-supertonic docker-build-depth-anything-cpp docker-build-moss-transcribe-cpp docker-build-privacy-filter ######################################################## ### Mock Backend for E2E Tests diff --git a/backend/README.md b/backend/README.md index 10c01d524800..0e92a0f03c58 100644 --- a/backend/README.md +++ b/backend/README.md @@ -46,6 +46,7 @@ The backend system provides language-specific Dockerfiles that handle the build - **vllm**: High-performance LLM inference - **mlx**: Apple Silicon optimization - **diffusers**: Stable Diffusion models +- **longcat-video**: CUDA text/image-to-video and speech-driven avatar generation - **Audio**: coqui, faster-whisper, kitten-tts - **Vision**: mlx-vlm, rfdetr - **Specialized**: rerankers, chatterbox, kokoro diff --git a/backend/backend.proto b/backend/backend.proto index 01c5b63a7b5c..ad62c6df07ae 100644 --- a/backend/backend.proto +++ b/backend/backend.proto @@ -577,6 +577,10 @@ message GenerateVideoRequest { float cfg_scale = 10; // Classifier-free guidance scale int32 step = 11; // Number of inference steps string dst = 12; // Output path for the generated video + string audio = 13; // Path to staged audio for audio-conditioned video + // Backend-specific per-request generation parameters. Values are strings + // and are validated/coerced by the selected backend. + map params = 14; } message TTSRequest { @@ -1256,4 +1260,3 @@ message ForwardReply { repeated ForwardHeader headers = 2; bytes body_chunk = 3; } - diff --git a/backend/index.yaml b/backend/index.yaml index 4d17bcc62600..3374a9d713fd 100644 --- a/backend/index.yaml +++ b/backend/index.yaml @@ -824,6 +824,30 @@ nvidia-cuda-12: "cuda12-vllm-omni" nvidia-cuda-13: "cuda13-vllm-omni" nvidia-l4t-cuda-13: "cuda13-nvidia-l4t-arm64-vllm-omni" +- &longcat-video + name: "longcat-video" + alias: "longcat-video" + license: mit + urls: + - https://github.com/meituan-longcat/LongCat-Video + tags: + - text-to-video + - image-to-video + - audio-to-video + - avatar-generation + - video-generation + - CUDA + icon: https://raw.githubusercontent.com/meituan-longcat/LongCat-Video/main/assets/longcat-video_logo.svg + description: | + LongCat-Video generation for text, image, and audio-conditioned avatars. + Supports LongCat-Video and LongCat-Video-Avatar-1.5, including multi-segment + talking-head continuation and an SDPA path for NVIDIA Blackwell ARM64 systems. + Requires Linux with an NVIDIA CUDA GPU; CPU, ROCm, and macOS are unsupported. + capabilities: + nvidia: "cuda12-longcat-video" + nvidia-cuda-12: "cuda12-longcat-video" + nvidia-cuda-13: "cuda13-longcat-video" + nvidia-l4t-cuda-13: "cuda13-nvidia-l4t-arm64-longcat-video" - &mlx name: "mlx" icon: https://avatars.githubusercontent.com/u/102832242?s=200&v=4 @@ -3605,6 +3629,44 @@ uri: "quay.io/go-skynet/local-ai-backends:master-gpu-rocm-hipblas-vllm-omni" mirrors: - localai/localai-backends:master-gpu-rocm-hipblas-vllm-omni +# longcat-video +- !!merge <<: *longcat-video + name: "longcat-video-development" + capabilities: + nvidia: "cuda12-longcat-video-development" + nvidia-cuda-12: "cuda12-longcat-video-development" + nvidia-cuda-13: "cuda13-longcat-video-development" + nvidia-l4t-cuda-13: "cuda13-nvidia-l4t-arm64-longcat-video-development" +- !!merge <<: *longcat-video + name: "cuda12-longcat-video" + uri: "quay.io/go-skynet/local-ai-backends:latest-gpu-nvidia-cuda-12-longcat-video" + mirrors: + - localai/localai-backends:latest-gpu-nvidia-cuda-12-longcat-video +- !!merge <<: *longcat-video + name: "cuda13-longcat-video" + uri: "quay.io/go-skynet/local-ai-backends:latest-gpu-nvidia-cuda-13-longcat-video" + mirrors: + - localai/localai-backends:latest-gpu-nvidia-cuda-13-longcat-video +- !!merge <<: *longcat-video + name: "cuda13-nvidia-l4t-arm64-longcat-video" + uri: "quay.io/go-skynet/local-ai-backends:latest-nvidia-l4t-cuda-13-arm64-longcat-video" + mirrors: + - localai/localai-backends:latest-nvidia-l4t-cuda-13-arm64-longcat-video +- !!merge <<: *longcat-video + name: "cuda12-longcat-video-development" + uri: "quay.io/go-skynet/local-ai-backends:master-gpu-nvidia-cuda-12-longcat-video" + mirrors: + - localai/localai-backends:master-gpu-nvidia-cuda-12-longcat-video +- !!merge <<: *longcat-video + name: "cuda13-longcat-video-development" + uri: "quay.io/go-skynet/local-ai-backends:master-gpu-nvidia-cuda-13-longcat-video" + mirrors: + - localai/localai-backends:master-gpu-nvidia-cuda-13-longcat-video +- !!merge <<: *longcat-video + name: "cuda13-nvidia-l4t-arm64-longcat-video-development" + uri: "quay.io/go-skynet/local-ai-backends:master-nvidia-l4t-cuda-13-arm64-longcat-video" + mirrors: + - localai/localai-backends:master-nvidia-l4t-cuda-13-arm64-longcat-video # rfdetr - !!merge <<: *rfdetr name: "rfdetr-development" diff --git a/backend/python/README.md b/backend/python/README.md index 45ee0e69c403..1864502f5925 100644 --- a/backend/python/README.md +++ b/backend/python/README.md @@ -27,6 +27,7 @@ The Python backends use a unified build system based on `libbackend.sh` that pro ### Computer Vision - **diffusers** - Stable Diffusion and image generation +- **longcat-video** - CUDA video and speech-driven avatar generation with LongCat-Video - **mlx-vlm** - Vision-language models for Apple Silicon - **rfdetr** - Object detection models diff --git a/backend/python/longcat-video/.gitignore b/backend/python/longcat-video/.gitignore new file mode 100644 index 000000000000..1ee12d07043c --- /dev/null +++ b/backend/python/longcat-video/.gitignore @@ -0,0 +1,6 @@ +backend_pb2.py +backend_pb2_grpc.py +lib/ +python/ +sources/ +venv/ diff --git a/backend/python/longcat-video/Makefile b/backend/python/longcat-video/Makefile new file mode 100644 index 000000000000..267ed8917ce7 --- /dev/null +++ b/backend/python/longcat-video/Makefile @@ -0,0 +1,36 @@ +# SPDX-License-Identifier: MIT + +LONGCAT_VIDEO_VERSION?=6b3f4b8582a8bc3f20f795735f5383716c4ba794 +LONGCAT_VIDEO_REPO?=https://github.com/meituan-longcat/LongCat-Video +LONGCAT_SOURCE_STAMP=sources/LongCat-Video/.localai-$(LONGCAT_VIDEO_VERSION) + +.PHONY: all +all: $(LONGCAT_SOURCE_STAMP) + bash install.sh + +$(LONGCAT_SOURCE_STAMP): patches/0001-sdpa-attention-fallback.patch + rm -rf sources/LongCat-Video + mkdir -p sources/LongCat-Video + cd sources/LongCat-Video && git init -q && \ + git remote add origin $(LONGCAT_VIDEO_REPO) && \ + git fetch --depth 1 origin $(LONGCAT_VIDEO_VERSION) && \ + git checkout --detach FETCH_HEAD && \ + git apply ../../patches/0001-sdpa-attention-fallback.patch && \ + rm -rf .git && \ + touch .localai-$(LONGCAT_VIDEO_VERSION) + +.PHONY: run +run: all + bash run.sh + +.PHONY: test +test: all + bash test.sh + +.PHONY: protogen-clean +protogen-clean: + $(RM) backend_pb2.py backend_pb2_grpc.py + +.PHONY: clean +clean: protogen-clean + rm -rf __pycache__ lib python sources venv diff --git a/backend/python/longcat-video/README.md b/backend/python/longcat-video/README.md new file mode 100644 index 000000000000..821de01303c4 --- /dev/null +++ b/backend/python/longcat-video/README.md @@ -0,0 +1,44 @@ +# LongCat Video backend + +This backend serves Meituan's `LongCat-Video` and +`LongCat-Video-Avatar-1.5` checkpoints through LocalAI's `GenerateVideo` +RPC. It supports: + +- text-to-video and image-to-video with `LongCat-Video`; +- audio + text-to-avatar and portrait + audio-to-avatar with Avatar 1.5; +- multi-segment avatar continuation for speech longer than one segment; +- PyTorch SDPA when FlashAttention is unavailable, including CUDA 13 ARM64 + systems such as NVIDIA DGX Spark. + +Install the `longcat-video` or `longcat-video-avatar-1.5` recipe from the +LocalAI Model Gallery. See the [LongCat user guide](../../../docs/content/features/longcat-video.md) +for Studio and API examples, hardware requirements, and manual configuration. + +The upstream source is pinned in `Makefile` and patched at build time. The +patch adds only the missing SDPA attention branches; model and source licenses +remain MIT. + +## Model options + +| Option | Default | Description | +| --- | --- | --- | +| `attention_backend` | `sdpa` | `sdpa`, `auto`, `flash2`, `flash3`, or `xformers`. The packaged backend guarantees only `sdpa`. | +| `use_distill` | `true` for Avatar, `false` for base | Loads the checkpoint's fast distillation LoRA. | +| `use_int8` | `false` | Loads Avatar 1.5's INT8 DiT. BF16 has a lower load-time peak on unified-memory systems. | +| `base_model` | `meituan-longcat/LongCat-Video` | Base components used by Avatar 1.5. | +| `max_segments` | `8` | Maximum avatar continuation segments accepted per request. | +| `resolution` | `480p` | Image-conditioned generation resolution (`480p` or `720p`). | + +Per-request `params` may set `num_segments`, `audio_guidance_scale`, +`offload_kv_cache`, `ref_img_index`, `mask_frame_range`, and `resolution`. + +Gallery and imported configs declare `known_input_modalities` and +`known_output_modalities`. Keep those declarations in manual configs as well; +they let model discovery distinguish base image-conditioned video from Avatar +audio conditioning without inspecting the backend or checkpoint name. + +LongCat is CUDA-only and very large. Avatar 1.5 also loads tokenizer, +text-encoder, and VAE components from the base checkpoint. Keep ample unified +memory and storage available; no CPU or macOS backend image is published. The +initial backend supports one GPU per process; tensor parallel sizes above one +are rejected explicitly. diff --git a/backend/python/longcat-video/backend.py b/backend/python/longcat-video/backend.py new file mode 100755 index 000000000000..62beb0a0e279 --- /dev/null +++ b/backend/python/longcat-video/backend.py @@ -0,0 +1,904 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: MIT + +import argparse +import datetime +import gc +import math +import os +import signal +import subprocess +import sys +import tempfile +import traceback +from concurrent import futures + +import grpc + +import backend_pb2 +import backend_pb2_grpc + +from longcat_utils import ( + BASE_MODEL_ID, + MODEL_KIND_AVATAR, + MODEL_KIND_BASE, + attention_overrides, + avatar_segments_for_duration, + avatar_segments_for_frames, + classify_model, + normalize_model_source, + normalize_num_frames, + parse_options, + require_bool, + require_float, + require_int, + validate_dimensions, +) + +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "common")) +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "common")) +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "sources", "LongCat-Video")) + +from grpc_auth import get_auth_interceptors + + +MAX_WORKERS = int(os.environ.get("PYTHON_GRPC_MAX_WORKERS", "1")) + +DEFAULT_NEGATIVE_PROMPT = ( + "Close-up, bright tones, overexposed, static, blurred details, subtitles, " + "paintings, low quality, JPEG compression residue, ugly, incomplete, extra " + "fingers, poorly drawn hands, poorly drawn faces, deformed, disfigured, " + "misshapen limbs, fused fingers, still picture, messy background, three legs, " + "many people in the background, walking backwards" +) + +LOAD_OPTIONS = { + "attention_backend", + "base_model", + "max_segments", + "resolution", + "use_distill", + "use_int8", +} + +REQUEST_PARAMS = { + "audio_guidance_scale", + "mask_frame_range", + "num_segments", + "offload_kv_cache", + "ref_img_index", + "resolution", +} + +BASE_CHECKPOINT_PATTERNS = [ + "config.json", + "model_index.json", + "dit/**", + "lora/cfg_step_lora.safetensors", + "scheduler/**", + "text_encoder/**", + "tokenizer/**", + "vae/**", +] + +AVATAR_BASE_PATTERNS = [ + "config.json", + "model_index.json", + "text_encoder/**", + "tokenizer/**", + "vae/**", +] + +AVATAR_COMMON_PATTERNS = [ + "config.json", + "lora/dmd_lora.safetensors", + "model_index.json", + "scheduler/**", + "whisper-large-v3/config.json", + "whisper-large-v3/model.safetensors", + "whisper-large-v3/preprocessor_config.json", +] + + +class BackendServicer(backend_pb2_grpc.BackendServicer): + def __init__(self): + self.model_kind = None + self.pipeline = None + self.options = {} + self.device_index = 0 + self.cp_split_hw = None + self._dist_store_dir = None + + def Health(self, request, context): + return backend_pb2.Reply(message=b"OK") + + def LoadModel(self, request, context): + model = request.Model + if request.ModelFile and os.path.isdir(request.ModelFile): + model = request.ModelFile + + model_kind = classify_model(model) + if model_kind is None: + return self._fail( + context, + grpc.StatusCode.INVALID_ARGUMENT, + "longcat-video only accepts LongCat-Video or LongCat-Video-Avatar-1.5 checkpoints", + ) + + try: + options = parse_options(request.Options) + unknown = sorted(set(options) - LOAD_OPTIONS) + if unknown: + raise ValueError(f"unknown model option(s): {', '.join(unknown)}") + + self._import_torch() + if not self.torch.cuda.is_available(): + return self._fail( + context, + grpc.StatusCode.FAILED_PRECONDITION, + "longcat-video requires an NVIDIA CUDA GPU", + ) + if request.TensorParallelSize > 1: + return self._fail( + context, + grpc.StatusCode.UNIMPLEMENTED, + "longcat-video currently supports one GPU per backend process", + ) + self._import_runtime() + + attention_name = str(options.get("attention_backend", "sdpa")).lower() + attention_overrides(attention_name) + resolution = str(options.get("resolution", "480p")).lower() + if resolution not in {"480p", "720p"}: + raise ValueError("resolution must be 480p or 720p") + + use_distill_default = model_kind == MODEL_KIND_AVATAR + use_distill = require_bool( + options.get("use_distill", use_distill_default), + "use_distill", + ) + use_int8 = require_bool(options.get("use_int8", False), "use_int8") + if model_kind == MODEL_KIND_BASE and use_int8: + raise ValueError( + "use_int8 is supported only by LongCat-Video-Avatar-1.5" + ) + + self.options = { + **options, + "attention_backend": attention_name, + "resolution": resolution, + "use_distill": use_distill, + "use_int8": use_int8, + "max_segments": require_int( + options.get("max_segments", 8), + "max_segments", + minimum=1, + maximum=64, + ), + } + + self._release_model() + self._ensure_distributed() + if model_kind == MODEL_KIND_BASE: + self._load_base_model(model) + else: + self._load_avatar_model(model) + self.model_kind = model_kind + print( + f"Loaded {normalize_model_source(model)} as {model_kind} " + f"with attention_backend={attention_name}", + file=sys.stderr, + ) + return backend_pb2.Result(message="Model loaded successfully", success=True) + except ValueError as err: + self._release_model() + return self._fail(context, grpc.StatusCode.INVALID_ARGUMENT, str(err)) + except Exception as err: + self._release_model() + print(f"Error loading LongCat model: {err}", file=sys.stderr) + traceback.print_exc() + return self._fail( + context, + grpc.StatusCode.INTERNAL, + f"failed to load LongCat model: {err}", + ) + + def Free(self, request, context): + self._release_model() + return backend_pb2.Result(message="Model released", success=True) + + def GenerateVideo(self, request, context): + if self.pipeline is None or self.model_kind is None: + return self._fail( + context, + grpc.StatusCode.FAILED_PRECONDITION, + "model is not loaded", + ) + if not request.prompt.strip(): + return self._fail( + context, + grpc.StatusCode.INVALID_ARGUMENT, + "prompt is required", + ) + if not request.dst: + return self._fail( + context, + grpc.StatusCode.INVALID_ARGUMENT, + "output destination is required", + ) + if request.end_image: + return self._fail( + context, + grpc.StatusCode.INVALID_ARGUMENT, + "longcat-video does not support end_image conditioning", + ) + + request_state = {"finished": False} + + def interrupt_if_cancelled(): + if not request_state["finished"] and self.pipeline is not None: + self.pipeline._interrupt = True + + try: + params = dict(request.params) + unknown = sorted(set(params) - REQUEST_PARAMS) + if unknown: + raise ValueError(f"unknown request param(s): {', '.join(unknown)}") + + os.makedirs(os.path.dirname(request.dst) or ".", mode=0o750, exist_ok=True) + if hasattr(context, "add_callback"): + context.add_callback(interrupt_if_cancelled) + + if request.start_image and not os.path.isfile(request.start_image): + raise ValueError("start_image is not a readable staged file") + if request.num_frames < 0: + raise ValueError("num_frames must not be negative") + + if self.model_kind == MODEL_KIND_BASE: + if request.audio: + raise ValueError( + "audio input requires a LongCat-Video-Avatar-1.5 model" + ) + self._generate_base(request, params) + else: + self._generate_avatar(request, params, context) + + return backend_pb2.Result( + message="Video generated successfully", success=True + ) + except ValueError as err: + return self._fail(context, grpc.StatusCode.INVALID_ARGUMENT, str(err)) + except Exception as err: + print(f"Error generating LongCat video: {err}", file=sys.stderr) + traceback.print_exc() + return self._fail( + context, + grpc.StatusCode.INTERNAL, + f"LongCat video generation failed: {err}", + ) + finally: + request_state["finished"] = True + if self.pipeline is not None: + self.pipeline._interrupt = False + + def _import_torch(self): + if hasattr(self, "torch"): + return + + import torch + + self.torch = torch + + def _import_runtime(self): + if hasattr(self, "LongCatVideoPipeline"): + return + + import imageio.v2 as imageio + import imageio_ffmpeg + import librosa + import numpy as np + import torch.distributed as dist + from diffusers.utils import load_image + from huggingface_hub import snapshot_download + from PIL import Image + from transformers import AutoTokenizer, UMT5EncoderModel + + from longcat_video.audio_process import ( + get_audio_encoder, + get_audio_feature_extractor, + ) + from longcat_video.context_parallel import context_parallel_util + from longcat_video.modules.autoencoder_kl_wan import AutoencoderKLWan + from longcat_video.modules.avatar.longcat_video_dit_avatar import ( + LongCatVideoAvatarTransformer3DModel, + ) + from longcat_video.modules.longcat_video_dit import ( + LongCatVideoTransformer3DModel, + ) + from longcat_video.modules.quantization import load_quantized_dit + from longcat_video.modules.scheduling_flow_match_euler_discrete import ( + FlowMatchEulerDiscreteScheduler, + ) + from longcat_video.pipeline_longcat_video import LongCatVideoPipeline + from longcat_video.pipeline_longcat_video_avatar import ( + LongCatVideoAvatarPipeline, + ) + + self.imageio = imageio + self.imageio_ffmpeg = imageio_ffmpeg + self.librosa = librosa + self.np = np + self.dist = dist + self.load_image = load_image + self.snapshot_download = snapshot_download + self.Image = Image + self.AutoTokenizer = AutoTokenizer + self.UMT5EncoderModel = UMT5EncoderModel + self.get_audio_encoder = get_audio_encoder + self.get_audio_feature_extractor = get_audio_feature_extractor + self.context_parallel_util = context_parallel_util + self.AutoencoderKLWan = AutoencoderKLWan + self.LongCatVideoAvatarTransformer3DModel = LongCatVideoAvatarTransformer3DModel + self.LongCatVideoTransformer3DModel = LongCatVideoTransformer3DModel + self.load_quantized_dit = load_quantized_dit + self.FlowMatchEulerDiscreteScheduler = FlowMatchEulerDiscreteScheduler + self.LongCatVideoPipeline = LongCatVideoPipeline + self.LongCatVideoAvatarPipeline = LongCatVideoAvatarPipeline + + def _ensure_distributed(self): + self.torch.cuda.set_device(self.device_index) + if not self.dist.is_initialized(): + self._dist_store_dir = tempfile.mkdtemp(prefix="localai-longcat-dist-") + init_file = os.path.join(self._dist_store_dir, "store") + self.dist.init_process_group( + backend="nccl", + init_method=f"file://{init_file}", + rank=0, + world_size=1, + timeout=datetime.timedelta(hours=24), + ) + self.context_parallel_util.init_context_parallel( + context_parallel_size=1, + global_rank=0, + world_size=1, + ) + self.cp_split_hw = self.context_parallel_util.get_optimal_split(1) + + def _resolve_checkpoint(self, model, patterns): + source = normalize_model_source(model) + if os.path.isdir(source): + return source + print(f"Downloading required files for {source}", file=sys.stderr) + return self.snapshot_download(repo_id=source, allow_patterns=patterns) + + def _load_base_model(self, model): + checkpoint = self._resolve_checkpoint(model, BASE_CHECKPOINT_PATTERNS) + dtype = self.torch.bfloat16 + overrides = attention_overrides(self.options["attention_backend"]) + + tokenizer = self.AutoTokenizer.from_pretrained( + checkpoint, + subfolder="tokenizer", + ) + text_encoder = self.UMT5EncoderModel.from_pretrained( + checkpoint, + subfolder="text_encoder", + torch_dtype=dtype, + low_cpu_mem_usage=True, + ) + vae = self.AutoencoderKLWan.from_pretrained( + checkpoint, + subfolder="vae", + torch_dtype=dtype, + low_cpu_mem_usage=True, + ) + scheduler = self.FlowMatchEulerDiscreteScheduler.from_pretrained( + checkpoint, + subfolder="scheduler", + ) + dit = self.LongCatVideoTransformer3DModel.from_pretrained( + checkpoint, + subfolder="dit", + cp_split_hw=self.cp_split_hw, + torch_dtype=dtype, + low_cpu_mem_usage=True, + **overrides, + ) + if self.options["use_distill"]: + dit.load_lora( + os.path.join(checkpoint, "lora", "cfg_step_lora.safetensors"), + "cfg_step_lora", + ) + dit.enable_loras(["cfg_step_lora"]) + + self.pipeline = self.LongCatVideoPipeline( + tokenizer=tokenizer, + text_encoder=text_encoder, + vae=vae, + scheduler=scheduler, + dit=dit, + ) + self.pipeline.to(self.device_index) + + def _load_avatar_model(self, model): + avatar_patterns = list(AVATAR_COMMON_PATTERNS) + model_subfolder = ( + "base_model_int8" if self.options["use_int8"] else "base_model" + ) + avatar_patterns.append(f"{model_subfolder}/**") + checkpoint = self._resolve_checkpoint(model, avatar_patterns) + + base_model = self.options.get("base_model") + if not base_model and os.path.isdir(normalize_model_source(model)): + sibling = os.path.join( + os.path.dirname(normalize_model_source(model)), "LongCat-Video" + ) + if os.path.isdir(sibling): + base_model = sibling + base_model = base_model or BASE_MODEL_ID + if classify_model(str(base_model)) != MODEL_KIND_BASE: + raise ValueError("base_model must point to a LongCat-Video checkpoint") + base_checkpoint = self._resolve_checkpoint(base_model, AVATAR_BASE_PATTERNS) + + dtype = self.torch.bfloat16 + overrides = attention_overrides(self.options["attention_backend"]) + tokenizer = self.AutoTokenizer.from_pretrained( + base_checkpoint, + subfolder="tokenizer", + ) + text_encoder = self.UMT5EncoderModel.from_pretrained( + base_checkpoint, + subfolder="text_encoder", + torch_dtype=dtype, + low_cpu_mem_usage=True, + ) + vae = self.AutoencoderKLWan.from_pretrained( + base_checkpoint, + subfolder="vae", + torch_dtype=dtype, + low_cpu_mem_usage=True, + ) + scheduler = self.FlowMatchEulerDiscreteScheduler.from_pretrained( + checkpoint, + subfolder="scheduler", + ) + + if self.options["use_int8"]: + previous_dtype = self.torch.get_default_dtype() + self.torch.set_default_dtype(dtype) + try: + dit = self.load_quantized_dit( + checkpoint, + subfolder="base_model_int8", + cp_split_hw=self.cp_split_hw, + **overrides, + ) + finally: + self.torch.set_default_dtype(previous_dtype) + else: + dit = self.LongCatVideoAvatarTransformer3DModel.from_pretrained( + checkpoint, + subfolder="base_model", + cp_split_hw=self.cp_split_hw, + torch_dtype=dtype, + low_cpu_mem_usage=True, + **overrides, + ) + + if self.options["use_distill"]: + dit.load_lora( + os.path.join(checkpoint, "lora", "dmd_lora.safetensors"), + "dmd", + multiplier=1.0, + lora_network_dim=128, + lora_network_alpha=64, + ) + dit.enable_loras(["dmd"]) + + audio_checkpoint = os.path.join(checkpoint, "whisper-large-v3") + audio_encoder = self.get_audio_encoder( + audio_checkpoint, + MODEL_KIND_AVATAR + "-v1.5", + ).to(self.device_index) + audio_feature_extractor = self.get_audio_feature_extractor( + audio_checkpoint, + MODEL_KIND_AVATAR + "-v1.5", + ) + self.pipeline = self.LongCatVideoAvatarPipeline( + tokenizer=tokenizer, + text_encoder=text_encoder, + vae=vae, + scheduler=scheduler, + dit=dit, + audio_encoder=audio_encoder, + audio_feature_extractor=audio_feature_extractor, + model_type="avatar-v1.5", + ) + self.pipeline.to(self.device_index) + + def _generate_base(self, request, params): + use_distill = self.options["use_distill"] + frames = normalize_num_frames(request.num_frames) + steps = ( + 16 + if use_distill + else require_int( + request.step or 50, + "step", + minimum=1, + maximum=200, + ) + ) + guidance_scale = ( + 1.0 + if use_distill + else require_float( + request.cfg_scale or 4.0, + "cfg_scale", + minimum=0.0, + maximum=30.0, + ) + ) + fps = require_int(request.fps or 15, "fps", minimum=1, maximum=60) + seed = request.seed if request.seed > 0 else 42 + negative_prompt = request.negative_prompt or DEFAULT_NEGATIVE_PROMPT + generator = self.torch.Generator(device=self.device_index).manual_seed(seed) + + if request.start_image: + resolution = self._resolution(params) + image = self.load_image(request.start_image) + output = self.pipeline.generate_i2v( + image=image, + prompt=request.prompt, + negative_prompt=negative_prompt, + resolution=resolution, + num_frames=frames, + num_inference_steps=steps, + use_distill=use_distill, + guidance_scale=guidance_scale, + generator=generator, + )[0] + else: + width, height = validate_dimensions(request.width, request.height) + output = self.pipeline.generate_t2v( + prompt=request.prompt, + negative_prompt=negative_prompt, + height=height, + width=width, + num_frames=frames, + num_inference_steps=steps, + use_distill=use_distill, + guidance_scale=guidance_scale, + generator=generator, + )[0] + + self._save_video(output, request.dst, fps) + + def _generate_avatar(self, request, params, context): + if not request.audio: + raise ValueError("audio is required for LongCat-Video-Avatar-1.5") + if not os.path.isfile(request.audio): + raise ValueError("audio input is not a readable staged file") + + use_distill = self.options["use_distill"] + steps = ( + 8 + if use_distill + else require_int( + request.step or 50, + "step", + minimum=1, + maximum=200, + ) + ) + text_guidance = ( + 1.0 + if use_distill + else require_float( + request.cfg_scale or 4.0, + "cfg_scale", + minimum=0.0, + maximum=30.0, + ) + ) + audio_guidance = ( + 1.0 + if use_distill + else require_float( + params.get("audio_guidance_scale", 4.0), + "audio_guidance_scale", + minimum=0.0, + maximum=20.0, + ) + ) + seed = request.seed if request.seed > 0 else 42 + generator = self.torch.Generator(device=self.device_index).manual_seed(seed) + negative_prompt = request.negative_prompt or DEFAULT_NEGATIVE_PROMPT + resolution = self._resolution(params) + + speech, sample_rate = self.librosa.load(request.audio, sr=16000, mono=True) + if speech.size == 0: + raise ValueError("audio contains no samples") + audio_duration = len(speech) / sample_rate + segments = self._avatar_segments(request, params, audio_duration) + + segment_frames = 93 + conditioning_frames = 13 + avatar_fps = 25 + generated_duration = ( + segment_frames + (segments - 1) * (segment_frames - conditioning_frames) + ) / avatar_fps + pad_samples = max( + 0, math.ceil((generated_duration - audio_duration) * sample_rate) + ) + if pad_samples: + speech = self.np.pad(speech, (0, pad_samples)) + + full_audio_embedding = self.pipeline.get_audio_embedding( + speech, + fps=avatar_fps, + device=self.device_index, + sample_rate=sample_rate, + model_type="avatar-v1.5", + ) + if not self.torch.isfinite(full_audio_embedding).all(): + raise ValueError("audio encoder returned non-finite values") + + indices = self.torch.arange(5) - 2 + + def audio_window(start_index): + centers = self.torch.arange( + start_index, + start_index + segment_frames, + ).unsqueeze(1) + indices.unsqueeze(0) + centers = self.torch.clamp( + centers, + min=0, + max=full_audio_embedding.shape[0] - 1, + ) + return full_audio_embedding[centers][None, ...].to(self.device_index) + + audio_start = 0 + common = { + "prompt": request.prompt, + "negative_prompt": negative_prompt, + "num_frames": segment_frames, + "num_inference_steps": steps, + "text_guidance_scale": text_guidance, + "audio_guidance_scale": audio_guidance, + "output_type": "both", + "generator": generator, + "audio_emb": audio_window(audio_start), + "use_distill": use_distill, + } + + if request.start_image: + output, latent = self.pipeline.generate_ai2v( + image=self.load_image(request.start_image), + resolution=resolution, + **common, + ) + else: + width, height = validate_dimensions(request.width, request.height) + output, latent = self.pipeline.generate_at2v( + height=height, + width=width, + **common, + ) + + video = self._frames_to_pil(output[0]) + width, height = video[0].size + current_video = video + reference_latent = latent[:, :, :1].clone() + all_frames = list(video) + + for segment in range(1, segments): + if hasattr(context, "is_active") and not context.is_active(): + raise RuntimeError("request was cancelled") + print( + f"Generating avatar segment {segment + 1}/{segments}", file=sys.stderr + ) + audio_start += segment_frames - conditioning_frames + output, latent = self.pipeline.generate_avc( + video=current_video, + video_latent=latent, + prompt=request.prompt, + negative_prompt=negative_prompt, + height=height, + width=width, + num_frames=segment_frames, + num_cond_frames=conditioning_frames, + num_inference_steps=steps, + text_guidance_scale=text_guidance, + audio_guidance_scale=audio_guidance, + generator=generator, + output_type="both", + use_kv_cache=True, + offload_kv_cache=require_bool( + params.get("offload_kv_cache", False), + "offload_kv_cache", + ), + enhance_hf=not use_distill, + audio_emb=audio_window(audio_start), + ref_latent=reference_latent, + ref_img_index=require_int( + params.get("ref_img_index", 10), + "ref_img_index", + minimum=-30, + maximum=30, + ), + mask_frame_range=require_int( + params.get("mask_frame_range", 3), + "mask_frame_range", + minimum=0, + maximum=32, + ), + use_distill=use_distill, + ) + current_video = self._frames_to_pil(output[0]) + all_frames.extend(current_video[conditioning_frames:]) + + self._save_avatar_video(all_frames, request.audio, request.dst, avatar_fps) + + def _avatar_segments(self, request, params, audio_duration): + if "num_segments" in params: + segments = require_int( + params["num_segments"], + "num_segments", + minimum=1, + ) + elif request.num_frames > 0: + segments = avatar_segments_for_frames(request.num_frames) + else: + segments = avatar_segments_for_duration(audio_duration) + + max_segments = self.options["max_segments"] + if segments > max_segments: + raise ValueError( + f"request needs {segments} avatar segments, but max_segments is {max_segments}; " + "trim the audio or raise the model's max_segments option" + ) + return segments + + def _resolution(self, params): + resolution = str(params.get("resolution", self.options["resolution"])).lower() + if resolution not in {"480p", "720p"}: + raise ValueError("resolution must be 480p or 720p") + return resolution + + def _frames_to_pil(self, frames): + images = [] + for frame in frames: + array = self.np.asarray(frame) + if self.np.issubdtype(array.dtype, self.np.floating): + array = self.np.clip(array, 0.0, 1.0) * 255 + images.append(self.Image.fromarray(array.astype(self.np.uint8))) + return images + + def _save_video(self, frames, path, fps): + writer = self.imageio.get_writer( + path, + format="FFMPEG", + mode="I", + fps=fps, + codec="libx264", + macro_block_size=1, + ffmpeg_params=[ + "-crf", + "18", + "-pix_fmt", + "yuv420p", + "-movflags", + "+faststart", + "-f", + "mp4", + ], + ) + try: + for frame in frames: + array = self.np.asarray(frame) + if self.np.issubdtype(array.dtype, self.np.floating): + array = self.np.clip(array, 0.0, 1.0) * 255 + writer.append_data(array.astype(self.np.uint8)) + finally: + writer.close() + + def _save_avatar_video(self, frames, audio_path, dst, fps): + output_dir = os.path.dirname(dst) or "." + handle, silent_path = tempfile.mkstemp( + prefix="longcat-silent-", + suffix=".mp4", + dir=output_dir, + ) + os.close(handle) + try: + self._save_video(frames, silent_path, fps) + command = [ + self.imageio_ffmpeg.get_ffmpeg_exe(), + "-y", + "-i", + silent_path, + "-i", + audio_path, + "-map", + "0:v:0", + "-map", + "1:a:0", + "-c:v", + "copy", + "-c:a", + "aac", + "-b:a", + "192k", + "-shortest", + "-movflags", + "+faststart", + "-f", + "mp4", + dst, + ] + subprocess.run( + command, + check=True, + stdout=subprocess.DEVNULL, + stderr=subprocess.PIPE, + text=True, + ) + except subprocess.CalledProcessError as err: + details = (err.stderr or "ffmpeg failed")[-2000:] + raise RuntimeError(f"failed to mux avatar audio: {details}") from err + finally: + try: + os.remove(silent_path) + except FileNotFoundError: + pass + + def _release_model(self): + self.pipeline = None + self.model_kind = None + gc.collect() + if hasattr(self, "torch") and self.torch.cuda.is_available(): + self.torch.cuda.empty_cache() + self.torch.cuda.ipc_collect() + + @staticmethod + def _fail(context, code, message): + if context is not None: + context.set_code(code) + context.set_details(message) + return backend_pb2.Result(message=message, success=False) + + +def serve(address): + server = grpc.server( + futures.ThreadPoolExecutor(max_workers=MAX_WORKERS), + options=[ + ("grpc.max_message_length", 64 * 1024 * 1024), + ("grpc.max_send_message_length", 64 * 1024 * 1024), + ("grpc.max_receive_message_length", 64 * 1024 * 1024), + ], + interceptors=get_auth_interceptors(), + ) + backend_pb2_grpc.add_BackendServicer_to_server(BackendServicer(), server) + server.add_insecure_port(address) + server.start() + print(f"LongCat Video backend listening on {address}", file=sys.stderr) + + def stop_server(signum, frame): + del signum, frame + server.stop(0) + + signal.signal(signal.SIGINT, stop_server) + signal.signal(signal.SIGTERM, stop_server) + server.wait_for_termination() + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description="Run the LongCat Video gRPC backend") + parser.add_argument( + "--addr", + default="localhost:50051", + help="address on which to serve the backend", + ) + arguments = parser.parse_args() + serve(arguments.addr) diff --git a/backend/python/longcat-video/install.sh b/backend/python/longcat-video/install.sh new file mode 100755 index 000000000000..4b0b2d00a005 --- /dev/null +++ b/backend/python/longcat-video/install.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: MIT +set -euo pipefail + +PYTHON_VERSION="3.12" +PYTHON_PATCH="12" +PY_STANDALONE_TAG="20251120" + +backend_dir=$(dirname "$0") +if [ -d "${backend_dir}/common" ]; then + source "${backend_dir}/common/libbackend.sh" +else + source "${backend_dir}/../common/libbackend.sh" +fi + +installRequirements diff --git a/backend/python/longcat-video/longcat_utils.py b/backend/python/longcat-video/longcat_utils.py new file mode 100644 index 000000000000..3d4ac10402b8 --- /dev/null +++ b/backend/python/longcat-video/longcat_utils.py @@ -0,0 +1,182 @@ +# SPDX-License-Identifier: MIT + +import json +import math +import os +from urllib.parse import urlparse + + +BASE_MODEL_ID = "meituan-longcat/LongCat-Video" +AVATAR_MODEL_ID = "meituan-longcat/LongCat-Video-Avatar-1.5" +MODEL_KIND_BASE = "base" +MODEL_KIND_AVATAR = "avatar" + +ATTENTION_OVERRIDES = { + "auto": {}, + "sdpa": { + "enable_flashattn2": False, + "enable_flashattn3": False, + "enable_xformers": False, + }, + "flash2": { + "enable_flashattn2": True, + "enable_flashattn3": False, + "enable_xformers": False, + }, + "flash3": { + "enable_flashattn2": False, + "enable_flashattn3": True, + "enable_xformers": False, + }, + "xformers": { + "enable_flashattn2": False, + "enable_flashattn3": False, + "enable_xformers": True, + }, +} + + +def parse_options(values): + options = {} + for raw in values: + if ":" not in raw: + options[raw.strip()] = True + continue + key, value = raw.split(":", 1) + key = key.strip() + value = value.strip() + if not key: + continue + lower = value.lower() + if lower in {"true", "false"}: + options[key] = lower == "true" + continue + try: + options[key] = int(value) + continue + except ValueError: + pass + try: + options[key] = float(value) + continue + except ValueError: + pass + options[key] = value + return options + + +def require_bool(value, name): + if isinstance(value, bool): + return value + if isinstance(value, str) and value.lower() in {"true", "false"}: + return value.lower() == "true" + raise ValueError(f"{name} must be true or false") + + +def require_int(value, name, minimum=None, maximum=None): + try: + parsed = int(value) + except (TypeError, ValueError) as err: + raise ValueError(f"{name} must be an integer") from err + if minimum is not None and parsed < minimum: + raise ValueError(f"{name} must be at least {minimum}") + if maximum is not None and parsed > maximum: + raise ValueError(f"{name} must be at most {maximum}") + return parsed + + +def require_float(value, name, minimum=None, maximum=None): + try: + parsed = float(value) + except (TypeError, ValueError) as err: + raise ValueError(f"{name} must be a number") from err + if minimum is not None and parsed < minimum: + raise ValueError(f"{name} must be at least {minimum}") + if maximum is not None and parsed > maximum: + raise ValueError(f"{name} must be at most {maximum}") + return parsed + + +def attention_overrides(name): + try: + return dict(ATTENTION_OVERRIDES[name]) + except KeyError as err: + choices = ", ".join(ATTENTION_OVERRIDES) + raise ValueError(f"attention_backend must be one of: {choices}") from err + + +def _model_name_from_directory(path): + for filename in ("model_index.json", "config.json"): + config_path = os.path.join(path, filename) + try: + with open(config_path, "r", encoding="utf-8") as config_file: + model_name = json.load(config_file).get("model_name", "") + except (FileNotFoundError, OSError, ValueError, TypeError): + continue + if model_name: + return model_name + return "" + + +def normalize_model_source(model): + value = model.rstrip("/") + for prefix in ("huggingface://", "hf://"): + if value.startswith(prefix): + return value[len(prefix) :] + parsed = urlparse(value) + if parsed.scheme in {"http", "https"} and parsed.netloc.lower() == "huggingface.co": + parts = [part for part in parsed.path.split("/") if part] + if len(parts) >= 2: + return "/".join(parts[:2]) + return value + + +def classify_model(model): + if not model: + return None + normalized = normalize_model_source(model) + if os.path.isdir(normalized): + name = _model_name_from_directory(normalized).lower() + if name == "longcat-video": + return MODEL_KIND_BASE + if name == "longcat-video-avatar-1.5": + return MODEL_KIND_AVATAR + return None + + normalized = normalized.lower() + if normalized == BASE_MODEL_ID.lower(): + return MODEL_KIND_BASE + if normalized == AVATAR_MODEL_ID.lower(): + return MODEL_KIND_AVATAR + return None + + +def normalize_num_frames(value, default=93): + frames = default if not value or value < 1 else value + return max(1, ((frames - 1) // 4) * 4 + 1) + + +def avatar_segments_for_frames(frames): + if not frames or frames <= 93: + return 1 + return 1 + math.ceil((frames - 93) / 80) + + +def avatar_segments_for_duration(duration_seconds, fps=25): + if duration_seconds <= 0: + return 1 + return avatar_segments_for_frames(math.ceil(duration_seconds * fps)) + + +def validate_dimensions(width, height): + width = width or 832 + height = height or 480 + if width < 256 or height < 256: + raise ValueError("width and height must each be at least 256") + if width > 1280 or height > 768: + raise ValueError("width and height must not exceed 1280x768") + if width % 16 != 0 or height % 16 != 0: + raise ValueError("width and height must be divisible by 16") + if width * height > 1280 * 768: + raise ValueError("requested video dimensions exceed the 1280x768 pixel limit") + return width, height diff --git a/backend/python/longcat-video/patches/0001-sdpa-attention-fallback.patch b/backend/python/longcat-video/patches/0001-sdpa-attention-fallback.patch new file mode 100644 index 000000000000..76829351f435 --- /dev/null +++ b/backend/python/longcat-video/patches/0001-sdpa-attention-fallback.patch @@ -0,0 +1,75 @@ +diff --git a/longcat_video/modules/attention.py b/longcat_video/modules/attention.py +index bb5630f..9b9f3cc 100644 +--- a/longcat_video/modules/attention.py ++++ b/longcat_video/modules/attention.py +@@ -2,6 +2,7 @@ from typing import List, Optional + + import torch + import torch.nn as nn ++import torch.nn.functional as F + + from einops import rearrange + +@@ -100,7 +101,8 @@ class Attention(nn.Module): + x = xformers.ops.memory_efficient_attention(q, k, v, attn_bias=None, op=None,) + x = rearrange(x, "B M H K -> B H M K") + else: +- raise RuntimeError("Unsupported attention operations.") ++ # Keep a dependency-free path for systems without optional kernels. ++ x = F.scaled_dot_product_attention(q, k, v, scale=self.scale) + + return x + +@@ -245,8 +247,22 @@ class MultiHeadCrossAttention(nn.Module): + attn_bias = xformers.ops.fmha.attn_bias.BlockDiagonalMask.from_seqlens([N] * B, kv_seqlen) + x = xformers.ops.memory_efficient_attention(q, k, v, attn_bias=attn_bias) + else: +- raise RuntimeError("Unsupported attention operations.") +- ++ # Preserve the variable-length block boundaries without materializing ++ # a dense attention mask. ++ blocks = [] ++ offset = 0 ++ for batch_index, key_count in enumerate(kv_seqlen): ++ query = q[0][batch_index * N:(batch_index + 1) * N] ++ key = k[0][offset:offset + key_count] ++ value = v[0][offset:offset + key_count] ++ output = F.scaled_dot_product_attention( ++ query.transpose(0, 1), ++ key.transpose(0, 1), ++ value.transpose(0, 1), ++ ) ++ blocks.append(output.transpose(0, 1)) ++ offset += key_count ++ x = torch.cat(blocks, dim=0) + + x = x.view(B, -1, C) + x = self.proj(x) +diff --git a/longcat_video/modules/avatar/attention.py b/longcat_video/modules/avatar/attention.py +index a169a7a..df9a469 100644 +--- a/longcat_video/modules/avatar/attention.py ++++ b/longcat_video/modules/avatar/attention.py +@@ -2,6 +2,7 @@ from typing import List, Optional + + import torch + import torch.nn as nn ++import torch.nn.functional as F + + from einops import rearrange + +@@ -111,7 +112,8 @@ class Attention(nn.Module): + x = xformers.ops.memory_efficient_attention(q, k, v, attn_bias=None, op=None,) + x = rearrange(x, "B M H K -> B H M K") + else: +- raise RuntimeError("Unsupported attention operations.") ++ # Keep a dependency-free path for systems without optional kernels. ++ x = F.scaled_dot_product_attention(q, k, v, scale=self.scale) + + return x + +@@ -429,2 +431,5 @@ class SingleStreamAttention(nn.Module): ++ else: ++ # This branch uses the native PyTorch kernel when optional kernels are off. ++ x = F.scaled_dot_product_attention(q, encoder_k, encoder_v, scale=self.scale) + + # linear transform diff --git a/backend/python/longcat-video/requirements-after.txt b/backend/python/longcat-video/requirements-after.txt new file mode 100644 index 000000000000..a9368375be0e --- /dev/null +++ b/backend/python/longcat-video/requirements-after.txt @@ -0,0 +1 @@ +accelerate diff --git a/backend/python/longcat-video/requirements-cpu.txt b/backend/python/longcat-video/requirements-cpu.txt new file mode 100644 index 000000000000..01311205e269 --- /dev/null +++ b/backend/python/longcat-video/requirements-cpu.txt @@ -0,0 +1,3 @@ +--index-url https://download.pytorch.org/whl/cpu +torch==2.12.1 +torchvision==0.27.1 diff --git a/backend/python/longcat-video/requirements-cublas12.txt b/backend/python/longcat-video/requirements-cublas12.txt new file mode 100644 index 000000000000..b3b28a042b96 --- /dev/null +++ b/backend/python/longcat-video/requirements-cublas12.txt @@ -0,0 +1,3 @@ +--index-url https://download.pytorch.org/whl/cu126 +torch==2.12.1 +torchvision==0.27.1 diff --git a/backend/python/longcat-video/requirements-cublas13.txt b/backend/python/longcat-video/requirements-cublas13.txt new file mode 100644 index 000000000000..c3fcbcaec5ac --- /dev/null +++ b/backend/python/longcat-video/requirements-cublas13.txt @@ -0,0 +1,3 @@ +--index-url https://download.pytorch.org/whl/cu130 +torch==2.12.1 +torchvision==0.27.1 diff --git a/backend/python/longcat-video/requirements-l4t13.txt b/backend/python/longcat-video/requirements-l4t13.txt new file mode 100644 index 000000000000..c3fcbcaec5ac --- /dev/null +++ b/backend/python/longcat-video/requirements-l4t13.txt @@ -0,0 +1,3 @@ +--index-url https://download.pytorch.org/whl/cu130 +torch==2.12.1 +torchvision==0.27.1 diff --git a/backend/python/longcat-video/requirements.txt b/backend/python/longcat-video/requirements.txt new file mode 100644 index 000000000000..e3ad4e3d6d76 --- /dev/null +++ b/backend/python/longcat-video/requirements.txt @@ -0,0 +1,23 @@ +certifi +diffusers==0.35.1 +einops==0.8.0 +ftfy==6.2.0 +grpcio==1.76.0 +huggingface-hub>=0.23,<1.0 +imageio==2.37.0 +imageio-ffmpeg==0.6.0 +librosa==0.11.0 +loguru==0.7.2 +numpy==1.26.4 +packaging +pillow +protobuf +pyloudnorm==0.1.1 +regex +safetensors +scipy==1.15.3 +sentencepiece +soundfile==0.13.1 +soxr==0.5.0.post1 +tqdm +transformers==4.41.0 diff --git a/backend/python/longcat-video/run.sh b/backend/python/longcat-video/run.sh new file mode 100755 index 000000000000..eaf36bda06cb --- /dev/null +++ b/backend/python/longcat-video/run.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: MIT +set -euo pipefail + +backend_dir=$(dirname "$0") +if [ -d "${backend_dir}/common" ]; then + source "${backend_dir}/common/libbackend.sh" +else + source "${backend_dir}/../common/libbackend.sh" +fi + +startBackend "$@" diff --git a/backend/python/longcat-video/test.py b/backend/python/longcat-video/test.py new file mode 100644 index 000000000000..13f32b8e5b6b --- /dev/null +++ b/backend/python/longcat-video/test.py @@ -0,0 +1,218 @@ +# SPDX-License-Identifier: MIT + +import importlib.util +import json +import os +import sys +import tempfile +import unittest + +BACKEND_DIR = os.path.dirname(os.path.abspath(__file__)) +sys.path.insert(0, BACKEND_DIR) + +# longcat-video is a backend directory, not an importable Python package name. +from longcat_utils import ( # noqa: E402 + MODEL_KIND_AVATAR, + MODEL_KIND_BASE, + attention_overrides, + avatar_segments_for_duration, + avatar_segments_for_frames, + classify_model, + normalize_model_source, + normalize_num_frames, + parse_options, + validate_dimensions, +) + + +SOURCE_DIR = os.path.join(BACKEND_DIR, "sources", "LongCat-Video") +try: + import torch + + sys.path.insert(0, SOURCE_DIR) + ATTENTION_TESTS_AVAILABLE = ( + os.path.isdir(SOURCE_DIR) and importlib.util.find_spec("triton") is not None + ) +except ImportError: + torch = None + ATTENTION_TESTS_AVAILABLE = False + +AVATAR_ATTENTION_TESTS_AVAILABLE = ATTENTION_TESTS_AVAILABLE and all( + importlib.util.find_spec(module) is not None + for module in ("pyloudnorm", "scipy", "torchvision") +) + + +class LongCatUtilsTest(unittest.TestCase): + def test_parse_options_preserves_colons_and_coerces_scalars(self): + options = parse_options( + [ + "use_distill:true", + "max_segments:4", + "audio_guidance_scale:3.5", + "source:https://example.com/model", + "flag", + ] + ) + + self.assertEqual(options["use_distill"], True) + self.assertEqual(options["max_segments"], 4) + self.assertEqual(options["audio_guidance_scale"], 3.5) + self.assertEqual(options["source"], "https://example.com/model") + self.assertEqual(options["flag"], True) + + def test_classify_model_accepts_only_supported_longcat_models(self): + cases = { + "meituan-longcat/LongCat-Video": MODEL_KIND_BASE, + "https://huggingface.co/meituan-longcat/LongCat-Video": MODEL_KIND_BASE, + "hf://meituan-longcat/LongCat-Video-Avatar-1.5": MODEL_KIND_AVATAR, + "other-org/LongCat-Video": None, + "meituan-longcat/LongCat-Video-Avatar": None, + "some-org/unrelated-model": None, + } + + for model, expected in cases.items(): + with self.subTest(model=model): + self.assertEqual(classify_model(model), expected) + + def test_classify_model_reads_local_checkpoint_metadata(self): + with tempfile.TemporaryDirectory() as directory: + with open( + os.path.join(directory, "model_index.json"), + "w", + encoding="utf-8", + ) as config_file: + json.dump({"model_name": "LongCat-Video-Avatar-1.5"}, config_file) + + self.assertEqual(classify_model(directory), MODEL_KIND_AVATAR) + + def test_normalize_model_source_handles_huggingface_uri_forms(self): + self.assertEqual( + normalize_model_source( + "https://huggingface.co/meituan-longcat/LongCat-Video/tree/main" + ), + "meituan-longcat/LongCat-Video", + ) + self.assertEqual( + normalize_model_source("huggingface://meituan-longcat/LongCat-Video"), + "meituan-longcat/LongCat-Video", + ) + + def test_frame_and_segment_rounding_matches_longcat_temporal_shape(self): + self.assertEqual(normalize_num_frames(94), 93) + self.assertEqual(normalize_num_frames(0), 93) + self.assertEqual(avatar_segments_for_frames(93), 1) + self.assertEqual(avatar_segments_for_frames(94), 2) + self.assertEqual(avatar_segments_for_frames(173), 2) + self.assertEqual(avatar_segments_for_frames(174), 3) + self.assertEqual(avatar_segments_for_duration(10.0), 3) + + def test_dimensions_are_bounded_and_aligned(self): + self.assertEqual(validate_dimensions(0, 0), (832, 480)) + self.assertEqual(validate_dimensions(512, 512), (512, 512)) + with self.assertRaisesRegex(ValueError, "divisible by 16"): + validate_dimensions(513, 512) + with self.assertRaisesRegex(ValueError, "must not exceed"): + validate_dimensions(1920, 1080) + + def test_attention_backend_validation(self): + self.assertEqual( + attention_overrides("sdpa"), + { + "enable_flashattn2": False, + "enable_flashattn3": False, + "enable_xformers": False, + }, + ) + with self.assertRaisesRegex(ValueError, "attention_backend"): + attention_overrides("unknown") + + +@unittest.skipUnless( + ATTENTION_TESTS_AVAILABLE, + "patched LongCat source and torch are required for attention tests", +) +class SDPAFallbackTest(unittest.TestCase): + def test_base_self_attention_matches_reference(self): + from longcat_video.modules.attention import Attention + + dim, heads, sequence = 64, 4, 32 + attention = Attention( + dim, + heads, + enable_flashattn2=False, + enable_flashattn3=False, + enable_xformers=False, + enable_bsa=False, + ).float() + query = torch.randn(2, heads, sequence, dim // heads) + key = torch.randn_like(query) + value = torch.randn_like(query) + + output = attention._process_attn(query, key, value, shape=(1, 1, sequence)) + reference = ( + torch.softmax( + (query @ key.transpose(-1, -2)) * attention.scale, + dim=-1, + ) + @ value + ) + + self.assertLess((output - reference).abs().max().item(), 1e-4) + + @unittest.skipUnless( + AVATAR_ATTENTION_TESTS_AVAILABLE, + "avatar audio dependencies are required for the avatar attention test", + ) + def test_avatar_self_attention_matches_reference(self): + from longcat_video.modules.avatar.attention import Attention + + dim, heads, sequence = 64, 4, 16 + attention = Attention( + dim, + heads, + enable_flashattn2=False, + enable_flashattn3=False, + enable_xformers=False, + ).float() + query = torch.randn(1, heads, sequence, dim // heads) + key = torch.randn_like(query) + value = torch.randn_like(query) + + output = attention._process_attn(query, key, value, shape=(1, 1, sequence)) + reference = ( + torch.softmax( + (query @ key.transpose(-1, -2)) * attention.scale, + dim=-1, + ) + @ value + ) + + self.assertLess((output - reference).abs().max().item(), 1e-4) + + def test_base_cross_attention_remains_block_diagonal(self): + from longcat_video.modules.attention import MultiHeadCrossAttention + + dim, heads = 64, 4 + attention = MultiHeadCrossAttention( + dim, + heads, + enable_flashattn2=False, + enable_flashattn3=False, + enable_xformers=False, + ).float() + query = torch.randn(2, 8, dim) + key_lengths = [5, 7] + condition = torch.randn(1, sum(key_lengths), dim) + + first = attention._process_cross_attn(query, condition, key_lengths) + changed = condition.clone() + changed[:, key_lengths[0] :] = torch.randn_like(changed[:, key_lengths[0] :]) + second = attention._process_cross_attn(query, changed, key_lengths) + + self.assertLess((first[0] - second[0]).abs().max().item(), 1e-5) + self.assertGreater((first[1] - second[1]).abs().max().item(), 1e-3) + + +if __name__ == "__main__": + unittest.main() diff --git a/backend/python/longcat-video/test.sh b/backend/python/longcat-video/test.sh new file mode 100755 index 000000000000..59cb0da3229c --- /dev/null +++ b/backend/python/longcat-video/test.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: MIT +set -euo pipefail + +backend_dir=$(dirname "$0") +if [ -d "${backend_dir}/common" ]; then + source "${backend_dir}/common/libbackend.sh" +else + source "${backend_dir}/../common/libbackend.sh" +fi + +runUnittests diff --git a/core/backend/video.go b/core/backend/video.go index e016d1a22a83..2702fefe9b8e 100644 --- a/core/backend/video.go +++ b/core/backend/video.go @@ -1,21 +1,37 @@ package backend import ( + "maps" "time" "github.com/mudler/LocalAI/core/config" "github.com/mudler/LocalAI/core/trace" - "github.com/mudler/LocalAI/pkg/grpc/proto" model "github.com/mudler/LocalAI/pkg/model" ) -func VideoGeneration(height, width int32, prompt, negativePrompt, startImage, endImage, dst string, numFrames, fps, seed int32, cfgScale float32, step int32, loader *model.ModelLoader, modelConfig config.ModelConfig, appConfig *config.ApplicationConfig) (func() error, error) { +// VideoGenerationOptions is the backend-neutral request passed to video generators. +// Media fields contain staged local paths by the time they reach this layer. +type VideoGenerationOptions struct { + Height int32 + Width int32 + Prompt string + NegativePrompt string + StartImage string + EndImage string + Audio string + Destination string + NumFrames int32 + FPS int32 + Seed int32 + CFGScale float32 + Step int32 + Params map[string]string +} +func VideoGeneration(options VideoGenerationOptions, loader *model.ModelLoader, modelConfig config.ModelConfig, appConfig *config.ApplicationConfig) (func() error, error) { opts := ModelOptions(modelConfig, appConfig) - inferenceModel, err := loader.Load( - opts..., - ) + inferenceModel, err := loader.Load(opts...) if err != nil { recordModelLoadFailure(appConfig, modelConfig.Name, modelConfig.Backend, err, nil) return nil, err @@ -25,19 +41,22 @@ func VideoGeneration(height, width int32, prompt, negativePrompt, startImage, en _, err := inferenceModel.GenerateVideo( appConfig.Context, &proto.GenerateVideoRequest{ - Height: height, - Width: width, - Prompt: prompt, - NegativePrompt: negativePrompt, - StartImage: startImage, - EndImage: endImage, - NumFrames: numFrames, - Fps: fps, - Seed: seed, - CfgScale: cfgScale, - Step: step, - Dst: dst, - }) + Height: options.Height, + Width: options.Width, + Prompt: options.Prompt, + NegativePrompt: options.NegativePrompt, + StartImage: options.StartImage, + EndImage: options.EndImage, + Audio: options.Audio, + NumFrames: options.NumFrames, + Fps: options.FPS, + Seed: options.Seed, + CfgScale: options.CFGScale, + Step: options.Step, + Dst: options.Destination, + Params: maps.Clone(options.Params), + }, + ) return err } @@ -45,15 +64,18 @@ func VideoGeneration(height, width int32, prompt, negativePrompt, startImage, en trace.InitBackendTracingIfEnabled(appConfig.TracingMaxItems, appConfig.TracingMaxBodyBytes) traceData := map[string]any{ - "prompt": prompt, - "negative_prompt": negativePrompt, - "height": height, - "width": width, - "num_frames": numFrames, - "fps": fps, - "seed": seed, - "cfg_scale": cfgScale, - "step": step, + "prompt": options.Prompt, + "negative_prompt": options.NegativePrompt, + "height": options.Height, + "width": options.Width, + "num_frames": options.NumFrames, + "fps": options.FPS, + "seed": options.Seed, + "cfg_scale": options.CFGScale, + "step": options.Step, + "has_start_image": options.StartImage != "", + "has_end_image": options.EndImage != "", + "has_audio": options.Audio != "", } startTime := time.Now() @@ -73,7 +95,7 @@ func VideoGeneration(height, width int32, prompt, negativePrompt, startImage, en Type: trace.BackendTraceVideoGeneration, ModelName: modelConfig.Name, Backend: modelConfig.Backend, - Summary: trace.TruncateString(prompt, 200), + Summary: trace.TruncateString(options.Prompt, 200), Error: errStr, Data: traceData, }) diff --git a/core/config/backend_capabilities.go b/core/config/backend_capabilities.go index d54463a8e4f9..e98d406c98ea 100644 --- a/core/config/backend_capabilities.go +++ b/core/config/backend_capabilities.go @@ -120,7 +120,7 @@ var UsecaseInfoMap = map[string]UsecaseInfo{ UsecaseVideo: { Flag: FLAG_VIDEO, GRPCMethod: MethodGenerateVideo, - Description: "Video generation via the GenerateVideo RPC.", + Description: "Video generation via the GenerateVideo RPC, with optional image or audio conditioning when supported by the backend.", }, UsecaseTranscript: { Flag: FLAG_TRANSCRIPT, @@ -303,6 +303,14 @@ var BackendCapabilities = map[string]BackendCapability{ DefaultUsecases: []string{UsecaseImage}, Description: "HuggingFace diffusers — Stable Diffusion, Flux, video generation", }, + "longcat-video": { + GRPCMethods: []GRPCMethod{MethodGenerateVideo}, + PossibleUsecases: []string{UsecaseVideo}, + DefaultUsecases: []string{UsecaseVideo}, + AcceptsImages: true, + AcceptsAudios: true, + Description: "LongCat-Video — text, image, and audio-conditioned avatar video generation on NVIDIA CUDA", + }, "stablediffusion": { GRPCMethods: []GRPCMethod{MethodGenerateImage}, PossibleUsecases: []string{UsecaseImage}, diff --git a/core/config/meta/constants.go b/core/config/meta/constants.go index 7fed6ba757ff..19eebcb50b88 100644 --- a/core/config/meta/constants.go +++ b/core/config/meta/constants.go @@ -79,6 +79,14 @@ var UsecaseOptions = []FieldOption{ {Value: "video", Label: "Video"}, } +// ModalityOptions enumerates the values accepted by known modality fields. +var ModalityOptions = []FieldOption{ + {Value: "text", Label: "Text"}, + {Value: "image", Label: "Image"}, + {Value: "audio", Label: "Audio"}, + {Value: "video", Label: "Video"}, +} + var DiffusersSchedulerOptions = []FieldOption{ {Value: "ddim", Label: "DDIM"}, {Value: "ddpm", Label: "DDPM"}, diff --git a/core/config/meta/registry.go b/core/config/meta/registry.go index 4fa555d65641..83785e695656 100644 --- a/core/config/meta/registry.go +++ b/core/config/meta/registry.go @@ -66,6 +66,22 @@ func DefaultRegistry() map[string]FieldMetaOverride { Options: UsecaseOptions, Order: 6, }, + "known_input_modalities": { + Section: "general", + Label: "Known Input Modalities", + Description: "Explicit input types this model accepts when use cases alone are not specific enough", + Component: "string-list", + Options: ModalityOptions, + Order: 7, + }, + "known_output_modalities": { + Section: "general", + Label: "Known Output Modalities", + Description: "Explicit output types this model produces when use cases alone are not specific enough", + Component: "string-list", + Options: ModalityOptions, + Order: 8, + }, // --- LLM --- "context_size": { diff --git a/core/config/model_capabilities.go b/core/config/model_capabilities.go index 51b2446754b1..79a117b4aeeb 100644 --- a/core/config/model_capabilities.go +++ b/core/config/model_capabilities.go @@ -1,5 +1,7 @@ package config +import "slices" + // This file is the single source of truth for deriving a model's user-facing // capabilities and input/output modalities from its ModelConfig. Both the // OpenAI-compatible /v1/models/capabilities endpoint and the Ollama-compatible @@ -7,14 +9,47 @@ package config // across clients. Keep the detection heuristics here rather than duplicating // them per endpoint. +// Canonical model modality values used by config declarations and discovery APIs. +const ( + ModalityText = "text" + ModalityImage = "image" + ModalityAudio = "audio" + ModalityVideo = "video" +) + +var modalityOrder = []string{ModalityText, ModalityImage, ModalityAudio, ModalityVideo} + +func declaredModalities(modalities []string) map[string]bool { + declared := make(map[string]bool, len(modalities)) + for _, modality := range modalities { + if slices.Contains(modalityOrder, modality) { + declared[modality] = true + } + } + return declared +} + +func orderedModalities(modalities map[string]bool) []string { + result := make([]string, 0, len(modalityOrder)) + for _, modality := range modalityOrder { + if modalities[modality] { + result = append(result, modality) + } + } + return result +} + // VisionSupported reports whether the model can accept image inputs. // // We deliberately avoid HasUsecases(FLAG_VISION): GuessUsecases has no // FLAG_VISION branch and reports true for any chat model, so it would paint // vision onto text-only models. Instead we look for explicit signals: the -// declared KnownUsecases bit, a multimodal projector, or a template/backend -// multimodal marker. +// declared input modality or KnownUsecases bit, a multimodal projector, or a +// template/backend multimodal marker. func (c *ModelConfig) VisionSupported() bool { + if slices.Contains(c.KnownInputModalities, ModalityImage) { + return true + } if c.KnownUsecases != nil && (*c.KnownUsecases&FLAG_VISION) == FLAG_VISION { return true } @@ -68,20 +103,21 @@ func (c *ModelConfig) ThinkingSupported() bool { } // AudioInputSupported reports whether a chat/generation model accepts audio as -// input (e.g. vLLM omni models). The signal is the vLLM per-prompt audio limit; -// there is no FLAG_* for "chat model that hears audio", which is exactly why a -// plain usecase list can't express it. Transcription models are handled -// separately in InputModalities via FLAG_TRANSCRIPT. +// input. Model configs can declare this directly; vLLM-family configs can also +// signal it through the per-prompt audio limit. Transcription models are +// handled separately in InputModalities via FLAG_TRANSCRIPT. func (c *ModelConfig) AudioInputSupported() bool { - return c.LimitMMPerPrompt.LimitAudioPerPrompt > 0 + return slices.Contains(c.KnownInputModalities, ModalityAudio) || + c.LimitMMPerPrompt.LimitAudioPerPrompt > 0 } // VideoInputSupported reports whether a chat/generation model accepts video as -// input. The signal is the vLLM per-prompt video limit. Note this is distinct -// from FLAG_VIDEO, which denotes video *generation* (diffusers) — an output -// modality, not an input one. +// input. Model configs can declare this directly; vLLM-family configs can also +// signal it through the per-prompt video limit. This is distinct from +// FLAG_VIDEO, which denotes video generation — an output modality. func (c *ModelConfig) VideoInputSupported() bool { - return c.LimitMMPerPrompt.LimitVideoPerPrompt > 0 + return slices.Contains(c.KnownInputModalities, ModalityVideo) || + c.LimitMMPerPrompt.LimitVideoPerPrompt > 0 } // Capabilities returns the ordered list of capability strings the model @@ -135,6 +171,7 @@ func (c *ModelConfig) Capabilities() []string { // attachment router consults to decide whether an image/audio/video file can be // handed to the active model directly. func (c *ModelConfig) InputModalities() []string { + modalities := declaredModalities(c.KnownInputModalities) imageGen := c.HasUsecases(FLAG_IMAGE) videoGen := c.HasUsecases(FLAG_VIDEO) chatish := c.HasUsecases(FLAG_CHAT) || c.HasUsecases(FLAG_COMPLETION) @@ -154,25 +191,17 @@ func (c *ModelConfig) InputModalities() []string { videoIn := c.VideoInputSupported() - var mods []string - if textIn { - mods = append(mods, "text") - } - if imageIn { - mods = append(mods, "image") - } - if audioIn { - mods = append(mods, "audio") - } - if videoIn { - mods = append(mods, "video") - } - return mods + modalities[ModalityText] = modalities[ModalityText] || textIn + modalities[ModalityImage] = modalities[ModalityImage] || imageIn + modalities[ModalityAudio] = modalities[ModalityAudio] || audioIn + modalities[ModalityVideo] = modalities[ModalityVideo] || videoIn + return orderedModalities(modalities) } // OutputModalities returns the set of modalities (text, image, audio, video) // the model produces, ordered text→image→audio→video. func (c *ModelConfig) OutputModalities() []string { + modalities := declaredModalities(c.KnownOutputModalities) textOut := c.HasUsecases(FLAG_CHAT) || c.HasUsecases(FLAG_COMPLETION) || c.HasUsecases(FLAG_EDIT) || c.HasUsecases(FLAG_TRANSCRIPT) imageOut := c.HasUsecases(FLAG_IMAGE) @@ -180,18 +209,9 @@ func (c *ModelConfig) OutputModalities() []string { c.HasUsecases(FLAG_AUDIO_TRANSFORM) || c.HasUsecases(FLAG_REALTIME_AUDIO) videoOut := c.HasUsecases(FLAG_VIDEO) - var mods []string - if textOut { - mods = append(mods, "text") - } - if imageOut { - mods = append(mods, "image") - } - if audioOut { - mods = append(mods, "audio") - } - if videoOut { - mods = append(mods, "video") - } - return mods + modalities[ModalityText] = modalities[ModalityText] || textOut + modalities[ModalityImage] = modalities[ModalityImage] || imageOut + modalities[ModalityAudio] = modalities[ModalityAudio] || audioOut + modalities[ModalityVideo] = modalities[ModalityVideo] || videoOut + return orderedModalities(modalities) } diff --git a/core/config/model_capabilities_test.go b/core/config/model_capabilities_test.go index 8aab180b2a89..545bad50bc81 100644 --- a/core/config/model_capabilities_test.go +++ b/core/config/model_capabilities_test.go @@ -21,6 +21,14 @@ var _ = Describe("Model capabilities derivation", func() { Expect(cfg.VisionSupported()).To(BeTrue()) }) + It("is true when image input is declared explicitly", func() { + cfg := &ModelConfig{ + KnownUsecases: usecaseBits(FLAG_CHAT), + KnownInputModalities: []string{ModalityText, ModalityImage}, + } + Expect(cfg.VisionSupported()).To(BeTrue()) + }) + It("is true when an mmproj projector is set", func() { cfg := &ModelConfig{KnownUsecases: usecaseBits(FLAG_CHAT), Backend: "llama.cpp"} cfg.MMProj = "mmproj.gguf" // promoted field from the embedded options struct @@ -37,6 +45,14 @@ var _ = Describe("Model capabilities derivation", func() { }) Describe("AudioInputSupported / VideoInputSupported", func() { + It("honors explicit model modality declarations", func() { + cfg := &ModelConfig{ + KnownInputModalities: []string{ModalityAudio, ModalityVideo}, + } + Expect(cfg.AudioInputSupported()).To(BeTrue()) + Expect(cfg.VideoInputSupported()).To(BeTrue()) + }) + It("detects vLLM omni audio input via limit_mm_per_prompt", func() { cfg := &ModelConfig{KnownUsecases: usecaseBits(FLAG_CHAT), Backend: "vllm"} cfg.LimitMMPerPrompt.LimitAudioPerPrompt = 1 @@ -93,6 +109,17 @@ var _ = Describe("Model capabilities derivation", func() { Expect(cfg.OutputModalities()).To(Equal([]string{"image"})) }) + It("conditioned video uses declared modalities without backend-specific inference", func() { + cfg := &ModelConfig{ + KnownUsecases: usecaseBits(FLAG_VIDEO), + KnownInputModalities: []string{ModalityAudio, ModalityImage, ModalityText, ModalityAudio, "unknown"}, + KnownOutputModalities: []string{ModalityVideo}, + } + Expect(cfg.Capabilities()).To(Equal([]string{UsecaseVideo})) + Expect(cfg.InputModalities()).To(Equal([]string{ModalityText, ModalityImage, ModalityAudio})) + Expect(cfg.OutputModalities()).To(Equal([]string{ModalityVideo})) + }) + It("a TTS model reads text and writes audio", func() { cfg := &ModelConfig{KnownUsecases: usecaseBits(FLAG_TTS), Backend: "piper"} Expect(cfg.Capabilities()).To(ContainElement(UsecaseTTS)) diff --git a/core/config/model_config.go b/core/config/model_config.go index 0038f4f8d118..5cdcbdd7c07b 100644 --- a/core/config/model_config.go +++ b/core/config/model_config.go @@ -52,7 +52,11 @@ type ModelConfig struct { TemplateConfig TemplateConfig `yaml:"template,omitempty" json:"template,omitempty"` KnownUsecaseStrings []string `yaml:"known_usecases,omitempty" json:"known_usecases,omitempty"` KnownUsecases *ModelConfigUsecase `yaml:"-" json:"-"` - Pipeline Pipeline `yaml:"pipeline,omitempty" json:"pipeline,omitempty"` + // KnownInputModalities and KnownOutputModalities describe model-specific I/O + // that usecases alone cannot express, such as image- or audio-conditioned video. + KnownInputModalities []string `yaml:"known_input_modalities,omitempty" json:"known_input_modalities,omitempty"` + KnownOutputModalities []string `yaml:"known_output_modalities,omitempty" json:"known_output_modalities,omitempty"` + Pipeline Pipeline `yaml:"pipeline,omitempty" json:"pipeline,omitempty"` PromptStrings, InputStrings []string `yaml:"-" json:"-"` InputToken [][]int `yaml:"-" json:"-"` diff --git a/core/gallery/importers/importers.go b/core/gallery/importers/importers.go index 3db8d88f7f9d..3fc40e02cf38 100644 --- a/core/gallery/importers/importers.go +++ b/core/gallery/importers/importers.go @@ -33,7 +33,7 @@ var ErrAmbiguousImport = errors.New("importer: ambiguous — specify preferences // pipeline_tag values. type AmbiguousImportError struct { // Modality is the importer modality key ("text", "asr", "tts", "image", - // "embeddings", "reranker", "detection"). Pre-mapped from the HF + // "video", "embeddings", "reranker", "detection"). Pre-mapped from the HF // pipeline_tag so the UI doesn't have to. Modality string // Candidates is the list of backend names whose Modality() matches — a @@ -144,6 +144,9 @@ var defaultImporters = []Importer{ // Image/Video (Batch 3) &StableDiffusionGGMLImporter{}, &ACEStepImporter{}, + // LongCat repositories carry generic Diffusers metadata, so this exact + // owner/repo matcher must run before DiffuserImporter. + &LongCatVideoImporter{}, // Text LLM (Batch 4) — VLLMOmniImporter must stay ahead of // VLLMImporter so Qwen Omni repos (which also carry tokenizer // files) route to vllm-omni rather than plain vllm. @@ -204,7 +207,7 @@ type Importer interface { // /backends/known to populate the import form dropdown. Name() string // Modality is the backend's primary modality ("text", "asr", "tts", - // "image", "embeddings", "reranker", "detection", "vad"). Used for + // "image", "video", "embeddings", "reranker", "detection", "vad"). Used for // grouping in the UI. Modality() string // AutoDetects is true when Match() can fire without an explicit diff --git a/core/gallery/importers/importers_test.go b/core/gallery/importers/importers_test.go index d29f59182eab..7e34b7b3ed47 100644 --- a/core/gallery/importers/importers_test.go +++ b/core/gallery/importers/importers_test.go @@ -15,6 +15,16 @@ import ( var _ = Describe("DiscoverModelConfig", func() { Context("With only a repository URI", func() { + It("should discover LongCat Avatar before the generic Diffusers importer", func() { + uri := "https://huggingface.co/meituan-longcat/LongCat-Video-Avatar-1.5" + + modelConfig, err := importers.DiscoverModelConfig(uri, json.RawMessage(`{}`)) + + Expect(err).ToNot(HaveOccurred(), fmt.Sprintf("Error: %v", err)) + Expect(modelConfig.ConfigFile).To(ContainSubstring("backend: longcat-video")) + Expect(modelConfig.ConfigFile).To(ContainSubstring("known_usecases:\n - video")) + }) + It("should discover and import using LlamaCPPImporter", func() { uri := "https://huggingface.co/mudler/LocalAI-functioncall-qwen2.5-7b-v0.5-Q4_K_M-GGUF" preferences := json.RawMessage(`{}`) @@ -242,7 +252,7 @@ var _ = Describe("DiscoverModelConfig", func() { for _, imp := range registry { names = append(names, imp.Name()) } - Expect(names).To(ContainElements("llama-cpp", "mlx", "vllm", "transformers", "diffusers")) + Expect(names).To(ContainElements("llama-cpp", "mlx", "vllm", "transformers", "diffusers", "longcat-video")) }) It("LlamaCPPImporter exposes name/modality/autodetect", func() { diff --git a/core/gallery/importers/longcat-video.go b/core/gallery/importers/longcat-video.go new file mode 100644 index 000000000000..f73ae9136ecc --- /dev/null +++ b/core/gallery/importers/longcat-video.go @@ -0,0 +1,127 @@ +package importers + +import ( + "encoding/json" + "path/filepath" + "strings" + + "github.com/mudler/LocalAI/core/config" + "github.com/mudler/LocalAI/core/gallery" + "github.com/mudler/LocalAI/core/schema" + "gopkg.in/yaml.v3" +) + +const ( + longCatOwner = "meituan-longcat" + longCatBaseRepo = "LongCat-Video" + longCatAvatarRepo = "LongCat-Video-Avatar-1.5" +) + +var _ Importer = &LongCatVideoImporter{} + +// LongCatVideoImporter is deliberately owner/repository-specific. LongCat +// checkpoints also contain generic Diffusers metadata, so broad file-based +// matching would let this backend claim unrelated video pipelines. +type LongCatVideoImporter struct{} + +func (i *LongCatVideoImporter) Name() string { return "longcat-video" } +func (i *LongCatVideoImporter) Modality() string { return "video" } +func (i *LongCatVideoImporter) AutoDetects() bool { return true } + +func isLongCatRepo(owner, repo string) bool { + repo = strings.Split(strings.Trim(repo, "/"), "/")[0] + return strings.EqualFold(owner, longCatOwner) && + (strings.EqualFold(repo, longCatBaseRepo) || strings.EqualFold(repo, longCatAvatarRepo)) +} + +func longCatModelID(details Details) (string, bool) { + if details.HuggingFace != nil && details.HuggingFace.ModelID != "" { + parts := strings.SplitN(details.HuggingFace.ModelID, "/", 2) + if len(parts) == 2 && isLongCatRepo(parts[0], parts[1]) { + repo := strings.Split(strings.Trim(parts[1], "/"), "/")[0] + return parts[0] + "/" + repo, true + } + } + if owner, repo, ok := HFOwnerRepoFromURI(details.URI); ok && isLongCatRepo(owner, repo) { + repo = strings.Split(strings.Trim(repo, "/"), "/")[0] + return owner + "/" + repo, true + } + return LocalModelPath(details.URI), false +} + +func (i *LongCatVideoImporter) Match(details Details) bool { + preferences, err := details.Preferences.MarshalJSON() + if err != nil { + return false + } + preferencesMap := make(map[string]any) + if len(preferences) > 0 { + if err := json.Unmarshal(preferences, &preferencesMap); err != nil { + return false + } + } + if backend, ok := preferencesMap["backend"].(string); ok { + return backend == i.Name() + } + + _, matched := longCatModelID(details) + return matched +} + +func (i *LongCatVideoImporter) Import(details Details) (gallery.ModelConfig, error) { + preferences, err := details.Preferences.MarshalJSON() + if err != nil { + return gallery.ModelConfig{}, err + } + preferencesMap := make(map[string]any) + if len(preferences) > 0 { + if err := json.Unmarshal(preferences, &preferencesMap); err != nil { + return gallery.ModelConfig{}, err + } + } + + model, canonical := longCatModelID(details) + name, _ := preferencesMap["name"].(string) + if name == "" { + if canonical { + name = strings.ToLower(filepath.Base(model)) + } else { + name = filepath.Base(strings.TrimSuffix(model, "/")) + } + } + + description, _ := preferencesMap["description"].(string) + if description == "" { + description = "Imported from " + details.URI + } + + options := []string{"attention_backend:sdpa"} + inputModalities := []string{config.ModalityText, config.ModalityImage} + if strings.EqualFold(filepath.Base(model), longCatAvatarRepo) { + options = append(options, "use_distill:true") + inputModalities = append(inputModalities, config.ModalityAudio) + } + + modelConfig := config.ModelConfig{ + Name: name, + Description: description, + Backend: i.Name(), + KnownUsecaseStrings: []string{config.UsecaseVideo}, + KnownInputModalities: inputModalities, + KnownOutputModalities: []string{config.ModalityVideo}, + Options: options, + PredictionOptions: schema.PredictionOptions{ + BasicModelRequest: schema.BasicModelRequest{Model: model}, + }, + } + + data, err := yaml.Marshal(modelConfig) + if err != nil { + return gallery.ModelConfig{}, err + } + return gallery.ModelConfig{ + Name: name, + Description: description, + ConfigFile: string(data), + }, nil +} diff --git a/core/gallery/importers/longcat-video_test.go b/core/gallery/importers/longcat-video_test.go new file mode 100644 index 000000000000..6abced791c88 --- /dev/null +++ b/core/gallery/importers/longcat-video_test.go @@ -0,0 +1,116 @@ +package importers_test + +import ( + "encoding/json" + + "github.com/mudler/LocalAI/core/config" + "github.com/mudler/LocalAI/core/gallery/importers" + hfapi "github.com/mudler/LocalAI/pkg/huggingface-api" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "gopkg.in/yaml.v3" +) + +func decodeLongCatModelConfig(data string) config.ModelConfig { + GinkgoHelper() + modelConfig := config.ModelConfig{} + Expect(yaml.Unmarshal([]byte(data), &modelConfig)).To(Succeed()) + return modelConfig +} + +var _ = Describe("LongCatVideoImporter", func() { + var importer *importers.LongCatVideoImporter + + BeforeEach(func() { + importer = &importers.LongCatVideoImporter{} + }) + + It("exposes video importer metadata", func() { + Expect(importer.Name()).To(Equal("longcat-video")) + Expect(importer.Modality()).To(Equal("video")) + Expect(importer.AutoDetects()).To(BeTrue()) + }) + + Describe("Match", func() { + It("matches both official repositories", func() { + for _, modelID := range []string{ + "meituan-longcat/LongCat-Video", + "meituan-longcat/LongCat-Video-Avatar-1.5", + } { + details := importers.Details{ + URI: "https://huggingface.co/" + modelID, + HuggingFace: &hfapi.ModelDetails{ + ModelID: modelID, + Author: "meituan-longcat", + }, + } + Expect(importer.Match(details)).To(BeTrue(), modelID) + } + }) + + It("matches official hf URI forms without metadata", func() { + Expect(importer.Match(importers.Details{ + URI: "https://huggingface.co/meituan-longcat/LongCat-Video-Avatar-1.5/tree/main/", + })).To(BeTrue()) + }) + + It("does not claim the same repository name under another owner", func() { + Expect(importer.Match(importers.Details{ + URI: "https://huggingface.co/other-org/LongCat-Video", + })).To(BeFalse()) + }) + + It("honors an explicit backend preference", func() { + Expect(importer.Match(importers.Details{ + URI: "/models/LongCat-Video", + Preferences: json.RawMessage(`{"backend":"longcat-video"}`), + })).To(BeTrue()) + Expect(importer.Match(importers.Details{ + URI: "hf://meituan-longcat/LongCat-Video", + Preferences: json.RawMessage(`{"backend":"diffusers"}`), + })).To(BeFalse()) + }) + }) + + Describe("Import", func() { + It("emits a base-model video configuration", func() { + modelConfig, err := importer.Import(importers.Details{ + URI: "https://huggingface.co/meituan-longcat/LongCat-Video", + }) + + Expect(err).NotTo(HaveOccurred()) + Expect(modelConfig.Name).To(Equal("longcat-video")) + Expect(modelConfig.ConfigFile).To(ContainSubstring("backend: longcat-video")) + Expect(modelConfig.ConfigFile).To(ContainSubstring("model: meituan-longcat/LongCat-Video")) + Expect(modelConfig.ConfigFile).To(ContainSubstring("- video")) + Expect(modelConfig.ConfigFile).To(ContainSubstring("attention_backend:sdpa")) + Expect(modelConfig.ConfigFile).NotTo(ContainSubstring("use_distill:true")) + + cfg := decodeLongCatModelConfig(modelConfig.ConfigFile) + Expect(cfg.KnownInputModalities).To(Equal([]string{ + config.ModalityText, + config.ModalityImage, + })) + Expect(cfg.KnownOutputModalities).To(Equal([]string{config.ModalityVideo})) + }) + + It("enables the distilled path for Avatar 1.5", func() { + modelConfig, err := importer.Import(importers.Details{ + URI: "hf://meituan-longcat/LongCat-Video-Avatar-1.5", + }) + + Expect(err).NotTo(HaveOccurred()) + Expect(modelConfig.Name).To(Equal("longcat-video-avatar-1.5")) + Expect(modelConfig.ConfigFile).To(ContainSubstring("model: meituan-longcat/LongCat-Video-Avatar-1.5")) + Expect(modelConfig.ConfigFile).To(ContainSubstring("use_distill:true")) + + cfg := decodeLongCatModelConfig(modelConfig.ConfigFile) + Expect(cfg.KnownInputModalities).To(Equal([]string{ + config.ModalityText, + config.ModalityImage, + config.ModalityAudio, + })) + Expect(cfg.KnownOutputModalities).To(Equal([]string{config.ModalityVideo})) + }) + }) +}) diff --git a/core/http/app_test.go b/core/http/app_test.go index 5917b034ae3c..fa6423292e7b 100644 --- a/core/http/app_test.go +++ b/core/http/app_test.go @@ -983,7 +983,22 @@ chat_template_kwargs: req.Header.Set("Authorization", bearerKey) resp, err = http.DefaultClient.Do(req) Expect(err).ToNot(HaveOccurred()) - Expect(resp.StatusCode).To(Equal(200)) + if resp.StatusCode == http.StatusBadRequest { + // The worker can finish between the status read and cancellation request. + resp, err = http.Get("http://127.0.0.1:9090/api/agent/jobs/" + jobID) + Expect(err).ToNot(HaveOccurred()) + Expect(resp.StatusCode).To(Equal(http.StatusOK)) + body, _ = io.ReadAll(resp.Body) + err = json.Unmarshal(body, &job) + Expect(err).ToNot(HaveOccurred()) + Expect(job.Status).To(Or( + Equal(schema.JobStatusCompleted), + Equal(schema.JobStatusFailed), + Equal(schema.JobStatusCancelled), + )) + } else { + Expect(resp.StatusCode).To(Equal(http.StatusOK)) + } } }) diff --git a/core/http/endpoints/localai/api_instructions.go b/core/http/endpoints/localai/api_instructions.go index 405921e5e589..13994d285f52 100644 --- a/core/http/endpoints/localai/api_instructions.go +++ b/core/http/endpoints/localai/api_instructions.go @@ -71,8 +71,9 @@ var instructionDefs = []instructionDef{ }, { Name: "video", - Description: "Video generation from text prompts", + Description: "Video generation from text prompts with optional image or audio conditioning", Tags: []string{"video"}, + Intro: "POST /video accepts start_image, end_image, and audio as public URL, base64, or data URI. Backend-specific tuning is passed as string values in params.", }, { Name: "face-recognition", diff --git a/core/http/endpoints/localai/video.go b/core/http/endpoints/localai/video.go index 65c140dc222f..9e6a33e10e8e 100644 --- a/core/http/endpoints/localai/video.go +++ b/core/http/endpoints/localai/video.go @@ -1,11 +1,12 @@ package localai import ( - "bufio" + "context" "encoding/base64" "encoding/json" "fmt" "io" + "net/http" "net/url" "os" "path/filepath" @@ -28,32 +29,105 @@ import ( "github.com/mudler/LocalAI/pkg/utils" ) -// Downloading user-supplied media URLs legitimately follows redirects (CDNs); -// WithFollowRedirects still strips any credential header on a cross-host hop. -var videoDownloadClient = httpclient.NewWithTimeout(30*time.Second, httpclient.WithFollowRedirects()) +const maxVideoInputBytes = 128 << 20 + +func newVideoDownloadClient() *http.Client { + client := httpclient.NewWithTimeout(30*time.Second, httpclient.WithFollowRedirects()) + checkRedirect := client.CheckRedirect + // Media CDNs commonly redirect, so validate every hop rather than trusting + // only the URL supplied by the caller. Keep the shared redirect policy too; + // it bounds the chain and strips credentials on cross-origin hops. + client.CheckRedirect = func(req *http.Request, via []*http.Request) error { + if err := utils.ValidateExternalURL(req.URL.String()); err != nil { + return fmt.Errorf("redirect URL validation failed: %w", err) + } + return checkRedirect(req, via) + } + return client +} + +var videoDownloadClient = newVideoDownloadClient() + +func openVideoMedia(ctx context.Context, ref string) (io.ReadCloser, int64, error) { + if strings.HasPrefix(ref, "http://") || strings.HasPrefix(ref, "https://") { + if err := utils.ValidateExternalURL(ref); err != nil { + return nil, 0, fmt.Errorf("URL validation failed: %w", err) + } + + req, err := http.NewRequestWithContext(ctx, http.MethodGet, ref, nil) + if err != nil { + return nil, 0, fmt.Errorf("creating download request: %w", err) + } + resp, err := videoDownloadClient.Do(req) + if err != nil { + return nil, 0, fmt.Errorf("downloading media: %w", err) + } + if resp.StatusCode < http.StatusOK || resp.StatusCode >= http.StatusMultipleChoices { + _ = resp.Body.Close() + return nil, 0, fmt.Errorf("media URL returned HTTP %d", resp.StatusCode) + } + return resp.Body, resp.ContentLength, nil + } -func downloadFile(url string) (string, error) { - if err := utils.ValidateExternalURL(url); err != nil { - return "", fmt.Errorf("URL validation failed: %w", err) + encoded := ref + if strings.HasPrefix(ref, "data:") { + comma := strings.IndexByte(ref, ',') + if comma < 0 || !strings.Contains(strings.ToLower(ref[:comma]), ";base64") { + return nil, 0, fmt.Errorf("data URI must contain a base64 payload") + } + encoded = ref[comma+1:] } + if encoded == "" { + return nil, 0, fmt.Errorf("media payload is empty") + } + + decodedSize := int64(base64.StdEncoding.DecodedLen(len(encoded))) + return io.NopCloser(base64.NewDecoder(base64.StdEncoding, strings.NewReader(encoded))), decodedSize, nil +} - // Get the data - resp, err := videoDownloadClient.Get(url) +func stageVideoMedia(ctx context.Context, directory, ref string) (string, error) { + return stageVideoMediaWithLimit(ctx, directory, ref, maxVideoInputBytes) +} + +func stageVideoMediaWithLimit(ctx context.Context, directory, ref string, maxBytes int64) (string, error) { + if ref == "" { + return "", nil + } + + source, declaredSize, err := openVideoMedia(ctx, ref) if err != nil { return "", err } - defer resp.Body.Close() + defer func() { _ = source.Close() }() + if declaredSize > maxBytes { + return "", fmt.Errorf("media exceeds the %d-byte limit", maxBytes) + } - // Create the file - out, err := os.CreateTemp("", "video") + output, err := os.CreateTemp(directory, "video-input-*") if err != nil { - return "", err + return "", fmt.Errorf("creating staged media file: %w", err) } - defer out.Close() + outputPath := output.Name() + keep := false + defer func() { + _ = output.Close() + if !keep { + _ = os.Remove(outputPath) + } + }() - // Write the body to file - _, err = io.Copy(out, resp.Body) - return out.Name(), err + written, err := io.Copy(output, io.LimitReader(source, maxBytes+1)) + if err != nil { + return "", fmt.Errorf("decoding media: %w", err) + } + if written > maxBytes { + return "", fmt.Errorf("media exceeds the %d-byte limit", maxBytes) + } + if err := output.Close(); err != nil { + return "", fmt.Errorf("closing staged media file: %w", err) + } + keep = true + return outputPath, nil } // @@ -72,7 +146,7 @@ func downloadFile(url string) (string, error) { * */ // VideoEndpoint -// @Summary Creates a video given a prompt. +// @Summary Creates a video from a prompt and optional image or audio conditioning. // @Tags video // @Param request body schema.VideoRequest true "query params" // @Success 200 {object} schema.OpenAIResponse "Response" @@ -91,63 +165,39 @@ func VideoEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, appConfi return echo.ErrBadRequest } - // Stage a base64- or URL-provided image into a temp file so the - // backend can read it as a path. Used for both start_image and - // (optional) end_image. Returns the temp file path, or "" if the - // input is empty. Caller is responsible for the defer-cleanup. - stageImage := func(ref string) (string, error) { - if ref == "" { - return "", nil - } - var fileData []byte - var err error - if strings.HasPrefix(ref, "http://") || strings.HasPrefix(ref, "https://") { - out, derr := downloadFile(ref) - if derr != nil { - return "", fmt.Errorf("failed downloading file: %w", derr) - } - defer os.RemoveAll(out) - fileData, err = os.ReadFile(out) - if err != nil { - return "", fmt.Errorf("failed reading file: %w", err) - } - } else { - fileData, err = base64.StdEncoding.DecodeString(ref) - if err != nil { - return "", err - } - } - outputFile, err := os.CreateTemp(appConfig.GeneratedContentDir, "b64") + stageInput := func(name, ref string) (string, error) { + path, err := stageVideoMedia(c.Request().Context(), appConfig.GeneratedContentDir, ref) if err != nil { - return "", err - } - writer := bufio.NewWriter(outputFile) - if _, err := writer.Write(fileData); err != nil { - outputFile.Close() - return "", err + return "", echo.NewHTTPError( + http.StatusBadRequest, + fmt.Sprintf("invalid %s: %v", name, err), + ) } - if err := writer.Flush(); err != nil { - outputFile.Close() - return "", err - } - outputFile.Close() - return outputFile.Name(), nil + return path, nil } - src, err := stageImage(input.StartImage) + src, err := stageInput("start_image", input.StartImage) if err != nil { return err } if src != "" { - defer os.RemoveAll(src) + defer func() { _ = os.Remove(src) }() } - endSrc, err := stageImage(input.EndImage) + endSrc, err := stageInput("end_image", input.EndImage) if err != nil { return err } if endSrc != "" { - defer os.RemoveAll(endSrc) + defer func() { _ = os.Remove(endSrc) }() + } + + audioSrc, err := stageInput("audio", input.Audio) + if err != nil { + return err + } + if audioSrc != "" { + defer func() { _ = os.Remove(audioSrc) }() } xlog.Debug("Parameter Config", "config", config) @@ -174,13 +224,19 @@ func VideoEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, appConfi tempDir := "" if !b64JSON { tempDir = filepath.Join(appConfig.GeneratedContentDir, "videos") + if err := os.MkdirAll(tempDir, 0o750); err != nil { + return err + } } // Create a temporary file outputFile, err := os.CreateTemp(tempDir, "b64") if err != nil { return err } - outputFile.Close() + if err := outputFile.Close(); err != nil { + _ = os.Remove(outputFile.Name()) + return err + } // TODO: use mime type to determine the extension output := outputFile.Name() + ".mp4" @@ -188,8 +244,15 @@ func VideoEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, appConfi // Rename the temporary file err = os.Rename(outputFile.Name(), output) if err != nil { + _ = os.Remove(outputFile.Name()) return err } + preserveOutput := false + defer func() { + if !preserveOutput { + _ = os.Remove(output) + } + }() baseURL := middleware.BaseURL(c) @@ -204,33 +267,36 @@ func VideoEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, appConfi "negative_prompt", input.NegativePrompt) fn, err := backend.VideoGeneration( - height, - width, - input.Prompt, - input.NegativePrompt, - src, - endSrc, - output, - input.NumFrames, - input.FPS, - input.Seed, - input.CFGScale, - input.Step, + backend.VideoGenerationOptions{ + Height: height, + Width: width, + Prompt: input.Prompt, + NegativePrompt: input.NegativePrompt, + StartImage: src, + EndImage: endSrc, + Audio: audioSrc, + Destination: output, + NumFrames: input.NumFrames, + FPS: input.FPS, + Seed: input.Seed, + CFGScale: input.CFGScale, + Step: input.Step, + Params: input.Params, + }, ml, *config, appConfig, ) if err != nil { - return err + return mapBackendError(err) } if err := fn(); err != nil { - return err + return mapBackendError(err) } item := &schema.Item{} if b64JSON { - defer os.RemoveAll(output) data, err := os.ReadFile(output) if err != nil { return err @@ -242,6 +308,7 @@ func VideoEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, appConfi if err != nil { return err } + preserveOutput = true } id := uuid.New().String() diff --git a/core/http/endpoints/localai/video_internal_test.go b/core/http/endpoints/localai/video_internal_test.go new file mode 100644 index 000000000000..383ae5fe4115 --- /dev/null +++ b/core/http/endpoints/localai/video_internal_test.go @@ -0,0 +1,68 @@ +package localai + +import ( + "context" + "encoding/base64" + "os" + "path/filepath" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("video media staging", func() { + It("stages raw base64 into a private temporary file", func() { + directory := GinkgoT().TempDir() + content := []byte("avatar audio") + + path, err := stageVideoMedia(context.Background(), directory, base64.StdEncoding.EncodeToString(content)) + + Expect(err).NotTo(HaveOccurred()) + DeferCleanup(os.Remove, path) + Expect(filepath.Dir(path)).To(Equal(directory)) + Expect(os.ReadFile(path)).To(Equal(content)) + info, err := os.Stat(path) + Expect(err).NotTo(HaveOccurred()) + Expect(info.Mode().Perm()).To(Equal(os.FileMode(0o600))) + }) + + It("accepts browser data URIs with codec parameters", func() { + content := []byte("recorded speech") + ref := "data:audio/webm;codecs=opus;base64," + base64.StdEncoding.EncodeToString(content) + + path, err := stageVideoMedia(context.Background(), GinkgoT().TempDir(), ref) + + Expect(err).NotTo(HaveOccurred()) + DeferCleanup(os.Remove, path) + Expect(os.ReadFile(path)).To(Equal(content)) + }) + + It("rejects malformed base64 and removes the partial file", func() { + directory := GinkgoT().TempDir() + + _, err := stageVideoMedia(context.Background(), directory, "not%%%base64") + + Expect(err).To(MatchError(ContainSubstring("decoding media"))) + entries, readErr := os.ReadDir(directory) + Expect(readErr).NotTo(HaveOccurred()) + Expect(entries).To(BeEmpty()) + }) + + It("enforces the configured streaming limit", func() { + directory := GinkgoT().TempDir() + encoded := base64.StdEncoding.EncodeToString([]byte("four")) + + _, err := stageVideoMediaWithLimit(context.Background(), directory, encoded, 3) + + Expect(err).To(MatchError(ContainSubstring("3-byte limit"))) + entries, readErr := os.ReadDir(directory) + Expect(readErr).NotTo(HaveOccurred()) + Expect(entries).To(BeEmpty()) + }) + + It("rejects non-base64 data URIs", func() { + _, err := stageVideoMedia(context.Background(), GinkgoT().TempDir(), "data:audio/wav,plain") + + Expect(err).To(MatchError(ContainSubstring("base64 payload"))) + }) +}) diff --git a/core/http/react-ui/e2e/import-form-ux-batch-e.spec.js b/core/http/react-ui/e2e/import-form-ux-batch-e.spec.js index 5532123a74dc..49881d367af9 100644 --- a/core/http/react-ui/e2e/import-form-ux-batch-e.spec.js +++ b/core/http/react-ui/e2e/import-form-ux-batch-e.spec.js @@ -23,6 +23,7 @@ const MOCK_BACKENDS = [ { name: 'kokoro', modality: 'tts', auto_detect: true, installed: true }, { name: 'whisper', modality: 'asr', auto_detect: true, installed: true }, { name: 'diffusers', modality: 'image', auto_detect: true, installed: false }, + { name: 'longcat-video', modality: 'video', auto_detect: true, installed: false }, { name: 'sentencetransformers', modality: 'embeddings', auto_detect: true, installed: true }, { name: 'rerankers', modality: 'reranker', auto_detect: true, installed: true }, { name: 'rfdetr', modality: 'detection', auto_detect: true, installed: true }, @@ -78,7 +79,7 @@ test.describe('Import form UX — Batch E (modality chip row)', () => { await page.locator('[data-testid="simple-options-toggle"]').click() await expect(chips(page)).toBeVisible() // Full set of chips renders. - for (const key of ['', 'text', 'asr', 'tts', 'image', 'embeddings', 'reranker', 'detection', 'vad']) { + for (const key of ['', 'text', 'asr', 'tts', 'image', 'video', 'embeddings', 'reranker', 'detection', 'vad']) { await expect(chip(page, key)).toBeVisible() } // "Any" is active by default. diff --git a/core/http/react-ui/e2e/media-history.spec.js b/core/http/react-ui/e2e/media-history.spec.js index 754393973490..7575388f750c 100644 --- a/core/http/react-ui/e2e/media-history.spec.js +++ b/core/http/react-ui/e2e/media-history.spec.js @@ -21,9 +21,10 @@ function mockImageGeneration(page, images) { }) } -function mockVideoGeneration(page, videos) { +function mockVideoGeneration(page, videos, onRequest) { return page.route('**/video', (route) => { if (route.request().method() !== 'POST') return route.continue() + onRequest?.(route.request().postDataJSON()) route.fulfill({ contentType: 'application/json', body: JSON.stringify({ @@ -254,4 +255,22 @@ test.describe('Media History - Video Generation', () => { await expect(page.getByTestId('media-history-item')).toHaveCount(1, { timeout: 10_000 }) await expect(page.getByTestId('media-history-item')).toContainText('a running cat') }) + + test('Avatar audio is forwarded to the video API', async ({ page }) => { + let requestBody + await mockVideoGeneration(page, [{ url: '/generated-videos/avatar.mp4' }], (body) => { requestBody = body }) + + await page.goto('/app/video') + await expect(page.getByRole('button', { name: 'test-video-model' })).toBeVisible({ timeout: 10_000 }) + await page.getByRole('button', { name: 'Reference media' }).click() + await page.getByLabel('Avatar audio', { exact: true }).setInputFiles({ + name: 'speech.wav', + mimeType: 'audio/wav', + buffer: Buffer.from('avatar speech'), + }) + await page.locator('.textarea').first().fill('a presenter speaking to camera') + await page.locator('button[type="submit"]').click() + + await expect.poll(() => requestBody?.audio).toBe(Buffer.from('avatar speech').toString('base64')) + }) }) diff --git a/core/http/react-ui/public/locales/de/importModel.json b/core/http/react-ui/public/locales/de/importModel.json index 67623b955e26..373cc309b3f4 100644 --- a/core/http/react-ui/public/locales/de/importModel.json +++ b/core/http/react-ui/public/locales/de/importModel.json @@ -72,6 +72,7 @@ "asr": "Spracherkennung", "tts": "Sprachsynthese", "image": "Bild / Video", + "video": "Videogenerierung", "embeddings": "Embeddings", "reranker": "Reranker", "detection": "Objekterkennung", diff --git a/core/http/react-ui/public/locales/de/media.json b/core/http/react-ui/public/locales/de/media.json index deb4d76314fe..096c74b365a3 100644 --- a/core/http/react-ui/public/locales/de/media.json +++ b/core/http/react-ui/public/locales/de/media.json @@ -52,7 +52,12 @@ "size": "Größe", "advanced": "Erweiterte Einstellungen", "seed": "Seed", - "seedPlaceholder": "Zufällig" + "seedPlaceholder": "Zufällig", + "frames": "Frames", + "referenceMedia": "Referenzmedien", + "startImage": "Startbild", + "endImage": "Endbild", + "avatarAudio": "Avatar-Audio" }, "actions": { "generate": "Video generieren", diff --git a/core/http/react-ui/public/locales/en/importModel.json b/core/http/react-ui/public/locales/en/importModel.json index 0a67d1613fba..ad040f3c2121 100644 --- a/core/http/react-ui/public/locales/en/importModel.json +++ b/core/http/react-ui/public/locales/en/importModel.json @@ -72,6 +72,7 @@ "asr": "Speech recognition", "tts": "Text-to-speech", "image": "Image / Video", + "video": "Video generation", "embeddings": "Embeddings", "reranker": "Rerankers", "detection": "Object detection", diff --git a/core/http/react-ui/public/locales/en/media.json b/core/http/react-ui/public/locales/en/media.json index 4cff8ffa4161..ae644a3fa106 100644 --- a/core/http/react-ui/public/locales/en/media.json +++ b/core/http/react-ui/public/locales/en/media.json @@ -52,7 +52,12 @@ "size": "Size", "advanced": "Advanced Settings", "seed": "Seed", - "seedPlaceholder": "Random" + "seedPlaceholder": "Random", + "frames": "Frames", + "referenceMedia": "Reference media", + "startImage": "Start image", + "endImage": "End image", + "avatarAudio": "Avatar audio" }, "actions": { "generate": "Generate", diff --git a/core/http/react-ui/public/locales/es/importModel.json b/core/http/react-ui/public/locales/es/importModel.json index 96e447a647d7..aa29731fbc49 100644 --- a/core/http/react-ui/public/locales/es/importModel.json +++ b/core/http/react-ui/public/locales/es/importModel.json @@ -72,6 +72,7 @@ "asr": "Reconocimiento de voz", "tts": "Texto a voz", "image": "Imagen / Video", + "video": "Generación de video", "embeddings": "Embeddings", "reranker": "Rerankers", "detection": "Detección de objetos", diff --git a/core/http/react-ui/public/locales/es/media.json b/core/http/react-ui/public/locales/es/media.json index 768b851a5274..1a36a3a24177 100644 --- a/core/http/react-ui/public/locales/es/media.json +++ b/core/http/react-ui/public/locales/es/media.json @@ -52,7 +52,12 @@ "size": "Tamaño", "advanced": "Configuración avanzada", "seed": "Seed", - "seedPlaceholder": "Aleatorio" + "seedPlaceholder": "Aleatorio", + "frames": "Fotogramas", + "referenceMedia": "Medios de referencia", + "startImage": "Imagen inicial", + "endImage": "Imagen final", + "avatarAudio": "Audio del avatar" }, "actions": { "generate": "Generar video", diff --git a/core/http/react-ui/public/locales/id/importModel.json b/core/http/react-ui/public/locales/id/importModel.json index a23333873bb7..bad709a868f4 100644 --- a/core/http/react-ui/public/locales/id/importModel.json +++ b/core/http/react-ui/public/locales/id/importModel.json @@ -72,6 +72,7 @@ "asr": "Pengenalan suara", "tts": "Text-to-speech", "image": "Gambar / Video", + "video": "Pembuatan video", "embeddings": "Embedding", "reranker": "Reranker", "detection": "Deteksi object", diff --git a/core/http/react-ui/public/locales/id/media.json b/core/http/react-ui/public/locales/id/media.json index 10350967b83a..50bbfa65b13b 100644 --- a/core/http/react-ui/public/locales/id/media.json +++ b/core/http/react-ui/public/locales/id/media.json @@ -52,7 +52,12 @@ "size": "Ukuran", "advanced": "Pengaturan Tingkat Lanjutan", "seed": "Seed", - "seedPlaceholder": "Acak" + "seedPlaceholder": "Acak", + "frames": "Frame", + "referenceMedia": "Media referensi", + "startImage": "Gambar awal", + "endImage": "Gambar akhir", + "avatarAudio": "Audio avatar" }, "actions": { "generate": "Hasilkan", diff --git a/core/http/react-ui/public/locales/it/importModel.json b/core/http/react-ui/public/locales/it/importModel.json index 925f7d0ad719..2d5b1391cf67 100644 --- a/core/http/react-ui/public/locales/it/importModel.json +++ b/core/http/react-ui/public/locales/it/importModel.json @@ -72,6 +72,7 @@ "asr": "Riconoscimento vocale", "tts": "Sintesi vocale", "image": "Immagine / Video", + "video": "Generazione video", "embeddings": "Embedding", "reranker": "Reranker", "detection": "Rilevamento oggetti", diff --git a/core/http/react-ui/public/locales/it/media.json b/core/http/react-ui/public/locales/it/media.json index 06adcf8ee785..b41629a1c000 100644 --- a/core/http/react-ui/public/locales/it/media.json +++ b/core/http/react-ui/public/locales/it/media.json @@ -52,7 +52,12 @@ "size": "Dimensione", "advanced": "Impostazioni avanzate", "seed": "Seed", - "seedPlaceholder": "Casuale" + "seedPlaceholder": "Casuale", + "frames": "Fotogrammi", + "referenceMedia": "Media di riferimento", + "startImage": "Immagine iniziale", + "endImage": "Immagine finale", + "avatarAudio": "Audio avatar" }, "actions": { "generate": "Genera video", diff --git a/core/http/react-ui/public/locales/ko/importModel.json b/core/http/react-ui/public/locales/ko/importModel.json index 5be2624e4257..8ad924e3db01 100644 --- a/core/http/react-ui/public/locales/ko/importModel.json +++ b/core/http/react-ui/public/locales/ko/importModel.json @@ -72,6 +72,7 @@ "asr": "음성 인식", "tts": "텍스트 음성 변환", "image": "이미지 / 비디오", + "video": "비디오 생성", "embeddings": "임베딩", "reranker": "리랭커", "detection": "객체 감지", diff --git a/core/http/react-ui/public/locales/ko/media.json b/core/http/react-ui/public/locales/ko/media.json index d17ef41f2979..a18abc0eb04c 100644 --- a/core/http/react-ui/public/locales/ko/media.json +++ b/core/http/react-ui/public/locales/ko/media.json @@ -52,7 +52,12 @@ "size": "크기", "advanced": "고급 설정", "seed": "시드", - "seedPlaceholder": "랜덤" + "seedPlaceholder": "랜덤", + "frames": "프레임", + "referenceMedia": "참조 미디어", + "startImage": "시작 이미지", + "endImage": "종료 이미지", + "avatarAudio": "아바타 오디오" }, "actions": { "generate": "생성", diff --git a/core/http/react-ui/public/locales/zh-CN/importModel.json b/core/http/react-ui/public/locales/zh-CN/importModel.json index 2255a071107f..f0800d08db78 100644 --- a/core/http/react-ui/public/locales/zh-CN/importModel.json +++ b/core/http/react-ui/public/locales/zh-CN/importModel.json @@ -72,6 +72,7 @@ "asr": "语音识别", "tts": "文字转语音", "image": "图像 / 视频", + "video": "视频生成", "embeddings": "嵌入", "reranker": "重排器", "detection": "对象检测", diff --git a/core/http/react-ui/public/locales/zh-CN/media.json b/core/http/react-ui/public/locales/zh-CN/media.json index 8feb8905765a..f395cc193b49 100644 --- a/core/http/react-ui/public/locales/zh-CN/media.json +++ b/core/http/react-ui/public/locales/zh-CN/media.json @@ -52,7 +52,12 @@ "size": "尺寸", "advanced": "高级设置", "seed": "随机种子", - "seedPlaceholder": "随机" + "seedPlaceholder": "随机", + "frames": "帧数", + "referenceMedia": "参考媒体", + "startImage": "起始图像", + "endImage": "结束图像", + "avatarAudio": "虚拟形象音频" }, "actions": { "generate": "生成视频", diff --git a/core/http/react-ui/src/App.css b/core/http/react-ui/src/App.css index d14aa8f08e7d..82c491f9c15e 100644 --- a/core/http/react-ui/src/App.css +++ b/core/http/react-ui/src/App.css @@ -2605,6 +2605,24 @@ select.input { user-select: none; } +button.collapsible-header { + width: 100%; + border: 0; + background: transparent; + font-family: inherit; + text-align: left; +} + +button.collapsible-header:hover { + color: var(--color-text-primary); +} + +button.collapsible-header:focus-visible { + border-radius: var(--radius-sm); + outline: 2px solid var(--color-primary); + outline-offset: 2px; +} + .collapsible-header i { transition: transform var(--duration-fast); } diff --git a/core/http/react-ui/src/components/ModalityChips.jsx b/core/http/react-ui/src/components/ModalityChips.jsx index cb6a27b5cb3d..52bfc48cc677 100644 --- a/core/http/react-ui/src/components/ModalityChips.jsx +++ b/core/http/react-ui/src/components/ModalityChips.jsx @@ -16,6 +16,7 @@ const CHIPS = [ { key: 'asr', label: 'Speech' }, { key: 'tts', label: 'TTS' }, { key: 'image', label: 'Image' }, + { key: 'video', label: 'Video' }, { key: 'embeddings', label: 'Embeddings' }, { key: 'reranker', label: 'Rerankers' }, { key: 'detection', label: 'Detection' }, diff --git a/core/http/react-ui/src/pages/ImportModel.jsx b/core/http/react-ui/src/pages/ImportModel.jsx index 6ba47ffe8273..5adc823032b0 100644 --- a/core/http/react-ui/src/pages/ImportModel.jsx +++ b/core/http/react-ui/src/pages/ImportModel.jsx @@ -16,7 +16,7 @@ const BACKENDS_FALLBACK_EMPTY = [] // Modality keys used as i18n keys under "modality.*" namespace; resolved // at render time inside `buildBackendOptions`. -const MODALITY_KEYS = ['text', 'asr', 'tts', 'image', 'embeddings', 'reranker', 'detection', 'vad'] +const MODALITY_KEYS = ['text', 'asr', 'tts', 'image', 'video', 'embeddings', 'reranker', 'detection', 'vad'] // buildBackendOptions groups known backends by modality and tags // auto_detect=false entries with a muted "manual pick" badge so users diff --git a/core/http/react-ui/src/pages/VideoGen.jsx b/core/http/react-ui/src/pages/VideoGen.jsx index fbe43209ea9a..d676b0fdc977 100644 --- a/core/http/react-ui/src/pages/VideoGen.jsx +++ b/core/http/react-ui/src/pages/VideoGen.jsx @@ -8,10 +8,11 @@ import LoadingSpinner from '../components/LoadingSpinner' import GenerationProgress from '../components/GenerationProgress' import ErrorWithTraceLink from '../components/ErrorWithTraceLink' import MediaHistory from '../components/MediaHistory' +import MediaInput from '../components/biometrics/MediaInput' import { videoApi, fileToBase64 } from '../utils/api' import { useMediaHistory } from '../hooks/useMediaHistory' -const SIZES = ['256x256', '512x512', '768x768', '1024x1024'] +const SIZES = ['256x256', '512x512', '768x768', '1024x1024', '832x480', '1280x720'] export default function VideoGen() { const { model: urlModel } = useParams() @@ -31,9 +32,10 @@ export default function VideoGen() { const [error, setError] = useState(null) const [videos, setVideos] = useState([]) const [showAdvanced, setShowAdvanced] = useState(false) - const [showImageInputs, setShowImageInputs] = useState(false) + const [showMediaInputs, setShowMediaInputs] = useState(false) const [startImage, setStartImage] = useState(null) const [endImage, setEndImage] = useState(null) + const [audioInput, setAudioInput] = useState(null) const { addEntry, selectEntry, selectedEntry, historyProps } = useMediaHistory('video') const handleGenerate = async (e) => { @@ -55,6 +57,7 @@ export default function VideoGen() { if (cfgScale) body.cfg_scale = parseFloat(cfgScale) if (startImage) body.start_image = startImage if (endImage) body.end_image = endImage + if (audioInput?.base64) body.audio = audioInput.base64 try { const data = await videoApi.generate(body) @@ -116,24 +119,46 @@ export default function VideoGen() { -
setShowAdvanced(!showAdvanced)}> - {t('video.labels.advanced')} -
+