feat(backends): add LongCat video and avatar generation

.github/backend-matrix.yml

@@ -478,6 +478,19 @@ include:
     dockerfile: "./backend/Dockerfile.python"
     context: "./"
     ubuntu-version: '2404'
+  - build-type: 'cublas'
+    cuda-major-version: "12"
+    cuda-minor-version: "8"
+    platforms: 'linux/amd64'
+    tag-latest: 'auto'
+    tag-suffix: '-gpu-nvidia-cuda-12-longcat-video'
+    runs-on: 'ubuntu-latest'
+    base-image: "ubuntu:24.04"
+    skip-drivers: 'false'
+    backend: "longcat-video"
+    dockerfile: "./backend/Dockerfile.python"
+    context: "./"
+    ubuntu-version: '2404'
   - build-type: 'cublas'
     cuda-major-version: "12"
     cuda-minor-version: "8"
@@ -1149,6 +1162,19 @@ include:
     dockerfile: "./backend/Dockerfile.python"
     context: "./"
     ubuntu-version: '2404'
+  - build-type: 'cublas'
+    cuda-major-version: "13"
+    cuda-minor-version: "0"
+    platforms: 'linux/amd64'
+    tag-latest: 'auto'
+    tag-suffix: '-gpu-nvidia-cuda-13-longcat-video'
+    runs-on: 'ubuntu-latest'
+    base-image: "ubuntu:24.04"
+    skip-drivers: 'false'
+    backend: "longcat-video"
+    dockerfile: "./backend/Dockerfile.python"
+    context: "./"
+    ubuntu-version: '2404'
   - build-type: 'cublas'
     cuda-major-version: "13"
     cuda-minor-version: "0"
@@ -1357,6 +1383,19 @@ include:
     backend: "vllm-omni"
     dockerfile: "./backend/Dockerfile.python"
     context: "./"
+  - build-type: 'l4t'
+    cuda-major-version: "13"
+    cuda-minor-version: "0"
+    platforms: 'linux/arm64'
+    tag-latest: 'auto'
+    tag-suffix: '-nvidia-l4t-cuda-13-arm64-longcat-video'
+    runs-on: 'ubuntu-24.04-arm'
+    base-image: "ubuntu:24.04"
+    skip-drivers: 'false'
+    ubuntu-version: '2404'
+    backend: "longcat-video"
+    dockerfile: "./backend/Dockerfile.python"
+    context: "./"
   - build-type: 'l4t'
     cuda-major-version: "13"
     cuda-minor-version: "0"

.github/workflows/bump_deps.yaml

@@ -26,6 +26,10 @@ jobs:
             variable: "DS4_VERSION"
             branch: "main"
             file: "backend/cpp/ds4/Makefile"
+          - repository: "meituan-longcat/LongCat-Video"
+            variable: "LONGCAT_VIDEO_VERSION"
+            branch: "main"
+            file: "backend/python/longcat-video/Makefile"
           - repository: "localai-org/privacy-filter.cpp"
             variable: "PRIVACY_FILTER_VERSION"
             branch: "master"

Makefile

@@ -1,5 +1,5 @@
 # Disable parallel execution for backend builds
-.NOTPARALLEL: backends/diffusers backends/llama-cpp backends/turboquant backends/outetts backends/piper backends/stablediffusion-ggml backends/whisper backends/crispasr backends/parakeet-cpp backends/moss-transcribe-cpp backends/faster-whisper backends/silero-vad backends/local-store backends/huggingface backends/rfdetr backends/rfdetr-cpp backends/insightface backends/speaker-recognition backends/kitten-tts backends/kokoro backends/chatterbox backends/llama-cpp-darwin backends/neutts build-darwin-python-backend build-darwin-go-backend backends/mlx backends/diffuser-darwin backends/mlx-vlm backends/mlx-audio backends/mlx-distributed backends/stablediffusion-ggml-darwin backends/vllm backends/vllm-omni backends/sglang backends/moonshine backends/pocket-tts backends/qwen-tts backends/faster-qwen3-tts backends/qwen-asr backends/nemo backends/voxcpm backends/whisperx backends/ace-step backends/acestep-cpp backends/fish-speech backends/voxtral backends/opus backends/trl backends/llama-cpp-quantization backends/kokoros backends/sam3-cpp backends/qwen3-tts-cpp backends/omnivoice-cpp backends/vibevoice-cpp backends/localvqe backends/tinygrad backends/sherpa-onnx backends/ds4 backends/ds4-darwin backends/liquid-audio backends/supertonic backends/depth-anything-cpp backends/privacy-filter backends/privacy-filter-darwin
+.NOTPARALLEL: backends/diffusers backends/llama-cpp backends/turboquant backends/outetts backends/piper backends/stablediffusion-ggml backends/whisper backends/crispasr backends/parakeet-cpp backends/moss-transcribe-cpp backends/faster-whisper backends/silero-vad backends/local-store backends/huggingface backends/rfdetr backends/rfdetr-cpp backends/insightface backends/speaker-recognition backends/kitten-tts backends/kokoro backends/chatterbox backends/llama-cpp-darwin backends/neutts build-darwin-python-backend build-darwin-go-backend backends/mlx backends/diffuser-darwin backends/mlx-vlm backends/mlx-audio backends/mlx-distributed backends/stablediffusion-ggml-darwin backends/vllm backends/vllm-omni backends/longcat-video backends/sglang backends/moonshine backends/pocket-tts backends/qwen-tts backends/faster-qwen3-tts backends/qwen-asr backends/nemo backends/voxcpm backends/whisperx backends/ace-step backends/acestep-cpp backends/fish-speech backends/voxtral backends/opus backends/trl backends/llama-cpp-quantization backends/kokoros backends/sam3-cpp backends/qwen3-tts-cpp backends/omnivoice-cpp backends/vibevoice-cpp backends/localvqe backends/tinygrad backends/sherpa-onnx backends/ds4 backends/ds4-darwin backends/liquid-audio backends/supertonic backends/depth-anything-cpp backends/privacy-filter backends/privacy-filter-darwin
 
 GOCMD=go
 GOTEST=$(GOCMD) test
@@ -565,6 +565,7 @@ prepare-test-extra: protogen-python
 	$(MAKE) -C backend/python/chatterbox
 	$(MAKE) -C backend/python/vllm
 	$(MAKE) -C backend/python/vllm-omni
+	$(MAKE) -C backend/python/longcat-video
 	$(MAKE) -C backend/python/sglang
 	$(MAKE) -C backend/python/vibevoice
 	$(MAKE) -C backend/python/liquid-audio
@@ -594,6 +595,7 @@ test-extra: prepare-test-extra
 	$(MAKE) -C backend/python/chatterbox test
 	$(MAKE) -C backend/python/vllm test
 	$(MAKE) -C backend/python/vllm-omni test
+	$(MAKE) -C backend/python/longcat-video test
 	$(MAKE) -C backend/python/vibevoice test
 	$(MAKE) -C backend/python/liquid-audio test
 	$(MAKE) -C backend/python/moonshine test
@@ -1254,6 +1256,7 @@ BACKEND_NEUTTS = neutts|python|.|false|true
 BACKEND_KOKORO = kokoro|python|.|false|true
 BACKEND_VLLM = vllm|python|.|false|true
 BACKEND_VLLM_OMNI = vllm-omni|python|.|false|true
+BACKEND_LONGCAT_VIDEO = longcat-video|python|.|--progress=plain|true
 BACKEND_SGLANG = sglang|python|.|false|true
 BACKEND_DIFFUSERS = diffusers|python|.|--progress=plain|true
 BACKEND_CHATTERBOX = chatterbox|python|.|false|true
@@ -1339,6 +1342,7 @@ $(eval $(call generate-docker-build-target,$(BACKEND_NEUTTS)))
 $(eval $(call generate-docker-build-target,$(BACKEND_KOKORO)))
 $(eval $(call generate-docker-build-target,$(BACKEND_VLLM)))
 $(eval $(call generate-docker-build-target,$(BACKEND_VLLM_OMNI)))
+$(eval $(call generate-docker-build-target,$(BACKEND_LONGCAT_VIDEO)))
 $(eval $(call generate-docker-build-target,$(BACKEND_SGLANG)))
 $(eval $(call generate-docker-build-target,$(BACKEND_DIFFUSERS)))
 $(eval $(call generate-docker-build-target,$(BACKEND_CHATTERBOX)))
@@ -1375,7 +1379,7 @@ $(eval $(call generate-docker-build-target,$(BACKEND_SUPERTONIC)))
 docker-save-%: backend-images
 	docker save local-ai-backend:$* -o backend-images/$*.tar
 
-docker-build-backends: docker-build-llama-cpp docker-build-ik-llama-cpp docker-build-turboquant docker-build-ds4 docker-build-rerankers docker-build-vllm docker-build-vllm-omni docker-build-sglang docker-build-transformers docker-build-outetts docker-build-diffusers docker-build-kokoro docker-build-faster-whisper docker-build-crispasr docker-build-coqui docker-build-chatterbox docker-build-vibevoice docker-build-liquid-audio docker-build-moonshine docker-build-pocket-tts docker-build-qwen-tts docker-build-fish-speech docker-build-faster-qwen3-tts docker-build-qwen-asr docker-build-nemo docker-build-voxcpm docker-build-whisperx docker-build-ace-step docker-build-acestep-cpp docker-build-voxtral docker-build-mlx-distributed docker-build-trl docker-build-llama-cpp-quantization docker-build-tinygrad docker-build-kokoros docker-build-sam3-cpp docker-build-rfdetr-cpp docker-build-qwen3-tts-cpp docker-build-omnivoice-cpp docker-build-vibevoice-cpp docker-build-localvqe docker-build-insightface docker-build-speaker-recognition docker-build-sherpa-onnx docker-build-cloud-proxy docker-build-supertonic docker-build-depth-anything-cpp docker-build-moss-transcribe-cpp docker-build-privacy-filter
+docker-build-backends: docker-build-llama-cpp docker-build-ik-llama-cpp docker-build-turboquant docker-build-ds4 docker-build-rerankers docker-build-vllm docker-build-vllm-omni docker-build-longcat-video docker-build-sglang docker-build-transformers docker-build-outetts docker-build-diffusers docker-build-kokoro docker-build-faster-whisper docker-build-crispasr docker-build-coqui docker-build-chatterbox docker-build-vibevoice docker-build-liquid-audio docker-build-moonshine docker-build-pocket-tts docker-build-qwen-tts docker-build-fish-speech docker-build-faster-qwen3-tts docker-build-qwen-asr docker-build-nemo docker-build-voxcpm docker-build-whisperx docker-build-ace-step docker-build-acestep-cpp docker-build-voxtral docker-build-mlx-distributed docker-build-trl docker-build-llama-cpp-quantization docker-build-tinygrad docker-build-kokoros docker-build-sam3-cpp docker-build-rfdetr-cpp docker-build-qwen3-tts-cpp docker-build-omnivoice-cpp docker-build-vibevoice-cpp docker-build-localvqe docker-build-insightface docker-build-speaker-recognition docker-build-sherpa-onnx docker-build-cloud-proxy docker-build-supertonic docker-build-depth-anything-cpp docker-build-moss-transcribe-cpp docker-build-privacy-filter
 
 ########################################################
 ### Mock Backend for E2E Tests

backend/README.md

@@ -46,6 +46,7 @@ The backend system provides language-specific Dockerfiles that handle the build
 - **vllm**: High-performance LLM inference
 - **mlx**: Apple Silicon optimization
 - **diffusers**: Stable Diffusion models
+- **longcat-video**: CUDA text/image-to-video and speech-driven avatar generation
 - **Audio**: coqui, faster-whisper, kitten-tts
 - **Vision**: mlx-vlm, rfdetr
 - **Specialized**: rerankers, chatterbox, kokoro

backend/backend.proto

@@ -577,6 +577,10 @@ message GenerateVideoRequest {
   float cfg_scale = 10;    // Classifier-free guidance scale
   int32 step = 11;         // Number of inference steps
   string dst = 12;        // Output path for the generated video
+  string audio = 13;      // Path to staged audio for audio-conditioned video
+  // Backend-specific per-request generation parameters. Values are strings
+  // and are validated/coerced by the selected backend.
+  map<string, string> params = 14;
 }
 
 message TTSRequest {
@@ -1256,4 +1260,3 @@ message ForwardReply {
   repeated ForwardHeader headers = 2;
   bytes body_chunk = 3;
 }
-

backend/index.yaml

@@ -372,7 +372,7 @@
   capabilities:
     nvidia: "cuda12-rfdetr"
     intel: "intel-rfdetr"
     #amd: "rocm-rfdetr"
     nvidia-l4t: "nvidia-l4t-arm64-rfdetr"
     metal: "metal-rfdetr"
     default: "cpu-rfdetr"
@@ -824,6 +824,30 @@
     nvidia-cuda-12: "cuda12-vllm-omni"
     nvidia-cuda-13: "cuda13-vllm-omni"
     nvidia-l4t-cuda-13: "cuda13-nvidia-l4t-arm64-vllm-omni"
+- &longcat-video
+  name: "longcat-video"
+  alias: "longcat-video"
+  license: mit
+  urls:
+    - https://github.com/meituan-longcat/LongCat-Video
+  tags:
+    - text-to-video
+    - image-to-video
+    - audio-to-video
+    - avatar-generation
+    - video-generation
+    - CUDA
+  icon: https://raw.githubusercontent.com/meituan-longcat/LongCat-Video/main/assets/longcat-video_logo.svg
+  description: |
+    LongCat-Video generation for text, image, and audio-conditioned avatars.
+    Supports LongCat-Video and LongCat-Video-Avatar-1.5, including multi-segment
+    talking-head continuation and an SDPA path for NVIDIA Blackwell ARM64 systems.
+    Requires Linux with an NVIDIA CUDA GPU; CPU, ROCm, and macOS are unsupported.
+  capabilities:
+    nvidia: "cuda12-longcat-video"
+    nvidia-cuda-12: "cuda12-longcat-video"
+    nvidia-cuda-13: "cuda13-longcat-video"
+    nvidia-l4t-cuda-13: "cuda13-nvidia-l4t-arm64-longcat-video"
 - &mlx
   name: "mlx"
   icon: https://avatars.githubusercontent.com/u/102832242?s=200&v=4
@@ -3605,13 +3629,51 @@
   uri: "quay.io/go-skynet/local-ai-backends:master-gpu-rocm-hipblas-vllm-omni"
   mirrors:
     - localai/localai-backends:master-gpu-rocm-hipblas-vllm-omni
+# longcat-video
+- !!merge <<: *longcat-video
+  name: "longcat-video-development"
+  capabilities:
+    nvidia: "cuda12-longcat-video-development"
+    nvidia-cuda-12: "cuda12-longcat-video-development"
+    nvidia-cuda-13: "cuda13-longcat-video-development"
+    nvidia-l4t-cuda-13: "cuda13-nvidia-l4t-arm64-longcat-video-development"
+- !!merge <<: *longcat-video
+  name: "cuda12-longcat-video"
+  uri: "quay.io/go-skynet/local-ai-backends:latest-gpu-nvidia-cuda-12-longcat-video"
+  mirrors:
+    - localai/localai-backends:latest-gpu-nvidia-cuda-12-longcat-video
+- !!merge <<: *longcat-video
+  name: "cuda13-longcat-video"
+  uri: "quay.io/go-skynet/local-ai-backends:latest-gpu-nvidia-cuda-13-longcat-video"
+  mirrors:
+    - localai/localai-backends:latest-gpu-nvidia-cuda-13-longcat-video
+- !!merge <<: *longcat-video
+  name: "cuda13-nvidia-l4t-arm64-longcat-video"
+  uri: "quay.io/go-skynet/local-ai-backends:latest-nvidia-l4t-cuda-13-arm64-longcat-video"
+  mirrors:
+    - localai/localai-backends:latest-nvidia-l4t-cuda-13-arm64-longcat-video
+- !!merge <<: *longcat-video
+  name: "cuda12-longcat-video-development"
+  uri: "quay.io/go-skynet/local-ai-backends:master-gpu-nvidia-cuda-12-longcat-video"
+  mirrors:
+    - localai/localai-backends:master-gpu-nvidia-cuda-12-longcat-video
+- !!merge <<: *longcat-video
+  name: "cuda13-longcat-video-development"
+  uri: "quay.io/go-skynet/local-ai-backends:master-gpu-nvidia-cuda-13-longcat-video"
+  mirrors:
+    - localai/localai-backends:master-gpu-nvidia-cuda-13-longcat-video
+- !!merge <<: *longcat-video
+  name: "cuda13-nvidia-l4t-arm64-longcat-video-development"
+  uri: "quay.io/go-skynet/local-ai-backends:master-nvidia-l4t-cuda-13-arm64-longcat-video"
+  mirrors:
+    - localai/localai-backends:master-nvidia-l4t-cuda-13-arm64-longcat-video
 # rfdetr
 - !!merge <<: *rfdetr
   name: "rfdetr-development"
   capabilities:
     nvidia: "cuda12-rfdetr-development"
     intel: "intel-rfdetr-development"
     #amd: "rocm-rfdetr-development"
     nvidia-l4t: "nvidia-l4t-arm64-rfdetr-development"
     metal: "metal-rfdetr-development"
     default: "cpu-rfdetr-development"

backend/python/README.md

@@ -27,6 +27,7 @@ The Python backends use a unified build system based on `libbackend.sh` that pro
 
 ### Computer Vision
 - **diffusers** - Stable Diffusion and image generation
+- **longcat-video** - CUDA video and speech-driven avatar generation with LongCat-Video
 - **mlx-vlm** - Vision-language models for Apple Silicon
 - **rfdetr** - Object detection models
 

backend/python/longcat-video/.gitignore

@@ -0,0 +1,6 @@
+backend_pb2.py
+backend_pb2_grpc.py
+lib/
+python/
+sources/
+venv/

backend/python/longcat-video/Makefile

@@ -0,0 +1,36 @@
+# SPDX-License-Identifier: MIT
+
+LONGCAT_VIDEO_VERSION?=6b3f4b8582a8bc3f20f795735f5383716c4ba794
+LONGCAT_VIDEO_REPO?=https://github.com/meituan-longcat/LongCat-Video
+LONGCAT_SOURCE_STAMP=sources/LongCat-Video/.localai-$(LONGCAT_VIDEO_VERSION)
+
+.PHONY: all
+all: $(LONGCAT_SOURCE_STAMP)
+	bash install.sh
+
+$(LONGCAT_SOURCE_STAMP): patches/0001-sdpa-attention-fallback.patch
+	rm -rf sources/LongCat-Video
+	mkdir -p sources/LongCat-Video
+	cd sources/LongCat-Video && git init -q && \
+		git remote add origin $(LONGCAT_VIDEO_REPO) && \
+		git fetch --depth 1 origin $(LONGCAT_VIDEO_VERSION) && \
+		git checkout --detach FETCH_HEAD && \
+		git apply ../../patches/0001-sdpa-attention-fallback.patch && \
+		rm -rf .git && \
+		touch .localai-$(LONGCAT_VIDEO_VERSION)
+
+.PHONY: run
+run: all
+	bash run.sh
+
+.PHONY: test
+test: all
+	bash test.sh
+
+.PHONY: protogen-clean
+protogen-clean:
+	$(RM) backend_pb2.py backend_pb2_grpc.py
+
+.PHONY: clean
+clean: protogen-clean
+	rm -rf __pycache__ lib python sources venv

backend/python/longcat-video/README.md

@@ -0,0 +1,44 @@
+# LongCat Video backend
+
+This backend serves Meituan's `LongCat-Video` and
+`LongCat-Video-Avatar-1.5` checkpoints through LocalAI's `GenerateVideo`
+RPC. It supports:
+
+- text-to-video and image-to-video with `LongCat-Video`;
+- audio + text-to-avatar and portrait + audio-to-avatar with Avatar 1.5;
+- multi-segment avatar continuation for speech longer than one segment;
+- PyTorch SDPA when FlashAttention is unavailable, including CUDA 13 ARM64
+  systems such as NVIDIA DGX Spark.
+
+Install the `longcat-video` or `longcat-video-avatar-1.5` recipe from the
+LocalAI Model Gallery. See the [LongCat user guide](../../../docs/content/features/longcat-video.md)
+for Studio and API examples, hardware requirements, and manual configuration.
+
+The upstream source is pinned in `Makefile` and patched at build time. The
+patch adds only the missing SDPA attention branches; model and source licenses
+remain MIT.
+
+## Model options
+
+| Option | Default | Description |
+| --- | --- | --- |
+| `attention_backend` | `sdpa` | `sdpa`, `auto`, `flash2`, `flash3`, or `xformers`. The packaged backend guarantees only `sdpa`. |
+| `use_distill` | `true` for Avatar, `false` for base | Loads the checkpoint's fast distillation LoRA. |
+| `use_int8` | `false` | Loads Avatar 1.5's INT8 DiT. BF16 has a lower load-time peak on unified-memory systems. |
+| `base_model` | `meituan-longcat/LongCat-Video` | Base components used by Avatar 1.5. |
+| `max_segments` | `8` | Maximum avatar continuation segments accepted per request. |
+| `resolution` | `480p` | Image-conditioned generation resolution (`480p` or `720p`). |
+
+Per-request `params` may set `num_segments`, `audio_guidance_scale`,
+`offload_kv_cache`, `ref_img_index`, `mask_frame_range`, and `resolution`.
+
+Gallery and imported configs declare `known_input_modalities` and
+`known_output_modalities`. Keep those declarations in manual configs as well;
+they let model discovery distinguish base image-conditioned video from Avatar
+audio conditioning without inspecting the backend or checkpoint name.
+
+LongCat is CUDA-only and very large. Avatar 1.5 also loads tokenizer,
+text-encoder, and VAE components from the base checkpoint. Keep ample unified
+memory and storage available; no CPU or macOS backend image is published. The
+initial backend supports one GPU per process; tensor parallel sizes above one
+are rejected explicitly.