From 6b6f42c11f084f627b987d80f5a9ad7b24aedb2d Mon Sep 17 00:00:00 2001 From: Ricardo-M-L Date: Sat, 24 Oct 2026 00:15:01 +0800 Subject: [PATCH] fix(video-agent-kit): gate subtitle codecs by render mode --- marketplace.json | 4 +- .../.claude-plugin/plugin.json | 2 +- .../video-agent-kit/.zcode-plugin/plugin.json | 2 +- plugins/video-agent-kit/README.md | 1 + .../video-agent-kit/mcp/ve_tools/subtitle.py | 18 +-- tests/test_video_agent_subtitle_render.py | 113 ++++++++++++++++++ 6 files changed, 129 insertions(+), 11 deletions(-) create mode 100644 tests/test_video_agent_subtitle_render.py diff --git a/marketplace.json b/marketplace.json index 884bfc8c..11438215 100644 --- a/marketplace.json +++ b/marketplace.json @@ -97,7 +97,7 @@ "en": "Automated video editing toolkit: standard cloud speech transcription and synthesis, full video frame extraction and understanding, local re-inspection, timeline, preview rendering, QC, and TTS.", "zh-CN": "自动化视频剪辑工具包: 标准云端语音转录与合成、完整视频抽帧理解、局部复看、时间线、预览渲染、QC 和 TTS。" }, - "version": "0.4.3", + "version": "0.4.4", "author": { "name": "Z.ai", "url": "https://z.ai" @@ -1411,4 +1411,4 @@ ] } ] -} \ No newline at end of file +} diff --git a/plugins/video-agent-kit/.claude-plugin/plugin.json b/plugins/video-agent-kit/.claude-plugin/plugin.json index c0c93dfe..b90561ca 100644 --- a/plugins/video-agent-kit/.claude-plugin/plugin.json +++ b/plugins/video-agent-kit/.claude-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "video-agent-kit", - "version": "0.4.3", + "version": "0.4.4", "description": "自动化视频剪辑工具包: 标准云端语音转录与合成、完整视频抽帧理解、局部复看、时间线、预览渲染、QC 和 TTS。", "author": { "name": "Z.ai", diff --git a/plugins/video-agent-kit/.zcode-plugin/plugin.json b/plugins/video-agent-kit/.zcode-plugin/plugin.json index a6179d52..7521928a 100644 --- a/plugins/video-agent-kit/.zcode-plugin/plugin.json +++ b/plugins/video-agent-kit/.zcode-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "video-agent-kit", - "version": "0.4.3", + "version": "0.4.4", "description": "自动化视频剪辑工具包: 标准云端语音转录与合成、完整视频抽帧理解、局部复看、时间线、预览渲染、QC 和 TTS。", "author": { "name": "Z.ai", diff --git a/plugins/video-agent-kit/README.md b/plugins/video-agent-kit/README.md index 4da1bd85..999ca5d4 100644 --- a/plugins/video-agent-kit/README.md +++ b/plugins/video-agent-kit/README.md @@ -16,6 +16,7 @@ - `video_ingest`：完整视频抽帧并返回 contact sheet，必须先跑；Claude Code 主 Agent 自己看图并形成理解。显式传入的 `transcript_path` 必须存在；不传 transcript 才表示允许无转录观察；在同一个 MCP 会话里，如果重新观察此前已绑定过 transcript 的同一视频指纹，会复用该视频对应的 transcript。同一路径的视频文件如果被覆盖或替换，必须重新调用 `video_ingest`。只有成功产出 observation package 后，工具才会把该视频记为已 full-ingest；抽帧失败不会推进会话状态。 - `video_watch_segment`：局部高 fps 复看；单窗口用 `start_time/end_time/fps`，多窗口用 `segments=[{start,end}, ...]` + `fps`，返回片段 contact sheet 和匹配转录。显式传 `video_path` 可直接复看任意存在的视频，不要求先跑完整 `video_ingest`。不带 `video_path` 时默认使用会话里的 active video；如果 active 文件被覆盖或替换，隐式复看会报错并要求显式传当前 `video_path` 或重新 `video_ingest`。显式传入的 `transcript_path` 必须存在；在同一个 MCP 会话里，如果显式传入此前已绑定过 transcript 的同一视频指纹且未传 transcript 参数，会复用该视频对应的 transcript。非数字时间参数会返回结构化错误；明显超过视频时长的窗口会报错，不会静默截断成较短观察。窗口台账按 `(start, end, fps)` 判重：只有时间和 fps 都相同的窗口才会被跳过，换一个 fps 重看同一窗口是新的观察；确有需要时可传 `force=true` 跳过判重。只有成功产出新 contact sheet 后，局部复看才会更新 active-video 会话状态；抽帧失败或 `skipped_duplicate` 不算新的视觉证据，多窗口部分失败时整体结果会带 `[ERROR]` 前缀汇总。 - `video_basic_operation`：OpenChatCut 风格的确定性基础操作封装，支持 `trim`、`splice`、`speed`、`crop`、`scale`、`rotate`、`flip`、`freeze_frame`。它适合源素材预处理或单步变换；timeline 成片仍应走 `validate_timeline` / `render_preview`。 +- `subtitle_render`：`mode="soft"` 复制原始视频和音频流，只需要 FFmpeg 的 `mov_text` 字幕编码器，不需要 libass 或媒体重编码器；`burn` 需要 `libx264` 和 `ass` 滤镜，`both` 还需要 `mov_text`。各模式在写出字幕和视频文件前检查所需能力。 - `speech_synthesize`：标准语音合成工具。配置 `VE_SPEECH_MCP_URL` 时调用远端 HTTP MCP；未配置时，在 ZCode 中优先使用宿主注入身份的官方通道，其他宿主回退到本地兼容后端。对外统一为 `cloud_tts`；`allowed_providers` 提供时是硬白名单。并发限制、限流、网络和服务端瞬时错误默认额外重试 3 次，可用 `retries` / `retry_count` 和 `retry_backoff_seconds` 覆盖；参数错误、缺凭据、格式不支持不会重试。`speed` 会映射为 `speech_rate`，也可显式传 `speech_rate` / `pitch_rate` / `loudness_rate` / `sample_rate` / `output_format`，本工具只暴露可被 ffprobe 稳定校验的 `wav` / `mp3` / `ogg_opus` 输出，并拒绝 `output_path` 后缀与 `output_format` 不一致。`tts_generate` 保留为兼容 alias。 - `validate_timeline`：校验 timeline 项目 JSON；依赖 ffprobe 检查源素材时长边界，时长优先读 format duration，缺失时回退到 stream duration，且同一源文件的时长探测会缓存。除了旧版线性 `clips[]`，也支持更接近剪辑软件项目文件的 `project` / `sequence` / `output_canvas` / `assets` / `tracks` / `markers` / `transitions` / clip effects 元数据校验。`tracks[]` timeline 中非视频轨的 clip（字幕 / overlay / callout 计划）可以用非空 `text` 代替媒体 `source`；视频 / main 轨 clip 和顶层 `clips[]` 仍必须有 `source`。 - `render_preview`：用 FFmpeg / ffprobe 渲染预览；render report 会记录 timeline hash 和 output hash。顶层 `clips[]` 仍按 legacy/debug 线性拼接渲染；项目式 `tracks[]` 会按 sequence 画布和 `timeline_start` 编译成基础多轨预览：主视频铺到画布并在片尾补少量末帧降低边界闪帧，源视频音频与 `audio/music/voiceover` 轨先烘焙成全片长度、已按时间线定位的 wav bed，再非归一化混音，`subtitle/text` 轨用 `drawtext` 烧录，`overlay/image` 轨按启用窗口叠加。父轨道没有 `type/name/track_type` 时不会被默认当作主视频轨；未知 track 类型会记录到 render plan 的 unsupported tracks 中，而不是声称完整渲染。 diff --git a/plugins/video-agent-kit/mcp/ve_tools/subtitle.py b/plugins/video-agent-kit/mcp/ve_tools/subtitle.py index 8d520ef9..341278c6 100644 --- a/plugins/video-agent-kit/mcp/ve_tools/subtitle.py +++ b/plugins/video-agent-kit/mcp/ve_tools/subtitle.py @@ -976,9 +976,17 @@ def subtitle_render(args: dict, ctx: RunContext) -> ToolResult: return ToolResult(text="[ERROR] subtitles_path is required (out/subtitles.json from subtitle_build)") if not shutil.which("ffmpeg"): return ToolResult(text="[ERROR] ffmpeg not found on PATH") - # Fail fast: subtitle burn-in needs libass (subtitles/ass filter); minimal - # builds drop it and only fail after reading the whole video. - cap_err = require_ffmpeg(encoders=("libx264", "aac"), filters=("subtitles",)) + mode = str(args.get("mode") or "burn").strip().lower() + if mode not in {"burn", "soft", "both"}: + return ToolResult(text="[ERROR] mode must be one of: burn, soft, both") + + # Soft subtitles copy the source streams; only burn-in needs libass and + # a video encoder. Check the actual filters/codecs used by each command. + encoders = ("libx264",) if mode in {"burn", "both"} else () + if mode in {"soft", "both"}: + encoders += ("mov_text",) + filters = ("ass",) if mode in {"burn", "both"} else () + cap_err = require_ffmpeg(encoders=encoders, filters=filters) if cap_err: return ToolResult(text=cap_err) video_path = ctx.resolve(args["video_path"]) @@ -988,10 +996,6 @@ def subtitle_render(args: dict, ctx: RunContext) -> ToolResult: if not subtitles_path.is_file(): return ToolResult(text=f"[ERROR] subtitles not found: {subtitles_path}") - mode = str(args.get("mode") or "burn").strip().lower() - if mode not in {"burn", "soft", "both"}: - return ToolResult(text="[ERROR] mode must be one of: burn, soft, both") - try: package = json.loads(subtitles_path.read_text(encoding="utf-8")) except Exception as exc: diff --git a/tests/test_video_agent_subtitle_render.py b/tests/test_video_agent_subtitle_render.py new file mode 100644 index 00000000..ab54ed39 --- /dev/null +++ b/tests/test_video_agent_subtitle_render.py @@ -0,0 +1,113 @@ +"""Subtitle render capabilities and real stream-copy mux regression coverage.""" + +import json +from contextlib import nullcontext +from pathlib import Path +import shutil +import subprocess +import sys +import tempfile +import unittest +from unittest.mock import patch + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "plugins/video-agent-kit/mcp")) +from ve_tools import subtitle # noqa: E402 +from ve_tools.run_context import RunContext # noqa: E402 + + +class SubtitleCapabilitiesTest(unittest.TestCase): + def test_capabilities_match_the_selected_render_mode(self): + for mode, encoders, filters in [ + (None, ("libx264",), ("ass",)), + ("soft", ("mov_text",), ()), + ("burn", ("libx264",), ("ass",)), + ("both", ("libx264", "mov_text"), ("ass",)), + ]: + with self.subTest(mode=mode), patch.object( + subtitle.shutil, "which", return_value="ffmpeg" + ), patch.object(subtitle, "require_ffmpeg", return_value="[ERROR] missing capability") as check: + result = subtitle.subtitle_render( + {"video_path": "source.mp4", "subtitles_path": "captions.json", "mode": mode}, + None, + ) + self.assertEqual(result.text, "[ERROR] missing capability") + check.assert_called_once_with(encoders=encoders, filters=filters) + + def test_invalid_mode_is_reported_before_capability_checks(self): + with patch.object(subtitle.shutil, "which", return_value="ffmpeg"), patch.object( + subtitle, "require_ffmpeg", return_value="[ERROR] missing capability" + ) as check: + result = subtitle.subtitle_render( + {"video_path": "source.mp4", "subtitles_path": "captions.json", "mode": "invalid"}, + None, + ) + self.assertIn("mode must be one of", result.text) + check.assert_not_called() + + +@unittest.skipUnless(shutil.which("ffmpeg") and shutil.which("ffprobe"), "requires ffmpeg and ffprobe") +class SoftSubtitleMuxTest(unittest.TestCase): + def test_soft_subtitles_keep_media_packets_without_burn_in_capabilities(self): + self.assert_soft_mux(minimal=True) + + def test_soft_subtitles_with_installed_ffmpeg_capabilities(self): + self.assert_soft_mux(minimal=False) + + def assert_soft_mux(self, minimal): + with tempfile.TemporaryDirectory() as tmp, patch.dict( + "os.environ", {"CLAUDE_PROJECT_DIR": tmp} + ): + directory = Path(tmp) + source = directory / "source.mp4" + subprocess.run( + ["ffmpeg", "-nostdin", "-v", "error", "-f", "lavfi", "-i", "color=size=64x64:rate=10", + "-f", "lavfi", "-i", "sine=frequency=440:sample_rate=44100", "-t", "1", + "-c:v", "mpeg4", "-c:a", "aac", str(source)], + check=True, capture_output=True, timeout=30, + ) + package = directory / "captions.json" + package.write_text(json.dumps({ + "language": "eng", + "cues": [{"start": 0, "end": 0.8, "text": "Hello subtitles"}], + "style": { + "font_file": str(directory / "unused-font.ttf"), "font_family_resolved": "Arial", + "font_size": 12, "video_width": 64, "video_height": 64, + "primary_colour": "&H00FFFFFF", "outline_colour": "&H00000000", + "back_colour": "&H80000000", "border_style": 1, "outline": 1, "shadow": 0, + "alignment": 2, "margin_l": 2, "margin_r": 2, "margin_v": 2, + }, + }), encoding="utf-8") + # Simulate a minimal FFmpeg capability listing, not the mux itself: + # the real command must only need mov_text, with no video/audio + # encoder or libass filter. Homebrew's minimal build has no libass. + capabilities = patch("ve_tools.ffproc._ffmpeg_feature_text", side_effect=lambda kind: + " S..... mov_text " if kind == "encoders" else "") if minimal else nullcontext() + with capabilities: + result = subtitle.subtitle_render({ + "video_path": str(source), "subtitles_path": str(package), "mode": "soft", + }, RunContext()) + self.assertFalse(result.text.startswith("[ERROR]"), result.text) + output = Path(result.data["outputs"]["soft"]) + for selector in ("v:0", "a:0"): + self.assertEqual(self.packet_hashes(source, selector), self.packet_hashes(output, selector)) + subtitles = subprocess.run( + ["ffmpeg", "-nostdin", "-v", "error", "-i", str(output), "-map", "0:s:0", "-f", "srt", "-"], + check=True, capture_output=True, text=True, timeout=30, + ).stdout + self.assertIn("00:00:00,000 --> 00:00:00,800", subtitles) + self.assertIn("Hello subtitles", subtitles) + + def packet_hashes(self, path, selector): + probe = subprocess.run( + ["ffprobe", "-v", "error", "-select_streams", selector, "-show_packets", + "-show_data_hash", "sha256", "-show_entries", "packet=data_hash", "-of", "json", str(path)], + check=True, capture_output=True, text=True, timeout=30, + ) + packets = json.loads(probe.stdout)["packets"] + self.assertTrue(packets) + return [packet["data_hash"] for packet in packets] + + +if __name__ == "__main__": + unittest.main()