From bbfe3a940f950118985f59dd4f55efa53b436a99 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Sun, 27 Sep 2026 05:19:55 +0800 Subject: [PATCH 1/2] Propose eight ReviewBot knowledge rules --- knowledge/repos/vllm-omni/benchmark/rules.md | 9 ++++++++- .../repos/vllm-omni/components/diffusion/rules-lora.md | 9 ++++++++- .../components/diffusion/rules-output-lifecycle.md | 9 ++++++++- .../repos/vllm-omni/components/distributed/rules.md | 9 ++++++++- .../components/scheduler/rules-shared-lifecycle.md | 9 ++++++++- .../vllm-omni/components/serving/rules-speech-output.md | 9 ++++++++- knowledge/repos/vllm-omni/models/cosmos3/rules.md | 9 ++++++++- knowledge/repos/vllm-omni/models/qwen-omni/rules.md | 9 ++++++++- 8 files changed, 64 insertions(+), 8 deletions(-) diff --git a/knowledge/repos/vllm-omni/benchmark/rules.md b/knowledge/repos/vllm-omni/benchmark/rules.md index abb51505..474f053c 100644 --- a/knowledge/repos/vllm-omni/benchmark/rules.md +++ b/knowledge/repos/vllm-omni/benchmark/rules.md @@ -1,7 +1,7 @@ --- title: "vLLM-Omni Benchmark 规则" created: 2026-09-05 -updated: 2026-09-22 +updated: 2026-09-26 type: rule tags: [vllm-omni, benchmark] sources: ["PR #7648", "PR #6817", tests/dfx/perf/scripts/run_benchmark.py, tests/benchmarks/test_omniinteract.py, "PR #7130", "PR #7259", "PR #7624", "PR #7504"] @@ -86,3 +86,10 @@ confidence: high - 强制：媒体路径经 PyAV（`av`）打开/seek/decode/resample；结果不得依赖主机安装的 ffmpeg 版本。帧抽取语义对齐“首个 PTS ≥ 目标时刻”；非 WAV 音频经 `AudioResampler` 并在循环后 `resample(None)` flush 尾部样本。 - 禁止：`subprocess` 调用 host `ffmpeg`/`ffprobe` 作为默认路径；把“本机 ffmpeg 能跑”当作可复现证据；在已知会挂起的旧 HEVC decoder 上无超时地阻塞 generate 阶段。 - 验收：覆盖 duration/JPEG/PCM 合同与失败 raise；至少用曾触发 host ffmpeg 4.4.2 挂起的样本证明不再无限忙等。^[PR #7504] + +## BENCH-1h — Buildkite perf 步骤必须按 JSON schema 选择 runner + +- 触发：修改 `.buildkite/**` 里 `run_benchmark.py` / `run_diffusion_benchmark.py` 的 `--test-config-file`，或把 perf JSON 在 `dataset` 与 `dataset_name` schema 之间迁移。 +- 强制：`run_diffusion_benchmark.py` 只跑 `is_diffusion_perf_config` 为真的 case（`benchmark_params[].dataset`）；`run_benchmark.py` 只跑 omni-bench case（`dataset_name`）。schema 过滤看字段，不看 `mark`。迁移 JSON 后，所有引用该文件的 CUDA/NPU/AMD 步骤必须一起换 runner，并改用对应的 `BENCHMARK_DIR` 与 artifact glob。 +- 禁止：一边已迁 omni-bench、一边仍调 diffusion runner（会 skip 全部 case，pytest 0 selected / exit 5）；把某一平台的 runner 修复外推为其他 pipeline 已对齐。 +- 验收：扫描全部 `.buildkite` 调用，断言 runner 与 JSON schema 一致；HunyuanVideo-1.5 t2v 等已迁 JSON 不得再回到 `run_diffusion_benchmark.py`。^[PR #8107] diff --git a/knowledge/repos/vllm-omni/components/diffusion/rules-lora.md b/knowledge/repos/vllm-omni/components/diffusion/rules-lora.md index d0278cb3..32ff7c92 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/rules-lora.md +++ b/knowledge/repos/vllm-omni/components/diffusion/rules-lora.md @@ -1,7 +1,7 @@ --- title: "Diffusion LoRA 规则" created: 2026-09-02 -updated: 2026-09-22 +updated: 2026-09-26 type: rule tags: [vllm-omni, components, diffusion] sources: ["PR #2783", docs/user_guide/diffusion/lora.md, vllm_omni/config/omni_config.py, vllm_omni/config/stage_config.py, vllm_omni/diffusion/data.py, vllm_omni/diffusion/lora/loader.py, vllm_omni/diffusion/lora/manager.py, vllm_omni/diffusion/lora/layers/base_linear.py, vllm_omni/diffusion/models/qwen_image/pipeline_qwen_image.py, vllm_omni/diffusion/models/wan2_2/pipeline_wan2_2.py, vllm_omni/diffusion/models/wan2_2/pipeline_wan2_2_i2v.py, vllm_omni/diffusion/utils/tf_utils.py, vllm_omni/diffusion/worker/diffusion_worker.py, vllm_omni/engine/async_omni_engine.py, vllm_omni/entrypoints/cli/serve.py, tests/diffusion/lora/test_loader.py, tests/diffusion/lora/test_lora_manager.py, tests/entrypoints/test_async_omni_diffusion_config.py, "PR #5500", "vllm_omni/diffusion/models/ltx2/ltx2_adapter_parser.py", "vllm_omni/diffusion/models/ltx2/ltx2_phase_adapter.py", "PR #6070", "PR #6476", "PR #6550", vllm_omni/diffusion/models/minimax_h3/lora.py, "PR #6268", benchmarks/kernels/benchmark_diffusion_lora_expand.py, tests/diffusion/lora/test_base_linear.py, "PR #7195", "PR #7349", "PR #5907"] @@ -158,3 +158,10 @@ confidence: high - 强制：manager 默认只扫描通用 diffusers 名 `transformer`、`transformer_2`、`unet`；模型专用属性（如 Bagel 的 `bagel`）必须由 pipeline 经 `_dit_modules` 或 `_lora_components` 声明。ModelOpt 默认只保留通用 attention 融合（`to_qkv`、`add_kv_proj`）；模型专用融合（如 Z-Image `w13`）必须写在模型 `packed_modules_mapping` 并由 adapter 合并。 - 禁止：把 `dit`/`bagel`/`w13` 等模型私有名硬编码进共享默认；在未扫描真实 denoiser 时宣称 adapter 已生效（零层绑定、输出等同 base）。 - 验收：Bagel/同类测试 pipeline 必须显式 `_lora_components` 才能发现层；Z-Image 映射在模型侧声明后仍可解析；去掉声明后不得靠框架默认偷跑。^[PR #5907] + +## DIFF-2ag4 — 任一未绑定 module 的 diffusion LoRA adapter 必须失败 + +- 触发：修改 `DiffusionLoRAManager._bind_adapter_weights` / `_activate_adapter`、fused/packed `lora_b` 切分，或把部分命中当成成功激活。 +- 强制：激活前必须证明 adapter 提供的每个 logical module 都已绑定；`bound_lora_names` 为空、或 `loras` 中仍有未绑定名，一律 `ValueError`，消息含 adapter id、`bound=k/n`、unbound 名和 expected target modules。fused-QKV / packed `output_slices` 对不上 `lora_b` 维度时立即 raise，不得 warning 后 `reset_lora` 并继续。空 adapter（`bound=0/0`）同样失败。稀疏 packed 目标（只供 Q 或 Q/V）在**已提供模块全部绑定**时合法，未提供的 slice 保持 `None`。失败必须走既有 activation cleanup：reset wrapper、清 active/suspended 身份。 +- 禁止：只拒绝零绑定却放行部分绑定（例如 `attn.to_out` 命中而 `attn.to_out.0` 被忽略）；把 fused 布局失败写成可跳过 warning；在部分 bind 后仍标 adapter active。 +- 验收：CPU 覆盖 unmatched 名、empty adapter、fused shape mismatch，以及从 active/suspended 切失败后的 rollback 与旧 adapter 再激活；packed 路径覆盖 Q、Q/V、Q/K/V 三种 name form。^[PR #8008] diff --git a/knowledge/repos/vllm-omni/components/diffusion/rules-output-lifecycle.md b/knowledge/repos/vllm-omni/components/diffusion/rules-output-lifecycle.md index e3ce91e7..33560b71 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/rules-output-lifecycle.md +++ b/knowledge/repos/vllm-omni/components/diffusion/rules-output-lifecycle.md @@ -1,7 +1,7 @@ --- title: "Diffusion output 与 multiprocess runtime 规则" created: 2026-09-02 -updated: 2026-09-22 +updated: 2026-09-26 type: rule tags: [vllm-omni, components, diffusion] sources: ["PR #5550", "PR #5864", "PR #5885", "PR #5978", "PR #6750", vllm_omni/diffusion/diffusion_engine.py, vllm_omni/diffusion/executor/multiproc_executor.py, vllm_omni/diffusion/inline_stage_diffusion_client.py, vllm_omni/diffusion/io_support.py, vllm_omni/diffusion/model_metadata.py, vllm_omni/diffusion/output_formatter.py, vllm_omni/diffusion/ipc.py, vllm_omni/diffusion/sched/request_scheduler.py, vllm_omni/diffusion/stage_diffusion_proc.py, vllm_omni/diffusion/utils/media_utils.py, vllm_omni/diffusion/worker/diffusion_worker.py, tests/diffusion/test_async_output_timeout.py, tests/diffusion/test_async_output_worker.py, tests/diffusion/test_diffusion_engine.py, tests/diffusion/test_diffusion_engine_cleanup.py, tests/diffusion/test_diffusion_ipc.py, tests/diffusion/test_inline_stage_diffusion_client.py, tests/diffusion/test_ipc_async.py, tests/diffusion/test_multiproc_engine_concurrency.py, tests/diffusion/test_result_pump.py, tests/diffusion/test_stage_diffusion_proc.py, tests/entrypoints/openai_api/test_video_server.py, "PR #6023", "PR #5983", "PR #4222", "PR #6094", "PR #6255", "PR #6288", "PR #6308", "PR #6499", "PR #6749", "PR #6847", "PR #6953", vllm_omni/diffusion/data.py, vllm_omni/diffusion/registry.py, vllm_omni/diffusion/models/diffusers_adapter/pipeline_utils.py, tests/diffusion/test_diffusion_output_formatter.py, tests/diffusion/test_diffusion_plugin_hooks.py, tests/entrypoints/openai_api/test_video_pipeline_capability.py, "PR #6294", "PR #6615", "PR #7126", "PR #7198"] @@ -217,3 +217,10 @@ confidence: high - 强制：含多 modality 的同一事件先全部 `validate_payload`，再入队；任一路不支持或非法不得部分改队列。`received_at`/boundary 时钟经 rank 同步，使 USP/SP 分片共享同一到达时刻。`lazy_initialize_session=False` 的 handler（如 camera）必须在首个 denoise chunk 前 `maybe_prepare_initial_session`，即使尚无 client enqueue。chunk apply 区分 `num_media_frames` 与 `num_latent_frames`，需要者才 `peek_chunk_media`。 - 禁止：先入队再校验导致半更新；把未注册 camera 当已可用；懒会话与必须预创建会话混用同一 skip 逻辑；用未同步的 per-rank `time.monotonic()` 驱动跨分片时间线。 - 验收：复合 prompt+camera 原子成功/失败、未注册拒绝、非懒会话在 chunk0 前存在、介质计数字段传播。^[PR #7198] + +## DIFF-1ak — inline DiffusionWorker 在 init_device 之后失败必须先 shutdown 再抛出 + +- 触发：修改 `DiffusionWorker.__init__` / `shutdown`、`WorkerWrapperBase` 自定义 pipeline 再初始化，或同一进程连续构造 sleep-enabled `uni`/`mp` worker。 +- 强制：`init_device()` 已占用 process-global distributed / CuMem 后，runner 解析、load、LoRA 或 wrapper 再初始化失败必须调用幂等 `shutdown()` 再重抛**原始**异常。`shutdown` 先断开 model/LoRA/profiler/sleep-buffer/request-state 引用，再停 offloader 与 KV-transfer，best-effort 清 connector、A2A workspace 与 distributed state，只释放本 worker `_owns_sleep_pool` 的池，并同步/回收后检查 allocator。cleanup 自己失败不得遮蔽原始 init 异常。 +- 禁止：构造失败后留下非空 model-parallel state，让下一实例撞 `model parallel state must be empty`；shutdown 后仍让 CuMem 跟踪的模型分配活着，让下一 sleep worker 撞 `Sleep mode can only be used for one instance per process`；依赖 wrapper 不可达来做确定性 teardown。 +- 验收：覆盖 init-after-`init_device` 失败、wrapper 再初始化失败、cleanup 异常不改写根因、shutdown 幂等与所有权；不得把单次 AMD custom-pipeline 变绿外推为所有平台已无 leftover mapping。^[PR #7986] diff --git a/knowledge/repos/vllm-omni/components/distributed/rules.md b/knowledge/repos/vllm-omni/components/distributed/rules.md index f938ba54..7b00d535 100644 --- a/knowledge/repos/vllm-omni/components/distributed/rules.md +++ b/knowledge/repos/vllm-omni/components/distributed/rules.md @@ -1,7 +1,7 @@ --- title: "Distributed 传输规则" created: 2026-08-05 -updated: 2026-09-22 +updated: 2026-09-26 type: rule tags: [vllm-omni, components, distributed] sources: ["PR #5744", "PR #5976", "PR #6001", "PR #6089", "PR #6834", vllm_omni/diffusion/distributed/parallel_state.py, tests/diffusion/distributed/test_expert_parallel_layout.py, vllm_omni/distributed/omni_connectors/adapter.py, vllm_omni/distributed/omni_connectors/kv_transfer_manager.py, vllm_omni/distributed/omni_connectors/transfer_adapter/chunk_transfer_adapter.py, vllm_omni/distributed/omni_connectors/transfer_adapter/base.py, vllm_omni/worker/omni_connector_model_runner_mixin.py, tests/distributed/omni_connectors/test_kv_recv_tp_consensus.py, tests/distributed/omni_connectors/test_chunk_transfer_adapter.py, tests/worker/test_omni_connector_mixin.py, "PR #5146", "PR #6021", "PR #6033", "PR #6360", "PR #6406", "PR #6626", "PR #6529", "PR #7136", "PR #6889", "PR #6093", "PR #7870", "PR #7166"] @@ -170,3 +170,10 @@ confidence: high - 强制：存在 stage `kv_transfer_config` 时必须 `async_chunk=False`。原生路径由 scheduler 拥有 diffusion KV 页；connector 失败 fail-close diffusion engine 后再拆页。非 caching P/D 池释放时按递增 `block_id` 归还，便于相邻页合并；deferred free 存逆序以匹配 vLLM drain。原生 transfer 边界用已确认 computed tokens，排除 async output placeholders；需要时短暂改写 request status/token 计数并在 `finally` 恢复。 - 禁止：`async_chunk=True` 搭配原生 KV transfer；把 eviction-priority free 顺序当成无缓存池的物理布局；用含 placeholder 的 token 计数作为 transfer 边界；connector 失败后仍继续拆页当成功。 - 验收：配置拒绝 async_chunk+kv_transfer；物理序 free；confirmed-token transfer 边界；失败 fail-close。^[PR #7166] + +## DIST-1m — SHM producer 只能按 put 的精确 key 回收，并有界 reap 跨进程已消费段 + +- 触发：修改 `SharedMemoryConnector` 的 `_pending_keys` / `cleanup` / `close` / `reap_consumed`，chunk adapter abort 后的 sender 清理，或 lock-file 与 `/dev/shm` 回收。 +- 强制：`cleanup` 只 unlink `put()` 当时登记的精确 key，不得按 request-id 做 `_` 前缀/后缀模糊匹配。receiver 读到 SHM 字节后即可标 consumed 并删 lock file，即使随后 deserialize 失败。producer 在 `put` 与空闲 save loop 上做有界 round-robin `reap_consumed`（丢弃对端已 unlink 的 key）。abort 后在 in-flight 排空再按 adapter 生成的精确 chunk key 入队清理；成功终态 payload 必须留给下游消费，且清理发生在 external-id 复用之前。receiver 用当前 registration identity 拒 stale work，不得堆积 cancellation tombstone。 +- 禁止：用 request-id 子串误删无关 SHM；把已跨进程消费的 key 永远留在 `_pending_keys`；deserialize 失败就留下 lock file；用 underscore-prefix 猜测所有权。 +- 验收：覆盖相似 request id、abort-during-put、external-id 复用、无效 payload 后 lock 消失、consumed-key 回收，以及取消后 `/dev/shm` 与 pending keys 归零。^[PR #8082] diff --git a/knowledge/repos/vllm-omni/components/scheduler/rules-shared-lifecycle.md b/knowledge/repos/vllm-omni/components/scheduler/rules-shared-lifecycle.md index 3fc8e01b..f01b0c1a 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/rules-shared-lifecycle.md +++ b/knowledge/repos/vllm-omni/components/scheduler/rules-shared-lifecycle.md @@ -1,7 +1,7 @@ --- title: "Scheduler shared lifecycle 规则" created: 2026-09-22 -updated: 2026-09-22 +updated: 2026-09-26 type: rule tags: [vllm-omni, components, scheduler] sources: ["PR #5461"] @@ -22,3 +22,10 @@ sources: ["PR #5461"] - 验收：同一组 contract 测试分别驱动 AR 与 generation 输入路径；断言 base dataclass 的 每个字段均被转交、现有 Omni entry 保持 identity、失败 KV load 终止输出、finished ID/ stats/events/cleanup 一致，并单独验证显式 abort policy。 ^[PR #5461] + +## SCHED-1c — 同 step 的 prefix-hit prefetch 必须在 publish 后按新 version 重规划 + +- 触发：修改 `OmniTensorPrefixCache` 的 same-step hit prefetch、`_hit_prefetch` / `_prefetch_queue`，或 `allocate_slots` 后同一次 `schedule()` 内后到请求命中先到请求本步才写入的 block。 +- 强制：B 在 `new_step_starts` 规划的 hit 可能读到 ABSENT mm 行，或回收块上上一 tenant 的 COMMITTED 行。A 在本步 `save_outputs` / `_publish_saved_step` 登记 write 后，必须丢掉 `reserved_version` 已不再匹配的 prefetch future（旧 fetch 可跑完但须 unregister pending read），再让 `_prefetch_hit_spans` 对现已 `IN_TRANSIT` 的行重规划。COW 保住旧 tenant 时，该 hit 也不得继续用旧 future。 +- 禁止：只因 `_hit_prefetch` 已有 future 就跳过 re-plan；把提前读到的 pool 行当成 B 的 merged prefix。 +- 验收：覆盖 fresh/reused block 与 immediate/deferred mm；断言 B 物化的 prefix 等于本步 A 刚写入的 hidden/mm，而不是 prefetch 当时的旧/空行。^[PR #7877] diff --git a/knowledge/repos/vllm-omni/components/serving/rules-speech-output.md b/knowledge/repos/vllm-omni/components/serving/rules-speech-output.md index 082b4301..1536b634 100644 --- a/knowledge/repos/vllm-omni/components/serving/rules-speech-output.md +++ b/knowledge/repos/vllm-omni/components/serving/rules-speech-output.md @@ -1,7 +1,7 @@ --- title: "Speech 输出采样率规则" created: 2026-09-05 -updated: 2026-09-22 +updated: 2026-09-26 type: rule tags: [vllm-omni, components, serving] sources: ["PR #6553", docs/serving/speech_api.md, vllm_omni/entrypoints/openai/audio_utils_mixin.py, vllm_omni/entrypoints/openai/protocol/audio.py, vllm_omni/entrypoints/openai/serving_speech.py, vllm_omni/entrypoints/openai/tts_adapters/base.py, vllm_omni/entrypoints/openai/tts_adapters/qwen3_tts.py, tests/entrypoints/openai_api/test_audio_format.py, tests/entrypoints/openai_api/test_serving_speech.py, tests/e2e/online_serving/test_qwen3_tts_customvoice_expansion.py, "PR #7499"] @@ -27,3 +27,10 @@ confidence: high - 强制：MOSS-TTS-Nano、Gepard 等 `async_chunk=false` 的稀疏音频模型由 adapter 在 `PreparedRequest.output_policy` 声明 `accumulate_nonstreaming`。serving 用 `request_id` 键存 policy，仅非流式路径在准备成功后写入，并在累积器 `pop` 消费；优先使用 FINAL_ONLY 已拼接波形，否则 `torch.cat` delta。streaming 永不读写该表。 - 禁止：在 `serving_speech.py` 增加 Gepard/MOSS 等模型名分支；在可失败步骤前写入 policy 导致泄漏；让并发请求共享可变 policy 槽位。 - 验收：adapter 设/不设 flag 的非流式拼接与 sentinel-only final；并发非流式隔离；streaming 路径无 policy 条目。^[PR #7499] + +## SERV-9d — speech 非合同异常必须映射为 HTTP 500 InternalServerError + +- 触发：修改 `OmniOpenAIServingSpeech.create_speech` 的通用异常处理、`create_error_response` 的默认 type/status，或 API server 把 `ErrorResponse` 转成 HTTP。 +- 强制：CUDA OOM、codec/`RuntimeError` 等非请求合同失败必须显式 `err_type="InternalServerError"` 与 `HTTPStatus.INTERNAL_SERVER_ERROR`。`ValueError` 及既有引擎合同错误保持 HTTP 400 / `BadRequestError`。API 包装层按 `ErrorInfo.code` / `type` 透传，不得把所有 `ErrorResponse` 压成 400。 +- 禁止：通用 `except` 省略 type/status，从而默认成 400；把 OOM 或内部 codec 失败写成客户端坏请求。 +- 验收：CPU mock 分别断言 OOM 与 unexpected `RuntimeError` 得到 500/`InternalServerError`，并参数化覆盖 API 包装对 400 与 500 的透传。^[PR #6487] diff --git a/knowledge/repos/vllm-omni/models/cosmos3/rules.md b/knowledge/repos/vllm-omni/models/cosmos3/rules.md index 821c9aeb..64ea99d2 100644 --- a/knowledge/repos/vllm-omni/models/cosmos3/rules.md +++ b/knowledge/repos/vllm-omni/models/cosmos3/rules.md @@ -1,7 +1,7 @@ --- title: "Cosmos3 规则" created: 2026-07-20 -updated: 2026-09-22 +updated: 2026-09-26 type: rule tags: [vllm-omni, models, diffusion] sources: ["PR #4657", "PR #5001", "PR #5634", "PR #6049", docs/features/session_state_manager.md, recipes/cosmos3/Cosmos3-Nano.md, vllm_omni/diffusion/models/cosmos3/, vllm_omni/model_extras/cosmos3.py, vllm_omni/model_extras/registry.py, vllm_omni/experimental/world_models/adapters/state_cosmos3_adapter.py, vllm_omni/platforms/rocm/platform.py, tests/diffusion/models/cosmos3/test_session_memory_equivalence.py, tests/diffusion/models/cosmos3/test_cosmos3_pipeline.py, "PR #6107", "PR #5614", "PR #6325", "PR #6913", "PR #6920", "PR #7427"] @@ -166,3 +166,10 @@ confidence: high - 强制：在加载 tokenizer/组件之前，若 distilled 且 `cfg_parallel_size > 1`，立即 `ValueError`，并指引 `--cfg-parallel-size 1` 与 `--ulysses-degree` 做多卡。distilled 路径强制 `guidance_scale=1.0`；调用方显式请求其他值时 `warning_once`，并说明 `negative_prompt` 不影响生成。 - 禁止：静默忽略 CFGP 导致无收益的重复实例；先加载权重再报并行非法；把 distilled 的 guidance override 写成仍消费 negative branch。 - 验收：参数化覆盖 distilled/non-distilled × `cfg_parallel_size`/`ulysses_degree`，断言 distilled+CFGP>1 在组件加载前失败；guidance resolver 覆盖显式非 1.0 的 warning 与强制 1.0。^[PR #7427] + +## COSMOS-7a — 共享 HF metadata 的 Cosmos3 拓扑只能经 deploy `pipeline:` 选择 + +- 触发：注册 Cosmos3 pipeline、写 `cosmos3_*.yaml`，或让 `--deploy-config` 的 stage/`model_config` 进入 Cosmos3 T2I/video/policy 启动。 +- 强制：T2I、video 与 policy checkpoint 共享 `model_type=cosmos3_omni` 和同一 `model_index.json` `_class_name`。`OMNI_PIPELINES` 不得注册可被该 metadata 自动命中的 `cosmos3_omni`。policy 用 `cosmos3_policy`，omni overlay 用 `cosmos3_omni_deploy`，两者都不声明 `hf_architectures` / `diffusers_class_name`，只经 YAML `pipeline:` 选中。单 stage overlay 必须 `async_chunk: false`；`guardrails: false` 等 stage extras 只有选中该 key 后才会合并。无 `--deploy-config` 时 T2I/video 仍走 CLI 单 stage fallback；`--no-guardrails` 仍是 CLI-only 路径。 +- 禁止：为了让 `--deploy-config` 生效而把 `cosmos3_omni` 做成 auto-detect（会吞掉 policy/video）；静默丢弃 deploy YAML 却宣称 guardrails/stage extras 已生效。 +- 验收：断言 `cosmos3_omni` 不在 registry、opt-in key 无 auto-capture 字段；`cosmos3_omni.yaml` 经 `merge_pipeline_deploy` 得到 `guardrails is False` 与 `final_output_type="video"`。^[PR #7971] diff --git a/knowledge/repos/vllm-omni/models/qwen-omni/rules.md b/knowledge/repos/vllm-omni/models/qwen-omni/rules.md index 6ab8d5c1..d0196552 100644 --- a/knowledge/repos/vllm-omni/models/qwen-omni/rules.md +++ b/knowledge/repos/vllm-omni/models/qwen-omni/rules.md @@ -1,7 +1,7 @@ --- title: "Qwen-Omni 规则" created: 2026-09-04 -updated: 2026-09-22 +updated: 2026-09-26 type: rule tags: [vllm-omni, models, qwen-omni] sources: ["PR #5687", "PR #6284", "PR #6449", "PR #4322", "PR #6748", "PR #6886", "PR #7019", vllm_omni/config/pipeline_registry.py, vllm_omni/deploy/qwen3_omni_moe.yaml, vllm_omni/deploy/qwen3_omni_moe_thinking.yaml, vllm_omni/engine/stage_init_utils.py, vllm_omni/model_executor/models/qwen2_5_omni/qwen2_5_omni.py, vllm_omni/model_executor/models/qwen3_omni/quantization.py, vllm_omni/model_executor/models/qwen3_omni/qwen3_omni.py, vllm_omni/model_executor/models/qwen3_omni/qwen3_omni_moe_thinker.py, vllm_omni/quantization/component_config.py, tests/config/test_config_factory.py, tests/diffusion/quantization/test_component_routing.py, tests/engine/test_stage_engine_args.py, tests/model_executor/models/qwen3_omni/test_qwen3_omni_quantization.py, "PR #7228", "PR #7200", "PR #7345", "PR #7340", "PR #7304"] @@ -115,3 +115,10 @@ confidence: high - 强制：vLLM 把 block 更新和 residual 分开，直到下一次 RMSNorm 才相加。有 residual 时捕获 `hidden_states + residual`；residual 为 `None` 时只 clone `hidden_states`。写入前 view 成 `(-1, hidden)`。捕获不得改变最终 Thinker 输出。 - 禁止：只复制 update 张量；把无 residual 的 embedding clone 和有 residual 的层当成同一公式；用捕获开关改变主 forward 数值。 - 验收：单 PP rank 上用会拆开 hidden/residual 的层断言捕获值等于逻辑状态（无 residual 为输入，下一层为 hidden+residual），且开启捕获后的最终输出与不捕获一致。^[PR #7304] + +## QOMNI-1i — thinker→talker 的 accept-hidden 层必须读 config，禁止写死 24 + +- 触发：修改 `thinker2talker_accync_chunk` / `thinker2talker_full_payload`、pooling layer lookup，或 Qwen3-Omni 变体的 `talker_config.accept_hidden_layer`。 +- 强制：embedding 固定层 0；accept-hidden 从 transfer manager 的 `_get_model_config()` 或 `config.hf_config.talker_config.accept_hidden_layer` 读取。layer dict 同时接受 int/str key；取 tensor 必须用 `is None`，不能对多元素 tensor 做布尔 `or`。full-payload 用 `hidden_states.layer_{idx}` 拼语义键，而不是写死 `layer_24`。 +- 禁止：把 30B 的 `24` 硬编码进共享 processor；用 truthiness 取 tensor；只改 30B 路径却声称任意 `accept_hidden_layer` 已兼容。 +- 验收：mock transfer_manager 分别覆盖 config mixin 与 adapter `config` 字段；断言非 24 的 accept-hidden 能取到对应层，缺失层返回 `None` 且不抛 `RuntimeError`。^[PR #5825] From 903c75e37027199f00a207d7df8519aa656a10bb Mon Sep 17 00:00:00 2001 From: tzhouam Date: Sun, 27 Sep 2026 06:27:58 +0800 Subject: [PATCH 2/2] Add page-level sources for proposed knowledge rules --- knowledge/repos/vllm-omni/benchmark/rules.md | 2 +- knowledge/repos/vllm-omni/components/diffusion/rules-lora.md | 2 +- .../vllm-omni/components/diffusion/rules-output-lifecycle.md | 2 +- knowledge/repos/vllm-omni/components/distributed/rules.md | 2 +- .../vllm-omni/components/scheduler/rules-shared-lifecycle.md | 2 +- .../repos/vllm-omni/components/serving/rules-speech-output.md | 2 +- knowledge/repos/vllm-omni/models/cosmos3/rules.md | 2 +- knowledge/repos/vllm-omni/models/qwen-omni/rules.md | 2 +- 8 files changed, 8 insertions(+), 8 deletions(-) diff --git a/knowledge/repos/vllm-omni/benchmark/rules.md b/knowledge/repos/vllm-omni/benchmark/rules.md index 474f053c..6a7a7fe8 100644 --- a/knowledge/repos/vllm-omni/benchmark/rules.md +++ b/knowledge/repos/vllm-omni/benchmark/rules.md @@ -4,7 +4,7 @@ created: 2026-09-05 updated: 2026-09-26 type: rule tags: [vllm-omni, benchmark] -sources: ["PR #7648", "PR #6817", tests/dfx/perf/scripts/run_benchmark.py, tests/benchmarks/test_omniinteract.py, "PR #7130", "PR #7259", "PR #7624", "PR #7504"] +sources: ["PR #7648", "PR #6817", tests/dfx/perf/scripts/run_benchmark.py, tests/benchmarks/test_omniinteract.py, "PR #7130", "PR #7259", "PR #7624", "PR #7504", "PR #8107"] confidence: high --- diff --git a/knowledge/repos/vllm-omni/components/diffusion/rules-lora.md b/knowledge/repos/vllm-omni/components/diffusion/rules-lora.md index 32ff7c92..f7b68130 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/rules-lora.md +++ b/knowledge/repos/vllm-omni/components/diffusion/rules-lora.md @@ -4,7 +4,7 @@ created: 2026-09-02 updated: 2026-09-26 type: rule tags: [vllm-omni, components, diffusion] -sources: ["PR #2783", docs/user_guide/diffusion/lora.md, vllm_omni/config/omni_config.py, vllm_omni/config/stage_config.py, vllm_omni/diffusion/data.py, vllm_omni/diffusion/lora/loader.py, vllm_omni/diffusion/lora/manager.py, vllm_omni/diffusion/lora/layers/base_linear.py, vllm_omni/diffusion/models/qwen_image/pipeline_qwen_image.py, vllm_omni/diffusion/models/wan2_2/pipeline_wan2_2.py, vllm_omni/diffusion/models/wan2_2/pipeline_wan2_2_i2v.py, vllm_omni/diffusion/utils/tf_utils.py, vllm_omni/diffusion/worker/diffusion_worker.py, vllm_omni/engine/async_omni_engine.py, vllm_omni/entrypoints/cli/serve.py, tests/diffusion/lora/test_loader.py, tests/diffusion/lora/test_lora_manager.py, tests/entrypoints/test_async_omni_diffusion_config.py, "PR #5500", "vllm_omni/diffusion/models/ltx2/ltx2_adapter_parser.py", "vllm_omni/diffusion/models/ltx2/ltx2_phase_adapter.py", "PR #6070", "PR #6476", "PR #6550", vllm_omni/diffusion/models/minimax_h3/lora.py, "PR #6268", benchmarks/kernels/benchmark_diffusion_lora_expand.py, tests/diffusion/lora/test_base_linear.py, "PR #7195", "PR #7349", "PR #5907"] +sources: ["PR #2783", docs/user_guide/diffusion/lora.md, vllm_omni/config/omni_config.py, vllm_omni/config/stage_config.py, vllm_omni/diffusion/data.py, vllm_omni/diffusion/lora/loader.py, vllm_omni/diffusion/lora/manager.py, vllm_omni/diffusion/lora/layers/base_linear.py, vllm_omni/diffusion/models/qwen_image/pipeline_qwen_image.py, vllm_omni/diffusion/models/wan2_2/pipeline_wan2_2.py, vllm_omni/diffusion/models/wan2_2/pipeline_wan2_2_i2v.py, vllm_omni/diffusion/utils/tf_utils.py, vllm_omni/diffusion/worker/diffusion_worker.py, vllm_omni/engine/async_omni_engine.py, vllm_omni/entrypoints/cli/serve.py, tests/diffusion/lora/test_loader.py, tests/diffusion/lora/test_lora_manager.py, tests/entrypoints/test_async_omni_diffusion_config.py, "PR #5500", "vllm_omni/diffusion/models/ltx2/ltx2_adapter_parser.py", "vllm_omni/diffusion/models/ltx2/ltx2_phase_adapter.py", "PR #6070", "PR #6476", "PR #6550", vllm_omni/diffusion/models/minimax_h3/lora.py, "PR #6268", benchmarks/kernels/benchmark_diffusion_lora_expand.py, tests/diffusion/lora/test_base_linear.py, "PR #7195", "PR #7349", "PR #5907", "PR #8008"] confidence: high --- diff --git a/knowledge/repos/vllm-omni/components/diffusion/rules-output-lifecycle.md b/knowledge/repos/vllm-omni/components/diffusion/rules-output-lifecycle.md index 33560b71..54aa85db 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/rules-output-lifecycle.md +++ b/knowledge/repos/vllm-omni/components/diffusion/rules-output-lifecycle.md @@ -4,7 +4,7 @@ created: 2026-09-02 updated: 2026-09-26 type: rule tags: [vllm-omni, components, diffusion] -sources: ["PR #5550", "PR #5864", "PR #5885", "PR #5978", "PR #6750", vllm_omni/diffusion/diffusion_engine.py, vllm_omni/diffusion/executor/multiproc_executor.py, vllm_omni/diffusion/inline_stage_diffusion_client.py, vllm_omni/diffusion/io_support.py, vllm_omni/diffusion/model_metadata.py, vllm_omni/diffusion/output_formatter.py, vllm_omni/diffusion/ipc.py, vllm_omni/diffusion/sched/request_scheduler.py, vllm_omni/diffusion/stage_diffusion_proc.py, vllm_omni/diffusion/utils/media_utils.py, vllm_omni/diffusion/worker/diffusion_worker.py, tests/diffusion/test_async_output_timeout.py, tests/diffusion/test_async_output_worker.py, tests/diffusion/test_diffusion_engine.py, tests/diffusion/test_diffusion_engine_cleanup.py, tests/diffusion/test_diffusion_ipc.py, tests/diffusion/test_inline_stage_diffusion_client.py, tests/diffusion/test_ipc_async.py, tests/diffusion/test_multiproc_engine_concurrency.py, tests/diffusion/test_result_pump.py, tests/diffusion/test_stage_diffusion_proc.py, tests/entrypoints/openai_api/test_video_server.py, "PR #6023", "PR #5983", "PR #4222", "PR #6094", "PR #6255", "PR #6288", "PR #6308", "PR #6499", "PR #6749", "PR #6847", "PR #6953", vllm_omni/diffusion/data.py, vllm_omni/diffusion/registry.py, vllm_omni/diffusion/models/diffusers_adapter/pipeline_utils.py, tests/diffusion/test_diffusion_output_formatter.py, tests/diffusion/test_diffusion_plugin_hooks.py, tests/entrypoints/openai_api/test_video_pipeline_capability.py, "PR #6294", "PR #6615", "PR #7126", "PR #7198"] +sources: ["PR #5550", "PR #5864", "PR #5885", "PR #5978", "PR #6750", vllm_omni/diffusion/diffusion_engine.py, vllm_omni/diffusion/executor/multiproc_executor.py, vllm_omni/diffusion/inline_stage_diffusion_client.py, vllm_omni/diffusion/io_support.py, vllm_omni/diffusion/model_metadata.py, vllm_omni/diffusion/output_formatter.py, vllm_omni/diffusion/ipc.py, vllm_omni/diffusion/sched/request_scheduler.py, vllm_omni/diffusion/stage_diffusion_proc.py, vllm_omni/diffusion/utils/media_utils.py, vllm_omni/diffusion/worker/diffusion_worker.py, tests/diffusion/test_async_output_timeout.py, tests/diffusion/test_async_output_worker.py, tests/diffusion/test_diffusion_engine.py, tests/diffusion/test_diffusion_engine_cleanup.py, tests/diffusion/test_diffusion_ipc.py, tests/diffusion/test_inline_stage_diffusion_client.py, tests/diffusion/test_ipc_async.py, tests/diffusion/test_multiproc_engine_concurrency.py, tests/diffusion/test_result_pump.py, tests/diffusion/test_stage_diffusion_proc.py, tests/entrypoints/openai_api/test_video_server.py, "PR #6023", "PR #5983", "PR #4222", "PR #6094", "PR #6255", "PR #6288", "PR #6308", "PR #6499", "PR #6749", "PR #6847", "PR #6953", vllm_omni/diffusion/data.py, vllm_omni/diffusion/registry.py, vllm_omni/diffusion/models/diffusers_adapter/pipeline_utils.py, tests/diffusion/test_diffusion_output_formatter.py, tests/diffusion/test_diffusion_plugin_hooks.py, tests/entrypoints/openai_api/test_video_pipeline_capability.py, "PR #6294", "PR #6615", "PR #7126", "PR #7198", "PR #7986"] confidence: high --- diff --git a/knowledge/repos/vllm-omni/components/distributed/rules.md b/knowledge/repos/vllm-omni/components/distributed/rules.md index 7b00d535..e5f33fe6 100644 --- a/knowledge/repos/vllm-omni/components/distributed/rules.md +++ b/knowledge/repos/vllm-omni/components/distributed/rules.md @@ -4,7 +4,7 @@ created: 2026-08-05 updated: 2026-09-26 type: rule tags: [vllm-omni, components, distributed] -sources: ["PR #5744", "PR #5976", "PR #6001", "PR #6089", "PR #6834", vllm_omni/diffusion/distributed/parallel_state.py, tests/diffusion/distributed/test_expert_parallel_layout.py, vllm_omni/distributed/omni_connectors/adapter.py, vllm_omni/distributed/omni_connectors/kv_transfer_manager.py, vllm_omni/distributed/omni_connectors/transfer_adapter/chunk_transfer_adapter.py, vllm_omni/distributed/omni_connectors/transfer_adapter/base.py, vllm_omni/worker/omni_connector_model_runner_mixin.py, tests/distributed/omni_connectors/test_kv_recv_tp_consensus.py, tests/distributed/omni_connectors/test_chunk_transfer_adapter.py, tests/worker/test_omni_connector_mixin.py, "PR #5146", "PR #6021", "PR #6033", "PR #6360", "PR #6406", "PR #6626", "PR #6529", "PR #7136", "PR #6889", "PR #6093", "PR #7870", "PR #7166"] +sources: ["PR #5744", "PR #5976", "PR #6001", "PR #6089", "PR #6834", vllm_omni/diffusion/distributed/parallel_state.py, tests/diffusion/distributed/test_expert_parallel_layout.py, vllm_omni/distributed/omni_connectors/adapter.py, vllm_omni/distributed/omni_connectors/kv_transfer_manager.py, vllm_omni/distributed/omni_connectors/transfer_adapter/chunk_transfer_adapter.py, vllm_omni/distributed/omni_connectors/transfer_adapter/base.py, vllm_omni/worker/omni_connector_model_runner_mixin.py, tests/distributed/omni_connectors/test_kv_recv_tp_consensus.py, tests/distributed/omni_connectors/test_chunk_transfer_adapter.py, tests/worker/test_omni_connector_mixin.py, "PR #5146", "PR #6021", "PR #6033", "PR #6360", "PR #6406", "PR #6626", "PR #6529", "PR #7136", "PR #6889", "PR #6093", "PR #7870", "PR #7166", "PR #8082"] confidence: high --- diff --git a/knowledge/repos/vllm-omni/components/scheduler/rules-shared-lifecycle.md b/knowledge/repos/vllm-omni/components/scheduler/rules-shared-lifecycle.md index f01b0c1a..55446591 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/rules-shared-lifecycle.md +++ b/knowledge/repos/vllm-omni/components/scheduler/rules-shared-lifecycle.md @@ -4,7 +4,7 @@ created: 2026-09-22 updated: 2026-09-26 type: rule tags: [vllm-omni, components, scheduler] -sources: ["PR #5461"] +sources: ["PR #5461", "PR #7877"] --- # Scheduler shared lifecycle 规则 diff --git a/knowledge/repos/vllm-omni/components/serving/rules-speech-output.md b/knowledge/repos/vllm-omni/components/serving/rules-speech-output.md index 1536b634..1458d0a7 100644 --- a/knowledge/repos/vllm-omni/components/serving/rules-speech-output.md +++ b/knowledge/repos/vllm-omni/components/serving/rules-speech-output.md @@ -4,7 +4,7 @@ created: 2026-09-05 updated: 2026-09-26 type: rule tags: [vllm-omni, components, serving] -sources: ["PR #6553", docs/serving/speech_api.md, vllm_omni/entrypoints/openai/audio_utils_mixin.py, vllm_omni/entrypoints/openai/protocol/audio.py, vllm_omni/entrypoints/openai/serving_speech.py, vllm_omni/entrypoints/openai/tts_adapters/base.py, vllm_omni/entrypoints/openai/tts_adapters/qwen3_tts.py, tests/entrypoints/openai_api/test_audio_format.py, tests/entrypoints/openai_api/test_serving_speech.py, tests/e2e/online_serving/test_qwen3_tts_customvoice_expansion.py, "PR #7499"] +sources: ["PR #6553", docs/serving/speech_api.md, vllm_omni/entrypoints/openai/audio_utils_mixin.py, vllm_omni/entrypoints/openai/protocol/audio.py, vllm_omni/entrypoints/openai/serving_speech.py, vllm_omni/entrypoints/openai/tts_adapters/base.py, vllm_omni/entrypoints/openai/tts_adapters/qwen3_tts.py, tests/entrypoints/openai_api/test_audio_format.py, tests/entrypoints/openai_api/test_serving_speech.py, tests/e2e/online_serving/test_qwen3_tts_customvoice_expansion.py, "PR #7499", "PR #6487"] confidence: high --- diff --git a/knowledge/repos/vllm-omni/models/cosmos3/rules.md b/knowledge/repos/vllm-omni/models/cosmos3/rules.md index 64ea99d2..ba36581b 100644 --- a/knowledge/repos/vllm-omni/models/cosmos3/rules.md +++ b/knowledge/repos/vllm-omni/models/cosmos3/rules.md @@ -4,7 +4,7 @@ created: 2026-07-20 updated: 2026-09-26 type: rule tags: [vllm-omni, models, diffusion] -sources: ["PR #4657", "PR #5001", "PR #5634", "PR #6049", docs/features/session_state_manager.md, recipes/cosmos3/Cosmos3-Nano.md, vllm_omni/diffusion/models/cosmos3/, vllm_omni/model_extras/cosmos3.py, vllm_omni/model_extras/registry.py, vllm_omni/experimental/world_models/adapters/state_cosmos3_adapter.py, vllm_omni/platforms/rocm/platform.py, tests/diffusion/models/cosmos3/test_session_memory_equivalence.py, tests/diffusion/models/cosmos3/test_cosmos3_pipeline.py, "PR #6107", "PR #5614", "PR #6325", "PR #6913", "PR #6920", "PR #7427"] +sources: ["PR #4657", "PR #5001", "PR #5634", "PR #6049", docs/features/session_state_manager.md, recipes/cosmos3/Cosmos3-Nano.md, vllm_omni/diffusion/models/cosmos3/, vllm_omni/model_extras/cosmos3.py, vllm_omni/model_extras/registry.py, vllm_omni/experimental/world_models/adapters/state_cosmos3_adapter.py, vllm_omni/platforms/rocm/platform.py, tests/diffusion/models/cosmos3/test_session_memory_equivalence.py, tests/diffusion/models/cosmos3/test_cosmos3_pipeline.py, "PR #6107", "PR #5614", "PR #6325", "PR #6913", "PR #6920", "PR #7427", "PR #7971"] confidence: high --- diff --git a/knowledge/repos/vllm-omni/models/qwen-omni/rules.md b/knowledge/repos/vllm-omni/models/qwen-omni/rules.md index d0196552..f42ecf77 100644 --- a/knowledge/repos/vllm-omni/models/qwen-omni/rules.md +++ b/knowledge/repos/vllm-omni/models/qwen-omni/rules.md @@ -4,7 +4,7 @@ created: 2026-09-04 updated: 2026-09-26 type: rule tags: [vllm-omni, models, qwen-omni] -sources: ["PR #5687", "PR #6284", "PR #6449", "PR #4322", "PR #6748", "PR #6886", "PR #7019", vllm_omni/config/pipeline_registry.py, vllm_omni/deploy/qwen3_omni_moe.yaml, vllm_omni/deploy/qwen3_omni_moe_thinking.yaml, vllm_omni/engine/stage_init_utils.py, vllm_omni/model_executor/models/qwen2_5_omni/qwen2_5_omni.py, vllm_omni/model_executor/models/qwen3_omni/quantization.py, vllm_omni/model_executor/models/qwen3_omni/qwen3_omni.py, vllm_omni/model_executor/models/qwen3_omni/qwen3_omni_moe_thinker.py, vllm_omni/quantization/component_config.py, tests/config/test_config_factory.py, tests/diffusion/quantization/test_component_routing.py, tests/engine/test_stage_engine_args.py, tests/model_executor/models/qwen3_omni/test_qwen3_omni_quantization.py, "PR #7228", "PR #7200", "PR #7345", "PR #7340", "PR #7304"] +sources: ["PR #5687", "PR #6284", "PR #6449", "PR #4322", "PR #6748", "PR #6886", "PR #7019", vllm_omni/config/pipeline_registry.py, vllm_omni/deploy/qwen3_omni_moe.yaml, vllm_omni/deploy/qwen3_omni_moe_thinking.yaml, vllm_omni/engine/stage_init_utils.py, vllm_omni/model_executor/models/qwen2_5_omni/qwen2_5_omni.py, vllm_omni/model_executor/models/qwen3_omni/quantization.py, vllm_omni/model_executor/models/qwen3_omni/qwen3_omni.py, vllm_omni/model_executor/models/qwen3_omni/qwen3_omni_moe_thinker.py, vllm_omni/quantization/component_config.py, tests/config/test_config_factory.py, tests/diffusion/quantization/test_component_routing.py, tests/engine/test_stage_engine_args.py, tests/model_executor/models/qwen3_omni/test_qwen3_omni_quantization.py, "PR #7228", "PR #7200", "PR #7345", "PR #7340", "PR #7304", "PR #5825"] confidence: high ---